
/*
 *         PVM version 3.3:  Parallel Virtual Machine System
 *               University of Tennessee, Knoxville TN.
 *           Oak Ridge National Laboratory, Oak Ridge TN.
 *                   Emory University, Atlanta GA.
 *      Authors:  A. L. Beguelin, J. J. Dongarra, G. A. Geist,
 *    W. C. Jiang, R. J. Manchek, B. K. Moore, and V. S. Sunderam
 *                   (C) 1992 All Rights Reserved
 *
 *                              NOTICE
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted
 * provided that the above copyright notice appear in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * Neither the Institutions (Emory University, Oak Ridge National
 * Laboratory, and University of Tennessee) nor the Authors make any
 * representations about the suitability of this software for any
 * purpose.  This software is provided ``as is'' without express or
 * implied warranty.
 *
 * PVM version 3 was funded in part by the U.S. Department of Energy,
 * the National Science Foundation and the State of Tennessee.
 */

/*
 *  pvmshmem.c
 *
 *  Shared-memory stuff.
 *
$Log: pvmshmem.c,v $
 * Revision 1.3  1994/11/08  15:36:49  manchek
 * shared memory damage control
 *
 * Revision 1.2  1994/06/04  21:44:25  manchek
 * updated header
 *
 * Revision 1.1  1994/06/03  20:38:26  manchek
 * Initial revision
 *
 */

#include <sys/param.h>
#include <fcntl.h>
#include <string.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/sem.h>
#ifdef IMA_SYMM
#include <parallel/parallel.h>
#endif
#include <errno.h>

#include "protoglarp.h"
#include "pvmalloc.h"
#include "listmac.h"
#include "pvmshmem.h"
#include "bfunc.h"

#ifndef min
#define min(a,b)	((a)<(b)?(a):(b))
#endif

extern int debugmask;				/* from pvmd.c or lpvmshmem.c */
extern int pvmpgsz;					/* from pvmdshmem.c or lpvmshmem.c */
extern int pgsz;					/* from pvmdshmem.c or lpvmshmem.c */
extern char *outmsgbuf;				/* from pvmdshmem.c or lpvmshmem.c */
extern int outbufsz;				/* from pvmdshmem.c or lpvmshmem.c */
extern char *infopage;				/* from pvmdshmem.c or lpvmshmem.c */
extern struct pidtid *pidtids;		/* from pvmdshmem.c or lpvmshmem.c */
extern int maxpidtid;				/* from pvmdshmem.c or lpvmshmem.c */
extern int shmbufsiz;				/* from pvmdshmem.c or lpvmshmem.c */
#ifdef IMA_POWER4
extern struct shmpghdr *globinfo;	/* from pvmdshmem.c or lpvmshmem.c */
#endif


/***************
 **  Globals  **
 **           **
 ***************/

struct peer *peers = 0;				/* tasks we're connected to */
int bufpageused = 0;				/* number of dirty pages in msg buf */


/***************
 **  Private  **
 **           **
 ***************/

static char rcsid[] = "$Id: pvmshmem.c,v 1.3 1994/11/08 15:36:49 manchek Exp $";
static char pvmtxt[512];			/* scratch for error log */
static int nxtpage = 0;				/* next free page in outgoing msg buf */


/*******************************************************
 **  Shared-memory data buffer manipulation routines  **
 **                                                   **
 *******************************************************/

/* initialize shared-memory msg buffer */
int
msgbufinit(buf)
	char *buf;
{
	char *p, *end;

	end = buf + shmbufsiz;
#ifndef IMA_KSR1
#ifdef IMA_SUNMP
	cond_init(&((struct msgboxhdr *)buf)->mb_cond, USYNC_PROCESS, 0);
#endif
	PAGEINITLOCK(&((struct msgboxhdr *)buf)->mb_lock);
#endif
	((struct msgboxhdr *)buf)->mb_read = -1;
	((struct msgboxhdr *)buf)->mb_last = -1;

	for (p = outmsgbuf; p < end; p += pvmpgsz) {
#if !defined(TEST_ADD) && !defined(IMA_KSR1)
		PAGEINITLOCK(&((struct shmpghdr *)p)->pg_lock);
#endif
		((struct shmpghdr *)p)->pg_priv = 0;
		((struct shmpghdr *)p)->pg_ref = 0;
	}

	if (debugmask & PDMNODE) {
		sprintf(pvmtxt, "msgbufinit() outmsgbuf=0x%x\n", outmsgbuf);
		pvmlogerror(pvmtxt);
		sprintf(pvmtxt, "msgbufinit() last buffer=0x%x\n", p - pvmpgsz);
		pvmlogerror(pvmtxt);
		sprintf(pvmtxt, "msgbufinit() pvmpgsz=%d\n", pvmpgsz);
		pvmlogerror(pvmtxt);
	}
	return 0;
}

/* 
 * Try to find a free page. Check page 0 first, and then the current one.
 * If neither is free, then move beyond the current page towards the end
 * of the buffer.
 */
char *
da_new(len)
	int len;
{
	char *p;
	int start;

	if (len > pvmpgsz - PVMPAGEHDR) {
		sprintf(pvmtxt, "da_new() len = %d: frag must fit in a page\n", len);
		pvmlogerror(pvmtxt);
		return 0;
	}

	p = outmsgbuf;
	if (nxtpage && !((struct shmpghdr *)p)->pg_ref)
		nxtpage = 0;		/* recycle to avoid page fault */
	else
		p += nxtpage*pvmpgsz;

	start = nxtpage;
	while (((struct shmpghdr *)p)->pg_ref) {
		if (++nxtpage == outbufsz) {
			nxtpage = 0;
			p = outmsgbuf;

		} else
			p += pvmpgsz;

		if (nxtpage == start) {		/* buffer full */
			p = TALLOC(len + PVMPAGEHDR, char, "data");
			if (p) {
#if !defined(IMA_KSR1) && !defined(TEST_ADD)
				PAGEINITLOCK(&((struct shmpghdr *)p)->pg_lock);
#endif
				((struct shmpghdr *)p)->pg_priv = 1;
			}
			break;
		}
	}

	if (nxtpage >= bufpageused)
		bufpageused = nxtpage + 1;
	if (p)
		((struct shmpghdr *)p)->pg_ref = 1;

/* XXX this returns nonzero if it fails, yuck */
	return (p + PVMPAGEHDR);
}

void
da_ref(p)
	char *p;
{
	p -= PVMPAGEHDR;
#ifdef TEST_ADD
	TEST_ADD(&((struct shmpghdr *)p)->pg_ref, 1);
#else
	PAGELOCK(&((struct shmpghdr *)p)->pg_lock);
	++((struct shmpghdr *)p)->pg_ref;
	PAGEUNLOCK(&((struct shmpghdr *)p)->pg_lock);
#endif
}


void
da_unref(p)
	char *p;
{
	p -= PVMPAGEHDR;
#ifdef TEST_ADD
	if (TEST_ADD(&((struct shmpghdr *)p)->pg_ref, -1L) < 0)
		pvmlogerror("ref count is negative!\n");
#else
	PAGELOCK(&((struct shmpghdr *)p)->pg_lock);
	if (--((struct shmpghdr *)p)->pg_ref < 0) {
		sprintf(pvmtxt, "ref = %d on page %d\n", 
			((struct shmpghdr *)p)->pg_ref, (p - outmsgbuf)/pgsz + 1);
		pvmlogerror(pvmtxt);
	}
	PAGEUNLOCK(&((struct shmpghdr *)p)->pg_lock);
#endif
	if (((struct shmpghdr *)p)->pg_priv && ((struct shmpghdr *)p)->pg_ref < 1)
		PVM_FREE(p);
}


void
peer_init()
{
	peers = TALLOC(1, struct peer, "peer");
	BZERO((char*)peers, sizeof(struct peer));
	peers->p_link = peers->p_rlink = peers;
}

/* attach peer's msg buffer; disconnect from inactive peers */
/* XXX this is a vicious hack - lazy cleanup -b */

struct peer *
peer_conn(tid)
	int tid;
{
	struct peer *pp, *pp2;
	char *p;

	for (pp = peers->p_link; pp != peers; ) {
		if (pp->p_tid == tid)		
			return pp;			/* already connected */
		pp2 = pp;
		pp = pp->p_link;
		if (pp2->p_tid == -1) {		/* used by pvmd */
			/* peer_detach(pp2); */
			shmdt(pp2->p_buf);
			LISTDELETE(pp2, p_link, p_rlink);
			PVM_FREE(pp2);
		}
	}

	if (pp == peers && tid) {
		int bufid, ntids, i;
#ifdef IMA_POWER4
		struct gtask *globtasks = (struct gtask *)(globinfo + 1);

		for (i = 0; globtasks[i].gt_tid; i++)
			if (globtasks[i].gt_tid == tid) {
				if (globtasks[i].gt_stat == ST_NOTREADY)
					return (struct peer *)0;
				if (globtasks[i].gt_stat == ST_SOCKET)
					return (struct peer *)-1L;
				break;
			}
		if (!globtasks[i].gt_tid)
			return (struct peer *)-1L;

		if ((bufid = shmget((key_t)tid, shmbufsiz, IPC_POWER4)) == -1) {
#else
		/* PAGELOCK(pvminfo); */
		ntids = min(maxpidtid, ((struct shmpghdr *)infopage)->pg_ref);
		for (i = 0; i < ntids; i++)
			if (pidtids[i].pt_tid == tid) {
				if (pidtids[i].pt_stat == ST_NOTREADY)
					return (struct peer *)0;
				if (pidtids[i].pt_stat == ST_SOCKET)
					return (struct peer *)-1L;
				break;
			}
		if (i == ntids)
			return (struct peer *)-1L;
		/* PAGEUNLOCK(pvminfo); */

		if ((bufid = shmget((key_t)tid, shmbufsiz, 0)) == -1) {
#endif
			/* if (errno != ENOENT) { */
				sprintf(pvmtxt, "peer_conn() shmget t%x", tid);
				pvmlogperror(pvmtxt);
			/* } */
			return (struct peer *)0;
		}
		while ((p = (char *)shmat(bufid, 0, 0)) == (char *)-1L) {
			sprintf(pvmtxt, "peer_conn() shmat to t%x", tid);
			pvmlogperror(pvmtxt);
			if (errno == EMFILE && (pp = peers->p_rlink) != peers) {
				shmdt(pp->p_buf);
				LISTDELETE(pp, p_link, p_rlink);
				PVM_FREE(pp);
				continue;
			}
			return (struct peer *)0;
		}
		if (!(pp = TALLOC(1, struct peer, "peer"))) {
			pvmlogerror("peer_conn() can't get memory\n");
			return (struct peer *)0;
		}
    	BZERO((char*)pp, sizeof(struct peer));
		pp->p_tid = tid;
		pp->p_buf = p;
		pp->p_shmid = bufid;
		pp->p_semid = -1;
		LISTPUTAFTER(peers, pp, p_link, p_rlink);
	}

	return pp;
}


/* wake up task */
void
peer_wake(pp)
	struct peer *pp;
{
	struct sembuf	sops;

	if (pp->p_semid == -1 
	&& (pp->p_semid = semget((key_t)pp->p_tid, 1, PERMS)) == -1) {
		sprintf(pvmtxt, "peer_wake(): semget t%x\n", pp->p_tid);
		pvmlogperror(pvmtxt);
		return;
	}
	sops.sem_num = 0;
	sops.sem_op = 1;
	sops.sem_flg = 0;
	if (semop(pp->p_semid, &sops, 1) == -1) {
		sprintf(pvmtxt, "peer_wake(): semop t%x\n", pp->p_tid);
		pvmlogperror(pvmtxt);
	}
}


/* detach peer's msg buffer, destroy if no one else is attached it */
peer_detach(pp)
	struct peer *pp;
{
	struct shmid_ds shmds;

	if (shmdt(pp->p_buf) == -1)
		pvmlogerror("peer_detach() shmdt");
	if (!shmctl(pp->p_shmid, IPC_STAT, &shmds) && shmds.shm_nattch == 0)
		shmctl(pp->p_shmid, IPC_RMID, (struct shmid_ds *)0);
/*
	if (shmdt(pp->p_buf) == -1)
		pvmlogerror("peer_detach() shmdt");
	if (shmctl(pp->p_shmid, IPC_STAT, &shmds) == -1)
		pvmlogperror("peer_detach() shmctl STAT");
	if (shmds.shm_nattch == 0 &&
	shmctl(pp->p_shmid, IPC_RMID, &shmds) == -1)
		pvmlogperror("peer_detach() shmctl RMID");
*/
}


void
peer_cleanup()
{
	struct peer *pp;

	for (pp = peers->p_link; pp != peers; pp = pp->p_link)
		peer_detach(pp);
/*
		shmdt(pp->p_buf);
*/
}


int
peer_dump()
{
	struct peer *pp;
	struct shmid_ds shmds;

	pvmlogerror("peer_dump()\n");
	for (pp = peers->p_link; pp != peers; pp = pp->p_link) {
		if (shmctl(pp->p_shmid, IPC_STAT, &shmds) == -1) {
			sprintf(pvmtxt, "peer_dump() shmctl STAT %d", pp->p_shmid);
			pvmlogperror(pvmtxt);

		} else {
			sprintf(pvmtxt, " t%x shmid=%d semid=%d shmlen=%d shmna=%d\n",
					pp->p_tid, pp->p_shmid, pp->p_semid, pp->p_dlen,
					(int)shmds.shm_nattch);
			pvmlogerror(pvmtxt);
		}
	}
}


