
/*
 *         PVM version 3.3:  Parallel Virtual Machine System
 *               University of Tennessee, Knoxville TN.
 *           Oak Ridge National Laboratory, Oak Ridge TN.
 *                   Emory University, Atlanta GA.
 *      Authors:  A. L. Beguelin, J. J. Dongarra, G. A. Geist,
 *    W. C. Jiang, R. J. Manchek, B. K. Moore, and V. S. Sunderam
 *                   (C) 1992 All Rights Reserved
 *
 *                              NOTICE
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted
 * provided that the above copyright notice appear in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * Neither the Institutions (Emory University, Oak Ridge National
 * Laboratory, and University of Tennessee) nor the Authors make any
 * representations about the suitability of this software for any
 * purpose.  This software is provided ``as is'' without express or
 * implied warranty.
 *
 * PVM version 3 was funded in part by the U.S. Department of Energy,
 * the National Science Foundation and the State of Tennessee.
 */

/*
 *	lpvmshmem.c
 *
 *	Libpvm core for MPP environment.
 *
$Log: lpvmshmem.c,v $
 * Revision 1.4  1994/11/07  22:42:39  manchek
 * general damage control and cleanup:
 * initialize variables
 * send null packets to wake up pvmd instead of reconnecting
 * clean up on catching SIGTERM
 *
 * Revision 1.3  1994/06/30  21:35:40  manchek
 * typo in peer_recv()
 *
 * Revision 1.2  1994/06/04  21:44:31  manchek
 * updated header.
 * changed TM_SET to TM_SETOPT
 *
 * Revision 1.1  1994/06/03  20:38:18  manchek
 * Initial revision
 *
 */

#include <sys/param.h>
#include <stdio.h>
#include <rpc/types.h>
#include <rpc/xdr.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/stat.h>
#include <fcntl.h>
#ifdef	SYSVSTR
#include <string.h>
#else
#include <strings.h>
#endif
#include <signal.h>
#include <errno.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/sem.h>
#include <sys/mman.h>
#include <unistd.h>
#ifdef IMA_SYMM
#include <parallel/parallel.h>
#endif
#ifdef IMA_KSR1
#include <pthread.h>
#endif
/*
#include <signal.h>
*/

#include <pvm3.h>
#include "global.h"
#include "tdpro.h"
#include "ddpro.h"
#include "pvmalloc.h"
#include "pvmdabuf.h"
#include "pvmfrag.h"
#include "pvmumbuf.h"
#include "listmac.h"
#include "pvmshmem.h"
#include "bfunc.h"
#include <pvmtev.h>
#include "tevmac.h"

/* task debug mask */

#define	TDMPACKET	1		/* packet tracing */
#define	TDMMESSAGE	2		/* message tracing */

#ifndef max
#define max(a,b)	((a)>(b)?(a):(b))
#endif

#ifndef min
#define min(a,b)	((a)<(b)?(a):(b))
#endif

char *getenv();

extern struct encvec *enctovec();
char *pvmgetpvmd();
char *pvmgethome();
char *pvmdsockfile();


/***************
 **  Globals  **
 **           **
 ***************/

extern int errno;					/* from libc */
extern char *sys_errlist[];
extern int sys_nerr;

extern int pvmrbufmid;				/* from pack.c */
extern int pvmsbufmid;				/* from pack.c */
extern int bufpageused;				/* from pvmshmem.c */

char *pvm_errlist[] = {					/* error messages for -pvm_errno */
		"Error 0",
		"Error 1",
	"Bad parameter",
	"Count mismatch",
		"Error 4",				/* not used */
	"End of buffer",
	"No such host",
	"No such file",
		"Error 8",				/* not used */
		"Error 9",				/* not used */
	"Malloc failed",
		"Error 11",				/* not used */
	"Can't decode message",
		"Error 13",				/* not used */
	"Can't contact local daemon",
	"No current buffer",
	"No such buffer",
	"Null group name",
	"Already in group",
	"No such group",
	"Not in group",
	"No such instance",
	"Host failed",
	"No parent task",
	"Not implemented",
	"Pvmd system error",
	"Version mismatch",
	"Out of resources",
	"Duplicate host",
	"Can't start pvmd",
	"Already in progress",
	"No such task",
	"No such entry",
	"Duplicate entry",
};

int pvmautoerr = 1;						/* whether to auto print err msgs */
int pvmcouttid = 0;                     /* child stdout dst and code */
int pvmcoutcod = 0;
int pvmctrctid = 0;                     /* child trace dst and code */
int pvmctrccod = 0;
int pvmmyndf = 0;						/* host native data enc, init XDR */
int pvmmyptid = -1;						/* parent task id */
int pvmmytid = -1;						/* this task id */
int pvmmyupid = -1;						/* process unix pid */
int pvmudpmtu = MAXFRAGSIZE;			/* max fragment size */
int pvmfrgsiz = MAXFRAGSIZE;			/* message frag length (to pack) */
int pvm_useruid = -1;					/* user's uid */
int shmbufsiz = 0;						/* shared-memory buffer size */

int pvm_errno = 0;						/* last libpvm error code */
int pvm_nerr = sizeof(pvm_errlist)
			/sizeof(pvm_errlist[0]);	/* exported num of errors */
int pvmschedtid = 0;                    /* scheduler task */
int pvmrescode = 0;                     /* allow use of reserved tids, codes */
struct umbuf *pvmrxlist = 0;			/* not-recvd msg list */
int pvmtidhmask = TIDHOST;				/* mask for host field of tids */
int pvmtidlmask = TIDLOCAL;				/* mask for local field of tids */
int pvmouttid = 0;                      /* stdout dst and code */
int pvmoutcod = 0;
int pvmtoplvl = 1;                      /* function called from outside lib */
int pvmtrctid = 0;                      /* trace dst and code */
int pvmtrccod = 0;
Pvmtmask pvmtrcmask;                    /* trace bitmask */
Pvmtmask pvmctrcmask;                   /* child trace bitmask */

int pgsz = 0;							/* system page size */
int pvmpgsz = 0;						/* PVM virtual page size */
char *outmsgbuf = 0;					/* my outgoing message buffer */
int outbufsz = 0;						/* how many frags in outgoing msg buf */
struct pidtid *pidtids = 0;				/* pid -> tid table */
#ifdef IMA_POWER4
struct shmpghdr *globinfo = 0;			/* global task info */
#endif
char *infopage = 0;						/* proto, NDF, pid-tid table */
int maxpidtid = 0;						/* size of pid-tid table */
int debugmask = 0;						/* which debugging info */


/***************
 **  Private  **
 **           **
 ***************/

static char rcsid[] = "$Id: lpvmshmem.c,v 1.4 1994/11/07 22:42:39 manchek Exp $";
static char pvmtxt[512];				/* scratch for error log */
static struct umbuf *rxfrag = 0;		/* not-assembled incm msgs */
static struct frag *rxbuf;				/* buffer for incoming packets */
static int pvmtrcmid = 0;               /* message buffer for trace */
static int pvmtrcsbf = 0;               /* existing other buffer during trace */

#ifdef IMA_SYMM
static int cpuonline;					/* the number of CPUs available */
static struct shpage *pvmfraginfo;		/* frag locks and ref counts */
#endif
static int myshmbufid;					/* ID of shared-memory buffer */
static int mysemid;						/* ID of semaphore to sleep on */
static char *pvminbox;					/* incoming message buffer */
static char *pvmdinbox;					/* pvmd's incoming message buffer */
static char *pvmdoutbuf;				/* pvmd's outgoing message buffer */
static int pvminboxsz;					/* size of incoming message buffer */
static int mypidtid;					/* my position in pid-tid table */
static int pvmdpid;						/* pvmd's Unix proc ID */
static struct sockaddr_in pvmdsad;		/* address of pvmd socket */
static int pvmdsock = -1;				/* pvmd socket descriptor */

#ifdef LOG
static FILE  *logfp = 0;				/* my own log file */
#endif

static int pvmrouteopt = PvmDontRoute;	/* task-task routing style */
static void (*pvmoldtermhdlr)() = 0;


/**************************
 **  Internal Functions  **
 **                      **
 **************************/

/*	bailout()
*
*	Called by low-level stuff in f.e. frag.c.  Don't really want to
*	bail in libpvm.
*/

void
pvmbailout(n)
	int n;
{
	n = n;	/* ayn rand was here */
}


/*	pvmlogerror()
*
*	Log a libpvm error message.  Prepends a string identifying the task.
*/

pvmlogerror(s)
	char *s;
{
	if (pvmmytid == -1)
		fprintf(stderr, "libpvm [pid%d]: %s", pvmmyupid, s);
	else 
#ifdef LOG
	{
		fprintf(logfp, "libpvm [t%x]: %s", pvmmytid, s);
		fflush(logfp);
	}
#else
		fprintf(stderr, "libpvm [t%x]: %s", pvmmytid, s);
#endif
}


/*	pvmlogperror()
*
*	Log a libpvm error message.  Prepends a string identifying the
*	task and appends the system error string for _errno.
*/

pvmlogperror(s)
	char *s;
{
	char *em;

	em = ((errno >= 0 && errno < sys_nerr)
		? sys_errlist[errno] : "Unknown Error");
	if (pvmmytid == -1)
		fprintf(stderr, "libpvm [pid%d]: %s: %s\n", pvmmyupid, s, em);
	else
		fprintf(stderr, "libpvm [t%x]: %s: %s\n", pvmmytid, s, em);
}


hdump(p, n, pad)
	char *p;	/* bytes */
	int n;		/* length */
	char *pad;
{
	int i;
	pad = pad ? pad : "";
	for (i = 0; n-- > 0; i = ++i & 15) {
		fprintf(stderr, "%s%02x%s",
			i ? "" : pad,
			0xff & *p++,
			n && i != 15 ? " " : "\n");
	}
}


/* get port # of pvmd socket */
void
getdsock()
{
	char buf[128];
	int d;
	int n;
	int try;
	char *p;

	if (!(p = getenv("PVMSOCK"))) {
		if (!(p = pvmdsockfile())) {
			pvmlogerror("getdsock() pvmdsockfile() failed\n");
			return;
		}
		if ((d = open(p, O_RDONLY, 0)) == -1) {
			pvmlogperror(p);
			return;
		}

		try = 3;
		do {
			if ((n = read(d, buf, sizeof(buf))) == -1) {
				pvmlogperror("getdsock() read addr file");
				return;
			}
			if (n != 13) {
				if (--try < 1) {
					pvmlogerror("getdsock() addr file: wrong length read\n");
					return;
				}
				sleep(1);   /* XXX hmm */
			}
		} while (n != 13);
		(void)close(d);
		buf[n] = 0;
		p = buf;
	}

	hex_inadport(p, &pvmdsad);
	pvmdsad.sin_family = AF_INET;
}


/*	prodpvmd()
*
*	wake up the pvmd, which is sleeping on sockets and not shared memory.
*	XXX this sucks.
*/

void
prodpvmd()
{
	static char dummy[TDFRAGHDR];

	if (pvmdsock == -1) {
		if ((pvmdsock = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
			pvmlogperror("prodpvmd() socket");
			return;
		}
		if (connect(pvmdsock, (struct sockaddr*)&pvmdsad, sizeof(pvmdsad))
		== -1) {
			pvmlogperror("prodpvmd() connect");
			return;
		}
		BZERO(dummy, sizeof(dummy));

	} else {
		write(pvmdsock, &dummy, TDFRAGHDR);
	}
}


/* check inbox; block on a semaphore if no message */
void
peer_wait()
{
	struct msgboxhdr *inbp = (struct msgboxhdr *)pvminbox;

#if defined(IMA_SUNMP)

	PAGELOCK(&inbp->mb_lock);
	while (inbp->mb_read == inbp->mb_last) 
		cond_wait(&inbp->mb_cond, &inbp->mb_lock);
	PAGEUNLOCK(&inbp->mb_lock);
#else
	struct sembuf   sops;

	sops.sem_num = 0;
	sops.sem_op = -1;
	sops.sem_flg = 0;
	while (inbp->mb_read == inbp->mb_last) 
		if (semop(mysemid, &sops, 1) == -1) {
			sprintf(pvmtxt, "peer_wait(): semop id = %d\n", mysemid);
			pvmlogperror(pvmtxt);
		}
#endif
}


/*  pvmmctl()
*
*   Entry points for libpvm control messages.
*
*   Called by mxinput() when a control message
*   (code between TC_FIRST and TC_LAST) is received.
*/

static int
pvmmctl(up)
	struct umbuf *up;
{
	char buf[512];          /* to convert sockaddr, misc */
	int rbf;                /* temp rx message storage */

	switch (up->ub_cod) {
	case TC_CONREQ:
		sprintf(pvmtxt, "pvmmctl() TCP conn request from t%x!\n", up->ub_src);
		pvmlogerror(pvmtxt);
		break;
	case TC_CONACK:
	case TC_TASKEXIT:
	case TC_NOOP:
		sprintf(pvmtxt, "pvmmctl() unexpected TC msg from t%x!\n", up->ub_src);
		pvmlogerror(pvmtxt);
		break;
	case TC_OUTPUT:
		rbf = pvm_setrbuf(up->ub_mid);
        pvmclaimo();
        pvm_setrbuf(rbf);
        break;
	case TC_SETTMASK:
        pvm_upkstr(buf);
        if (strlen(buf) + 1 == TEV_MASK_LENGTH)
            BCOPY(buf, pvmtrcmask, TEV_MASK_LENGTH);
        else
            pvmlogerror("pvmmctl() bogus trace mask\n");
        break;

    default:
        sprintf(pvmtxt, "pvmmctl() from t%x code=%d ?\n",
                up->ub_src, up->ub_cod);
        pvmlogerror(pvmtxt);
        break;
    }

    umbuf_free(up->ub_mid);
    return 0;
}
		

/*	mroute()
*
*   Route a message to a destination.
*   Returns when
*       outgoing message (if any) fully sent and
*       (timed out (tmout) or
*           at least one message fully received)
*   Returns >=0 the number of complete messages downloaded, or
*   negative on error.
*/

int
mroute(mid, dtid, code, tmout)
	int mid;				/* message */
	int dtid;				/* dest */
	int code;				/* type code */
	struct timeval *tmout;	/* get at least one message */
{
	struct umbuf *txup;			/* tx message or null */
	struct frag *txfp = 0;		/* cur tx frag or null */
	int gotem = 0;				/* count complete msgs downloaded */
	int block;					/* get at least one message */
	int loopcount = 0;
	struct msgboxhdr *inbp;		/* incoming box */
	struct timeval tnow, tstop;

	/* XXX do we really have to do this? */
	if ((dtid == TIDPVMD && code == TM_MCA) || dtid == TIDGID)
		return node_mcast(mid, dtid, code);

	if (tmout) {
		if (tmout->tv_sec || tmout->tv_usec) {
			gettimeofday(&tnow, (struct timezone *)0);
			tstop.tv_sec = tnow.tv_sec + tmout->tv_sec;
			tstop.tv_usec = tnow.tv_usec + tmout->tv_usec;
			block = 1;
		} else
			block = 0;
	} else {
		block = 1;
		tstop.tv_sec = -1;
		tstop.tv_usec = -1;
	}

	if (txup = midtobuf(mid)) {
		txfp = txup->ub_frag->fr_link;
		txfp = txfp->fr_buf ? txfp : 0;
	}

	inbp = (struct msgboxhdr *)pvminbox;

	do {
		if (block && tstop.tv_sec != -1) {
			gettimeofday(&tnow, (struct timezone *)0);
			if (tnow.tv_sec > tstop.tv_sec 
			|| (tnow.tv_sec == tstop.tv_sec && tnow.tv_usec >= tstop.tv_usec))
				break;
		}

		if (loopcount++ > BUSYWAIT && !txfp && tstop.tv_sec == -1) {
			inbp->mb_sleep = 1;
			(void)peer_wait();
			inbp->mb_sleep = 0;
			loopcount = 0;
		}

/*
#ifdef TEST_ADD
		if (inbp->mb_read != TEST_ADD(&inbp->mb_last,0))
#else
		PAGELOCK(&inbp->mb_lock);		
		if (inbp->mb_read != inbp->mb_last) {
			PAGEUNLOCK(&inbp->mb_lock);
#endif
			if (peer_recv(&gotem) == -1)
				return -1;
		}
#ifndef TEST_ADD
		PAGEUNLOCK(&inbp->mb_lock);
#endif
*/
		if (inbp->mb_read != inbp->mb_last && peer_recv(&gotem) == -1)
			return -1;

		if (txfp && peer_send(txup, txfp, dtid, code)) {
			txfp = txfp->fr_link;
			if (!txfp->fr_buf)
				txfp = 0;
		}

	} while (txfp || (block && !gotem));

	return gotem;
}


/* receives a fragment from another process */
int
peer_recv(gotem)
	int *gotem;
{
	struct umbuf *rxup;			/* rx message */
	struct umbuf *up;
	struct frag *fp;
	char *cp;
	int src;
	int ff;
	struct peer *pp;
	int next;					/* frag being received */
	struct shmpkhdr *inmsgs;	/* incoming messages */
	struct msgboxhdr *inbp;		/* incoming box */

	inbp = (struct msgboxhdr *)pvminbox;
	inmsgs = (struct shmpkhdr *)(inbp + 1);

	next = (inbp->mb_read + 1) % pvminboxsz;
	src = inmsgs[next].ph_src;
	fp = fr_new(0);
	if ((src & ~pvmtidhmask) != TIDPVMD) {
		if (!(pp = peer_conn(src)) || pp == (struct peer *)-1L) {
			sprintf(pvmtxt, "mroute() can't connect to src t%x\n", 
				inmsgs[next].ph_src);
			pvmlogerror(pvmtxt);
			return PvmSysErr;
		}
		fp->fr_dat = pp->p_buf + INBOXPAGE*pgsz + inmsgs[next].ph_dat;
	} else
		fp->fr_dat = pvmdoutbuf + inmsgs[next].ph_dat;
	fp->fr_buf = fp->fr_dat - (inmsgs[next].ph_dat & (pgsz-1)) 
				+ PVMPAGEHDR;
	fp->fr_max = pvmfrgsiz;
	ff = inmsgs[next].ph_flag;

	cp = fp->fr_dat;
	src = pvmget32(cp + 4);
	fp->fr_len = pvmget32(cp + 8);
	fp->fr_max = pvmfrgsiz;
	/* ff = pvmget8(cp + 12); */
	fp->fr_dat += TDFRAGHDR;
/*
	fprintf(stderr, "mroute() frag src t%x len %d ff %d\n",
				src, fp->fr_len, ff);
*/
	if (debugmask & TDMPACKET) {
		sprintf(pvmtxt, "mroute() src t%x len %d dst t%x flag %d\n",
			src, fp->fr_len, inmsgs[next].ph_dst, ff);
		pvmlogerror(pvmtxt);
	}
	/*
	* if start of message, make new umbuf, add to frag pile
	*/
	if (ff & FFSOM) {
		cp += TDFRAGHDR;
		fp->fr_len -= TTMSGHDR;
		fp->fr_dat += TTMSGHDR;
		rxup = midtobuf(umbuf_new());
		rxup->ub_cod = pvmget32(cp);
		rxup->ub_enc = pvmget32(cp + 4);
		rxup->ub_src = src;
		LISTPUTBEFORE(rxfrag, rxup, ub_link, ub_rlink);
	}

	/* locate frag's message */

	for (rxup = rxfrag->ub_link; rxup != rxfrag; rxup = rxup->ub_link)
		if (rxup->ub_src == src)
			break;

	if (rxup == rxfrag) {	/* uh oh, no message for it */
		pvmlogerror("mroute() frag with no message\n");
		fr_unref(fp);

	} else {
		LISTPUTBEFORE(rxup->ub_frag, fp, fr_link, fr_rlink);
		rxup->ub_len += fp->fr_len;
	/*
	* if end of message, move to rxlist and count it
	*/
		if (ff & FFEOM) {
			LISTDELETE(rxup, ub_link, ub_rlink);
			rxup->ub_codef = enctovec(rxup->ub_enc);
			LISTPUTBEFORE(pvmrxlist, rxup, ub_link, ub_rlink);
			(*gotem)++;
			/* XXX */
			if (rxup->ub_cod >= (int)TC_FIRST && rxup->ub_cod <= (int)TC_LAST) {
				rxup = rxup->ub_rlink;
				pvmmctl(rxup->ub_link);
			}
		}
	}
/*
#ifndef TEST_ADD
	PAGELOCK(&inbp->mb_lock);
#endif
*/
	inbp->mb_read = next;
	return 0;
}


/* sends a fragment to another process; returns 1 if sent, 0 otherwise */
int
peer_send(txup, txfp, dtid, code)
    struct umbuf *txup;     /* tx message or null */
    struct frag *txfp;      /* cur tx frag or null */
    int dtid;               /* dest */
    int code;               /* type code */
{
    char *txcp = 0;             /* point to remainder of txfp */
    int txtogo = 0;             /* len of txfp */
	char *cp;
    int i;
    int ff;
	int loc;					/* location of data in outgoing buffer */
	int len = 0;				/* len of txfp */
	int next;					/* frag being received */
	struct peer *pp;
	struct shmpkhdr *dmsgs;	
	struct msgboxhdr *dboxp;	/* receiving box of peer */
	int onemsg = 1;

	if (!txfp->fr_u.dab) {
		pvmlogerror("InPlace not implemented!\n");
		return 0;
	}
#ifdef IMA_POWER4
	if ((dtid & ~pvmtidhmask) != TIDPVMD)		/* to another task */
#else
	if ((dtid & pvmtidhmask) == (pvmmytid & pvmtidhmask)
	&& (dtid & ~pvmtidhmask) != TIDPVMD)		/* to local task */
#endif
	{
		if (!(pp = peer_conn(dtid)))
			return 0;
		if (pp != (struct peer *)-1L)
			dboxp = (struct msgboxhdr *)pp->p_buf;
		else
			dboxp = (struct msgboxhdr *)pvmdinbox;
	} else
		dboxp = (struct msgboxhdr *)pvmdinbox;
/*
#ifdef TEST_ADD
	if ((TEST_ADD(&dboxp->mb_last,0) + 1) % pvminboxsz 
	== TEST_ADD(&dboxp->mb_read,0))
		return 0;
#else
	PAGELOCK(&dboxp->mb_lock);			
	if ((dboxp->mb_last + 1) % pvminboxsz == dboxp->mb_read) {
		PAGEUNLOCK(&dboxp->mb_lock);
		return 0;
	}
	PAGEUNLOCK(&dboxp->mb_lock);
#endif
*/
	if ((dboxp->mb_last + 1) % pvminboxsz == dboxp->mb_read) 
		return 0;		/* full */

	txcp = txfp->fr_dat;
	len = txfp->fr_len;
	/*
	* if this is first frag, prepend t-t header
	*/
	ff = 0;
	if (txfp->fr_rlink == txup->ub_frag) {
		txcp -= TTMSGHDR;
		len += TTMSGHDR;
		pvmput32(txcp, code);
		pvmput32(txcp + 4, txup->ub_enc);
		ff = FFSOM;
	}
	if (txfp->fr_link == txup->ub_frag)
		ff |= FFEOM;
	/*
	* prepend t-d header
	*/
	txcp -= TDFRAGHDR;
	pvmput32(txcp, dtid);
	pvmput32(txcp + 4, pvmmytid);
	pvmput32(txcp + 8, len);
	pvmput8(txcp + 12, ff);
	len += TDFRAGHDR;

/* XXX oh really, just kill me.  this tests whether the pg is private  -b */
	if ((loc = txcp - outmsgbuf) > outbufsz*pvmpgsz || loc < 0) {

		cp = 0;
		do {
			if (cp)
				da_unref(cp);
			if (onemsg) {
				onemsg = 0;
				if (debugmask & TDMPACKET)
					pvmlogerror("peer_send(): outgoing buffer full\n");
			}
			cp = da_new(len	+ (DDFRAGHDR > TDFRAGHDR ? 
				DDFRAGHDR - TDFRAGHDR : 0));
		} while ((loc = cp - outmsgbuf) > outbufsz*pvmpgsz || loc < 0);	

		txfp->fr_dat = cp;
		if (DDFRAGHDR > TDFRAGHDR)
			txfp->fr_dat += DDFRAGHDR - TDFRAGHDR;
		BCOPY(txcp, txfp->fr_dat, len); 
/*
		BCOPY(txfp->fr_buf - PVMPAGEHDR, cp - PVMPAGEHDR, 
			len + PVMPAGEHDR + txfp->fr_dat - cp);
*/
		da_unref(txfp->fr_buf);
		txfp->fr_buf = cp;
		txcp = txfp->fr_dat;
	}

	if (debugmask & TDMPACKET) {
		sprintf(pvmtxt, "mroute() dst t%x len %d page %d flag %d\n", 
			dtid, txfp->fr_len, loc/pgsz + 1, ff);
		pvmlogerror(pvmtxt);
	}

	dmsgs = (struct shmpkhdr *)(dboxp + 1);
	da_ref(txfp->fr_buf);
	PAGELOCK(&dboxp->mb_lock);
	next = (dboxp->mb_last + 1) % pvminboxsz;
	PAGEUNLOCK(&dboxp->mb_lock);
	while (next == dboxp->mb_read) ;	/* full */
	PAGELOCK(&dboxp->mb_lock);
	dmsgs[next].ph_src = pvmmytid;
	dmsgs[next].ph_dst = dtid;
	dmsgs[next].ph_dat = loc;
	dmsgs[next].ph_flag = ff;
/*
	if (dboxp->mb_last == dboxp->mb_read
	&& dboxp != (struct msgboxhdr *)pvmdinbox)
*/
	dboxp->mb_last = next;
	if (dboxp != (struct msgboxhdr *)pvmdinbox && dboxp->mb_sleep)
#ifdef IMA_SUNMP
		cond_signal(&dboxp->mb_cond);
#else
		peer_wake(pp);
#endif
	PAGEUNLOCK(&dboxp->mb_lock);
	/* wake up pvmd */
	if (dboxp == (struct msgboxhdr *)pvmdinbox
	&& (dboxp->mb_last + pvminboxsz - 1) % pvminboxsz == dboxp->mb_read)
		(void)prodpvmd();

	return 1;
}


int
node_mcast(mid, dtid, code)
	int mid;    /* message id */
	int dtid;   /* destination */
	int code;   /* type */
{
	int i;
	long count = 0;
	int cc = 0;
	static int *tids;       /* intended recipients of multicast message */
	static int ntask;       /* number of tids */
	int dummy;
	static struct timeval ztv = { 0, 0 };

	/* intercept multicast info */

	if (dtid == TIDPVMD) {
		int sbf = mid;

		pvm_setrbuf(mid);
		pvm_upkint(&ntask, 1, 1);
		tids = TALLOC(ntask, int, "tids");
		pvm_upkint(tids, ntask, 1);
		/* sbf = pvm_setsbuf(pvm_mkbuf(PvmDataFoo)); */
		pvm_setsbuf(pvm_mkbuf(PvmDataFoo));
		dummy = TIDGID;
		pvm_pkint(&dummy, 1, 1);
		pvm_setrbuf(pvm_setsbuf(sbf));
		return 0;
	}

	for (i = 0; i < ntask; i++)
		cc = mroute(mid, tids[i], code, &ztv);
	PVM_FREE(tids);
	ntask = 0;

	return cc;
}


/*	msendrecv()
*
*	Single op to send a system message (usually to our pvmd) and get
*	the reply.
*	Returns message handle or negative if error.
*/

int
msendrecv(other, code)
	int other;				/* dst, src tid */
	int code;				/* message code */
{
	int cc;
	struct umbuf *up;

	if (pvmsbufmid <= 0)
		return PvmNoBuf;

	/* send code to other */
	if (debugmask & TDMMESSAGE) {
		sprintf(pvmtxt, "msendrecv() to t%x code %d\n", other, code);
		pvmlogerror(pvmtxt);
	}
	if ((cc = mroute(pvmsbufmid, other, code, (struct timeval *)0)) < 0)
		return cc;

	if (code == TM_MCA)		/* for node_mcast() */
		return 1;

	/* recv code from other */
	for (up = pvmrxlist->ub_link; 1; up = up->ub_link) {
		if (up == pvmrxlist) {
			up = up->ub_rlink;
			if ((cc = mroute(0, 0, 0, (struct timeval *)0)) < 0)
				return cc;
			up = up->ub_link;
		}

		if (debugmask & TDMMESSAGE) {
			sprintf(pvmtxt, "msendrecv() cmp from t%x code %d\n",
					up->ub_src, up->ub_cod);
			pvmlogerror(pvmtxt);
		}
		if (up->ub_src == other && up->ub_cod == code)
			break;
	}
	LISTDELETE(up, ub_link, ub_rlink);
	if (pvmrbufmid > 0)
		umbuf_free(pvmrbufmid);
	pvmrbufmid = 0;
	if (cc = pvm_setrbuf(up->ub_mid))
		return cc;
	return up->ub_mid;
}


static void
catch_kill(sig)
	int sig;
{
	if (pvmoldtermhdlr)
		pvmoldtermhdlr(sig);
	pvmendtask();
	exit(sig);
}


/*	pvmbeatask()
*
*	Initialize libpvm, config process as a task.
*	This is called as the first step of each libpvm function so no
*	explicit initialization is required.
*
*	Returns 0 if okay, else error code.
*/

int
pvmbeatask()
{
	struct shmpkhdr *inmsgs;
	struct pidtidhdr *pvminfo;
	struct msgboxhdr *dboxp;		/* receiving box of pvmd */
	int next;
	static int altpid = -1;         /* pid of ancestor forked by pvmd */
	int msgcnt;
	char *msgbuf;					/* my message buffer */
	int bufid;
	int i;
	int pid;
	int bid;
	union semun {
		int val;
		struct semid_ds *buf;
		ushort *array;
	} sunion;
#ifdef LOG
	char fname[32];
#endif

	if (pvmmytid != -1)
		return 0;

	if ((pvm_useruid = getuid()) == -1) {
		pvmlogerror("pvmbeatask() can't getuid()\n");
		return PvmSysErr;
	}
	pvmmyupid = getpid();

#ifdef LOG
sprintf(fname, "/tmp/pvmt.%d", pvm_useruid);
if ((logfp = fopen(fname, "a")) == NULL)
	pvmlogerror("pvmbeatask() can't open log file\n");
#endif

    /*
    * get expected pid from environment in case we were started by
    * the pvmd and someone forked again
    */

    if (altpid == -1) {
        char *p;

        if (p = getenv("PVMEPID"))
            altpid = atoi(p);
        else
            altpid = 0;
    }

	pgsz = sysconf(_SC_PAGESIZE);
	pvmpgsz = FRAGPAGE*pgsz;
	pvmfrgsiz = pvmpgsz - PVMPAGEHDR;
	pvminboxsz = 
		(INBOXPAGE*pgsz - sizeof(struct msgboxhdr))/sizeof(struct shmpkhdr);

	pvmoldtermhdlr = signal(SIGTERM, catch_kill);

	/*
	*	initialize received-message list and fragment reassembly list
	*/

	rxfrag = TALLOC(1, struct umbuf, "umb");
	BZERO((char*)rxfrag, sizeof(struct umbuf));
	rxfrag->ub_link = rxfrag->ub_rlink = rxfrag;

	pvmrxlist = TALLOC(1, struct umbuf, "umb");
	BZERO((char*)pvmrxlist, sizeof(struct umbuf));
	pvmrxlist->ub_link = pvmrxlist->ub_rlink = pvmrxlist;

	peer_init();

	if ((bufid = shmget((key_t)(TIDPVMD + pvm_useruid), 0, PERMS)) 
	== -1) {
		pvmlogperror("pvmbeatask() shmget: can't connect to pvmd");
		return PvmSysErr;
	}
	if ((pvmdinbox = (char *)shmat(bufid, 0, 0)) == (char *)-1L) {
		pvmlogperror("pvmbeatask() shmat pvmd");
		return PvmSysErr;
	}
	infopage = pvmdinbox + INBOXPAGE*pgsz;
	pvmdoutbuf = infopage + pgsz;

	dboxp = (struct msgboxhdr *)pvmdinbox;
	inmsgs = (struct shmpkhdr *)(dboxp + 1);
	PAGELOCK(&dboxp->mb_lock);
/*
#ifdef TEST_ADD
	while ((TEST_ADD(&dboxp->mb_last,0) + 1) % pvminboxsz
	== TEST_ADD(&dboxp->mb_read,0))
		;
#else
*/
	while ((next = (dboxp->mb_last + 1) % pvminboxsz) == dboxp->mb_read) {
		;
/*
		PAGEUNLOCK(&dboxp->mb_lock);
		PAGELOCK(&dboxp->mb_lock);
#endif
*/
	}
	inmsgs[next].ph_src = pvmmyupid;
	inmsgs[next].ph_dst = altpid;
	inmsgs[next].ph_dat = -1;
	dboxp->mb_last = next;
	PAGEUNLOCK(&dboxp->mb_lock);

	pvminfo = (struct pidtidhdr *)(infopage + PVMPAGEHDR);
	pvmdpid = pvminfo->i_dpid;
	(void)getdsock();
	(void)prodpvmd();

	if (pvminfo->i_proto != TDPROTOCOL) {
		sprintf(pvmtxt, "beatask() t-d protocol mismatch (%d/%d)\n",
			TDPROTOCOL, pvminfo[0]);
		pvmlogerror(pvmtxt);
		return PvmSysErr;
	}
	pvmmyndf = pvminfo->i_ndf;
	shmbufsiz = pvminfo->i_bufsiz;
	outbufsz = (shmbufsiz - INBOXPAGE*pgsz)/pvmpgsz;
	pidtids = (struct pidtid *)(pvminfo + 1);
	maxpidtid = (pgsz - sizeof(struct pidtidhdr))/sizeof(struct pidtid);

#ifdef IMA_POWER4
	if ((bid = shmget((key_t)pvm_useruid, pgsz, IPC_POWER4)) == -1)
		pvmlogperror("pvmbeatask() shmget: can't find global table");
	if ((globinfo = (struct shmpghdr *)shmat(bid, 0, 0))
	== (struct shmpghdr *)-1)
		pvmlogperror("pvmbeatask() shmget: can't attach global table");
#endif

	pid = altpid ? altpid : pvmmyupid;
	while (pvmmytid == -1) {
		int ntids;			/* number of entries in pid-tid table */

		PAGELOCK(&((struct shmpghdr *)infopage)->pg_lock);
		ntids = min(maxpidtid, ((struct shmpghdr *)infopage)->pg_ref);
		for (i = 0; i < ntids; i++)
			if (pidtids[i].pt_pid == pid) {
				pvmmytid = pidtids[i].pt_tid;
				pvmmyptid = pidtids[i].pt_ptid;
				/* pidtids[i].pt_pid = pvmmyupid; */
				mypidtid = i;
				break;
			}
		PAGEUNLOCK(&((struct shmpghdr *)infopage)->pg_lock);
	}

#ifdef IMA_POWER4
	if ((myshmbufid = 
	shmget((key_t)pvmmytid, shmbufsiz, IPC_POWER4|IPC_CREAT|IPC_EXCL|PERMS)) 
	== -1) {
		pvmlogperror("pvmbeatask() shmget: can't alloc msg buf");
		return PvmSysErr;
	}
#else
	if ((myshmbufid = 
	shmget((key_t)pvmmytid, shmbufsiz, IPC_CREAT|IPC_EXCL|PERMS)) == -1) {
		pvmlogperror("pvmbeatask() shmget: can't alloc msg buf");
		return PvmSysErr;
	}
#endif /*IMA_POWER4*/
	if ((pvminbox = (char *)shmat(myshmbufid, 0, 0)) == (char *)-1L) {
		pvmlogperror("pvmbeatask() shmat");
		return PvmSysErr;
	}
#if defined(IMA_SGIMP) || defined(IMA_SGIMP64) || defined(IMA_ALPHAMP)
	if ((mysemid = semget((key_t)pvmmytid, 1, IPC_CREAT|IPC_EXCL|PERMS)) == -1){
		pvmlogperror("pvmbeatask() semget: can't create semaphore");
		return PvmSysErr;
	}
/*
	sunion.val = 0;
	if (semctl(mysemid, 0, SETVAL, sunion) == -1) {
		pvmlogperror("pvmbeatask() semctl: can't reset semaphore");
		return PvmSysErr;
	}
*/
#endif /*IMA_SGIMP || IMA_SGIMP64 || IMA_ALPHAMP*/
	outmsgbuf = pvminbox + INBOXPAGE*pgsz;
	msgbufinit(pvminbox);
	/* XXX PAGELOCK(pvminfo); */
	pidtids[mypidtid].pt_stat = ST_SHMEM;
	/* XXX PAGEUNLOCK(pvminfo); */

	return 0;
}


int
pvmendtask()
{
	char *p;
	int i;
	struct shmid_ds shmds;

	if (pvmsbufmid > 0)
		umbuf_free(pvmsbufmid);
	if (pvmrbufmid > 0)
		umbuf_free(pvmrbufmid);

	/*
	* hang around until all outgoing messages are received
	* XXX this loses if the receiving task dies
	*/

	for (p = outmsgbuf, i = 0; i < bufpageused; i++, p += FRAGPAGE * pgsz) {
		while (((struct shmpghdr *)p)->pg_ref > 0)
			sleep(1);
	}

	/* PAGELOCK(pvminfo->i_pghd); */
	pidtids[mypidtid].pt_stat = ST_EXIT;
	/* PAGEUNLOCK(pvminfo->i_pghd); */
	shmdt(pvminbox);
/*
	if (shmctl(myshmbufid, IPC_STAT, &shmds) == -1)
		pvmlogperror("pvmendtask() shmctl STAT");
	if (shmds.shm_nattch == 0 &&
*/
#ifdef IMA_POWER4
	shmdt((char*)globinfo);
#endif
	if (shmctl(myshmbufid, IPC_RMID, (struct shmid_ds *)0) == -1)
		pvmlogperror("pvmendtask() shmctl RMID");
#if defined(IMA_SGIMP) || defined(IMA_SGIMP64) || defined(IMA_ALPHAMP)
	if (semctl(mysemid, 0, IPC_RMID) == -1)
		pvmlogperror("pvmendtask() semctl RMID");
#endif

	if (pvmmytid != -1)
		pvmmytid = -1;
	if (pvmdsock != -1)
		(void)close(pvmdsock);

	peer_cleanup();

	/* XXX free rxfrag and rxlist */
#ifdef LOG
fclose(logfp);
#endif

	return 0;
}


/************************
 **  Libpvm Functions  **
 **                    **
 ************************/


int
pvm_getopt(what)
	int what;
{
	int rc = 0;
	int err = 0;
	int x;

	if (x = pvmtoplvl) {
		pvmtoplvl = 0;
		if (pvmmytid != -1 && TEV_DO_TRACE(TEV_GETOPT0)) {
			pvm_pkint(&what, 1, 1);
			TEV_FIN;
		}
	}

	switch (what) {
	case PvmRoute:
		rc = pvmrouteopt;
		break;

	case PvmDebugMask:
		rc = debugmask;
		break;

	case PvmAutoErr:
		rc = pvmautoerr;
		break;

	case PvmOutputTid:
		rc = pvmcouttid;
		break;

	case PvmOutputCode:
		rc = pvmcoutcod;
		break;

	case PvmTraceTid:
		rc = pvmctrctid;
		break;

	case PvmTraceCode:
		rc = pvmctrccod;
		break;

	case PvmFragSize:
		break;

	case PvmResvTids:
		rc = pvmrescode;
		break;

	case PvmSelfOutputTid:
		rc = pvmouttid;
		break;

	case PvmSelfOutputCode:
		rc = pvmoutcod;
		break;

	case PvmSelfTraceTid:
		rc = pvmtrctid;
		break;

	case PvmSelfTraceCode:
		rc = pvmtrccod;
		break;

	default:
		err = 1;
		break;
	}

	if (x) {
		if (pvmmytid != -1 && TEV_DO_TRACE(TEV_GETOPT1)) {
			pvm_pkint(&rc, 1, 1);
			TEV_FIN;
		}
		pvmtoplvl = x;
	}

	if (err)
		return lpvmerr("pvm_getopt", PvmBadParam);
	return rc;
}


int
pvm_setopt(what, val)
	int what;
	int val;
{
	int rc = 0;
	int err = 0;
	int sbf, rbf;
	int x;
	char buf[16];

	if (x = pvmtoplvl) {
		pvmtoplvl = 0;
		if (pvmmytid != -1 && TEV_DO_TRACE(TEV_SETOPT0)) {
			pvm_pkint(&what, 1, 1);
			pvm_pkint(&val, 1, 1);
			TEV_FIN;
		}
	}

	switch (what) {
	case PvmRoute:
		switch (val) {
		case PvmDontRoute:
		case PvmAllowDirect:
		case PvmRouteDirect:
			rc = pvmrouteopt;
			pvmrouteopt = val;
			break;

		default:
			rc = PvmBadParam;
			err = 1;
			break;
		}
		break;

	case PvmDebugMask:
		rc = debugmask;
		debugmask = val;
		break;

	case PvmAutoErr:
		rc = pvmautoerr;
		pvmautoerr = val;
		break;

	case PvmOutputTid:
		if (val && val != pvmmytid
		&& (val != pvmouttid || pvmcoutcod != pvmoutcod)) {
			rc = PvmBadParam;
			err = 1;

		} else {
			rc = pvmcouttid;
			pvmcouttid = val;
		}
		break;

	case PvmOutputCode:
		if (pvmcouttid && pvmcouttid != pvmmytid && val != pvmoutcod) {
			rc = PvmBadParam;
			err = 1;

		} else {
			rc = pvmcoutcod;
			pvmcoutcod = val;
		}
		break;

	case PvmTraceTid:
		if (val && val != pvmmytid
		&& (val != pvmtrctid || pvmctrccod != pvmtrccod)) {
			rc = PvmBadParam;
			err = 1;

		} else {
			rc = pvmctrctid;
			pvmctrctid = val;
		}
		break;

	case PvmTraceCode:
		if (pvmctrctid && pvmctrctid != pvmmytid && val != pvmtrccod) {
			rc = PvmBadParam;
			err = 1;

		} else {
			rc = pvmctrccod;
			pvmctrccod = val;
		}
		break;

	case PvmFragSize:
/*
		if (val < TDFRAGHDR + TTMSGHDR + 4 || val > 1048576) {
*/
			rc = PvmBadParam;
			err = 1;
/*
		} else {
			rc = pvmfrgsiz;
			pvmfrgsiz = val;
		}
*/
		break;

	case PvmResvTids:
		rc = pvmrescode;
		pvmrescode = val;
		break;

	case PvmSelfOutputTid:
		sbf = pvm_setsbuf(pvm_mkbuf(PvmDataFoo));
		rbf = pvm_setrbuf(0);
		what = TS_OUTTID;
		pvm_pkint(&what, 1, 1);
		sprintf(buf, "%x", 0xffffffff & val);
		pvm_pkstr(buf);
		if ((rc = msendrecv(TIDPVMD, TM_SETOPT)) > 0) {
			pvm_freebuf(pvm_setrbuf(rbf));
			rc = pvmouttid;
			pvmouttid = val;
			pvmcouttid = pvmouttid;
			pvmcoutcod = pvmoutcod;

		} else {
			pvm_setrbuf(rbf);
			err = 1;
		}
		pvm_freebuf(pvm_setsbuf(sbf));
		break;

	case PvmSelfOutputCode:
		sbf = pvm_setsbuf(pvm_mkbuf(PvmDataFoo));
		rbf = pvm_setrbuf(0);
		what = TS_OUTCOD;
		pvm_pkint(&what, 1, 1);
		sprintf(buf, "%x", 0xffffffff & val);
		pvm_pkstr(buf);
		if ((rc = msendrecv(TIDPVMD, TM_SETOPT)) > 0) {
			pvm_freebuf(pvm_setrbuf(rbf));
			rc = pvmoutcod;
			pvmoutcod = val;
			pvmcouttid = pvmouttid;
			pvmcoutcod = pvmoutcod;

		} else {
			pvm_setrbuf(rbf);
			err = 1;
		}
		pvm_freebuf(pvm_setsbuf(sbf));
		break;

	case PvmSelfTraceTid:
		sbf = pvm_setsbuf(pvm_mkbuf(PvmDataFoo));
		rbf = pvm_setrbuf(0);
		what = TS_TRCTID;
		pvm_pkint(&what, 1, 1);
		sprintf(buf, "%x", 0xffffffff & val);
		pvm_pkstr(buf);
		if ((rc = msendrecv(TIDPVMD, TM_SETOPT)) > 0) {
			pvm_freebuf(pvm_setrbuf(rbf));
			rc = pvmtrctid;
			pvmtrctid = val;
			pvmctrctid = pvmtrctid;
			pvmctrccod = pvmtrccod;

		} else {
			pvm_setrbuf(rbf);
			err = 1;
		}
		pvm_freebuf(pvm_setsbuf(sbf));
		break;

	case PvmSelfTraceCode:
		sbf = pvm_setsbuf(pvm_mkbuf(PvmDataFoo));
		rbf = pvm_setrbuf(0);
		what = TS_TRCCOD;
		pvm_pkint(&what, 1, 1);
		sprintf(buf, "%x", 0xffffffff & val);
		pvm_pkstr(buf);
		if ((rc = msendrecv(TIDPVMD, TM_SETOPT)) > 0) {
			pvm_freebuf(pvm_setrbuf(rbf));
			rc = pvmtrccod;
			pvmtrccod = val;
			pvmctrctid = pvmtrctid;
			pvmctrccod = pvmtrccod;

		} else {
			pvm_setrbuf(rbf);
			err = 1;
		}
		pvm_freebuf(pvm_setsbuf(sbf));
		break;

	default:
		rc = PvmBadParam;
		err = 1;
		break;
	}

	if (x) {
		if (pvmmytid != -1 && TEV_DO_TRACE(TEV_SETOPT1)) {
			pvm_pkint(&rc, 1, 1);
			TEV_FIN;
		}
		pvmtoplvl = x;
	}

	if (err)
		return lpvmerr("pvm_setopt", rc);
	return rc;
}


int
pvm_perror(s)
	char *s;
{
	if (pvmmytid == -1)
		fprintf(stderr, "libpvm [pid%d]: ", pvmmyupid);
	else
		fprintf(stderr, "libpvm [t%x]: ", pvmmytid);
	fprintf(stderr, "%s: %s\n",
		(s ? s : "(null)"),
		(pvm_errno <= 0 && pvm_errno > -pvm_nerr
				? pvm_errlist[-pvm_errno] : "Unknown Error"));
	return 0;
}


int
pvm_getfds(fds)		/* XXX this function kinda sucks */
	int **fds;			/* fd list return */
{
	int cc;

	cc = PvmNotImpl;
	return (cc < 0 ? lpvmerr("pvm_getfds", cc) : cc);
}


int
pvm_start_pvmd(argc, argv, block)
	int argc;		/* number of args to pass to pvmd (>= 0) */
	char **argv;	/* args for pvmd or null */
	int block;		/* if true, don't return until add hosts are started */
{
	char *sfn;
	struct stat sb;
	int cc;
	char *fn;			/* file to exec */
	char **av;
	int x;

	if (x = pvmtoplvl) {
		pvmtoplvl = 0;
		if (pvmmytid != -1 && TEV_DO_TRACE(TEV_START_PVMD0)) {
			pvm_pkint(&argc, 1, 1);
			pvm_pkint(&block, 1, 1);
			for (cc = 0; cc < argc; cc++)
				pvm_pkstr(argv[cc]);
			TEV_FIN;
		}
	}

	if (argc < 0 || !argv)
		argc = 0;

	if ((pvm_useruid = getuid()) == -1) {
		pvmlogerror("can't getuid()\n");
		cc = PvmSysErr;
		goto bail;
	}

	if (!(sfn = pvmdsockfile())) {
		pvmlogerror("pvm_start_pvmd() pvmdsockfile() failed\n");
		cc = PvmSysErr;
		goto bail;
	}

	if (stat(sfn, &sb) != -1) {
		cc = PvmDupHost;
		goto bail;
	}

	fn = pvmgetpvmd();

	av = TALLOC(argc + 2, char *, "argv");
	if (argc > 0)
		BCOPY((char *)&argv[0], (char *)&av[1], argc * sizeof(char*));
	av[0] = fn;
	av[argc + 1] = 0;

	if (!fork()) {
		if (!fork()) {
			for (cc = getdtablesize(); cc-- > 0; )
				(void)close(cc);
			(void)open("/dev/null", O_RDONLY, 0);
			(void)open("/dev/null", O_WRONLY, 0);
			(void)dup2(1, 2);
			execvp(av[0], av);
		}
		_exit(0);
	}
	(void)wait(0);

	PVM_FREE(av);

	for (cc = 8; cc > 0 && stat(sfn, &sb) == -1; cc--)
		sleep(1);

	if (cc <= 0) {
		cc = PvmCantStart;
		goto bail;
	}
	if (cc = BEATASK)
		goto bail;

	if (block) {
		struct pvmhostinfo *hip;
		int t = 1;

		pvm_config((int*)0, (int*)0, &hip);
		while ((cc = pvm_addhosts(&hip[0].hi_name, 1, (int*)0)) == PvmAlready) {
			sleep(t);
			if (t < 8)
				t *= 2;
		}
		if (cc != PvmDupHost)
			goto bail;
		cc = BEATASK;
	}

bail:

	if (x) {
		if (TEV_DO_TRACE(TEV_START_PVMD1)) {
			pvm_pkint(&cc, 1, 1);
			TEV_FIN;
		}
		pvmtoplvl = x;
	}

	return (cc < 0 ? lpvmerr("pvm_start_pvmd", cc) : 0);
}


/*	tev_begin()
*
*	Start a trace event.  Initialize a message, write time and event kind.
*/

int
tev_begin(kind)
	int kind;
{
	struct timeval now;

	gettimeofday(&now, (struct timezone *)0);
	pvmtrcmid = pvm_mkbuf(PvmDataFoo);
	pvmtrcsbf = pvm_setsbuf(pvmtrcmid);
	pvm_pkint((int *)&now.tv_sec, 1, 1);
	pvm_pkint((int *)&now.tv_usec, 1, 1);
	pvm_pkint(&pvmmytid, 1, 1);
	pvm_pkint(&kind, 1, 1);
/*
	fprintf(stderr, "tev_begin() mid %d\n", pvmtrcmid);
*/
	return 1;
}


/*	tev_fin()
*
*	End a trace event.  Send message.
*/

int
tev_fin()
{
	int routetmp;
	static struct timeval ztv = { 0, 0 };

/*
	fprintf(stderr, "tev_fin() mid %d\n", pvmtrcmid);
*/
	if (pvmmytid != pvmtrctid) {
		if ((routetmp = pvmrouteopt) == PvmRouteDirect)
			pvmrouteopt = PvmAllowDirect;
		mroute(pvmtrcmid,  pvmtrctid, pvmtrccod, &ztv);
		pvmrouteopt = routetmp;
	}
	pvm_setsbuf(pvmtrcsbf);
	pvmtrcsbf = 0;
	pvm_freebuf(pvmtrcmid);
	pvmtrcmid = 0;
	return 0;
}


/*	tev_do_trace()
*
*	Export TEV_DO_TRACE() so user code doesn't have to import
*	masks and trace tid, etc.
*/

int
tev_do_trace(kind)
	int kind;
{
	return TEV_DO_TRACE(kind);
}


int
pvm_precv(tid, tag, cp, len, dt, rtid, rtag, rlen)
	int tid;
	int tag;
	void *cp;
	int len;
	int dt;
	int *rtid;
	int *rtag;
	int *rlen;
{
	int rbf;
	int cc = 0;
	int l;

	switch (dt) {

	case PVM_BYTE:
		len *= sizeof(char);
		break;

	case PVM_SHORT:
	case PVM_USHORT:
		len *= sizeof(short);
		break;

	case PVM_INT:
	case PVM_UINT:
		len *= sizeof(int);
		break;

	case PVM_LONG:
	case PVM_ULONG:
		len *= sizeof(long);
		break;

	case PVM_FLOAT:
		len *= sizeof(float);
		break;

	case PVM_CPLX:
		len *= sizeof(float) * 2;
		break;

	case PVM_DOUBLE:
		len *= sizeof(double);
		break;

	case PVM_DCPLX:
		len *= sizeof(double) * 2;
		break;

	default:
		cc = PvmBadParam;
		break;
	}

	if (!cc) {
		rbf = pvm_setrbuf(0);
		cc = pvm_recv(tid, tag);
		if (cc > 0) {
			pvm_bufinfo(cc, &l, rtag, rtid);
			if (rlen)
				*rlen = l;
			if (l < len)
				len = l;
			pvm_upkbyte((char *)cp, len, 1);
			pvm_freebuf(cc);
			cc = 0;
		}
		pvm_setrbuf(rbf);
	}
	if (cc < 0)
		lpvmerr("pvm_precv", cc);
	return cc;
}


/* find dynamic buffer */
char *
dynbuf(tid, len)
	int tid;
	int len;
{
	struct peer *pp;
	int fd;
	char fname[32];
	struct shmpghdr *ph;

	while (!(pp = peer_conn(tid)))
		;
	if (len > SHMBUFSIZE && len > pp->p_dlen && pp->p_dbuf) {
		munmap((caddr_t)pp->p_dbuf, SHMBUFSIZE);
		pp->p_dbuf = 0;
	}

	if (!(ph = (struct shmpghdr *)pp->p_dbuf)) {
		sprintf(fname, PVMSHMFILE, tid);
		if ((fd = open(fname, O_CREAT|O_RDWR, 0600)) == -1 ||
		(pp->p_dbuf = (char *)mmap(0, max(len,SHMBUFSIZE), PROT_READ|PROT_WRITE,
#if defined(IMA_SGIMP) || defined(IMA_SGIMP64)
		MAP_SHARED|MAP_AUTOGROW, fd, 0)) == (char *)-1L)
#else
		MAP_SHARED, fd, 0)) == (char *)-1)
#endif
		{
			pvmlogperror(fname);
			return (char *)-1L;
		}
#ifdef IMA_SUNMP
		/* fill buffer with 0's */
		lseek(fd, len - 1, SEEK_SET);
		write(fd, fname, 1);
#endif
		close(fd);
		pp->p_dlen = len;
	} else if (ph->pg_ref) {	/* previous msg has not been recv'd */
		while (ph->pg_ref)
			(void)peer_wait();
#ifdef IMA_SUNMP
	} else if (pp->p_dlen < len) {
		if ((fd = open(fname, O_CREAT|O_RDWR, 0600)) == -1) {
			pvmlogperror(fname);
			return (char *)-1L;
		}
		lseek(fd, len - 1, SEEK_SET);
		write(fd, fname, 1);
		close(fd);
		pp->p_dlen = len;
#endif
	}

	return pp->p_dbuf;
}

int
pvm_psend(tid, tag, cp, len, dt)
	int tid;
	int tag;
	void *cp;
	int len;
	int dt;
{
	int sbf;
	int cc = 0;

	switch (dt) {

	case PVM_BYTE:
		len *= sizeof(char);
		break;

	case PVM_SHORT:
	case PVM_USHORT:
		len *= sizeof(short);
		break;

	case PVM_INT:
	case PVM_UINT:
		len *= sizeof(int);
		break;

	case PVM_LONG:
	case PVM_ULONG:
		len *= sizeof(long);
		break;

	case PVM_FLOAT:
		len *= sizeof(float);
		break;

	case PVM_CPLX:
		len *= sizeof(float) * 2;
		break;

	case PVM_DOUBLE:
		len *= sizeof(double);
		break;

	case PVM_DCPLX:
		len *= sizeof(double) * 2;
		break;

	default:
		cc = PvmBadParam;
		break;
	}

	if (!cc) {
#if 0
		if ((tid & pvmtidhmask) == (pvmmytid & pvmtidhmask)
		&& (tid & ~pvmtidhmask) != TIDPVMD) {		/* to local task */
			char *dbuf;

			len += sizeof(struct shmpghdr);
			if ((dbuf = dynbuf(tid, len)) != (char *)-1L) {
				BCOPY(cp, dbuf, len);

			} else
				cc = PvmSysErr;

		} else 
#endif /*0*/
		{
			sbf = pvm_setsbuf(pvm_mkbuf(PvmDataRaw));
			pvm_pkbyte((char *)cp, len, 1);
			if ((cc = pvm_send(tid, tag)) > 0)
				cc = 0;
			pvm_freebuf(pvm_setsbuf(sbf));
		}
	}
	if (cc < 0)
		lpvmerr("pvm_psend", cc);
	return cc;
}

