
/*
 *         PVM version 3.3:  Parallel Virtual Machine System
 *               University of Tennessee, Knoxville TN.
 *           Oak Ridge National Laboratory, Oak Ridge TN.
 *                   Emory University, Atlanta GA.
 *      Authors:  A. L. Beguelin, J. J. Dongarra, G. A. Geist,
 *    W. C. Jiang, R. J. Manchek, B. K. Moore, and V. S. Sunderam
 *                   (C) 1992 All Rights Reserved
 *
 *                              NOTICE
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted
 * provided that the above copyright notice appear in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * Neither the Institutions (Emory University, Oak Ridge National
 * Laboratory, and University of Tennessee) nor the Authors make any
 * representations about the suitability of this software for any
 * purpose.  This software is provided ``as is'' without express or
 * implied warranty.
 *
 * PVM version 3 was funded in part by the U.S. Department of Energy,
 * the National Science Foundation and the State of Tennessee.
 */

/*
 *	mpp.c
 *
 *  MPP interface.
 *
 *		void mpp_init(int argc, char **argv):	
 *			Initialization. Create a table to keep track of active nodes.
 *			argc, argv: passed from main.
 *
 *		int mpp_load(int flags, char *name, char *argv, int count, int tids[],
 *					int ptid):
 *			Load executable onto nodes; create new entries in task table,
 *			encode node number and process type into task IDs, etc.
 *
 *				Construction of Task ID:
 *
 *				 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1
 *				+-+-+---------------------+-+-----------+---------------------+
 *				|s|g|		host index	  |n| instance  |	node # (2048)	  |
 *				+-+-+---------------------+-+-----------+---------------------+
 *
 *				The "n" bit is set for node task but clear for host task.
 *
 *			flags:	exec options;
 *			name:	executable to be loaded;
 *			argv:	command line argument for executable
 *			count:	number of tasks to be created;
 *			tids:	array to store new task IDs;
 *			ptid:	parent task ID.
 *
 *			mpp_new(int count, int ptid):		
 *				Allocate a set of nodes. (called by mpp_load())
 *				count: number of nodes;  ptid: parent task ID.
 *
 *		void mpp_output():	
 *			Send all pending packets to nodes via native send. Node number
 *			and process type are extracted from task ID.
 *
 *		int mpp_mcast(int src, struct pkt pp, int tids[], int ntask):	
 *			Global send.
 *			src:	source task ID;
 *			pp:		packet;
 *			tids:	list of destination task IDs;
 *			ntask:	how many.
 *
 *		int mpp_probe():	
 *			Probe for pending packets from nodes (non-blocking). Returns
 *			1 if packets are dectected, otherwise 0.
 *
 *		void mpp_input():	
 *			Receive pending packets (from nodes) via native recv.
 *
 *		struct task *mpp_find(int pid):
 *			Find a task in task table by its Unix pid.
 *
 *		void mpp_free(int tid):
 *			Remove node/process-type from active list.
 *			tid: task ID.
 *
 */

/*
 * Jul 12 23:57:07 EDT 1993
 *      deleted loclinput(), and merged loclinpkt() into pvmd.c
 *
$Log: pvmdmimd.c,v $
 * Revision 1.4  1994/11/07  21:30:45  manchek
 * Modify mpp_output() and mpp_mcast() to send a null packet to alert precv().
 * Modify mpp_input() to handle DataInPlace properly.
 * mpp_new() should return PvmOutOfRes when it runs out of nodes.
 * Fix a bug in the way ptype is computed in mpp_mcast()
 *
 * Revision 1.3  1994/06/03  20:54:52  manchek
 * version 3.3.0
 *
 * Revision 1.2  1993/12/20  15:39:49  manchek
 * patch 6 from wcj
 *
 * Revision 1.1  1993/08/30  23:36:09  manchek
 * Initial revision
 *
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/stat.h>
#include <errno.h>
#include <stdio.h>
#ifdef  SYSVSTR
#include <string.h>
#else
#include <strings.h>
#endif
#include <nx.h>

#include "pvm3.h"
#include "global.h"
#include "ddpro.h"
#include "tdpro.h"
#include "protoglarp.h"
#include "pvmalloc.h"
#include "host.h"
#include "pvmfrag.h"
#include "mesg.h"
#include "pkt.h"
#include "task.h"
#include "listmac.h"
#include "pvmdmp.h"
#include "pvmmimd.h"
#include "bfunc.h"

#ifndef min
#define min(a,b)	((a)<(b)?(a):(b))
#endif

/* Global */

extern char **environ;

extern int debugmask;			/* from pvmd.c */
extern char **epaths;			/* from pvmd.c */
extern int myhostpart;			/* from pvmd.c */
extern int myndf;				/* from pvmd.c */
extern struct htab *hosts;		/* from pvmd.c */
extern int tidhmask;			/* from pvmd.c */
extern int ourudpmtu;			/* from pvmd.c */

int partsize;					/* size of partition */
int tidtmask = TIDPTYPE;		/* mask for ptype field of tids */
int tidnmask = TIDNODE;			/* mask for node field of tids */

/* private */

static char rcsid[] = "$Id: pvmdmimd.c,v 1.4 1994/11/07 21:30:45 manchek Exp $";
static struct nodeset *busynodes;	/* active nodes; ordered by proc type */
static char etext[512];			/* scratch for error log */
static int ptypemask;			/* mask; we use these bits of ptype in tids */
static long isendmid = -1;		/* msg ID returned by isend() */
static struct pkt *outpkt = 0;	/* packet being sent */


void
mpp_init(argc, argv)
	int *argc;
	char **argv;
{
	if ((partsize = nx_initve((char *)0, 0, "", argc, argv)) < 0) {
		pvmlogperror("mpp_init() nx_initve\n");
		pvmbailout();
	}
	sprintf(etext, "using %d nodes\n", partsize);
	pvmlogerror(etext);

	busynodes = TALLOC(1, struct nodeset, "nsets");
	BZERO((char*)busynodes, sizeof(struct nodeset));
	busynodes->n_link = busynodes;
	busynodes->n_rlink = busynodes;

	ptypemask = tidtmask >> (ffs(tidtmask) - 1);
}

/*
 * allocate a set of nodes; assign ptype sequentially, but give the same
 * ptype to all siblings if possible (so they can send msg directly)
 */
struct nodeset *
mpp_new(count, ptid)
	int count;		/* number of nodes requested */
	int ptid;		/* parent's tid */
{
	struct nodeset *sp, *newp;
	int last = -1;
	long ptype = busynodes->n_ptype;
	int i;

ptype = 0;

	if (!(newp = TALLOC(1, struct nodeset, "nsets"))) {
		pvmlogerror("nodes_new() can't get memory\n");
		pvmbailout(0);
	}
	BZERO((char*)newp, sizeof(struct nodeset));
	
	newp->n_size = count;

	for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link) {
		if (sp->n_ptype == ptype) {
/*
			if (sp->n_ptid != ptid) {
				ptype = sp->n_ptype + 1;
				last = -1;
				continue;
			}
*/
			if (sp->n_first - last > count)
				goto done;
			else {
				last = sp->n_first + sp->n_size - 1;
				if (sp->n_link == busynodes && partsize - last > count)
					goto done;
			}
		} else if (sp->n_ptype > ptype) {
			if (partsize - last > count)
				goto done;
			else {
				ptype = sp->n_ptype;	/* go to the next layer */
				last = -1;
			}
		}
	}

#if 0
	if ((sp = busynodes->n_rlink) != busynodes) {
		last = -1;
		ptype = sp->n_ptype + 1;
	}
	for (i = 1; i < NUMPTYPE; i++, ptype++) {
		for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link)
			if ((sp->n_ptype & ptypemask) == (ptype & ptypemask))
			/* these bits are used in tid's type field; must be unique */
				break;
		if (sp == busynodes)
			goto done;		
	}
	pvmlogerror("mpp_new() out of descriptors: too many spawns\n");
#endif /*0*/

	if (partsize - last <= count) {
		pvmlogerror("mpp_new() not enough nodes in partition\n");
		return (struct nodeset *)0;
	}

done:
	if (debugmask & PDMNODE) {
		sprintf(etext, "mpp_new() %d nodes %d ... ptype=%d ptid=%x\n",
			count, last+1, ptype, ptid);
		pvmlogerror(etext);
	}
	newp->n_first = last + 1;
	newp->n_ptype = ptype;
	newp->n_alive = count;
	newp->n_ptid = ptid;
	LISTPUTAFTER(sp, newp, n_link, n_rlink);

	return newp;
}

/* remove nodes/ptype from active list */
void
mpp_free(tid)
	int tid;
{
	struct nodeset *sp;
	int node;
	int ptype;

	if (!TIDISNODE(tid))
		return;

	node = tid & tidnmask;
	ptype = TIDTOTYPE(tid);
	for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link) {
		if ((sp->n_ptype & ptypemask) == ptype && node >= sp->n_first
				&& node - sp->n_first < sp->n_size) {

			if (debugmask & PDMNODE) {
				sprintf(etext, "mpp_free() t%x type=%ld alive=%d\n",
					tid, sp->n_ptype, sp->n_alive);
				pvmlogerror(etext);
			}
			if (--sp->n_alive == 0) {
#if 0
				if (busynodes->n_ptype <= sp->n_ptype)
				/* ptype cannot be recycled */
					busynodes->n_ptype = sp->n_ptype + 1;
#endif
				LISTDELETE(sp, n_link, n_rlink);
				PVM_FREE(sp);
			}
			return;
		}
	}
	sprintf(etext, "mpp_free() t%x not active\n", tid);
	pvmlogerror(etext);
}

/* load executable onto the given set of nodes */
int
mpp_load(flags, name, argv, count, tids, ptid)
	int flags;              /* exec options */
	char *name;             /* executable */
	char **argv;            /* arg list (argv[-1] must be there) */
	int count;				/* how many */
	int tids[];				/* array to store new tids */
	int ptid;				/* parent task ID */
{
    char path[MAXPATHLEN];
    struct stat sb;
    char **ep;
	static int first = 1;
	int j;
	int pvminfo[SIZEHINFO];	/* proto, myset, parent tid, frag size, NDF */
	int ptypepart;
	struct task *tp;
	long *pids = 0;			/* array of OSF/1 process IDs */
	int err = 0;
	long *nodes = 0;
	struct nodeset *sp;
	long nnodes;			/* number of nodes to needed */

    for (ep = epaths; *ep; ep++) {
    	/* search for file */
        (void)strcpy(path, *ep);
        (void)strcat(path, "/");
        (void)strncat(path, name, sizeof(path) - strlen(path) - 1);

        if (stat(path, &sb) == -1
                || ((sb.st_mode & S_IFMT) != S_IFREG)
                || !(sb.st_mode & S_IEXEC)) {
            if (debugmask & PDMTASK) {
                sprintf(etext, "mpp_load() stat failed <%s>\n", path);
                pvmlogerror(etext);
            }
            continue;
        }

		pids = TALLOC(partsize, long, "pids");
		if (!(sp = mpp_new(count, ptid))) {
			err = PvmOutOfRes;
			goto done;
		}
		nnodes = (count == partsize) ? -1 : count;
		nodes = TALLOC(count, long, "nodes");
		for (j = 0; j < count; j++)
			nodes[j] = sp->n_first + j;

		err = nx_loadve(nodes, nnodes, sp->n_ptype, pids, path, argv, environ);
		if (err < count) {
			sprintf(etext, "mpp_load() loaded only %d <%s>\n", err, path);
			pvmlogerror(etext);
			err = PvmDSysErr;
			goto done;
		}
		if (first) {
			if (_setptype(PVMDPTYPE) < 0)
        		pvmlogperror("mpp_load() setptype to PVMDPTYPE");
			first = 0;
		}

		ptypepart = ((sp->n_ptype & ptypemask) << (ffs(tidtmask) - 1)) 
					| TIDONNODE;
		pvminfo[0] = TDPROTOCOL;
		pvminfo[1] = myhostpart + ptypepart;
		pvminfo[2] = ptid;
		pvminfo[3] = MAXFRAGSIZE;
		pvminfo[4] = myndf;
		if (count == partsize)
			err = _csend(PMTHOST, pvminfo, sizeof(pvminfo), -1, sp->n_ptype);
		else {
		if (sp->n_ptype != PVMDPTYPE && _setptype(sp->n_ptype) < 0)
			pvmlogperror("mpp_load() setptype");
		if (_gsendx(PMTHOST, pvminfo, sizeof(pvminfo), nodes, (long)count)
				< 0) {
			pvmlogperror("mpp_load() send");
			err = PvmDSysErr;
			goto done;
		}
		if (sp->n_ptype != PVMDPTYPE)
			_setptype(PVMDPTYPE);
		}

		if (nodes)
			PVM_FREE(nodes);

		if (debugmask & PDMTASK) {
			sprintf(etext, "mpp_load() %d type=%ld ptid=%x pid%ld... t%x...\n",
				count, sp->n_ptype, ptid, pids[0], 
				myhostpart + ptypepart + sp->n_first);
			pvmlogerror(etext);
		}

		/* create new task structs */

		for (j = 0; j < count; j++) {
			if (pids[j] > 0) {
				tp = task_new(myhostpart + ptypepart + sp->n_first + j);
				task_setpid(tp, pids[j]);
/*
sprintf(etext, "mpp_load() pid=%ld\n", pids[j]);
pvmlogerror(etext);
*/
				tp->t_a_out = STRALLOC(name);
				tp->t_ptid = ptid;
				tp->t_flag |= TF_CONN;		/* no need for the auth crap */
				tids[j] = tp->t_tid;
			} else
				tids[j] = PvmDSysErr;
		}
		if (pids)
			PVM_FREE(pids);
		return 0;
	}
	if (debugmask & PDMTASK) {
		sprintf(etext, "mpp_load() didn't find <%s>\n", name);
		pvmlogerror(etext);
	}
	err = PvmNoFile;

done:
	for (j = 0; j < count; j++)
		tids[j] = err;
	if (pids)
		PVM_FREE(pids);
	return err;
}


/* input from node tasks */
void
mpp_input()
{
	char *cp;
	struct pkt *pp = 0;
	int hdrdiff = 0;		/* extra space needed for DD frag header */
	int len;

	if ((len = _infocount()) < 0)
		pvmlogerror("mpp_input() infocount");
	if (DDFRAGHDR > TDFRAGHDR)
		hdrdiff = DDFRAGHDR - TDFRAGHDR;
	pp = pk_new(len + hdrdiff);
	pp->pk_len = len;
	pp->pk_dat += hdrdiff;
	if (_crecv(-1, pp->pk_dat, (long)len) < 0)
		pvmlogperror("mpp_input() crecv");
	else {
		cp = pp->pk_dat;
		if ((len = pvmget32(cp + 8) + TDFRAGHDR) != pp->pk_len) {
			struct pkt *hdr;

			/* inplace frag, head & body sent separately */
			hdr = pp;
			pp = pk_new(len + hdrdiff);
			pp->pk_dat += hdrdiff;
			BCOPY(hdr->pk_dat, pp->pk_dat, hdr->pk_len);
			pp->pk_len = len;
			if (_crecvx(PMTPACK, pp->pk_dat + hdr->pk_len, 
			(long)(len - hdr->pk_len), _infonode(), 0, (long*)0) < 0) {
				pvmlogperror("mpp_input() crecv body");
				return;
			}
			pk_free(hdr);
		}
		loclinpkt(0, pp);
	}
}


/* use gsendx() to multicast */
/* XXX can't multicast to tasks with different ptypes */
int
mpp_mcast(pp, tids, ntask)
	struct pkt *pp;	/* packet to send */
	int tids[];		/* target tasks */
	int ntask;		/* how many */
{
	int i;
	long count = 0;
	struct nodeset *sp;
	long ptype = -1;
	int ptypepart;
	long *nodes = 0;
	char *cp;

	nodes = TALLOC(ntask, long, "nodes");
	for (i = 0; i < ntask; i++) {
/*
		if ((tids[i] & tidtmask) != ptypepart 
				|| (tids[i] & tidhmask) != myhostpart)
*/
		if ((tids[i] & tidhmask) != myhostpart)
			continue;
        if (!TIDISNODE(tids[i])) {      /* Arrrgh!!! */
            struct pkt *pp2;
            struct task *tp;

            if (tp = task_find(tids[i])) {
                pp2 = pk_new(0);
                pp2->pk_src = pp->pk_src;
                pp2->pk_dst = pp->pk_dst;
                pp2->pk_flag = pp->pk_flag;
                pp2->pk_buf = pp->pk_buf;
                pp2->pk_max = pp->pk_max;
                pp2->pk_dat = pp->pk_dat;
                pp2->pk_len = pp->pk_len;
                da_ref(pp->pk_buf);
                pkt_to_task(tp, pp2);
            }
            continue;
        } 
		if (ptype == -1) {
			ptype = TIDTOTYPE(tids[i]);
			ptypepart = tids[i] & tidtmask;
		}
		if (ptype != TIDTOTYPE(tids[i]))
			pvmlogerror("mpp_mcast(): multiple ptypes? Oh no ..\n");
		nodes[count++] = tids[i] & tidnmask;
	}
	for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link)
		if ((sp->n_ptype & ptypemask) == ptype) {
			ptype = sp->n_ptype;
			break;
		}
	if (sp == busynodes) {
		sprintf(etext, "mpp_mcast() pkt from t%x scrapped (no cube)\n", 
		pp->pk_src);
		pvmlogerror(etext);
		return -1;
	}

	if (count) {
		if (debugmask & PDMMESSAGE) {
			sprintf(etext, "mpp_mcast() pkt from t%x len=%d to type=%ld %ld\n",
					pp->pk_src, pp->pk_len, ptype, count);
			pvmlogerror(etext);
		}
		cp = (pp->pk_dat -= TDFRAGHDR);
        pvmput32(cp, pp->pk_dst);
        pvmput32(cp + 4, pp->pk_src);
        pvmput32(cp + 8, pp->pk_len);
        pvmput8(cp + 12, pp->pk_flag & (FFSOM|FFEOM));
        pp->pk_len += TDFRAGHDR;
		if (ptype != PVMDPTYPE && _setptype(ptype) < 0)
			pvmlogperror("mpp_mcast() setptype");
		/* inform precv/recv of a packed message */
		if (_gsendx(PMTPACK, cp, 0, nodes, count) < 0)
			pvmlogperror("mpp_mcast() gsendx null msg");
		if (_gsendx(PMTPACK, pp->pk_dat, (long)pp->pk_len, nodes, count) < 0)
			pvmlogperror("mpp_mcast() gsendx");
		if (ptype != PVMDPTYPE)
			_setptype(PVMDPTYPE);
	}
	PVM_FREE(nodes);
	return 0;
}


/* output to node tasks */
void
mpp_output(tp, pp)
	struct task *tp;
	struct pkt *pp;
{
    char *cp;
	long node;				/* node number */
	long ptype;				/* process type */
	struct nodeset *sp;

	node = tp->t_tid & tidnmask;
	ptype = (tp->t_tid & tidtmask) >> (ffs(tidtmask) - 1);
	for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link)
		if ((sp->n_ptype & ptypemask) == ptype) {
			ptype = sp->n_ptype;
			break;
		}
	if (sp == busynodes) {
		sprintf(etext, "mpp_output() pkt to t%x scrapped (no ptype %ld)\n",
			tp->t_tid, ptype);
		pvmlogerror(etext);
		pk_free(pp);
		goto done;
	}

	if (debugmask & PDMPACKET) {
		sprintf(etext,
			"mpp_output() src t%x dst t%x ff %x len %d ptype %d\n",
			pp->pk_src, pp->pk_dst, pp->pk_flag & (FFSOM|FFEOM),
			pp->pk_len, ptype);
		pvmlogerror(etext);
	}
	cp = (pp->pk_dat -= TDFRAGHDR);
	pvmput32(cp, pp->pk_dst);
	pvmput32(cp + 4, pp->pk_src);
	pvmput32(cp + 8, pp->pk_len);
	pvmput8(cp + 12, pp->pk_flag & (FFSOM|FFEOM));
	pp->pk_len += TDFRAGHDR;

	if (isendmid >= 0) {
		_msgwait(isendmid);
		if (outpkt) {
			pk_free(outpkt);
			outpkt = 0;
		}
	}
	/* inform precv/recv of a packed message */
	if (_csend(PMTPACK, cp, 0, node, ptype) < 0) {
		sprintf(etext,"mpp_output() can't send null msg to t%x", tp->t_tid);
		pvmlogperror(etext);
	}
	if ((isendmid = _isend(PMTPACK, cp, (long)pp->pk_len, node, ptype)) < 0) {
		sprintf(etext,"mpp_output() can't send to t%x", tp->t_tid);
		pvmlogperror(etext);
		pk_free(pp);
		mpp_free(tp->t_tid);
		tp = tp->t_rlink;
		task_cleanup(tp->t_link);
		task_free(tp->t_link);
	} else 
		outpkt = pp;

done:
	if (tp->t_flag & TF_CLOSE) {
	    mpp_free(tp->t_tid);
		tp = tp->t_rlink;
/* XXX tm_exit() also calls task_cleanup(); should only be done once */
		task_cleanup(tp->t_link);
		task_free(tp->t_link);
	}
}


/* probe for messages from node tasks */
int
mpp_probe()
{
	if (busynodes->n_link != busynodes)
		return (_iprobe(-1));
	else	/* task queue empty */
		return 0;
}


/* break pkt into smaller fragments and put them on txq */
/*
pkt_to_task(tp, pp)
	struct task *tp;
	struct pkt *pp;
{
	struct pkt *pp2;
	int lim = ourudpmtu - TDFRAGHDR;
	char *cp = pp->pk_dat;
	int togo;
	int n;
	int ff = pp->pk_flag & FFSOM;
	int fe = pp->pk_flag & FFEOM;

	for (togo = pp->pk_len; togo > 0; togo -= n) {
		n = min(togo, lim);
		pp2 = pk_new(0);
		pp2->pk_src = pp->pk_src;
		pp2->pk_dst = pp->pk_dst;
		if (n == togo)
			ff |= fe;
		pp2->pk_flag = ff;
		ff = 0;
		pp2->pk_buf = pp->pk_buf;
		pp2->pk_max = pp->pk_max;
		pp2->pk_dat = cp;
		pp2->pk_len = n;
		da_ref(pp->pk_buf);
		cp += n;
		LISTPUTBEFORE(tp->t_txq, pp2, pk_link, pk_rlink);
	}
	pk_free(pp);
	return 0;
}
*/
