
/*
 *         PVM version 3.3:  Parallel Virtual Machine System
 *               University of Tennessee, Knoxville TN.
 *           Oak Ridge National Laboratory, Oak Ridge TN.
 *                   Emory University, Atlanta GA.
 *      Authors:  A. L. Beguelin, J. J. Dongarra, G. A. Geist,
 *    W. C. Jiang, R. J. Manchek, B. K. Moore, and V. S. Sunderam
 *                   (C) 1992 All Rights Reserved
 *
 *                              NOTICE
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted
 * provided that the above copyright notice appear in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * Neither the Institutions (Emory University, Oak Ridge National
 * Laboratory, and University of Tennessee) nor the Authors make any
 * representations about the suitability of this software for any
 * purpose.  This software is provided ``as is'' without express or
 * implied warranty.
 *
 * PVM version 3 was funded in part by the U.S. Department of Energy,
 * the National Science Foundation and the State of Tennessee.
 */

/*
 *  pvmdmimd.c:  routines for creating and managing node tasks
 *
 *
 * July 8 14:13:54 EDT 1993
 * 		fixed bug in mpp_mcast which caused it to fail when count is
 *		not the power of 2
 * 
 * Jul 12 23:23:06 EDT 1993
 *		deleted loclinput(), and merged loclinpkt() into pvmd.c
 *
$Log: pvmdmimd.c,v $
 * Revision 1.4  1994/11/07  21:21:28  manchek
 * Modify mpp_input() to handle DataInPlace properly.
 * Fix a bug in the way cube id is computed in mpp_mcast()
 *
 * Revision 1.3  1994/06/03  20:54:44  manchek
 * version 3.3.0
 *
 * Revision 1.2  1993/12/20  15:39:50  manchek
 * patch 6 from wcj
 *
 * Revision 1.1  1993/08/30  23:35:43  manchek
 * Initial revision
 *
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/stat.h>
#include <errno.h>
#include <stdio.h>
#include <signal.h>
#ifdef  SYSVSTR
#include <string.h>
#else
#include <strings.h>
#endif
#include <cube.h>
#include "pvm3.h"
#include "global.h"
#include "ddpro.h"
#include "tdpro.h"
#include "protoglarp.h"
#include "pvmalloc.h"
#include "host.h"
#include "pvmfrag.h"
#include "mesg.h"
#include "pkt.h"
#include "task.h"
#include "listmac.h"
#include "pvmdmp.h"
#include "pvmmimd.h"
#include "bfunc.h"

#ifndef MAXPATHLEN
#define MAXPATHLEN  CANBSIZ
#endif

#define PVMBINPATH		"pvm3/bin"

#ifndef min
#define min(a,b)	((a)<(b)?(a):(b))
#endif

char *getenv();

/* global */

extern void     pvmbailout();
extern short    qm_cubeid;		/* from libhost.a */
extern int      debugmask;		/* from pvmd.c */
extern char 	**epaths;   	/* from pvmd.c */
extern int 		myhostpart;		/* from pvmd.c */
extern int 		myndf;			/* from pvmd.c */
extern struct htab *hosts;		/* from pvmd.c */
extern int		tidhmask;		/* from lpvm.c */
extern struct task *locltasks;	/* from task.c */
extern int 		log_fd;			/* from pvmlog.c */
extern int		ourudpmtu;		/* from pvmd.c */

int tidtmask = TIDPTYPE;     	/* mask for cube field of tids */
int tidnmask = TIDNODE;   		/* mask for node field of tids */

struct nodeset *busynodes = 0;	/* allocated cubes */


/* private */

static char rcsid[] = "$Id: pvmdmimd.c,v 1.4 1994/11/07 21:21:28 manchek Exp $";
static char     etext[512];		/* scratch for error log */
static long 	isendmid = -1;	/* msg ID returned by isend() */
static struct pkt *outpkt = 0;	/* packet being sent */

/* make list head */
void
mpp_init(argc, argv)
	int *argc;
	char **argv;
{
	busynodes = TALLOC(1, struct nodeset, "cube");
	BZERO((char *) busynodes, sizeof(struct nodeset));
	busynodes->n_link = busynodes;
	busynodes->n_rlink = busynodes;
}

/* get a new cube and create a new entry in cube table; return 0 if failed */
struct nodeset *
mpp_new(count, ptid)
{
	struct nodeset  *sp;
	static int serial;		/* serial number of cube; must be < 99999 */
	char cname[CUBENAMELEN];
	char cubetype[8];

	MAKECUBENAME(cname, serial);
	/* redirect stdout (from node tasks) to log file */
	if (log_fd != -1 && !serial)
		dup2(log_fd, 1); 
	serial++;
	sprintf(cubetype, "%d", count);

	if (_getcube(cname, cubetype, NULL, 0, NULL) < 0) {
		pvmlogperror("mpp_new(): getcube");
		return (struct nodeset *)0;
	}
	if (_setpid(PVMDPTYPE) < 0)
		pvmlogperror("mpp_new(): can't setpid\n");
	if (!(sp = TALLOC(1, struct nodeset, "cube"))) {
		pvmlogerror("mpp_new() can't get memory\n");
		pvmbailout(0);
	}
	BZERO((char *) sp, sizeof(struct nodeset));
	sp->n_cid = qm_cubeid;
	sp->n_name = STRALLOC(cname);
	sp->n_ptid = ptid;
	sp->n_alive = sp->n_size = count;
	LISTPUTBEFORE(busynodes, sp, n_link, n_rlink);
	return sp;
}

/* remove a cube struct from active list and free all associated storage */
void
mpp_free(tid)
	int tid;
{
	struct nodeset *sp;
	int cid;

	cid = (tid & tidtmask) >> (ffs(tidtmask)-1);
	for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link)
		if (sp->n_cid == cid)
			break;
	if (sp != busynodes && --sp->n_alive == 0) {
		if (debugmask & PDMNODE) {
			sprintf(etext, "mpp_free() release c%d\n", cid);
			pvmlogerror(etext);
		}
		if (sp->n_cid != qm_cubeid)
			_attachcube(sp->n_name);
		_killcube(-1, 0);
		_relcube(sp->n_name);
		if (sp->n_link && sp->n_rlink)
			LISTDELETE(sp, n_link, n_rlink);
		if (sp->n_name)
			PVM_FREE(sp->n_name);
		PVM_FREE(sp);
	}
/*
sprintf(etext, "alive=%d\n", sp->n_alive);
pvmlogerror(etext);
*/
}

mpp_load(flags, name, argv, count, tids, ptid)
    int flags;              /* exec options */
    char *name;             /* executable */
    char **argv;            /* arg list (argv[-1] must be there) */
    int count;              /* how many */
    int tids[];             /* array to store new tids */
    int ptid;               /* parent task ID */
{
	char path[MAXPATHLEN];
	struct stat sb;
	char **ep;
	int err;
	struct nodeset *sp;
	int pvminfo[5];		/* proto, myset, parent tid, frag size, NDF */
	int mysetpart;
	int i;
	struct task *tp;

	for (ep = epaths; *ep; ep++) {
		if ((*ep)[0] != '/') {
			(void) strcpy(path, getenv("HOME"));	/* needs absolute path */
			(void) strcat(path, "/");
			(void) strcat(path, *ep);
		} else
			(void) strcpy(path, *ep);
		(void) strcat(path, "/");
		(void) strncat(path, name, sizeof(path) - strlen(path) - 1);

		if (stat(path, &sb) == -1
		    || ((sb.st_mode & S_IFMT) != S_IFREG)
		    || !(sb.st_mode & S_IEXEC)) {
			continue;
		}

		/* file found */
		if (!(sp = mpp_new(count, ptid))) {
			err = PvmOutOfRes;
			goto done;
		}
		mysetpart = myhostpart + (qm_cubeid << (ffs(tidtmask) - 1)) | TIDONNODE;
		pvminfo[0] = TDPROTOCOL;
		pvminfo[1] = mysetpart;
		pvminfo[2] = ptid;
		pvminfo[3] = MAXFRAGSIZE;
		pvminfo[4] = myndf;
		HTOCL(pvminfo,5);

		/* load processes */
		if (count == (1 << (ffs(count) - 1))) { 		/* power of 2 */
			if (_load(path, -1, 0) < 0) {
				pvmlogperror("mpp_load(): load");
				_relcube(sp->n_name);
				err = PvmDSysErr;
				goto done;
			}
			if (_csend(PMTHOST, pvminfo, sizeof(pvminfo), -1, 0) < 0)
				pvmlogperror("mpp_load(): csend");
		} else {
			for (i = 0; i < count; i++) {
				if (_load(path, (long)i, 0) < 0) {
					sprintf(etext, "loading node %d", i);
					pvmlogperror(etext);
					err = PvmDSysErr;
					goto done;
				}
				if (_csend(PMTHOST, pvminfo, sizeof(pvminfo), (long)i, 0) < 0)
						pvmlogperror("csend");
			}
		}

		/* create new task structs */
		for (i = 0; i < count; i++) {
			if (_crecv(PMTPHYS, &tids[i], sizeof(int)) < 0) {
				pvmlogperror("mpp_load(): crecv tids");
				tids[i] = PvmDSysErr;
			} else {
				CTOHL(&tids[i],1);
				tids[i] += mysetpart;
				tp = task_new(tids[i]);
				tp->t_a_out = STRALLOC(name);
				tp->t_ptid = ptid;
				tp->t_flag |= TF_CONN;      /* no need for the auth crap */
			}
		}
		if (debugmask & PDMTASK) {
			sprintf(etext, "mpp_load() %d node#%d... cubeid=%d ptid=%d\n", 
				count, tids[0]-mysetpart, sp->n_cid, ptid);
			pvmlogerror(etext);
		}
		return 0;
	}
	if (debugmask & PDMTASK) {
		sprintf(etext, "mpp_load() didn't find <%s> in %s\n", name, *epaths);
		pvmlogerror(etext);
	}
	err = PvmNoFile;

done:
	for (i = 0; i < count; i++)
		tids[i] = err;
	return err;
}


/* input from node tasks */
void
mpp_input()
{
	char *cp;
	struct pkt *pp = 0;
	int hdrdiff = 0;		/* extra space needed for DD frag header */
	int len;

	if ((len = _infocount()) < 0)
		pvmlogerror("mpp_input() infocount");
	if (DDFRAGHDR > TDFRAGHDR)
		hdrdiff = DDFRAGHDR - TDFRAGHDR;
	pp = pk_new(len + hdrdiff);
	pp->pk_len = len;
	pp->pk_dat += hdrdiff;
	if (_crecv(-1, pp->pk_dat, (long)len) < 0)
		pvmlogperror("mpp_input() crecv");
	else {
		cp = pp->pk_dat;
		if ((len = pvmget32(cp + 8) + TDFRAGHDR) != pp->pk_len) {
			struct pkt *hdr;

			/* inplace frag, head & body sent separately */
			hdr = pp;
			pp = pk_new(len + hdrdiff);
			pp->pk_dat += hdrdiff;
			BCOPY(hdr->pk_dat, pp->pk_dat, hdr->pk_len);
			pp->pk_len = len;
			/* type is virtual node number */
			if (_crecv(_infonode(), pp->pk_dat + hdr->pk_len,
			(long)(len - hdr->pk_len)) < 0) {
				pvmlogperror("mpp_input() crecv body");
				return;
			}
			pk_free(hdr);
		}
	loclinpkt(0, pp);
}
}


/* use csend() to multicast */
/* XXX cannot mcast to multiple cubes */
int
mpp_mcast(pp, tids, ntask)
	struct pkt *pp;	/* packet to send */
	int tids[];		/* target tasks */
	int ntask;		/* how many */
{
	int i;
	int count = 0;
	struct nodeset *sp;
	short cid = -1;
	int cubepart;
	long *nodes = 0;
	char *cp;

	nodes = TALLOC(ntask, long, "nodes");
	for (i = 0; i < ntask; i++) {
/*
		if ((tids[i] & tidtmask) != cubepart 
				|| (tids[i] & tidhmask) != myhostpart)
*/
		if ((tids[i] & tidhmask) != myhostpart)
			continue;
        if (!TIDISNODE(tids[i])) {      /* Arrrgh!!! */
            struct pkt *pp2;
            struct task *tp;

            if (tp = task_find(tids[i])) {
                pp2 = pk_new(0);
                pp2->pk_src = pp->pk_src;
                pp2->pk_dst = pp->pk_dst;
                pp2->pk_flag = pp->pk_flag;
                pp2->pk_buf = pp->pk_buf;
                pp2->pk_max = pp->pk_max;
                pp2->pk_dat = pp->pk_dat;
                pp2->pk_len = pp->pk_len;
                da_ref(pp->pk_buf);
                pkt_to_task(tp, pp2);
            }
			continue;
		}
		if (cid == -1) {
			cid = TIDTOTYPE(tids[i]);
			cubepart = tids[i] & tidtmask;
		}
		if (cid != TIDTOTYPE(tids[i]))
			pvmlogerror("mpp_mcast(): multiple cubes? Not implemented ...\n");
		nodes[count++] = tids[i] & tidnmask;
	}
	for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link)
		if (sp->n_cid == cid) {
			if (cid != qm_cubeid &&
			(_attachcube(sp->n_name) < 0 || _setpid(PVMDPTYPE) < 0))
				pvmlogperror("mpp_output() attachcube/setpid");
			break;
		}
	if (sp == busynodes) {
		sprintf(etext, "mpp_mcast() pkt from t%x scrapped (no cube)\n", 
		pp->pk_src);
		pvmlogerror(etext);
		return -1;
	}

	if (count) {
		cp = (pp->pk_dat -= TDFRAGHDR);
        pvmput32(cp, pp->pk_dst);
        pvmput32(cp + 4, pp->pk_src);
        pvmput32(cp + 8, pp->pk_len);
        pvmput8(cp + 12, pp->pk_flag & (FFSOM|FFEOM));
        pp->pk_len += TDFRAGHDR;
		if (count == sp->n_size && count == (1 << (ffs(count) - 1))) {
			if (_csend(PMTPACK, pp->pk_dat, (long)pp->pk_len, -1, 0) < 0)
				pvmlogperror("mpp_mcast() global csend");
		} else {
			for (i = 0; i < count; i++)
				if (_csend(PMTPACK, pp->pk_dat, (long)pp->pk_len, nodes[i], 0)
						< 0)
					pvmlogperror("mpp_mcast() csend");
		}
		if (debugmask & PDMMESSAGE) {
			sprintf(etext, "mpp_mcast() pkt from t%x len=%d to cube=%d %d\n",
				pp->pk_src, pp->pk_len, cid, count);
			pvmlogerror(etext);
		}
		PVM_FREE(nodes);
	}
	return 0;
}


/* output to node tasks */
void
mpp_output(tp, pp)
	struct task *tp;
	struct pkt *pp;
{
    char *cp;
	long node;				/* node number */
	int cid;				/* cube ID */
	struct nodeset *sp;

	node = tp->t_tid & tidnmask;
	cid = (tp->t_tid & tidtmask) >> (ffs(tidtmask) - 1);
	if (cid != qm_cubeid) {
		for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link)
			if (sp->n_cid == cid) {
				if (_attachcube(sp->n_name) < 0 || _setpid(PVMDPTYPE) < 0)
					pvmlogperror("mpp_output() attachcube/setpid");
				break;
			}
		if (sp == busynodes) {
			sprintf(etext, "mpp_output() pkt to t%x scrapped (no cube %d)\n",
				tp->t_tid, cid);
			pvmlogerror(etext);
			pk_free(pp);
			goto done;
		}
	}
	if (debugmask & PDMPACKET) {
		sprintf(etext,
			"mpp_output() src t%x dst t%x ff %x len %d cube=%ld\n",
			pp->pk_src, pp->pk_dst, pp->pk_flag & (FFSOM|FFEOM),
			pp->pk_len, cid);
		pvmlogerror(etext);
	}
	cp = (pp->pk_dat -= TDFRAGHDR);
	pvmput32(cp, pp->pk_dst);
	pvmput32(cp + 4, pp->pk_src);
	pvmput32(cp + 8, pp->pk_len);
	pvmput8(cp + 12, pp->pk_flag & (FFSOM|FFEOM));
	pp->pk_len += TDFRAGHDR;

	if (isendmid >= 0) {
		_msgwait(isendmid);
		if (outpkt) {
			pk_free(outpkt); 
			outpkt = 0;
		}
	}
	if ((isendmid = _isend(PMTPACK, cp, (long)pp->pk_len, node, 0)) < 0) {
		sprintf(etext,"mpp_output() can't send to t%x", tp->t_tid);
		pvmlogperror(etext);
		pk_free(pp);
		mpp_free(tp->t_tid);
		tp = tp->t_rlink;
		task_cleanup(tp->t_link);
		task_free(tp->t_link);
	} else 
		outpkt = pp;

done:
	if (tp->t_flag & TF_CLOSE) {
		mpp_free(tp->t_tid);
		tp = tp->t_rlink;
/* XXX tm_exit() also calls task_cleanup(); should only be done once */
		task_cleanup(tp->t_link);
		task_free(tp->t_link);
	}
}

/* probe for messages from node tasks */
int
mpp_probe()
{
	struct nodeset *sp;

	for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link) {
		if (sp->n_cid != qm_cubeid)
			if (_attachcube(sp->n_name) < 0 || _setpid(PVMDPTYPE) < 0)
				pvmlogperror("mpp_probe() attachcube/setpid");
		if (_iprobe(-1) == 1)
			return 1;
	}
	return 0;
}

void
mpp_kill(tp, signum)
	struct task *tp;
	int signum;
{
	struct nodeset *sp;
	int cid;

	if (TIDISNODE(tp->t_tid)) {
		if (signum == SIGKILL || signum == SIGTERM) {
			if ((cid = (tp->t_tid & tidtmask) >> (ffs(tidtmask) - 1)) 
					!= qm_cubeid) {
				for (sp = busynodes->n_link; sp != busynodes; sp = sp->n_link)
					if (sp->n_cid == cid) {
						if (_attachcube(sp->n_name) < 0 
								|| _setpid(PVMDPTYPE) < 0)
							pvmlogperror("mpp_kill() attachcube/setpid");
						break;
					}
			}
			if (_killproc(tp->t_tid & tidnmask, 0) < 0)
				pvmlogperror("mpp_kill() killproc");
			if (debugmask & PDMNODE) {
				sprintf(etext, "mpp_kill() zap t%x on c%d\n", tp->t_tid, cid);
				pvmlogerror(etext);
			}
       	    mpp_free(tp->t_tid);
			tp = tp->t_rlink;
/* XXX tm_exit() also calls task_cleanup(); should only be done once */
			task_cleanup(tp->t_link);
			task_free(tp->t_link);
		} else {
			sprintf(etext,"mpp_kill() signal %d to node t%x ignored\n",
				signum, tp->t_tid);
			pvmlogerror(etext);
		}
	} else
		(void)kill(tp->t_pid, signum);
}


/* break pkt into smaller fragments and put them on txq */
/*
pkt_to_task(tp, pp)
	struct task *tp;
	struct pkt *pp;
{
	struct pkt *pp2;
	int lim = ourudpmtu - TDFRAGHDR;
	char *cp = pp->pk_dat;
	int togo;
	int n;
	int ff = pp->pk_flag & FFSOM;
	int fe = pp->pk_flag & FFEOM;

	for (togo = pp->pk_len; togo > 0; togo -= n) {
		n = min(togo, lim);
		pp2 = pk_new(0);
		pp2->pk_src = pp->pk_src;
		pp2->pk_dst = pp->pk_dst;
		if (n == togo)
			ff |= fe;
		pp2->pk_flag = ff;
		ff = 0;
		pp2->pk_buf = pp->pk_buf;
		pp2->pk_max = pp->pk_max;
		pp2->pk_dat = cp;
		pp2->pk_len = n;
		da_ref(pp->pk_buf);
		cp += n;
		LISTPUTBEFORE(tp->t_txq, pp2, pk_link, pk_rlink);
	}
	pk_free(pp);
	return 0;
}
*/
