
/*
 *         PVM version 3.3:  Parallel Virtual Machine System
 *               University of Tennessee, Knoxville TN.
 *           Oak Ridge National Laboratory, Oak Ridge TN.
 *                   Emory University, Atlanta GA.
 *      Authors:  A. L. Beguelin, J. J. Dongarra, G. A. Geist,
 *    W. C. Jiang, R. J. Manchek, B. K. Moore, and V. S. Sunderam
 *                   (C) 1992 All Rights Reserved
 *
 *                              NOTICE
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted
 * provided that the above copyright notice appear in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * Neither the Institutions (Emory University, Oak Ridge National
 * Laboratory, and University of Tennessee) nor the Authors make any
 * representations about the suitability of this software for any
 * purpose.  This software is provided ``as is'' without express or
 * implied warranty.
 *
 * PVM version 3 was funded in part by the U.S. Department of Energy,
 * the National Science Foundation and the State of Tennessee.
 */

/*
 *	ddpro.c
 *
 *	Entry points for messages from network.
 *
$Log: ddpro.c,v $
 * Revision 1.7  1994/10/15  19:02:36  manchek
 * cast message tags for comparison as integer.
 * send output and trace open messages for dmp and shmem ports.
 * check newhosts when deleting host
 *
 * Revision 1.6  1994/07/18  19:18:10  manchek
 * hostfailentry() no longer matches pvmd' to t0.
 * wa_dep wasn't propogated from hoststart to htupd
 *
 * Revision 1.5  1994/06/30  21:33:46  manchek
 * addhosts() uses PVM_DPATH envar
 *
 * Revision 1.4  1994/06/21  18:29:21  manchek
 * subscript arith in dmname() broke with opt
 *
 * Revision 1.3  1994/06/04  22:33:16  manchek
 * missed resetting busyadding in hostfailentry(WT_HTUPD)
 *
 * Revision 1.2  1994/06/03  20:38:12  manchek
 * version 3.3.0
 *
 * Revision 1.1  1993/08/30  23:26:47  manchek
 * Initial revision
 *
 */


#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#ifdef	SYSVSTR
#include <string.h>
#else
#include <strings.h>
#endif
#include <netdb.h>
#include <errno.h>
#include <stdio.h>

#include "global.h"
#include "fromlib.h"
#include "tdpro.h"
#include "ddpro.h"
#include <pvmsdpro.h>
#include "protoglarp.h"
#include "pvmalloc.h"
#include "host.h"
#include "mesg.h"
#include "waitc.h"
#include "task.h"
#include "listmac.h"
#if defined(IMA_PGON) || defined(IMA_I860) || defined(IMA_CM5) || defined(SHMEM)
#include "pvmdmp.h"
#endif
#include "bfunc.h"


extern void pvmbailout();
extern char *inadport_hex();
extern char *getenv();
char **colonsep();
char *varsub();

extern int debugmask;				/* from pvmd.c */
extern char **epaths;				/* from pvmd.c */
extern char *debugger;				/* from pvmd.c */
extern struct htab *filehosts;		/* from pvmd.c */
extern struct htab *hosts;			/* from pvmd.c */
extern int hostertid;				/* from pvmd.c */
extern struct task *locltasks;		/* from task.c */
extern char *myarchname;			/* from pvmd.c */
extern int myhostpart;				/* from pvmd.c */
extern int mytid;					/* from pvmd.c */
extern struct htab *newhosts;		/* from pvmd.c */
extern int pprime;					/* from pvmd.c */
extern int runstate;				/* from pvmd.c */
extern int pvmschedtid;				/* from pvmd.c */
extern int tidhmask;				/* from pvmd.c */
extern int tidlmask;				/* from pvmd.c */
extern struct waitc *waitlist;		/* from waitc.c */

int busyadding = 0;					/* lock for addhost op */
int hosterwid = 0;					/* wid waiting on hoster */


/***************
 **  Private  **
 **           **
 ***************/

static char rcsid[] = "$Id: ddpro.c,v 1.7 1994/10/15 19:02:36 manchek Exp $";
static char pvmtxt[512];			/* scratch for error log */

int dm_add();
int dm_addack();
int dm_db();
int dm_dback();
int dm_delhost();
int dm_delhostack();
int dm_halt();
int dm_hostsync();
int dm_hostsyncack();
int dm_htcommit();
int dm_htdel();
int dm_htupd();
int dm_htupdack();
int dm_mca();
int dm_notify();
int dm_notifyack();
int dm_null();
int dm_pstat();
int dm_pstatack();
int dm_sendsig();
int dm_slconf();
int dm_exec();
int dm_execack();
int dm_startack();
int dm_task();
int dm_taskack();
int dm_taskout();

int (*netswitch[])() = {
	dm_add, dm_addack,
	dm_exec, dm_execack,
	dm_sendsig,
	dm_htupd, dm_htupdack,
	dm_htcommit,
	dm_slconf,
	dm_startack,
	dm_task, dm_taskack,
	dm_delhost, dm_delhostack,
	dm_null,
	dm_taskout,
	dm_pstat, dm_pstatack,
	dm_halt,
	dm_mca,
	dm_notify, dm_notifyack,
	dm_db, dm_dback,
	dm_htdel,
	dm_hostsync, dm_hostsyncack,
};


char *dm_names[] = {
	"dm_add", "dm_addack",
	"dm_exec", "dm_execack",
	"dm_sendsig",
	"dm_htupd", "dm_htupdack",
	"dm_htcommit",
	"dm_slconf",
	"dm_startack",
	"dm_task", "dm_taskack",
	"dm_delhost", "dm_delhostack",
	"dm_null",
	"dm_taskout",
	"dm_pstat", "dm_pstatack",
	"dm_halt",
	"dm_mca",
	"dm_notify", "dm_notifyack",
	"dm_db", "dm_dback",
	"dm_htdel",
	"dm_hostsync", "dm_hostsyncack",
};

char *
dmname(code)
	int code;
{
	if (code < (int)DM_FIRST || code > (int)DM_LAST)
		return "unknown";
	code -= DM_FIRST;
	return dm_names[code];
}


/*	hostfailentry()
*
*	We've decided a remote pvmd has failed.
*	Wake up any wait context waiting on that host (different action
*	for each kind of wait).
*	Send a HTDEL message to remaining hosts except us.
*/

int
hostfailentry(hp)
	struct hostd *hp;
{
	int hpart = hp->hd_hostpart;
	struct waitc *wp, *wp2;
	struct mesg *mp;

	if (debugmask & PDMHOST) {
		sprintf(pvmtxt, "hostfailentry() host %s\n", hp->hd_name);
		pvmlogerror(pvmtxt);
		hd_dump(hp);
	}

	if (hp == hosts->ht_hosts[hosts->ht_master]) {
		pvmlogerror("hostfailentry() lost master host, we're screwwwed\n");
		pvmbailout(0);
	}

	/*
	* if we're master pvmd, send HT_DEL message to all others
	*/

	if (hp->hd_hostpart && hosts->ht_master == hosts->ht_local) {
		struct hostd *hp2;
		int hh;

		mp = mesg_new(0);
		mp->m_cod = DM_HTDEL;
		pkint(mp, hosts->ht_serial);
		pkint(mp, hp->hd_hostpart);
		for (hh = hosts->ht_last; hh > 0; hh--)
			if (hh != hosts->ht_local
			&& (hp2 = hosts->ht_hosts[hh]) && hp2 != hp) {
				mp->m_ref++;
				mp->m_dst = hp2->hd_hostpart | TIDPVMD;
				sendmessage(mp);
			}
		mesg_unref(mp);

		/* inform the scheduler too */

		if (pvmschedtid) {
			mp = mesg_new(0);
			mp->m_cod = SM_HOSTX;
			mp->m_dst = pvmschedtid;
			pkint(mp, hp->hd_hostpart | TIDPVMD);
			sendmessage(mp);
		}
	}

	for (wp = waitlist->wa_link; wp != waitlist; wp = wp->wa_link) {
		if (wp->wa_on && (wp->wa_on & TIDHOST) == hpart) {
			switch (wp->wa_kind) {

			case WT_ADDHOST:	/* the master must have died */
			case WT_DELHOST:	/* the master must have died */

				sprintf(pvmtxt, "hostfailentry() can't deal with wait kind %d\n",
						wp->wa_kind);
				pvmlogerror(pvmtxt);
				break;

			case WT_HTUPD:
				if (wp->wa_peer == wp) {
					int hh;

					mp = mesg_new(0);
					mp->m_cod = DM_HTCOMMIT;

					for (hh = hosts->ht_last; hh > 0; hh--)
						if (hh != hosts->ht_local && (hp = hosts->ht_hosts[hh])) {
							mp->m_ref++;
							mp->m_dst = hp->hd_hostpart | TIDPVMD;
							sendmessage(mp);
						}
					mesg_unref(mp);

					busyadding = 0;
					sendmessage(wp->wa_mesg);
					wp->wa_mesg = 0;
				}
				break;

			case WT_SPAWN:
				{
					struct waitc_spawn *wxp;
					int v;

					wxp = (struct waitc_spawn*)wp->wa_spec;

	/* mark tasks assigned to this host as failed */

					for (v = wxp->w_veclen; v-- > 0; )
						if (wxp->w_vec[v] == hp->hd_hostpart)
							wxp->w_vec[v] = PvmHostFail;

					ht_delete(wxp->w_ht, hp);

	/* is this was the last wait, reply to task */

					if (wp->wa_peer == wp)
						assign_tasks(wp);
				}
				break;

			case WT_TASK:

	/* send message if we're the last waiter */

				if (wp->wa_peer == wp) {
					mp = wp->wa_mesg;
					mp->m_ref++;
					sendmessage(mp);
				}
				break;

			case WT_HOSTSTART:

	/* reply to waiter */

				busyadding = 0;
				if (wp->wa_spec) {
					free_waitc_add((struct waitc_add *)wp->wa_spec);
					wp->wa_spec = 0;
				}
				pkint(wp->wa_mesg, PvmDSysErr);
				sendmessage(wp->wa_mesg);
				wp->wa_mesg = 0;
				break;

			case WT_TASKX:
				mp = mesg_new(0);
				mp->m_dst = wp->wa_tid;
				mp->m_cod = wp->wa_dep;
				pkint(mp, wp->wa_on);
				sendmessage(mp);
				break;

			case WT_PSTAT:
			case WT_MSTAT:
				mp = mesg_new(0);
				mp->m_dst = wp->wa_tid;
				if (wp->wa_kind == WT_PSTAT)
					mp->m_cod = TM_PSTAT;
				else
					mp->m_cod = TM_MSTAT;
				pkint(mp, PvmHostFail);
				sendmessage(mp);
				break;

			case WT_HOSTF:
				mp = mesg_new(0);
				mp->m_dst = wp->wa_tid;
				mp->m_cod = wp->wa_dep;
				pkint(mp, wp->wa_on);
				sendmessage(mp);
				break;

			case WT_HOSTSYNC:
				pkint(wp->wa_mesg, PvmHostFail);
				sendmessage(wp->wa_mesg);
				wp->wa_mesg = 0;
				break;

			default:
				sprintf(pvmtxt, "hostfailentry() alien wait kind %d\n",
						wp->wa_kind);
				pvmlogerror(pvmtxt);
				break;
			}

			wp2 = wp->wa_rlink;
			wait_delete(wp);
			wp = wp2;
		}
	}
	return 0;
}


int
netentry(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int c = mp->m_cod;

	if (debugmask & PDMMESSAGE) {
		sprintf(pvmtxt,
				"hostentry() from host %s src t%x dst t%x cod %s wid %d\n",
				hp->hd_name, mp->m_src, mp->m_dst, dmname(c),
				mp->m_wid);
		pvmlogerror(pvmtxt);
/*
		pvmhdump(mp->m_frag->fr_link->fr_dat, mp->m_frag->fr_link->fr_len,
				"hostentry() ");
*/
	}

	if (c < (int)DM_FIRST || c > (int)DM_LAST) {
		sprintf(pvmtxt, "netentry() message from t%x with bogus code %d\n",
				mp->m_src, c);
		pvmlogerror(pvmtxt);
		goto bail;
	}

	c -= DM_FIRST;
	(netswitch[c])(hp, mp);

bail:
	mesg_unref(mp);
	return 0;
}


/*********************************
 **  Pvmd message entry points  **
 **                             **
 *********************************/

/*	hostids_new()
*
*	Get a list of new host tids (ones not it user in host table).
*	Wrap around to lowest number when we reach the maximum.
*	Return as many as are available.
*/

int
hostids_new(num, tids)
	int *num;			/* count requested, returned */
	int *tids;			/* return tids */
{
	static int lasthh = 1;
	int oldhh;
	int maxhostid = tidhmask >> (ffs(tidhmask) - 1);
	int i;

	oldhh = lasthh;

	/* find next free hostid */

	for (i = 0; i < *num; i++) {
		if (++lasthh > maxhostid)
			lasthh = 1;
		while ((lasthh <= hosts->ht_last && hosts->ht_hosts[lasthh])) {
			if (++lasthh > maxhostid)
				lasthh = 1;
			if (lasthh == oldhh)
				goto done;
		}
		tids[i] = lasthh << (ffs(tidhmask) - 1);
/*
		sprintf(pvmtxt, "hostids_new() tid %x\n", tids[i]);
		pvmlogerror(pvmtxt);
*/
	}

done:
	return *num = i;
}


free_waitc_add(wxp)
	struct waitc_add *wxp;
{
	int i;

	for (i = 0; i < wxp->w_num && wxp->w_hosts[i]; i++)
		hd_unref(wxp->w_hosts[i]);
	PVM_FREE(wxp->w_hosts);
	PVM_FREE(wxp);
	return 0;
}


/*	addhosts()
*
*	Add hosts requested with DM_ADD or SM_ADD.  Call the hoster or
*	start 'em.
*/

int
addhosts(mp, rmp)
	struct mesg *mp;	/* the request message */
	struct mesg *rmp;	/* reply message blank */
{
	struct hostd *hp, *hp2;
	struct mesg *mp2;
	struct waitc *wp = 0;
	struct waitc_add *wxp = 0;
	int i, j;
	int count;
	int ngood;
	int ntid;
	struct hostent *he;
	int maxhostid = (tidhmask >> ffs(tidhmask) - 1);
	int hh;
	int pid;
	int *tids;
	char *pvmdpath;
	char buf[512];

	/*
	* have to lock this for 2 reasons:
	*  1. system can't handle overlapping host table updates,
	*  2. the new host tids aren't reserved
	*/
	if (busyadding) {
		pvmlogerror("addhosts() already adding new hosts\n");
		pkint(rmp, PvmAlready);
		sendmessage(rmp);
		return 0;
	}

	busyadding = 1;

	/* sanity check count */

	if (upkint(mp, &count) || count < 1 || count > maxhostid) {
		pvmlogerror("addhosts() bad msg format\n");
		goto bad;
	}

	/*
	* make wait context, extract host list from message,
	*/

	wp = wait_new(WT_HOSTSTART);
	wp->wa_tid = mp->m_src;
	wp->wa_dep = mp->m_wid;
	wxp = TALLOC(1, struct waitc_add, "waix");
	wxp->w_num = count;
	wxp->w_hosts = TALLOC(count, struct hostd *, "waiv");
	BZERO((char*)wxp->w_hosts, count * sizeof(struct hostd *));
	wp->wa_spec = (void *)wxp;

	for (i = 0; i < count; i++) {
		hp = hd_new(0);
		wxp->w_hosts[i] = hp;
		if (upkstr(mp, buf, sizeof(buf))) {
			pvmlogerror("addhosts() bad msg format\n");
			goto bad;
		}
		if (parsehost(buf, hp)) {
			hp->hd_err = PvmBadParam;

		} else {

		/* Set unspecified fields from hostfile if available */

			if (filehosts &&
					((hp2 = nametohost(filehosts, hp->hd_name))
					|| (hp2 = filehosts->ht_hosts[0])))
				applydefaults(hp, hp2);
		}
	}

	/*
	* lookup IP addresses  XXX we already have some of them
	*/

	ngood = 0;
	for (i = 0; i < count; i++) {
		hp = wxp->w_hosts[i];
		if (hp->hd_err)
			continue;

		if (he = gethostbyname(hp->hd_aname ? hp->hd_aname : hp->hd_name)) {
			BCOPY(he->h_addr_list[0], (char*)&hp->hd_sad.sin_addr,
					sizeof(struct in_addr));

		} else {
			if (debugmask & PDMSTARTUP) {
				sprintf(pvmtxt,
						"start_slaves() can't gethostbyname: %s\n",
						hp->hd_name);
				pvmlogerror(pvmtxt);
			}
			hp->hd_err = PvmNoHost;
			continue;
		}

	/* make sure it's not already configured */

		for (hh = hosts->ht_last; hh > 0; hh--)
			if ((hp2 = hosts->ht_hosts[hh])
			&& (hp2->hd_sad.sin_addr.s_addr == hp->hd_sad.sin_addr.s_addr)) {
				hp->hd_err = PvmDupHost;
				break;
			}
		if (hp->hd_err)
			continue;

	/* make sure new ones aren't duplicated */

		for (j = i; j-- > 0; )
			if (hp->hd_sad.sin_addr.s_addr
			== wxp->w_hosts[j]->hd_sad.sin_addr.s_addr) {
				hp->hd_err = PvmDupHost;
				break;
			}
		if (hp->hd_err)
			continue;

		ngood++;
	}

	/*
	* assign tids  XXX these are unreserved until hosts are added...
	*/

	ntid = ngood;
	tids = TALLOC(ngood, int, "xxx");
	hostids_new(&ntid, tids);
	if (ntid < ngood) {
		pvmlogerror("addhosts() out of hostids\n");
		ngood = ntid;
	}
	for (j = i = 0; i < count; i++) {
		hp = wxp->w_hosts[i];
		if (hp->hd_err)
			continue;
		if (j < ntid)
			hp->hd_hostpart = tids[j++];
		else
			hp->hd_err = PvmOutOfRes;
	}
	PVM_FREE(tids);

/*
	if (!ngood) {
		XXX don't really need to send the message
	}
*/

	/* keep stub reply message to caller */

	wp->wa_mesg = rmp;

	/* make request message and send to hoster or pvmd' */

	mp2 = mesg_new(0);
	mp2->m_wid = wp->wa_wid;
	pkint(mp2, ngood);
	if (!(pvmdpath = getenv("PVM_DPATH")))
		pvmdpath = PVMDPATH;
	for (i = 0; i < count; i++) {
		hp = wxp->w_hosts[i];
		if (hp->hd_err)
			continue;
		pkint(mp2, hp->hd_hostpart);
		pkstr(mp2, hp->hd_sopts ? hp->hd_sopts : "");
		if (hp->hd_login)
			sprintf(buf, "%s@%s", hp->hd_login,
					(hp->hd_aname ? hp->hd_aname : hp->hd_name));
		else
			strcpy(buf, (hp->hd_aname ? hp->hd_aname : hp->hd_name));
		pkstr(mp2, buf);
		(void)sprintf(buf, "%s -s -d%x -n%s %d %s %d",
				(hp->hd_dpath ? hp->hd_dpath : pvmdpath),
				debugmask,
				hp->hd_name,
				hosts->ht_master,
				inadport_hex(&hosts->ht_hosts[hosts->ht_master]->hd_sad),
				hosts->ht_hosts[hosts->ht_master]->hd_mtu);
		(void)sprintf(buf + strlen(buf), " %d %s",
				((hp->hd_hostpart & tidhmask) >> (ffs(tidhmask) - 1)),
				inadport_hex(&hp->hd_sad));
		pkstr(mp2, buf);
	}
	mp2->m_cod = SM_STHOST;

	if (hostertid) {
		hosterwid = wp->wa_wid;
		mp2->m_dst = hostertid;
		wp->wa_on = hostertid;
		sendmessage(mp2);

	} else {
		wp->wa_on = TIDPVMD;

		if (pid = fork()) {		/* still us */
			if (pid == -1) {
	/* nack request if can't fork */
				pvmlogperror("addhosts() fork");
				goto bad;

			} else {
				pprime = pid;
				mesg_unref(mp2);
			}

		} else {				/* pvmd' to do the startup */
			beprime();
			mesg_rewind(mp2);
			hoster(mp2);
		}
	}

	return 0;

bad:
	if (wxp)
		free_waitc_add(wxp);
	if (wp) {
		wp->wa_mesg = 0;	/* shared with rmp */
		wait_delete(wp);
	}
	busyadding = 0;
	pkint(rmp, -1);
	sendmessage(rmp);
	return 0;
}


/*	dm_add()
*
*	(Master pvmd) gets request to add new hosts.
*
*	DM_ADD(wid) {
*		int count
*		string name[count]
*	}
*/

int
dm_add(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct mesg *rmp;

	rmp = mesg_new(0);
	rmp->m_dst = mp->m_src;
	rmp->m_cod = DM_ADDACK;
	rmp->m_wid = mp->m_wid;
	addhosts(mp, rmp);
	return 0;
}


/*	dm_addack()
*
*	Reply to DM_ADD operation.
*
*	DM_ADDACK(wid_rtn) {
*		int nhosts			// or error code
*		int narches			// if nhosts >= 0
*		{
*			int tid			// or error code
*			string name
*			string arch
*			int mtu
*			int speed
*		} [nhosts]
*	}
*/

int
dm_addack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;

	if (!(wp = wait_get(hp, mp, WT_ADDHOST)))
		return 0;
	mp->m_ref++;
	mp->m_dst = wp->wa_tid;
	mp->m_cod = TM_ADDHOST;
	mp->m_src = mytid;
	sendmessage(mp);
	wait_delete(wp);
	return 0;
}


/*	exectasks()
*
*	Start tasks requested with DM_EXEC or SM_EXEC.
*	Call the tasker or start 'em.
*	We try to fork/exec given number of tasks.  On first error, fill
*	remainder of tid array with error code.
*	Return tid array to caller.
*/

int
exectasks(mp, rmp, schtid)
	struct mesg *mp;	/* the request message */
	struct mesg *rmp;	/* reply message blank */
	int schtid;			/* scheduler for new tasks */
{
	int ptid;				/* parent task id */
	char *file = 0;			/* file to exec */
	int flags;
	int count;				/* number of tasks */
	int nargs;				/* exec args */
	char **av = 0;
	int outtid, outcod;		/* stdout dst and code */
	int trctid, trccod;		/* trace dst and code */
	int nenv;				/* num of envars */
	char **env = 0;			/* envars */
	int *tids = 0;			/* list of tids from forkexec */
	struct mesg *mp2;		/* reply message hdl */
	struct task *tp = 0;
	int err = 0;			/* error code from forkexec */
	int i;
	struct timeval now;

	/* unpack message */

	if (upkuint(mp, &ptid)
	|| upkstralloc(mp, &file)
	|| upkint(mp, &flags)
	|| upkint(mp, &count)
	|| upkint(mp, &nargs))
		goto bad;

	nargs += 2;
	av = TALLOC(nargs + 1, char*, "argv");
	av++;
	BZERO((char*)av, nargs * sizeof(char*));
	av[0] = file;
	file = 0;
	av[--nargs] = 0;
	for (i = 1; i < nargs; i++)
		if (upkstralloc(mp, &av[i]))
			goto bad;

	if (upkuint(mp, &outtid)
	|| upkuint(mp, &outcod)
	|| upkuint(mp, &trctid)
	|| upkuint(mp, &trccod)
	|| upkuint(mp, &nenv))
		goto bad;
	BZERO((char*)(env = TALLOC((nenv + 1), char*, "env")),
			(nenv + 1) * sizeof(char*));
	for (i = 0; i < nenv; i++)
		if (upkstralloc(mp, &env[i]))
			goto bad;

	if (count < 1)
		goto bad;

	tids = TALLOC(count, int, "tids");

#if defined(IMA_PGON) || defined(IMA_I860) || defined(IMA_CM5) || defined(SHMEM)

#ifndef SHMEM
/* XXX only writing the first tid, oops. */
	if (flags & PvmMppFront) {
		/* spawn on service node */
		if (err = forkexec(flags, av[0], av, nenv, env, &tp)) {
			tids[0] = err;
		} else {
			tp->t_ptid = ptid;
			tids[0] = tp->t_tid;
		}
	} else
#endif /*SHMEM*/
		mpp_load(flags, av[0], av, count, tids, ptid);

	for (i = 0; i < count; i++) {
		if (tids[i] < 0)
			break;
		tp = task_find(tids[i]);
		tp->t_outtid = outtid;
		tp->t_outcod = outcod;
		tp->t_trctid = trctid;
		tp->t_trccod = trccod;
		tp->t_sched = schtid;
		if (trctid) {
			mp2 = mesg_new(0);
			mp2->m_cod = trccod;
			mp2->m_dst = trctid;
			gettimeofday(&now, (struct timezone*)0);
			pkint(mp2, (int)now.tv_sec);
			pkint(mp2, (int)now.tv_usec);
			pkint(mp2, tp->t_tid);
			pkint(mp2, TEV_NEWTASK);
			pkint(mp2, ptid);
			pkint(mp2, flags);
			pkstr(mp2, av[0]);
			sendmessage(mp2);
		}
		if (outtid) {
			mp2 = mesg_new(0);
			mp2->m_cod = outcod;
			mp2->m_dst = outtid;
			pkint(mp2, tp->t_tid);
			pkint(mp2, -2);
			pkint(mp2, ptid);
			sendmessage(mp2);
		}
	}

#else /*defined(IMA_PGON) || defined(IMA_I860) || defined(IMA_CM5) || defined(SHMEM)*/

	for (i = 0; i < count; i++) {
		if (err) {
			tids[i] = err;

		} else {
			if (err = forkexec(flags, av[0], av, nenv, env, &tp)) {
				tids[i] = err;

			} else {
				tp->t_ptid = ptid;
				tp->t_outtid = outtid;
				tp->t_outcod = outcod;
				tp->t_trctid = trctid;
				tp->t_trccod = trccod;
				tp->t_sched = schtid;
				tids[i] = tp->t_tid;
				if (trctid) {
					mp2 = mesg_new(0);
					mp2->m_cod = trccod;
					mp2->m_dst = trctid;
					gettimeofday(&now, (struct timezone*)0);
					pkint(mp2, (int)now.tv_sec);
					pkint(mp2, (int)now.tv_usec);
					pkint(mp2, tp->t_tid);
					pkint(mp2, TEV_NEWTASK);
					pkint(mp2, ptid);
					pkint(mp2, flags);
					pkstr(mp2, av[0]);
					sendmessage(mp2);
				}
				if (outtid) {
					mp2 = mesg_new(0);
					mp2->m_cod = outcod;
					mp2->m_dst = outtid;
					pkint(mp2, tp->t_tid);
					pkint(mp2, -2);
					pkint(mp2, ptid);
					sendmessage(mp2);
				}
			}
		}
	}

#endif /*defined(IMA_PGON) || defined(IMA_I860) || defined(IMA_CM5) || defined(SHMEM)*/

	pkint(rmp, count);
	for (i = 0; i < count; i++)
		pkint(rmp, tids[i]);
	sendmessage(rmp);
	goto cleanup;

bad:
	sprintf(pvmtxt, "exectasks() from t%x bad msg format\n", mp->m_src);
	pvmlogerror(pvmtxt);

cleanup:
	if (tids)
		PVM_FREE(tids);
	if (file)
		PVM_FREE(file);
	if (av) {
		for (i = 0; i < nargs; i++)
			if (av[i])
				PVM_FREE(av[i]);
		av--;
		PVM_FREE(av);
	}
	if (env) {
		for (i = 0; i < nenv; i++)
			if (env[i])
				PVM_FREE(env[i]);
		PVM_FREE(env);
	}
	return 0;
}


/*	dm_exec()
*
*	Request to start task processes.
*
*	DM_EXEC(wid) {
*		int ptid
*		string file
*		int flags
*		int count
*		int nargs
*		string argv[nargs]
*		int outtid
*		int outcod
*		int trctid
*		int trccod
*		int nenv
*		string env[nenv]
*	}
*/

int
dm_exec(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct mesg *rmp;

	hp = hp;

	rmp = mesg_new(0);
	rmp->m_dst = mp->m_src;
	rmp->m_cod = DM_EXECACK;
	rmp->m_wid = mp->m_wid;
	exectasks(mp, rmp, pvmschedtid);
	return 0;
}


/*	dm_execack()
*
*	Reply to DM_EXEC op.
*
*	DM_EXECACK(wid_rtn) {
*		int count
*		int tids[count]
*	}
*/

int
dm_execack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;
	struct waitc_spawn *wxp;
	int rcnt;				/* num of tids+errors returned */
	int tid;
	int v;
	int err = 0;

	if (!(wp = wait_get(hp, mp, WT_SPAWN)))
		return 0;

	wxp = (struct waitc_spawn*)wp->wa_spec;
	upkint(mp, &rcnt);
	v = wxp->w_veclen;

	/*
	* unpack tids and place in result vector where hostpart is now
	*/

/*
	sprintf(pvmtxt, "dm_execack() hp %x vec len %d, repl len %d\n",
			hp->hd_hostpart, v, rcnt);
	pvmlogerror(pvmtxt);
*/
	while (rcnt-- > 0) {
		upkint(mp, &tid);
		if (tid < 0)
			err++;
		while (--v >= 0)
			if (wxp->w_vec[v] == hp->hd_hostpart)
				break;
		if (v < 0) {
/* XXX aargh */
			pvmlogerror("dm_execack() too many tids for result vector, hosing the request\n");
			wait_delete(wp);
			return 0;
		}
		wxp->w_vec[v] = tid;
	}

	if (err)
		ht_delete(wxp->w_ht, hp);

	/*
	* if everyone has checked in, either restart the failed ones
	* or reply to the caller.
	*/

	if (wp->wa_peer == wp)
		assign_tasks(wp);
	wait_delete(wp);
	return 0;
}


/*	dm_sendsig()
*
*	Request to send signal to local task.
*
*	DM_SENDSIG {
*		int tid
*		int signum
*	}
*/

int
dm_sendsig(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int tid;
	int signum;
	struct task *tp;

	hp = hp;

	if (upkuint(mp, &tid) || upkint(mp, &signum)) {
		pvmlogerror("dm_sendsig() bad msg format\n");
		return 0;
	}
	if (tp = task_find(tid)) {
#if defined(IMA_I860) || defined(IMA_CM5)
		(void)mpp_kill(tp, signum);

#else
		if (tp->t_pid)
			(void)kill(tp->t_pid, signum);

		else {
			sprintf(pvmtxt, "dm_sendsig() signal for t%x scrapped (pid = 0)\n",
					tid);
			pvmlogerror(pvmtxt);
		}
#endif

	} else
		if (debugmask & (PDMTASK|PDMAPPL)) {
			sprintf(pvmtxt, "dm_sendsig() signal for t%x scrapped\n", tid);
			pvmlogerror(pvmtxt);
		}
	return 0;
}


/*	dm_htupd()
*
*	Host table update phase 1 - (Non-master) pvmd is notified of new
*	host table.
*
*	DM_HTUPD(wid) {
*		int serial
*		int master
*		int console
*		int count
*		{
*			int index
*			string name
*			string arch
*			string hexipaddr
*			int mtu
*			int speed
*		} [count]
*	}
*/

int
dm_htupd(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int count;			/* number of hosts in message */
	int hh;
	char buf[16];		/* for converting sockaddr */
	struct mesg *mp2;

	/* unpack new host table params */

	newhosts = ht_new(1);
	newhosts->ht_local = hosts->ht_local;
	upkint(mp, &newhosts->ht_serial);
	upkint(mp, &newhosts->ht_master);
	upkint(mp, &newhosts->ht_cons);

	/* add current hosts to the table */

	ht_merge(newhosts, hosts);

	/* unpack new hosts and add to table */

	/* XXX if we find a host already in table we should kill it with
	   XXX hostfail and put the new one in its place */

	upkint(mp, &count);
	while (count-- > 0) {
		upkint(mp, &hh);
		hp = hd_new(hh);
		upkstralloc(mp, &hp->hd_name);
		upkstralloc(mp, &hp->hd_arch);
		upkstr(mp, buf, sizeof(buf));
		hex_inadport(buf, &hp->hd_sad);
		upkint(mp, &hp->hd_mtu);
		upkint(mp, &hp->hd_speed);
		ht_insert(newhosts, hp);
		hd_unref(hp);
	}

	if (debugmask & PDMHOST) {
		pvmlogerror("dm_htupd() new host table:\n");
		ht_dump(newhosts);
	}
	runstate = PVMDHTUPD;

	/* reply to sender that we have new host table */

	mp2 = mesg_new(0);
	mp2->m_dst = mp->m_src;
	mp2->m_cod = DM_HTUPDACK;
	mp2->m_wid = mp->m_wid;
	sendmessage(mp2);
	return 0;
}


/*	dm_htupdack()
*
*	Reply to DM_HTUPD op.
*
*	DM_HTUPDACK(wid_rtn) { }
*/

int
dm_htupdack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;

	if (!(wp = wait_get(hp, mp, WT_HTUPD)))
		return 0;

	/* is this is the last host checking in, send ht commit */

	if (wp->wa_peer == wp) {
		int hh;

		mp = mesg_new(0);
		mp->m_cod = DM_HTCOMMIT;

		for (hh = hosts->ht_last; hh > 0; hh--)
			if (hh != hosts->ht_local && (hp = hosts->ht_hosts[hh])) {
				mp->m_ref++;
				mp->m_dst = hp->hd_hostpart | TIDPVMD;
				sendmessage(mp);
			}
		mesg_unref(mp);

		busyadding = 0;
		sendmessage(wp->wa_mesg);
		wp->wa_mesg = 0;
	}
	wait_delete(wp);
	return 0;
}


/*	ht_diff()
*
*	Generates a host table containing entries in ht2 but not in ht1.
*/

struct htab *
ht_diff(htp2, htp1)
	struct htab *htp2;		/* more */
	struct htab *htp1;		/* less */
{
	struct htab *htp;
	int hh;

	htp = ht_new(1);
	for (hh = htp2->ht_last; hh > 0; hh--)
		if (htp2->ht_hosts[hh] && !htp2->ht_hosts[hh]->hd_err
		&& (hh > htp1->ht_last || !htp1->ht_hosts[hh]))
			ht_insert(htp, htp2->ht_hosts[hh]);
	return htp;
}


/*	gotnewhosts()
*
*	Used to wake up any waitcs of kind WT_HOSTA.
*	Sends a message containing count and list of new d-tids.
*/

gotnewhosts(htp2, htp1)
	struct htab *htp2;		/* new host table */
	struct htab *htp1;		/* old host table */
{
	struct mesg *mp;
	struct htab *htp;
	struct waitc *wp, *wp2;
	int hh;

	mp = 0;
	for (wp = waitlist->wa_link; wp != waitlist; wp = wp2) {
		wp2 = wp->wa_link;
		if (wp->wa_kind == WT_HOSTA) {
			if (!mp) {
/* XXX shouldn't diff host tables and pack message several times, oops */
				mp = mesg_new(0);
				htp = ht_diff(htp2, htp1);
				pkint(mp, htp->ht_cnt);
				for (hh = htp->ht_last; hh > 0; hh--)
					if (htp->ht_hosts[hh])
						pkint(mp, htp->ht_hosts[hh]->hd_hostpart);
				ht_free(htp);
			}
			mp->m_ref++;
			mp->m_dst = wp->wa_tid;
			mp->m_cod = wp->wa_dep;
			sendmessage(mp);
			if (wp->wa_count != -1 && --wp->wa_count < 1)
				wait_delete(wp);
		}
	}
	if (mp)
		mesg_unref(mp);
	return 0;
}


/*	dm_htcommit()
*
*	Host table update phase 2 - commit to new host table.
*
*	DM_HTCOMMIT { }
*/

int
dm_htcommit(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct htab *htp;

	if (hp != hosts->ht_hosts[hosts->ht_master]) {
		sprintf(pvmtxt, "dm_htcommit() from t%x (not master)?\n",
				mp->m_src);
		pvmlogerror(pvmtxt);
	}

	if (newhosts) {
		htp = hosts;
		hosts = newhosts;
		newhosts = 0;
		if (debugmask & PDMHOST) {
			sprintf(pvmtxt,
			"dm_htcommit() committing from host table serial %d to %d\n",
					htp->ht_serial, hosts->ht_serial);
			pvmlogerror(pvmtxt);
		}

		gotnewhosts(hosts, htp);
		ht_free(htp);
		runstate = PVMDNORMAL;

	} else {
		pvmlogerror("dm_htcommit() no new host table pending?\n");
	}
	return 0;
}


/*	dm_slconf()
*
*	Pvmd gets config info (from master).
*	One of these should arrive before a pvmd is fully up, even
*	before the host table update, to set private params.
*	More may be used later to set random knobs.
*
*	DM_SLCONF {
*		{
*			int fieldtype		// from ddpro.h
*			string value
*		} []					// implied
*	}
*/

int
dm_slconf(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int t;				/* field type */
	char *s, *s2;

	mp = mp;
	hp = hp;

	while (!upkint(mp, &t) && !upkstralloc(mp, &s)) {
		switch (t) {

		case DM_SLCONF_EP:
			if (debugmask & (PDMTASK|PDMSTARTUP)) {
				sprintf(pvmtxt, "dm_slconf() ep<%s>\n", s);
				pvmlogerror(pvmtxt);
			}
			epaths = colonsep(varsub(s));
			PVM_FREE(s);
			break;

		case DM_SLCONF_BP:
			if (debugmask & PDMSTARTUP) {
				sprintf(pvmtxt, "dm_slconf() bp<%s>\n", s);
				pvmlogerror(pvmtxt);
			}
			debugger = varsub(s);
			PVM_FREE(s);
			break;

		case DM_SLCONF_WD:
			if (debugmask & (PDMTASK|PDMSTARTUP)) {
				sprintf(pvmtxt, "dm_slconf() wd<%s>\n", s);
				pvmlogerror(pvmtxt);
			}
			s2 = varsub(s);
			if (chdir(s2) == -1)
				pvmlogperror(s2);
			PVM_FREE(s);
			PVM_FREE(s2);
			break;

		case DM_SLCONF_SCHED:
			if (debugmask & (PDMSCHED|PDMSTARTUP)) {
				sprintf(pvmtxt, "dm_slconf() sched<t%x>\n", pvmschedtid);
				pvmlogerror(pvmtxt);
			}
			pvmschedtid = pvmxtoi(s);
			break;

		default:
			if (strlen(s) > sizeof(pvmtxt)-50)
				s[sizeof(pvmtxt)-50] = 0;
			sprintf(pvmtxt, "dm_slconf() ? type %d val <%s>\n", t, s);
			pvmlogerror(pvmtxt);
			PVM_FREE(s);
			break;
		}
	}
	return 0;
}


/*	errnamecode()
*
*	Map an error code name to the code.
*	XXX Should this be done for real or killed?  Hurt me.
*/

static char *errnames[] = {
	"PvmOk",
	"",
	"PvmBadParam",
	"PvmMismatch",
	"",
	"PvmNoData",
	"PvmNoHost",
	"PvmNoFile",
	"",
	"",
	"PvmNoMem",
	"",
	"PvmBadMsg",
	"",
	"PvmSysErr",
	"PvmNoBuf",
	"PvmNoSuchBuf",
	"PvmNullGroup",
	"PvmDupGroup",
	"PvmNoGroup",
	"PvmNotInGroup",
	"PvmNoInst",
	"PvmHostFail",
	"PvmNoParent",
	"PvmNotImpl",
	"PvmDSysErr",
	"PvmBadVersion",
	"PvmOutOfRes",
	"PvmDupHost",
	"PvmCantStart",
	"PvmAlready",
	"PvmNoTask",
	"PvmNoEntry",
	"PvmDupEntry",
	0
};

int
errnamecode(s)
	char *s;
{
	int i;

	for (i = 0; errnames[i]; i++)
		if (!strcmp(s, errnames[i]))
			return -i;
	return PvmCantStart;
}


/*	startack()
*
*	Take results from pvmd' or hoster.  Update the machine config
*	if any were successful.  Reply to the DM_ADD or SM_ADD request.
*/

int
startack(wp, mp)
	struct waitc *wp;		/* wait context on hoster */
	struct mesg *mp;
{
	struct hostd *hp;
	struct mesg *mp2;
	struct waitc *wp2;		/* seed waitc for htupd peer group */
	struct waitc *wp3;
	int count;				/* num of new hosts */
	int happy;				/* num of happy new hosts */
	struct waitc_add *wxp;
	char *av[16];			/* for reply parsing */
	int ac;
	int ver;
	int i, j;
	int t;
	int hh;
	char buf[512];

	/*
	* unpack startup results, update hosts in wait context
	*/

	wxp = (struct waitc_add *)wp->wa_spec;
	count = wxp->w_num;
	upkint(mp, &j);

	while (j-- > 0) {
		if (upkuint(mp, &t) || upkstr(mp, buf, sizeof(buf))) {
			pvmlogerror("startack() bad message format\n");
			pkint(wp->wa_mesg, PvmDSysErr);
			sendmessage(wp->wa_mesg);
			wp->wa_mesg = 0;
			wait_delete(wp);
			busyadding = 0;
			return 0;
		}
		for (i = count; i-- > 0 && wxp->w_hosts[i]->hd_hostpart != t; ) ;
		if (i < 0) {
			sprintf(pvmtxt, "startack() what? some random tid %x\n", t);
			pvmlogerror(pvmtxt);
			pkint(wp->wa_mesg, PvmDSysErr);
			sendmessage(wp->wa_mesg);
			wp->wa_mesg = 0;
			wait_delete(wp);
			busyadding = 0;
			return 0;
		}
		hp = wxp->w_hosts[i];
		ac = sizeof(av)/sizeof(av[0]);
		if (crunchzap(buf, &ac, av) || ac != 4) {
			sprintf(pvmtxt, "startack() host %s expected version\n",
					hp->hd_name);
			pvmlogerror(pvmtxt);
			hp->hd_err = errnamecode(buf);
			continue;
		}

		ver = atoi(av[0]);
		if (ver != DDPROTOCOL) {
			sprintf(pvmtxt,
					"slave_exec() host %s d-d protocol mismatch (%d/%d)\n",
					hp->hd_name, ver, DDPROTOCOL);
			pvmlogerror(pvmtxt);
			hp->hd_err = PvmBadVersion;
			continue;
		}

		hp->hd_arch = STRALLOC(av[1]);
		hex_inadport(av[2], &hp->hd_sad);
		hp->hd_mtu = atoi(av[3]);
	}

	/*
	* update reply message to add-host requestor
	*/

	mp2 = wp->wa_mesg;
	pkint(mp2, count);
	pkint(mp2, 0);	/* XXX narches = 0 for now */
	for (i = 0; i < count; i++) {
		hp = wxp->w_hosts[i];
		if (hp->hd_err) {
			pkint(mp2, hp->hd_err);
			pkstr(mp2, "");
			pkstr(mp2, "");
			pkint(mp2, 0);

		} else {
			pkint(mp2, hp->hd_hostpart);
			pkstr(mp2, hp->hd_name);
			pkstr(mp2, hp->hd_arch);
			pkint(mp2, hp->hd_speed);
		}
	}

	/*
	* delete broken ones, done now if none succeeded,
	* otherwise done when host table update is complete.
	*/

	for (j = i = 0; i < count; i++)
		if (!wxp->w_hosts[i]->hd_err) {
			hp = wxp->w_hosts[i];
			wxp->w_hosts[i] = 0;
			wxp->w_hosts[j++] = hp;

		} else {
			hd_unref(wxp->w_hosts[i]);
			wxp->w_hosts[i] = 0;
		}
	count = j;

	if (count < 1) {
		busyadding = 0;
		sendmessage(wp->wa_mesg);
		wp->wa_mesg = 0;
		free_waitc_add(wxp);
		wait_delete(wp);
		return 0;
	}

	wp2 = wait_new(WT_HTUPD);
	wp2->wa_dep = wp->wa_dep;
	wp2->wa_mesg = wp->wa_mesg;
	wp->wa_mesg = 0;

	/*
	* make next host table
	*/

	newhosts = ht_new(1);
	newhosts->ht_serial = hosts->ht_serial + 1;
	newhosts->ht_master = hosts->ht_master;
	newhosts->ht_cons = hosts->ht_cons;
	newhosts->ht_local = hosts->ht_local;

	for (i = 0; i < count; i++)
		ht_insert(newhosts, wxp->w_hosts[i]);

	free_waitc_add(wxp);
	wait_delete(wp);
	wp = 0;

	runstate = PVMDHTUPD;

	/*
	* send DM_SLCONF message to each new host
	*/

	for (hh = newhosts->ht_last; hh > 0; hh--)
		if (hp = newhosts->ht_hosts[hh]) {
			mp2 = mesg_new(0);
			mp2->m_cod = DM_SLCONF;
			mp2->m_dst = hp->hd_hostpart | TIDPVMD;
			if (hp->hd_epath) {
				pkint(mp2, DM_SLCONF_EP);
				pkstr(mp2, hp->hd_epath);
			}
			if (hp->hd_bpath) {
				pkint(mp2, DM_SLCONF_BP);
				pkstr(mp2, hp->hd_bpath);
			}
			if (hp->hd_wdir) {
				pkint(mp2, DM_SLCONF_WD);
				pkstr(mp2, hp->hd_wdir);
			}
			if (pvmschedtid) {
				sprintf(buf, "%x", pvmschedtid);
				pkint(mp2, DM_SLCONF_SCHED);
				pkstr(mp2, buf);
			}
			sendmessage(mp2);
		}

	/*
	* create host table update message containing all current hosts
	* plus new ones, send to each new host.
	*/

	mp2 = mesg_new(0);
	mp2->m_cod = DM_HTUPD;
	pkint(mp2, newhosts->ht_serial);
	pkint(mp2, newhosts->ht_master);
	pkint(mp2, newhosts->ht_cons);
	pkint(mp2, hosts->ht_cnt + newhosts->ht_cnt);
	for (hh = hosts->ht_last; hh > 0; hh--)
		if (hp = hosts->ht_hosts[hh]) {
			pkint(mp2, hh);
			pkstr(mp2, hp->hd_name);
			pkstr(mp2, hp->hd_arch);
			pkstr(mp2, inadport_hex(&hp->hd_sad));
			pkint(mp2, hp->hd_mtu);
			pkint(mp2, hp->hd_speed);
		}
	for (hh = newhosts->ht_last; hh > 0; hh--)
		if (hp = newhosts->ht_hosts[hh]) {
			pkint(mp2, hh);
			pkstr(mp2, hp->hd_name);
			pkstr(mp2, hp->hd_arch);
			pkstr(mp2, inadport_hex(&hp->hd_sad));
			pkint(mp2, hp->hd_mtu);
			pkint(mp2, hp->hd_speed);
		}

	for (hh = newhosts->ht_last; hh > 0; hh--)
		if (hp = newhosts->ht_hosts[hh]) {
			mp2->m_ref++;
			mp2->m_dst = hp->hd_hostpart | TIDPVMD;
			wp3 = wait_new(WT_HTUPD);
			wp3->wa_dep = wp2->wa_dep;
			wp2->wa_mesg->m_ref++;
			wp3->wa_mesg = wp2->wa_mesg;
			wp3->wa_on = hp->hd_hostpart;
			LISTPUTBEFORE(wp2, wp3, wa_peer, wa_rpeer);
			mp2->m_wid = wp3->wa_wid;
			sendmessage(mp2);
		}
	mesg_unref(mp2);

	/*
	* create host table update message containing happy new hosts,
	* send to each old host.
	*/

	mp2 = mesg_new(0);
	mp2->m_cod = DM_HTUPD;
	pkint(mp2, newhosts->ht_serial);
	pkint(mp2, newhosts->ht_master);
	pkint(mp2, newhosts->ht_cons);
	pkint(mp2, newhosts->ht_cnt);
	for (hh = newhosts->ht_last; hh > 0; hh--)
		if (hp = newhosts->ht_hosts[hh]) {
			pkint(mp2, hh);
			pkstr(mp2, hp->hd_name);
			pkstr(mp2, hp->hd_arch);
			pkstr(mp2, inadport_hex(&hp->hd_sad));
			pkint(mp2, hp->hd_mtu);
			pkint(mp2, hp->hd_speed);
		}

	for (hh = hosts->ht_last; hh > 0; hh--)
		if (hh != hosts->ht_local && (hp = hosts->ht_hosts[hh])) {
			mp2->m_ref++;
			mp2->m_dst = hp->hd_hostpart | TIDPVMD;
			wp3 = wait_new(WT_HTUPD);
			wp3->wa_dep = wp2->wa_dep;
			wp2->wa_mesg->m_ref++;
			wp3->wa_mesg = wp2->wa_mesg;
			wp3->wa_on = hp->hd_hostpart;
			LISTPUTBEFORE(wp2, wp3, wa_peer, wa_rpeer);
			mp2->m_wid = wp3->wa_wid;
			sendmessage(mp2);
		}
	mesg_unref(mp2);

	/*
	* update our host table
	*/

	gotnewhosts(newhosts, hosts);

	/* XXX returning to normal state right here is a hack, we should
	   XXX wait for all the htupdacks to come back but we need the
	   XXX regular message service, hostfail entry, etc. */

	for (hh = newhosts->ht_last; hh > 0; hh--)
		if ((hp = newhosts->ht_hosts[hh]) && !hp->hd_err)
			ht_insert(hosts, hp);
	hosts->ht_serial = newhosts->ht_serial;

	if (debugmask & PDMHOST) {
		pvmlogerror("startack() committing to new host table:\n");
		ht_dump(hosts);
	}
	runstate = PVMDNORMAL;
	ht_free(newhosts);
	newhosts = 0;

	/* if peered waitcs on htupdack already finished, send the reply */

	if (wp2->wa_peer == wp2) {
		busyadding = 0;
		sendmessage(wp2->wa_mesg);
		wp2->wa_mesg = 0;
	}
	wait_delete(wp2);

	return 0;
}


/*	dm_startack()
*
*	The results come back from the pvmd' hoster to pvmd[master].
*
*	DM_STARTACK(wid_rtn) {
*		int count
*		{
*			int tid
*			string result		// line from slave pvmd or error token
*		} [count]
*	}
*/

int
dm_startack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;		/* wait context on pvmd' */

	if (!(wp = wait_get(hp, mp, WT_HOSTSTART)))
		return 0;

	finack_to_host(hp);

	startack(wp, mp);

	return 0;
}


/*	dm_task()
*
*	(Remote or local) pvmd requests information about local task(s).
*
*	DM_TASK(wid) {
*		int where			// 0 for all, hostpart, or full tid
*	}
*/

int
dm_task(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct task *tp;
	struct mesg *mp2;
	int where;

	hp = hp;

	if (upkuint(mp, &where)) {
		pvmlogerror("dm_task() bad msg format\n");
		return 0;
	}

	/* pack list of local tasks and reply to waiter */

	mp2 = mesg_new(0);
	mp2->m_dst = mp->m_src;
	mp2->m_cod = DM_TASKACK;
	mp2->m_wid = mp->m_wid;
	if (where & tidlmask) {
		if (tp = task_find(where)) {
			pkint(mp2, tp->t_tid);
			pkint(mp2, tp->t_ptid);
			pkint(mp2, myhostpart);
			pkint(mp2, tp->t_flag);
			pkstr(mp2, tp->t_a_out ? tp->t_a_out : "");
			pkint(mp2, tp->t_pid);
		}

	} else {
		for (tp = locltasks->t_link; tp != locltasks; tp = tp->t_link) {
			pkint(mp2, tp->t_tid);
			pkint(mp2, tp->t_ptid);
			pkint(mp2, myhostpart);
			pkint(mp2, tp->t_flag);
			pkstr(mp2, (tp->t_a_out ? tp->t_a_out : ""));
			pkint(mp2, tp->t_pid);
		}
	}
	sendmessage(mp2);
	return 0;
}


/*	dm_taskack()
*
*	Reply to DM_TASK op.
*
*	DM_TASKACK(wid_rtn) {
*		{
*			int tid
*			int ptid
*			int hostpart
*			int flag
*			string a_out
*			int pid
*		} []				// implied
*	}
*/

int
dm_taskack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;
	struct mesg *mp2;
	int i;
	char *p;

	if (!(wp = wait_get(hp, mp, WT_TASK)))
		return 0;

	/* append data to waiting message */

	mp2 = wp->wa_mesg;
	while (!upkint(mp, &i)) {
		pkint(mp2, i);			/* tid */
		upkint(mp, &i);			/* ptid */
		pkint(mp2, i);
		upkint(mp, &i);			/* host */
		pkint(mp2, i);
		upkint(mp, &i);			/* flag */
		pkint(mp2, i);
		upkstralloc(mp, &p);	/* a.out name */
		pkstr(mp2, p);
		PVM_FREE(p);
		upkint(mp, &i);			/* pid */
		pkint(mp2, i);
	}

	/* send message if we're the last waiter */

	if (wp->wa_peer == wp) {
		mp2->m_ref++;
		sendmessage(mp2);

	}
	wait_delete(wp);
	return 0;
}


/*	dm_delhost()
*
*	(Master pvmd) gets request to delete hosts.
*
*	DM_DELHOST(wid) {
*		int count
*		string names[count]
*	}
*/

int
dm_delhost(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int count;
	char buf[128];
	struct mesg *mp2;		/* DELHOSTACK message */
	struct mesg *mp3;		/* HTDEL message */
	struct htab *ht_del;	/* hosts to delete */
	struct htab *ht_save;	/* remaining hosts */
	int hh;

/* XXX danger, this doesn't check if already doing a host add/delete */

	/* sanity check count */

	if (upkint(mp, &count)) {
		pvmlogerror("dm_delhost() bad msg format\n");
		return 0;
	}
	if (count < 1 || count > (tidhmask >> (ffs(tidhmask) - 1))) {
		pvmlogerror("dm_delhost() bad count\n");
		return 0;
	}

	/*
	* read host names from message, generate delete and save sets
	* and a DELHOSTACK reply message with result code for each host.
	* set SHUTDOWN flag for each host in delete set.
	*/

	ht_del = ht_new(1);
	ht_save = ht_new(1);
	ht_merge(ht_save, hosts);

	mp2 = mesg_new(0);
	mp2->m_cod = DM_DELHOSTACK;
	mp2->m_wid = mp->m_wid;
	mp2->m_dst = mp->m_src;

	mp3 = mesg_new(0);
	mp3->m_cod = DM_HTDEL;
	pkint(mp3, hosts->ht_serial);

	pkint(mp2, count);
	while (count-- > 0) {
		upkstr(mp, buf, sizeof(buf));
		if (hp = nametohost(hosts, buf)) {
			if (tidtohost(ht_del, hp->hd_hostpart)) {
				pkint(mp2, PvmDupHost);

			} else {
				if (hp->hd_hostpart == myhostpart)
					pkint(mp2, PvmBadParam);

				else {
					ht_insert(ht_del, hp);
					ht_delete(ht_save, hp);
					pkint(mp3, hp->hd_hostpart);
					fin_to_host(hp);
					pkint(mp2, 0);
				}
			}

		} else
			pkint(mp2, PvmNoHost);
	}

	/*
	* send HTDEL message to all hosts in ht_save set except us
	*/

	for (hh = ht_save->ht_last; hh > 0; hh--)
		if (hh != hosts->ht_local && (hp = ht_save->ht_hosts[hh])) {
			mp3->m_ref++;
			mp3->m_dst = hp->hd_hostpart | TIDPVMD;
			sendmessage(mp3);
		}
	mesg_unref(mp3);

	/* reply to host that requested DELHOST operation */

	sendmessage(mp2);

	ht_free(ht_del);
	ht_free(ht_save);
	return 0;
}


/*	dm_delhostack()
*
*	Reply to DM_DELHOST operation.
*
*	DM_DELHOSTACK(wid_rtn) {
*		int count			// or negative for error
*		int status[count]	// status of each host
*	}
*/

int
dm_delhostack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;

	if (!(wp = wait_get(hp, mp, WT_DELHOST)))
		return 0;

	mp->m_ref++;
	mp->m_src = mytid;
	mp->m_dst = wp->wa_tid;
	mp->m_cod = TM_DELHOST;
	sendmessage(mp);
	wait_delete(wp);
	return 0;
}


/*	dm_null()
*
*	No-op message.
*
*	DM_NULL { }
*/

int
dm_null(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	hp = hp;
	mp = mp;
/*
	sprintf(pvmtxt, "dm_null() from %s\n", hp->hd_name);
	pvmlogerror(pvmtxt);
*/
	return 0;
}


/*	dm_taskout()
*
*	Stdout data from a task.
*
*	DM_TASKOUT {
*		int tid
*		int length
*		char data[length]
*	}
*/

int
dm_taskout(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int tid;
	int l;
	char *p, *q, c;
	char buf2[32];
	char buf[4100];		 /* a bit bigger than in pvmd.c`loclstout() */

	hp = hp;

	if (upkuint(mp, &tid) || upkint(mp, &l) || l < 1)
		return 0;

	/* unpack data, leaving room at head of buffer */

	p = buf + 32;
	if (l > sizeof(buf) - (p - buf) - 2)
		l = sizeof(buf) - (p - buf) - 2;
	byteupk(mp, p, l, 1, 1);

	/* ends with "\n\0" */
	if (p[l - 1] != '\n')
		p[l++] = '\n';
	p[l] = 0;

	sprintf(buf2, "[t%x] ", tid);
	l = strlen(buf2);
	while (*p) {
		for (q = p; *q++ != '\n'; ) ;
		c = *q;
		*q = 0;
		BCOPY(buf2, p - l, l);
		pvmlogerror(p - l);
		*q = c;
		p = q;
	}
	return 0;
}


/*	dm_pstat()
*
*	(Remote or local) pvmd requests status of a local task.
*
*	DM_PSTAT(wid) {
*		int tid
*	}
*/

int
dm_pstat(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int tid;
	struct mesg *mp2;

	hp = hp;
	upkuint(mp, &tid);
	if (tid == (myhostpart | TIDPVMD) || task_find(tid))
		tid = 0;
	else
		tid = PvmNoTask;
	mp2 = mesg_new(0);
	mp2->m_dst = mp->m_src;
	mp2->m_cod = DM_PSTATACK;
	mp2->m_wid = mp->m_wid;
	pkint(mp2, tid);
	sendmessage(mp2);
	return 0;
}


/*	dm_pstatack()
*
*	Reply to DM_PSTAT op.
*
*	DM_PSTATACK(wid_rtn) {
*		int status
*	}
*/

int
dm_pstatack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;

	if (!(wp = wait_get(hp, mp, 0)))
		return 0;

	if (wp->wa_kind == WT_PSTAT)
		mp->m_cod = TM_PSTAT;
	else
		mp->m_cod = TM_MSTAT;
	mp->m_ref++;
	mp->m_src = mytid;
	mp->m_dst = wp->wa_tid;
	sendmessage(mp);
	wait_delete(wp);
	return 0;
}


/*	dm_halt()
*
*	Task has requested whole machine to halt.
*
*	DM_HALT { }
*/

int
dm_halt(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int hh;

	sprintf(pvmtxt, "dm_halt() from (%s), halting...\n",
			hp->hd_name);
	pvmlogerror(pvmtxt);
	for (hh = hosts->ht_last; hh > 0; hh--) {
		if (hh == hosts->ht_local || !(hp = hosts->ht_hosts[hh]))
			continue;
		finack_to_host(hp);
	}
	runstate = PVMDHALTING;
	return 0;
}


/*	dm_mca()
*
*	Remote pvmd defines a new (single-use) multicast address, to be used
*	by a subsequent message.
*
*	DM_MCA {
*		int gtid			// multicast tid
*		int count			// number of addresses
*		int tids[count]		// addresses
*	}
*/

int
dm_mca(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct mca *mcap;
	int i;

	/* unpack struct mca from message */

	mcap = mca_new();
	upkuint(mp, &mcap->mc_tid);
	upkint(mp, &mcap->mc_ndst);
	mcap->mc_dsts = TALLOC(mcap->mc_ndst, int, "mcad");
	for (i = 0; i < mcap->mc_ndst; i++)
		upkuint(mp, &mcap->mc_dsts[i]);

	/* put on list of mcas of src host */

	LISTPUTBEFORE(hp->hd_mcas, mcap, mc_link, mc_rlink);

	if (debugmask & PDMMESSAGE) {
		sprintf(pvmtxt, "dm_mca() mca %x from %s\n", mcap->mc_tid, hp->hd_name);
		pvmlogerror(pvmtxt);
	}
	return 0;
}


/*	dm_notify()
*
*	Remote pvmd requests to be notified on an event.
*
*	DM_NOTIFY(wid) {
*		int what			// event type XXX currenty PvmTaskExit
*		int tid				// address
*	}
*/

int
dm_notify(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int what, tid;
	struct waitc *wp;
	struct mesg *mp2;

	hp = hp;

	upkint(mp, &what);	/* XXX currently PvmTaskExit */
	upkuint(mp, &tid);

	if (task_find(tid)) {
		wp = wait_new(WT_TASKX);
		wp->wa_on = tid;
		wp->wa_tid = mp->m_src;
		wp->wa_dep = mp->m_wid;

	} else {
		mp2 = mesg_new(0);
		mp2->m_dst = mp->m_src;
		mp2->m_cod = DM_NOTIFYACK;
		mp2->m_wid = mp->m_wid;
		sendmessage(mp2);
	}
	return 0;
}


/*	dm_notifyack()
*
*	Reply to DM_NOTIFY op.
*
*	DM_NOTIFYACK(wid_rtn) { }
*/

int
dm_notifyack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;

	hp = hp;

	if (!(wp = wait_get((struct hostd*)0, mp, 0)))
		return 0;

	if (wp->wa_tid) {
		mp = mesg_new(0);
		mp->m_dst = wp->wa_tid;
		mp->m_cod = wp->wa_dep;
		pkint(mp, wp->wa_on);
		sendmessage(mp);
	}

	wait_delete(wp);
	return 0;
}


/*	dm_db()
*
*	(Remote or local) pvmd requests a database op.
*
*	DM_DB(wid) {
*		int opcode		// insert, delete, lookup
*		string name
*		int index
*		int data
*	}
*/

int
dm_db(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int opcode;			/* op requested */
	char *name = 0;		/* class name */
	int req;			/* index requested */
	int data;			/* data to put or get */
	struct mesg *mp2;

	hp = hp;

	if (upkint(mp, &opcode) || upkstralloc(mp, &name) || upkint(mp, &req))
		goto badformat;

	switch (opcode) {

	case TMDB_INSERT:
		if (upkint(mp, &data))
			goto badformat;
		if ((req = nmd_insert(name, req, data)) == -1)
			req = PvmDupEntry;
		break;

	case TMDB_DELETE:
		if ((req = nmd_delete(name, req)) == -1)
			req = PvmNoEntry;
		break;

	case TMDB_LOOKUP:
		if ((req = nmd_lookup(name, req, &data)) == -1)
			req = PvmNoEntry;
		break;

	default:
		goto badformat;
		break;
	}

	mp2 = mesg_new(0);
	mp2->m_dst = mp->m_src;
	mp2->m_cod = DM_DBACK;
	mp2->m_wid = mp->m_wid;
	pkint(mp2, req);
	pkint(mp2, data);
	sendmessage(mp2);
	PVM_FREE(name);
	return 0;

badformat:
	pvmlogerror("dm_db() bad msg format\n");

	if (name)
		PVM_FREE(name);
	return 0;
}


/*	dm_dback()
*
*	Reply to DM_DB op.
*
*	DM_DBACK(wid_rtn) {
*		int index		// and status
*		int data		// if 'lookup'
*	}
*/

int
dm_dback(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;

	if (!(wp = wait_get(hp, mp, WT_DB)))
		return 0;

	mp->m_ref++;
	mp->m_src = mytid;
	mp->m_dst = wp->wa_tid;
	mp->m_cod = TM_DB;
	sendmessage(mp);
	wait_delete(wp);
	return 0;
}


int
dm_htdel(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	int serial;
	int tid;

	if (hp != hosts->ht_hosts[hosts->ht_master]) {
		sprintf(pvmtxt, "dm_htdel() from t%x (not master)?\n",
				mp->m_src);
		pvmlogerror(pvmtxt);
		return 0;
	}
	if (upkint(mp, &serial)) {
		pvmlogerror("dm_htdel() bad format\n");
		return 0;
	}
	if (serial != hosts->ht_serial) {
		sprintf(pvmtxt, "dm_htdel() for serial %d, current is %d?\n",
				serial, hosts->ht_serial);
		pvmlogerror(pvmtxt);
		return 0;
	}
	while (!upkuint(mp, &tid)) {
		if (hp = tidtohost(hosts, tid)) {
			if (debugmask & PDMHOST) {
				sprintf(pvmtxt, "dm_htdel() host %s\n", hp->hd_name);
				pvmlogerror(pvmtxt);
			}
			hostfailentry(hp);
			ht_delete(hosts, hp);
			if (newhosts)
				ht_delete(newhosts, hp);
		}
	}
	return 0;
}


/*	dm_hostsync()
*
*	Request time of day clock sample
*
*	DM_HOSTSYNC(wid) { }
*/

int
dm_hostsync(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct mesg *mp2;
	struct timeval now;

	mp = mp;
	hp = hp;

	mp2 = mesg_new(0);
	mp2->m_dst = mp->m_src;
	mp2->m_cod = DM_HOSTSYNCACK;
	mp2->m_wid = mp->m_wid;
	gettimeofday(&now, (struct timezone *)0);
	pkint(mp2, (int)now.tv_sec);
	pkint(mp2, (int)now.tv_usec);
	sendmessage(mp2);
	return 0;
}


/*	dm_hostsyncack()
*
*	Clock sample comes back
*
*	DM_HOSTSYNCACK(wid_rtn) {
*		int sec
*		int usec
*	}
*/

int
dm_hostsyncack(hp, mp)
	struct hostd *hp;
	struct mesg *mp;
{
	struct waitc *wp;
	struct mesg *mp2;
	int i;

	if (!(wp = wait_get(hp, mp, WT_HOSTSYNC)))
		return 0;

	mp2 = wp->wa_mesg;
	pkint(mp2, 0);
	upkuint(mp, &i);
	pkint(mp2, i);
	upkuint(mp, &i);
	pkint(mp2, i);
	sendmessage(mp2);

	wp->wa_mesg = 0;
	wait_delete(wp);
	return 0;
}


