
/*
 *         PVM version 3.3:  Parallel Virtual Machine System
 *               University of Tennessee, Knoxville TN.
 *           Oak Ridge National Laboratory, Oak Ridge TN.
 *                   Emory University, Atlanta GA.
 *      Authors:  A. L. Beguelin, J. J. Dongarra, G. A. Geist,
 *    W. C. Jiang, R. J. Manchek, B. K. Moore, and V. S. Sunderam
 *                   (C) 1992 All Rights Reserved
 *
 *                              NOTICE
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted
 * provided that the above copyright notice appear in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * Neither the Institutions (Emory University, Oak Ridge National
 * Laboratory, and University of Tennessee) nor the Authors make any
 * representations about the suitability of this software for any
 * purpose.  This software is provided ``as is'' without express or
 * implied warranty.
 *
 * PVM version 3 was funded in part by the U.S. Department of Energy,
 * the National Science Foundation and the State of Tennessee.
 */

/*
 * pvmhost.c
 *
 * Special task that relays messages between pvmd and node tasks.
 * It runs on the frontend and connects to pvmd via sockets, but
 * communicates with the nodes using CMMD send/receive.
 *
 *
 * Jul 10 16:50:52 1993
 * 		changed mc_dsts[] to short from int.
 *
$Log: pvmhost.c,v $
 * Revision 1.5  1994/11/09  14:26:05  manchek
 * more mpp fixes
 *
 * Revision 1.4  1994/11/07  21:15:30  manchek
 * Send a null packet to alert precv().
 * Handle DataInPlace properly
 *
 * Revision 1.3  1994/06/03  20:54:29  manchek
 * version 3.3.0
 *
 * Revision 1.2  1993/12/20  15:39:48  manchek
 * patch 6 from wcj
 *
 * Revision 1.1  1993/08/30  23:35:09  manchek
 * Initial revision
 *
 */
#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <errno.h>
#include <cm/cmmd.h>

#include "global.h"
#include "tdpro.h"
#include "protoglarp.h"
#include "pvmalloc.h"
#include "pvmfrag.h"
#include "pvmmimd.h"
#include "bfunc.h"

#define TVCLEAR(tvp)    ((tvp)->tv_sec = (tvp)->tv_usec = 0)
#ifndef min
#define min(a,b)    ((a)<(b)?(a):(b))
#endif

char *getenv();

/* from lpvm.c */
extern int pvmmyptid;
extern int pvmmytid;
extern int pvmudpmtu;
extern int pvmmyndf;
extern int pvmtidhmask;

static char rcsid[] = "$Id: pvmhost.c,v 1.5 1994/11/09 14:26:05 manchek Exp $";
static char etext[512];         /* scratch for error log */
static int tidnmask = TIDNODE;
static int tidtmask = TIDPTYPE;


main(argc, argv)
	int argc;
	char **argv;
{
	int pvminfo[SIZEHINFO];    /* ntask, hostpart, ptid, MTU, NDF */
	char *p;
	fd_set wrk_rfds, wrk_wfds, rfds, wfds;
	int nfds;
	struct timeval tout;
	int *socks;
	int	dsock;					/* pvmd socket */
	struct frag *frpvmd = 0;	/* (small) frag from pvmd */
	struct frag *topvmd = 0;	/* (big) frag to pvmd */
	struct frag *frtask = 0;	/* (big) frag from task */
	struct frag *totask;		/* (small) frag being sent to task */
	char *txcp = 0;             /* point to remainder of topvmd */
    int txtogo = 0;             /* len of remainder of topvmd */
	int toread;					/* number of bytes to be read from pvmd */
	int frtogo;					/* len of remainder of a fragment */
	int topvmd_dst;				/* dst of fragment being sent to pvmd */ 
	int topvmd_src;				/* src of fragment being sent to pvmd */
	int len;
	int topvmd_ff;				/* ff of fragment being sent to pvmd */
	int	dst;					/* dst of fragment being sent to node */
	int	node;					/* node number */
	CMMD_mcb rmid;				/* msg IDs returned by async recv */
	int n;
	int typepart = TIDONNODE;
	char *cp;
	short *mc_dsts;				/* array to store multicast targets */
	int mc_ndst;				/* how many of them */
	int countdown;


	pvmbeatask();

	CMMD_enable();

	pvm_getfds(&socks);
	dsock = socks[0];
	nfds = dsock + 1;

	if (p = getenv("PVMENTASK")) {
		n = atoi(p);
		pvminfo[0] = (n & tidnmask) + 1;	/* number of tasks to spawn */
		typepart += n & tidtmask;
	} else
		pvminfo[0] = CMMD_partition_size();
/*
sprintf(etext, "pvmhost: type = %d", typepart);
pvmlogerror(etext);
*/
	pvminfo[1] = (pvmmytid & pvmtidhmask) + typepart;
	pvminfo[2] = pvmmyptid;
	pvminfo[3] = MAXFRAGSIZE;
	pvminfo[4] = pvmmyndf;
	CMMD_bc_from_host(pvminfo, sizeof(pvminfo));

	frtask = fr_new(MAXFRAGSIZE);
	if ((rmid = ASYNCRECV(frtask->fr_dat, frtask->fr_max)) < 0) {
		pvmlogperror("pvmhost: ASYNCRECV");
		pvm_exit();
		exit(0);
	}

	FD_ZERO(&wrk_rfds);
	FD_ZERO(&wrk_wfds);
	FD_SET(dsock, &wrk_rfds);

	while (1) {			/* ferry messages between pvmd and tasks */
		
		tout.tv_sec = 0;
		tout.tv_usec = 0;
		if (!topvmd && ASYNCDONE(rmid)) {
			topvmd = frtask;
			txtogo = CMMD_mcb_bytes(rmid);		/* including header */
/*
sprintf(etext, "pvmhost: recv %d", txtogo);
pvmlogerror(etext);
*/
			txcp = topvmd->fr_dat;
			if ((len = pvmget32(txcp + 8) + TDFRAGHDR) != txtogo) {
			/* inplace data, head & body sent separately */

				if (len > MAXFRAGSIZE) {
					struct frag *hdr = topvmd;
					
					topvmd = fr_new(len);
					BCOPY(hdr->fr_dat, topvmd->fr_dat, txtogo);
					fr_unref(hdr);
				}
				CMMD_receive_block(CMMD_mcb_source(rmid), CMMD_ANY_TAG,
				topvmd->fr_dat + txtogo, len - txtogo);
				txtogo = len;
			}
			frtask = fr_new(MAXFRAGSIZE);
			/* ready for the next message */
			CMMD_free_mcb(rmid);
			rmid = ASYNCRECV(frtask->fr_dat, frtask->fr_max);
			FD_SET(dsock, &wrk_wfds);
			topvmd_dst = pvmget32(txcp);
			topvmd_src = pvmget32(txcp + 4);
			topvmd_ff = pvmget8(txcp + 12);
			if (txtogo > pvmudpmtu) {
				frtogo = pvmudpmtu;
				pvmput32(txcp + 8, frtogo - TDFRAGHDR);
				pvmput8(txcp + 12, (topvmd_ff & FFSOM));
			} else
				frtogo = txtogo;
		}
		rfds = wrk_rfds;
		wfds = wrk_wfds;

		if (select(nfds, &rfds, &wfds, (fd_set*)0, &tout) == -1 
		&& errno != EINTR) {
			pvmlogperror("pvmhost: select");
			exit(0);
		}

		if (FD_ISSET(dsock, &rfds)) {

			if (!frpvmd) {
				frpvmd = fr_new(MAXFRAGSIZE);
				toread = TDFRAGHDR;
			}
			n = read(dsock, frpvmd->fr_dat + frpvmd->fr_len, toread);
/*
sprintf(etext, "pvmhost: read %d", n);
pvmlogerror(etext);
*/
			if (n == -1 && errno != EWOULDBLOCK && errno != EINTR) {
				pvmlogperror("pvmhost: read pvmd sock");
				exit(0);
			}
			if (!n) {
				pvmlogerror("pvmhost: read EOF on pvmd sock\n");
				exit(0);
			}

			if ((frpvmd->fr_len += n) < TDFRAGHDR) {
				toread -= n;
				continue;
			}
			n = TDFRAGHDR + pvmget32(frpvmd->fr_dat + 8);   /* header + body */
			if (frpvmd->fr_len < n) {
				toread = n - frpvmd->fr_len;
				continue;
			}

			dst = pvmget32(frpvmd->fr_dat);
			node = dst & tidnmask;
			/* inform precv/recv of a packed message */
			while (CMMD_send_noblock(node, PMTPACK, &node, 0) == CMMD_ERRVAL) {
				sprintf(etext, "pvmhost: can't send null msg to t%x", dst);
				pvmlogperror(etext);
				CMMD_poll_for_services();
			}
			while (CMMD_send_noblock(node, PMTPACK, frpvmd->fr_dat,
			frpvmd->fr_len) == CMMD_ERRVAL) {
				sprintf(etext, "pvmhost: can't send to t%x", dst);
				pvmlogperror(etext);
				CMMD_poll_for_services();
			}
			fr_unref(frpvmd);
			frpvmd = 0;
/*
sprintf(etext, "pvmhost: sent %d to node %d\n", frpvmd->fr_len, (dst & tidnmask));
pvmlogerror(etext);
*/
				
		}

		if (FD_ISSET(dsock, &wfds)) {
			n = write(dsock, txcp, frtogo);
/*
sprintf(etext, "pvmhost: wrote %d to pvmd\n", n);
pvmlogerror(etext);
*/
			if (n == -1 && errno != EWOULDBLOCK && errno != EINTR) {
				pvmlogperror("pvmhost: write pvmd sock");
				exit(0);
			}
			if (n > 0 && (txtogo -= n) > 0) {
				txcp += n;
				if (!(frtogo -= n)) {
					int ff = 0;

					/* new fragment */
					txcp -= TDFRAGHDR;
					txtogo += TDFRAGHDR;
					if (txtogo > pvmudpmtu)
					/* big frag needs to be broken up */
						frtogo = pvmudpmtu;
					else {
						frtogo = txtogo;
						ff = topvmd_ff & FFEOM;
					}
					pvmput32(txcp, topvmd_dst);
					pvmput32(txcp + 4, topvmd_src);
					pvmput32(txcp + 8, frtogo - TDFRAGHDR);
					pvmput8(txcp + 12, ff);
				}
			}
			if (!txtogo) {		/* entire message sent */
				FD_CLR(dsock, &wrk_wfds);
				fr_unref(topvmd);
				topvmd = 0;
			}
		}
		countdown = 1024;
		while (CMMD_poll_for_services() && countdown--)
			;
	}
}


/* 
 * make ntask copies of multicast fragment and send them to the nodes
 * listed in tids[]
 */
/*
cm_mcast(fp, nodes, ntask, tidpart)
	struct frag *fp;	* fragment to send *
	short nodes[];		* targets *
	int ntask;			* how many of them *
	int tidpart;		* hostpart + typepart *
{
	int i;
	char *cp = fp->fr_dat;
	int dst;
	
	for (i = 0; i < ntask; i++) {
		dst = tidpart + nodes[i];
		pvmput32(cp, dst);
		while (CMMD_send_noblock(nodes[i], PMTPACK, fp->fr_dat,
		fp->fr_len) == CMMD_ERRVAL) {
			sprintf(etext, "pvmhost: cm_mcast() can't send to t%x", dst);
			pvmlogperror(etext);
			CMMD_poll_for_services();
		}
	}
}
*/
