#include "rompi.h" enum {	TAGcomm = 0,	TAGtag,	TAGsource,	TAG}; /* simple MPI for Plan 9 torus driver. This is a first cut. *//* rank: rather than have to ask a daemon for rank, we're going to  * require that processes be started as follows:  * rank = z * (xsize + ysize) + y * xsize + x * Thus, rank0 is (0,0,0) * This disallows the dynamic stuff but nobody in their right mind uses * that anyway. We're going to avoid the DCMF entirely and C++ of course. */struct bufpacket {	struct  bufpacket *next, *prev;	unsigned long tag[TAG];	void *userdata; /* for irecv */	int count; /* for irecv */	MPI_Datatype datatype;	int num; /* redundant, but we'll figure it out later. */	int source;	int itag;	MPI_Comm comm;	MPI_Status status;	int xmit; /* if this was an xmit request. */	unsigned char data[];}; /* these are rounded to 64 byte boundaries for now. */struct ringbuf {	unsigned long tag[TAG];	void *userdata; /* for irecv */	int count; /* number of units; bytecount is computed using datatype*/	MPI_Datatype datatype;	int source;	int itag;	MPI_Comm comm;	MPI_Status status;	int xmit; /* if this was an xmit request. */	int done;};  int myproc = -1, nproc = -1, maxrank = -1;int rompidebug = 0; /* 16: print # packets read, 2: recv, 1: send */char *myname = nil;extern int torusfd;struct bufpacket buffered = {&buffered, &buffered}; void torussend(void *buf, int length, int x, int y, int z, int deposit, void *tag, int taglen);int torusrecv(void *buf, long length, void *tag, long taglen);void torusinit(int *pmyproc, const int pnprocs);void ranktoxyz(int rank, int *x, int *y, int *z);int xyztorank(int x, int y, int z);extern int x, y, z, xsize, ysize, zsize;u64int	tbget(void); voidbuf_add(struct bufpacket *new){//print("Add %p: %p %p %p %p: ", new, buffered.next->prev, buffered.next, &buffered, buffered.next);	buffered.next->prev = new;	new->next = buffered.next;	new->prev = &buffered;	buffered.next = new;//print("Added: %p %p %p %p\n", buffered.next->prev, buffered.next, &buffered, buffered.next);} voidbuf_del(struct bufpacket *r){//print("DEL: %p, %p %p\n", r, r->next, r->prev);//print("DEL %p: %p %p: ", r, r->next->prev, r->prev->next);	r->next->prev = r->prev;	r->prev->next = r->next;//print(": %p %p\n", r->next->prev, r->prev->next);} intbuf_empty(void){	return buffered.next == &buffered;} voidpanic(char *s){    void exits(char *);	char *m = smprint("Fatal: %s", s);	print("%s\n", m);	exits(m);} int MPI_Abort( MPI_Comm comm, int errorcode ){	panic("MPI_Abort: aborting as abort is not implemented\n");	exit(errorcode);} int MPI_Comm_dup (         MPI_Comm comm,         MPI_Comm *comm_out ){	/* yes well ... we have no real use for this yet. Maybe never. 	 * we can hope so anyway.	 */	*comm_out = comm;	return MPI_SUCCESS;}int MPI_Comm_split ( MPI_Comm comm, int color, int key, MPI_Comm *comm_out ){	panic("no split yet. Or ever. \n");	return -1;} int MPI_Comm_free ( MPI_Comm *commp ){	/* that was easy */	return MPI_SUCCESS;}/* we require that myproc be the first arg and nproc the second. */intMPI_Init(int *argc, char ***argv){	int xyztorank(int x, int y, int z);	extern int x, y, z;	char **av = *argv;	myname = av[0];//print( "myname %s av[1] %s\n", myname, av[1]);	nproc = strtoul(av[1], 0, 10);	/* fix up args so av[0] is my name */	av[1] = av[0];	*argc = *argc - 1;	*argv = av + 1;	torusinit(&myproc, nproc);//print( "x %d y %d z %d myproc %d xyztorank() %d\n", x, y, z, myproc, xyztorank(x,y,z)); 	maxrank = nproc - 1; 	/* test */	if (myproc != xyztorank(x,y,z)) {		print( "myproc %d != xyztorank %d ", myproc, xyztorank(x,y,z));		panic("bail");	}	return MPI_SUCCESS;} intMPI_Finalize(void){	return MPI_SUCCESS;} intMPI_Comm_size(MPI_Comm x, int *s){	switch(x) {		case MPI_COMM_WORLD:			*s = nproc;			break;		default:			*s = nproc;//			panic("comm size only implements MPI_COMM_WORLD\n");	}	return MPI_SUCCESS;} intMPI_Comm_rank(MPI_Comm x, int *s){	switch(x) {		case MPI_COMM_WORLD:			*s = myproc;			break;		default:			panic("comm rank only implements MPI_COMM_WORLD\n");	}	return MPI_SUCCESS;} doubleMPI_Wtime(void){	unsigned long long t;	double f;	static int fd = -1;	extern int pread(int, void *, int, unsigned long long); 	if (fd < 0) {		fd = open("/dev/bintime", 0);		if (fd < 0)			panic("Can't open /dev/bintime");	} 	pread(fd, &t, sizeof(t), 0ULL);	f = t;	f = f/1000000000.0;	return f;} /* the size of an mpi data type is the data type value >> 8 & 0xff. Just assign to u8int after the >> */ /* if we like this function merge it in with the one below later. */intMPI_Sendxyz( void *buf, int num, MPI_Datatype datatype, int x, int y, int z, int hint,              int tag, MPI_Comm comm ){	unsigned char nbytes = datatype >> 8;	unsigned long torustag[TAG];	int count;	count = num * nbytes;	switch(comm) {		case MPI_COMM_WORLD:			break;		default:			panic("MPI_Send only implements MPI_COMM_WORLD\n");	} 	torustag[TAGcomm] = comm;	torustag[TAGtag] = tag;	torustag[TAGsource] = myproc;	torussend(buf, count, x, y, z, hint, torustag, sizeof(torustag)); 	return MPI_SUCCESS;} intMPI_Send( void *buf, int num, MPI_Datatype datatype, int dest,               int tag, MPI_Comm comm ){	unsigned char nbytes = datatype >> 8;	unsigned long torustag[TAG];	int count;	int x, y, z;	count = num * nbytes;	switch(comm) {		case MPI_COMM_WORLD:			break;		default:			panic("MPI_Send only implements MPI_COMM_WORLD\n");	} 	torustag[TAGcomm] = comm;	torustag[TAGtag] = tag;	torustag[TAGsource] = myproc;	ranktoxyz(dest, &x, &y, &z);	torussend(buf, count, x, y, z, 0, torustag, sizeof(torustag)); 	return MPI_SUCCESS;} /* we don't do anything with the request yet. Sorry. */int MPI_Isend( void *buf, int num, MPI_Datatype datatype, int dest, int tag,               MPI_Comm comm, MPI_Request *request ){	unsigned char nbytes = datatype >> 8;	unsigned long torustag[TAG];	int count;	int x, y, z;	struct bufpacket *b; 	b = calloc(sizeof(*b), 1);	if (! b) {		panic("calloc in recv returned 0");	}	count = num * nbytes;	torustag[TAGcomm] = comm;	torustag[TAGtag] = tag;	torustag[TAGsource] = myproc;	ranktoxyz(dest, &x, &y, &z);	torussend(buf, count, x, y, z, 0, torustag, sizeof(torustag)); 	b->xmit = 1;	*request = (MPI_Request )b;		return MPI_SUCCESS;}/* the kernel is catching these things.  * Just create the descriptors and save them in the request array.  */intMPI_Irecv( void *buf, int num, MPI_Datatype datatype, int source,               int tag, MPI_Comm comm, MPI_Request *request ){	unsigned char nbytes = datatype >> 8;	int count; 	count = num * nbytes;	struct bufpacket *b; 	b = calloc(sizeof(*b), 1);	if (! b) {		panic("calloc in recv returned 0");	}	b->userdata = buf;	b->count = count;	b->num = num;	b->source = source;	b->itag = tag;	b->comm = comm; 	*request = (MPI_Request )b;		return MPI_SUCCESS;} int MPI_Waitall(        int count,         MPI_Request array_of_requests[],         MPI_Status array_of_statuses[] ){ 	/* just allocate a bufpacket with room for headers and data and such  -- make it a nice page-multiple*/	struct bufpacket *req;	MPI_Request *curreq;	for(curreq = array_of_requests; curreq - array_of_requests < count; curreq++) {		req = (struct bufpacket *) *curreq;		/* if it's a send, we're done; no way to check yet */		if (req->xmit)			continue;		MPI_Recv(req->userdata, req->num, req->datatype, 				req->source, req->itag, req->comm, &req->status);		free(req);	} 	if (rompidebug &2)print("MPI WaitAll: done\n");	return MPI_SUCCESS;} /* todo: Should have implemented recv as a special kind of irecv+waitall.  * reason is that irecv is the common case. OH well, live and learn. */intMPI_Recv( void *buf, int num, MPI_Datatype datatype, int source,               int tag, MPI_Comm comm, MPI_Status *status ){	static int nthpacket = 0;	unsigned char nbytes = datatype >> 8;	int count;	int size;	count = num * nbytes;	/* just allocate a bufpacket with room for headers and data and such  -- make it a nice page-multiple*/	struct bufpacket *b;	switch(comm) {		case MPI_COMM_WORLD:			break;		default:			panic("MPI_Recv only implements MPI_COMM_WORLD\n");	} //print("WANT %d, buffer %s\n", source, buf_empty() ? "empty":"");	/* Well, first, let's see if it's here somewhere. */	for(b = buffered.next; b != &buffered; b = b->next) {		if (rompidebug &2) print("Check %p tag %ld source %ld\n", b, b->tag[TAGtag] , b->tag[TAGsource]);		if ((b->tag[TAGcomm] == comm) && (b->tag[TAGtag] == tag  || tag == MPI_ANY_TAG) && (b->tag[TAGsource] == source || source == MPI_ANY_SOURCE)){			buf_del(b);			goto got;			}	}//print("GET IT!\n");	/* get it */	while (1) {		b = calloc(1048576 + 4096, 1);		if (! b) {			panic("calloc in recv returned 0");		} 		/* for MPI, you pretty much have to take the worst case. It might be Some Other Packet *///print("IR %p\n", b);		size = torusrecv(b->data, 1048576, b->tag, sizeof(b->tag));		if (size < 0)			panic("torusrecv -1: %r");		nthpacket++;		if (rompidebug & 16)			print("%d'th packet\n", nthpacket);		/* tag matching. */		if (rompidebug &2){			int rx, ry, rz;			ranktoxyz(b->tag[TAGsource], &rx, &ry, &rz);			print("Want (%x,%d,%d): REcv comm %lx tag %ld source %ld(%d, %d, %d)\n", comm, tag, source, b->tag[TAGcomm],b->tag[TAGtag], b->tag[TAGsource], rx, ry, rz);		}		if ((b->tag[TAGcomm] == comm) && (b->tag[TAGtag] == tag  || tag == MPI_ANY_TAG) && (b->tag[TAGsource] == source || source == MPI_ANY_SOURCE))			break;		/* for someone else ... *///print("Add %d: ", b->tag[TAGsource]);		buf_add(b);	}		 got:	if (rompidebug &2)print("MPI recv: got it\n");	status->count = count;	status->cancelled = 0;	status->MPI_SOURCE = b->tag[TAGsource];	status->MPI_TAG = b->tag[TAGtag];	status->MPI_ERROR = 0;	if (rompidebug &2)print("MPI recv: memcpy(%p,%p,%d)\n", buf, b->data, count);	memcpy(buf, b->data, count);	//free(b);	return MPI_SUCCESS;} /* the size of an mpi data type is the data type value >> 8 & 0xff. Just assign to u8int after the >> *//* ah, MPIsendbuf	address of send buffer (choice) count	number of elements in send buffer (integer) datatype	data type of elements of send buffer (handle) op	reduce operation (handle) root	rank of root process (integer) comm	communicator (handle)  */ intreduce_end_slow ( void *buf, int num, MPI_Datatype datatype, int root,                MPI_Comm comm,  MPI_Status *status){	int tox, toy, toz;	int torank, fromrank;  	unsigned long torustag[TAG];	int reducetag = 0xcafebabe; 	torustag[TAGcomm] = comm;	torustag[TAGtag] = reducetag;	torustag[TAGsource] = myproc; 	if(rompidebug&8) print("%lld reduce_end: %d(%d, %d, %d)\n", tbget(), myproc, x, y, z); 	/* the easy cases: z > 0 and the origin */	if (z) {		/* get the sum from rank x,y,0 */		fromrank = xyztorank(x,y,0);		if(rompidebug&4) print("%lld reduce_end: %d(%d,%d,%d): wait from %d(%d, %d, %d)\n", tbget(), myproc, x, y, z, fromrank, x, 0, 0);		if (rompidebug&128) print("DOT:%lld b%d->b%d;\n", tbget(), fromrank, myproc);		MPI_Recv(buf, num, datatype, fromrank, reducetag, MPI_COMM_WORLD, status);	} else if (! myproc) {		for(tox = 1; tox < xsize; tox++){			torank = xyztorank(tox, 0, 0);			MPI_Send(buf, num, datatype, torank, 1, comm);		}		for(toy = 1; toy < ysize; toy++){			torank = xyztorank(0, toy, 0);			MPI_Send(buf, num, datatype, torank, 1, comm);		}		for(toz = 1; toz < zsize; toz++){			torank = xyztorank(0, 0, toz);			MPI_Send(buf, num, datatype, torank, 1, comm);		}	} else {		if (! y) {			/* get the sum from rank 0 */			fromrank=xyztorank(0,0,0);			if(rompidebug&4) print("%lld reduce_end: %d(%d,%d,%d): wait from %d(%d, %d, %d)\n", tbget(), myproc, x, y, z, fromrank, x, 0, 0);			if (rompidebug&128) print("DOT:%lld b%d->b%d;\n", tbget(), fromrank, myproc);			MPI_Recv(buf, num, datatype, fromrank, reducetag, MPI_COMM_WORLD, status);			/* send out our y >0 data */			for(toy = 1; toy < ysize; toy++){				torank = xyztorank(x, toy, 0);				MPI_Send(buf, num, datatype, torank, 1, comm);			}			for(toz = 1; toz < zsize; toz++){				torank = xyztorank(x, 0, toz);				MPI_Send(buf, num, datatype, torank, 1, comm);			}		} else {			/* get the sum from rank x,0,0 */			fromrank=xyztorank(x,0,0);			if(rompidebug&4) print("%lld reduce_end: %d(%d,%d,%d): wait from %d(%d, %d, %d)\n", tbget(), myproc, x, y, z, fromrank, x, 0, 0);			if (rompidebug&128) print("DOT:%lld b%d->b%d;\n", tbget(), fromrank, myproc);			MPI_Recv(buf, num, datatype, fromrank, reducetag, MPI_COMM_WORLD, status);			/* send out our z >0 data */			for(toz = 1; toz < zsize; toz++){				torank = xyztorank(x, y, toz);				MPI_Send(buf, num, datatype, torank, 1, comm);			}		}	}	return MPI_SUCCESS; }int reduce_end ( void *buf, int num, MPI_Datatype datatype, int root,                MPI_Comm comm,  MPI_Status *status){	int fromrank;	unsigned long torustag[TAG];	int reducetag = 1; 	torustag[TAGcomm] = comm;	torustag[TAGtag] = reducetag;	torustag[TAGsource] = myproc; 	if(rompidebug&8) print("%lld reduce_end: %d(%d, %d, %d)\n", tbget(), myproc, x, y, z); 	//rompidebug |= 2;	/* the easy cases: z > 0 and the origin */	if (z) {		/* get the sum from rank x,y,0 */		fromrank = xyztorank(x,y,0);

Lines 1–500 of 695. Load more