/* * * Copyright (C) 2007-2009, IBM Corporation,  *                     Eric Van Hensbergen (bergevan@us.ibm.com) * * Description: torus device interface * * All rights reserved * */ /* * TO DO *	+ ilocks on tx/rx state (eg, for SMP) *	- `reception descriptor' is misleading: it's a block of chunks filled as a fifo *          will need to do something for variable-length packets *	- fifo map (eg, set_inj_fifo_map) *	- set or inspect hint bits to determine routing *	- self-addressed messages *	+ check each software ring against interrupt state instead of scanning bit sets *	+ eliminate in32/out32 for memory-mapped access *	- inspect and eliminate values from torus.h */ #include "u.h"#include "../port/lib.h"#include "mem.h"#include "dat.h"#include "fns.h"#include "../port/error.h" #include "ureg.h" #include "bgp_personality.h"#include "bgcns.h"#include "io.h" #include "frag.h" /* Torus definitions from Linux which might need to be cleaned up */#include "torus.h" #define NEXT(x, l)	(((x)+1)%(l))#define PREV(x, l)	(((x) == 0) ? (l)-1: (x)-1) enum {						/* DCRs (page 23) */	Dreset		= 0xd00,	/* DMA Reset */	Dbasectl	= 0xd01,	/* Base Controls */	Didmamin	= 0xd02,	/* Injection Min Valid Addr */	Didmamax	= 0xd03,	/* Injection Max Valid Addr */	Didmatlb	= 0xd12,	/* Injection Range TLB Attributes */	Drdmatlb	= 0xd13,	/* Reception Range TLB Attributes */	Drdmamin	= 0xd14,	/* Reception Min Valid Addr */	Drdmamax	= 0xd15,	/* Reception Max Valid Addr */	Dififowm0	= 0xd24,	/* Torus fifo injection threshold */	Dififowm1	= 0xd25,	/* Torus fifo injection threshold */	Dilocalwm0	= 0xd26,	/* Local inj fifo threshold */	Drfifowm0	= 0xd27,	/* Torus fifo reception threshold */	Drlocalwm0	= 0xd2b,	/* Local rec fifo threshold */	Dififoctl	= 0xd2c,	/* Injection fifo enable */	Drfifoctl	= 0xd30,	/* Reception fifo enable */	Dififopri	= 0xd32,	/* Injection fifo priority */	Diquanta	= 0xd37,	/* Injection Service Quanta */	Drthresh	= 0xd3a,	/* Threshold for recv fifo type 0 (type 1 next) */	Dififomap	= 0xd3c,	/* imFifo to torus inj fifo map */	Dififolocal	= 0xd5c,	/* imFifo is used for local copies */	Drfifomap0	= 0xd60,	/* reception fifo to rmfifo map */	Drfifomap1	= 0xd61,	/* reception fifo to rmfifo map */	Drclrmask	= 0xd68,	/* Recption DMA Fifo Clear enable */	Drhdrclrmask= 0xd6c,	/* Reception DMA header fifo clear */	Drintron	= 0xd71,	/* Reception fifo intr enable */	Dichzintron	= 0xd76,	/* Injection Ctr Hits Zero Intr Enable */	Dfatalon	= 0xd7e,	/* Fatal Error Enable 0 */	Derrthresh	= 0xd89,	/* Thresholds for SRAM ECC errors */	Dififoce	= 0xd8a,	/* Injection fifo ce (start/end) */	Dclear0		= 0xdb1,	/* Clear_0 */	Dclear1		= 0xdb2,	/* Clear_1 */}; enum {						/* Dbasectl */	Rusedma		= IB(0),	/* =1, use DMA; =0, core does the work */	Rstorehdr	= IB(1),	/* =1, store DMA headers in FIFO (debug) */	Rnoprefetch	= IB(2),	/* =1, disable torus prefetch (0 is default) */	RL3burst	= IB(3),	/* enable burst reads to L3 */	Rinjenable	= IB(4),	/* enable injection unit */	Rrcvenable	= IB(5),	/* enable reception unit */	Rimfuenable	= IB(6),	/* enable imfu_arb fifo/counter */	Rrmfuenable	= IB(7),	/* enable rmfu_arb fifo/counter */	RL3noprefetch= IB(8),	/* disable L3 read prefetch for DMA */	Rrdmastop	= IB(28),	/* =1, rdma stop when fifo full; =0, rdma retry when full */	Rrcvsticky	= IB(29),	/* =1, fifo cross threshold not sticky; =0, is sticky (default) */	Rinjsticky	= IB(30),	/* =1, fifo cross threshold not sticky, ... */}; enum {	X		= 0,			/* dimension */	Y		= 1,	Z		= 2,	N		= 3, 	Xp		= 0,			/* direction */	Xm		= 1,	Yp		= 2,	Ym		= 3,	Zp		= 4,	Zm		= 5,	Nd		= 6, 	Hp		= 7,}; enum {	Torushdrlen=	8,	/* hardware header only */	Tpkthdrlen=		16,	/* both hardware and software headers */ 	PayloadSize	= 256-Tpkthdrlen,	Chunk		= 32,			/* Chunk size (bytes) for size field */	Quad		= 16,			/* 128 bits */ 	Ngroup		= 4,			/* groups */	Ninjfifo	= 32,			/* injection fifos (per group) */	Nrcvfifo	= 8,			/* receive fifos (per group) */	Ncounter	= 64,			/* counters per group, for each of injection and reception */	Npid		= 4,			/* [Pid0, Pid1] */	Nregion		= 8,			/* DMA regions (mainly for put/get modes) */ 	/* software choices */	Ntx			= 256,		/* injection fifo entries */	Nrx			= 512,		/* reception fifo entries */ 	KernelGroup		= Ngroup-1,		/* arbitrary, until ron gets started */	KernelPid		= Npid-1,	KernelTxFifo	= Ninjfifo-1,	KernelRxFifo	= Nrcvfifo-1,	KernelRxRegion	= Nregion-1,	/* not really used, see init_rx_queue below */}; /* * Normal (memory FIFO mode) packet header. * The hardware requires an 8-byte header * of which the last two bytes are reserved (they contain a sequence * number and a header checksum inserted by the hardware). * The next 8 bytes are a software header in memory FIFO * mode, which we use for the fragmentation header, * but other formats are possible in the direct-put/remote-get modes. */typedef struct Tpkt Tpkt;struct Tpkt {	/* hardware header */	u8int	sk;					/* Skip Checksum Control */	u8int	hint;				/* Hint|Dp|Pid0 */	u8int	size;				/* Size|Pid1|Dm|Dy|VC */	u8int	dst[N];				/* Destination Coordinates */	u8int	_6_[2];				/* reserved */ 	/* hardware and software headers */	union{		struct{	/* direct-put DMA mode */			u8int	paoff[4];		/* put address offset (updated by hw for large packets) */			u8int	rdma;		/* RDMA counter ID */			u8int	valid;		/* valid bytes in payload (updated by hw) */			u8int	rgflags;		/* remote get flags */			u8int	idma;		/* IDMA FIFO ID */		};		struct{	/* memory FIFO mode */			u8int	putoff[4];		/* put offset (updated by hw) */			u8int	other[4];		/* software */		};		Hdr;		/* our software header (frag.h), variant of memory FIFO */	};	u8int	payload[];}; /* * SKIP is a field in .sk giving the number of 2-bytes * to skip from the top of the packet before including * the packet bytes into the running checksum. * SIZE is a field in .size giving the size of the * packet in 32-byte 'chunks'. * * I realize using this format for the bitfields is * overkill, but bear with me for consistency. */#define SKIP(n)		F(n, 1, 7)#define SIZE(n)		F(n, 5, 3)#define PID0(n)	((n)&2? Pid0: 0)#define PID1(n)	((n)&1? Pid1: 0) enum {	Sk		= 0x01,			/* Skip Checksum */ 	Pid0	= 0x01,			/* Destination Group FIFO MSb */	Dp		= 0x02,			/* Multicast Deposit */	Hzm		= 0x04,			/* Z- Hint */	Hzp		= 0x08,			/* Z+ Hint */	Hym		= 0x10,			/* Y- Hint */	Hyp		= 0x20,			/* Y+ Hint */	Hxm		= 0x40,			/* X- Hint */	Hxp		= 0x80,			/* X+ Hint */ 	Vcd0	= 0x00,			/* Dynamic 0 VC */	Vcd1	= 0x01,			/* Dynamic 1 VC */	Vcbn	= 0x02,			/* Deterministic Bubble VC */	Vcbp	= 0x03,			/* Deterministic Priority VC */	Dy		= 0x04,			/* Dynamic Routing */	Dm		= 0x08,			/* DMA Mode */	Pid1	= 0x10,			/* Destination Group FIFO LSb */}; /* * Reception FIFO is a ring of full packets with hw/sw headers */typedef union Rcvbuf Rcvbuf;union Rcvbuf {	Tpkt;	u8int packet[256];}; /* * Injection descriptors (p. 14) * * Injection FIFO is a ring of DMA control data * (with reference to payload) and packet headers, * payload elsewhere */typedef struct Injdesc Injdesc;struct Injdesc {	u32int	flags;		/* low-order byte: B(0)-B(5) unused; */						/* B(6) pre-fetch on local memmove; */						/* B(7) 1=local, 0=torus */	u32int	counter;	/* low-order byte: injection counter ID */	u32int	base;		/* base physical address (>>4) of payload */	u32int	length;		/* count in bytes */	Tpkt	hdrs;		/* MAC-level header, then software header */}; /* * Injection and Reception FIFO (ring) */typedef struct Fifo Fifo;struct Fifo {	u32int	start;		/* 32-byte aligned */	u32int	end;	u32int	head;		/* 16-byte aligned */	u32int	tail;		/* address to write new descriptor (host owned) */}; /* * DMA Fifos are controlled through a memory-mapped region */typedef struct Fifoctl Fifoctl;struct Fifoctl {	struct {		Fifo	fifo[Ninjfifo];				// 0 - 1ff		u32int	empty;						// 200		u32int	__unused0;					// 204		u32int	avail;						// 208		u32int	__unused1;					// 20c		u32int	threshold;					// 210		u32int	__unused2;					// 214		u32int	clear_threshold;			// 218		u32int	__unused3;					// 21c		u32int	dma_active;					// 220		u32int	dma_activate;				// 224		u32int	dma_deactivate;				// 228		u8int	__unused4[0x100-0x2c];		// 22c - 2ff 		u32int	counter_enabled[2];			// 300		u32int	counter_enable[2];			// 308		u32int	counter_disable[2];			// 310		u32int	__unused5[2];				// 318		u32int	counter_hit_zero[2];		// 320		u32int	counter_clear_hit_zero[2];	// 328		u32int	counter_group_status;		// 330		u8int	__unused6[0x400-0x334];		// 334 - 3ff 		struct {			u32int	counter;			u32int	increment;			u32int	base;			u32int	__unused;		} counter[Ncounter];				// 400 - 7ff	} inj; 	struct {		Fifo	fifo[Nrcvfifo+1];			// 800 - 87f (rcv fifos), 											// 880-88f (special header fifo)		u8int	__unused0[0x900-0x890];		// 890 - 900 		u32int	glob_ints[16];				// 900 - 93f		u8int	__unused1[0xa00-0x940];		// 940 - 9ff 		u32int	notempty[2];				// a00		u32int	available[2];				// a08		u32int	threshold[2];				// a10		u32int	clear_threshold[2];			// a18		u8int	__unused2[0xb00 - 0xa20];	// a20 - aff 		u32int	counter_enabled[2];			// b00		u32int	counter_enable[2];			// b08		u32int	counter_disable[2];			// b10		u32int	__unused3[2];				// b18		u32int	counter_hit_zero[2];		// b20		u32int	counter_clear_hit_zero[2];	// b28		u32int	counter_group_status;		// b30		u8int	__unused4[0xc00 - 0xb34];	// b34 - bff				struct {			u32int	counter;			u32int	increment;			u32int	base;			u32int	limit;		} counter[Ncounter];				// c00 - fff	} rcv;}; /* Software structure to help track tx/rx rings */typedef struct TxRing TxRing;struct TxRing {//	Lock	lock; 	uint	active;	Injdesc*	desc;		/* descriptor ring (in virtual space) */	Block	**blocks;		/* blocks currently on ring */	Fifo*	fifo;			/* hardware memory fifo assigned to ring */	u32int	start;			/* main memory copy of fifo->start */	uint	tail_idx;	uint	pending_idx; 	u32int*	incr;		/* address of increment register for counter */	u32int	ctrid;		/* counter ID for descriptor (group<<6 | fifo-index) */ 	int	ntd;			/* number of transmit descriptors */	int	np;				/* number of packets in fifo */		int full;			/* number of times the txring was full */		QLock	rl;	Rendez	r;		/* wait for free space */ 	int	group;	int	index;	TxRing*	next;}; typedef struct RxRing RxRing;struct RxRing {//	Lock	lock; 	uint	active;	Rcvbuf*	desc;		/* packet buffer ring (in virtual space) */	Fifo*	fifo;		/* hardware memory fifo assigned to ring */	u32int	start;		/* main memory copy of fifo->start */	uint	head_idx; 	/* statistics */	u32int	dropped;	u32int	delivered; 	int	nrd;			/* number of reception descriptors */ 	int	group;	int	index;	RxRing*	next;}; typedef struct Torus Torus;struct Torus {	u8int	addr[N];		/* of this node */	u8int	size[N];		/* of the torus */	u8int	torus[N];		/* in that dimension (!mesh) */	u8int	cp[N];			/* cutoff in [XYZ]p */	u8int	cm[N];			/* cutoff in [XYZ]m */ 	u32int	zmzpymypxmxp;	/* reset FIFOs */	u32int	xpxmypymzpzm;	/* ditto, in IBM field order */ 	QLock	alock;	int	ref; 	Queue*	rcvq; 	/*	 * these counters will need to become per DMA group	 * (eg, in TxRing or RxRing) when more than one fifo is used,	 * to allow concurrent access	 */	uint	rcvintr;	uint	rcvpackets;	uint	rcvtoss;	uint	rcvqfull;	uint	rcvqpass;	uint	rcvqbread;	uint	rcvqread;	uint	rcvpause;	uint	rcvresume;	uint  rcvring;	uint  rcvringdrop;	uint	aror;	uint	activemsg;	uint	activemsgreply;	uint	activemsgmm;	/* last active message x,y,z -- since iprint does not work */	uint ax, ay, az;	uint	lastactiveptr;	uint	injintr;	uint	injgetfifo;	uint	injgetstatus;	uint  injring;	uint	memfifofull;	uint	unreachable;	uint	self;	uint	injpackets;	uint	unaligned; 	Fifoctl*	dma;				/* pointer to memory-mapped hw control structure */	TxRing	txr[Ngroup][Ninjfifo];	/* software data for injection */	RxRing	rxr[Ngroup][Nrcvfifo];	/* software data for reception */ 	Lock	txlock;					/* protect all groups for now */	Lock	rxlock;					/* protect all groups for now */ 	/* reception counters aren't needed for memory FIFO mode */ 	Frags*	frags;}; enum {	DbgRingR=	1<<5,	DbgRingX=	1<<4,	DbgDMA=	1<<2,	DbgSetup=	1<<1,	DbgIO=		1<<0,}; typedef struct Ring Ring; struct Ring {	u32int tag[3];	void *userdata; /* for irecv */	u32int count; /* number of units; bytecount is computed using datatype*/	u32int datatype;	u32int source;	u32int itag;	u32int comm;	u32int status;	u32int xmit; /* if this was an xmit request. */	u32int done;	u32int x, y, z;}; struct Ringb {	u8int *base;	int count;}; /* active message ring.  * received messages from kernel  increment prod * processes eat messages and increment con * if prod-cons is ever > size, then the kernel does two things:  * 1. no longer dq messages, so that nobody can send us any -- this back pressure will  * back up to senders via the torus hardware * 2. no longer allow messages to be sent. You have to be a good boy.  * We'll see how this goes. But for the breadth first search we have to do something like this * as their current code is not scalable -- they allocate a buffer per mpi rank, and that  * just won't do.  I pad to make sure we don't get cache wars -- procs run on core > 0. * ROUND 1: no flow control as described -- let's just go for bandwidth and see what we get.  * it need not be high -- latency is more important just now.  */struct AmRing {        u8int *base;        int size;        int prod;        u8int _116[128-12];        int con;        u8int _124[128-4];}; /* active messages -- if this is on, the rx interrupt will manage the message and not enqueue it *//* enabled by writing a to /dev/torusctl and disabled by writing i to /dev/torusctl */int activemessages = 0; struct AmRing *activering;static u8int *onering = nil;static int ringsize;static int debug_torus = 1; static Torus torus;int torus_dump_cons = 1; static void torusdisable(int cause);static void torusinject(Torus*, TxRing*, Block*);static int quicktorusinject(Torus *t, TxRing *tx, Tpkt *pkt, void *data, int kaddr, int block); /* sorry for the ugliness, I'll search and replace it eventually */#define DUMP_DCR(desc, x) p = seprint(p, e, " " desc "(%ux): %ux\n", (unsigned int)x, dcrget(x)) typedef struct Dcr Dcr;struct Dcr {	u32int	dcr;

Lines 1–500 of 2362. Load more