lael-labs/linux

drivers/infiniband/core/rw.c+11−13

 	return false;
 }
-static inline u32 rdma_rw_max_sge(struct ib_device *dev,
-		enum dma_data_direction dir)
-{
-	return dir == DMA_TO_DEVICE ?
-		dev->attrs.max_sge : dev->attrs.max_sge_rd;
-}
 static inline u32 rdma_rw_fr_page_list_len(struct ib_device *dev)
 {
 	/* arbitrary limit to avoid allocating gigantic resources */
 	return min_t(u32, dev->attrs.max_fast_reg_page_list_len, 256);
 }
+/* Caller must have zero-initialized *reg. */
 static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,
 		struct rdma_rw_reg_ctx *reg, struct scatterlist *sg,
 		u32 sg_cnt, u32 offset)
 		u8 port_num, struct scatterlist *sg, u32 sg_cnt, u32 offset,
 		u64 remote_addr, u32 rkey, enum dma_data_direction dir)
 {
+	struct rdma_rw_reg_ctx *prev = NULL;
 	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device);
 	int i, j, ret = 0, count = 0;
 	}
 	for (i = 0; i < ctx->nr_ops; i++) {
-		struct rdma_rw_reg_ctx *prev = i ? &ctx->reg[i - 1] : NULL;
 		struct rdma_rw_reg_ctx *reg = &ctx->reg[i];
 		u32 nents = min(sg_cnt, pages_per_mr);
 		sg_cnt -= nents;
 		for (j = 0; j < nents; j++)
 			sg = sg_next(sg);
+		prev = reg;
 		offset = 0;
 	}
+	if (prev)
+		prev->wr.wr.next = NULL;
 	ctx->type = RDMA_RW_MR;
 	return count;
 		u64 remote_addr, u32 rkey, enum dma_data_direction dir)
 {
 	struct ib_device *dev = qp->pd->device;
-	u32 max_sge = rdma_rw_max_sge(dev, dir);
+	u32 max_sge = dir == DMA_TO_DEVICE ? qp->max_write_sge :
+		      qp->max_read_sge;
 	struct ib_sge *sge;
 	u32 total_len = 0, i, j;
 			rdma_wr->wr.opcode = IB_WR_RDMA_READ;
 		rdma_wr->remote_addr = remote_addr + total_len;
 		rdma_wr->rkey = rkey;
+		rdma_wr->wr.num_sge = nr_sge;
 		rdma_wr->wr.sg_list = sge;
 		for (j = 0; j < nr_sge; j++, sg = sg_next(sg)) {
-			rdma_wr->wr.num_sge++;
 			sge->addr = ib_sg_dma_address(dev, sg) + offset;
 			sge->length = ib_sg_dma_len(dev, sg) - offset;
 			sge->lkey = qp->pd->local_dma_lkey;
 			offset = 0;
 		}
-		if (i + 1 < ctx->nr_ops)
+		rdma_wr->wr.next = i + 1 < ctx->nr_ops ?
-			rdma_wr->wr.next = &ctx->map.wrs[i + 1].wr;
+			&ctx->map.wrs[i + 1].wr : NULL;
 	}
 	ctx->type = RDMA_RW_MULTI_WR;

drivers/infiniband/core/verbs.c+9−0

@@ -825,6 +825,15 @@ struct ib_qp ib_create_qp(struct ib_pd pd, · 824 unmodified lines
825	}	825	}
826	}	826	}
827		827
		828	/*
		829	* Note: all hw drivers guarantee that max_send_sge is lower than
		830	* the device RDMA WRITE SGE limit but not all hw drivers ensure that
		831	* max_send_sge <= max_sge_rd.
		832	*/
		833	qp->max_write_sge = qp_init_attr->cap.max_send_sge;
		834	qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
		835	device->attrs.max_sge_rd);
		836
828	return qp;	837	return qp;
829	}	838	}
830	EXPORT_SYMBOL(ib_create_qp);	839	EXPORT_SYMBOL(ib_create_qp);

drivers/infiniband/hw/hfi1/Kconfig+2−1

 config INFINIBAND_HFI1
 	tristate "Intel OPA Gen1 support"
-	depends on X86_64 && INFINIBAND_RDMAVT
+	depends on X86_64 && INFINIBAND_RDMAVT && I2C
 	select MMU_NOTIFIER
 	select CRC32
+	select I2C_ALGOBIT
 	---help---
 	This is a low-level driver for Intel OPA Gen1 adapter.
 config HFI1_DEBUG_SDMA_ORDER

drivers/infiniband/hw/hfi1/Makefile+1−1

@@ -10,7 +10,7 @@ obj-$(CONFIG_INFINIBAND_HFI1) += hfi1.o · 9 unmodified lines
10	hfi1-y := affinity.o chip.o device.o driver.o efivar.o \	10	hfi1-y := affinity.o chip.o device.o driver.o efivar.o \
11	eprom.o file_ops.o firmware.o \	11	eprom.o file_ops.o firmware.o \
12	init.o intr.o mad.o mmu_rb.o pcie.o pio.o pio_copy.o platform.o \	12	init.o intr.o mad.o mmu_rb.o pcie.o pio.o pio_copy.o platform.o \
13	qp.o qsfp.o rc.o ruc.o sdma.o sysfs.o trace.o twsi.o \	13	qp.o qsfp.o rc.o ruc.o sdma.o sysfs.o trace.o \
14	uc.o ud.o user_exp_rcv.o user_pages.o user_sdma.o verbs.o \	14	uc.o ud.o user_exp_rcv.o user_pages.o user_sdma.o verbs.o \
15	verbs_txreq.o	15	verbs_txreq.o
16	hfi1-$(CONFIG_DEBUG_FS) += debugfs.o	16	hfi1-$(CONFIG_DEBUG_FS) += debugfs.o

drivers/infiniband/hw/hfi1/affinity.c+441−126

@@ -47,12 +47,18 @@ · 46 unmodified lines
47	#include <linux/topology.h>	47	#include <linux/topology.h>
48	#include <linux/cpumask.h>	48	#include <linux/cpumask.h>
49	#include <linux/module.h>	49	#include <linux/module.h>
		50	#include <linux/cpumask.h>
50		51
51	#include "hfi.h"	52	#include "hfi.h"
52	#include "affinity.h"	53	#include "affinity.h"
53	#include "sdma.h"	54	#include "sdma.h"
54	#include "trace.h"	55	#include "trace.h"
55		56
		57	struct hfi1_affinity_node_list node_affinity = {
		58	.list = LIST_HEAD_INIT(node_affinity.list),
		59	.lock = __SPIN_LOCK_UNLOCKED(&node_affinity.lock),
		60	};
		61
56	/* Name of IRQ types, indexed by enum irq_type */	62	/* Name of IRQ types, indexed by enum irq_type */
57	static const char * const irq_type_names[] = {	63	static const char * const irq_type_names[] = {
58	"SDMA",	64	"SDMA",
@@ -61,6 +67,9 @@ static const char * const irq_type_names[] = { · 2 unmodified lines
61	"OTHER",	67	"OTHER",
62	};	68	};
63		69
		70	/* Per NUMA node count of HFI devices */
		71	static unsigned int *hfi1_per_node_cntr;
		72
64	static inline void init_cpu_mask_set(struct cpu_mask_set *set)	73	static inline void init_cpu_mask_set(struct cpu_mask_set *set)
65	{	74	{
66	cpumask_clear(&set->mask);	75	cpumask_clear(&set->mask);
@@ -69,47 +78,136 @@ static inline void init_cpu_mask_set(struct cpu_mask_set *set) · 2 unmodified lines
69	}	78	}
70		79
71	/* Initialize non-HT cpu cores mask */	80	/* Initialize non-HT cpu cores mask */
72	int init_real_cpu_mask(struct hfi1_devdata *dd)	81	void init_real_cpu_mask(void)
73	{	82	{
74	struct hfi1_affinity *info;
75	int possible, curr_cpu, i, ht;	83	int possible, curr_cpu, i, ht;
76		84
77	info = kzalloc(sizeof(*info), GFP_KERNEL);	85	cpumask_clear(&node_affinity.real_cpu_mask);
78	if (!info)
79	return -ENOMEM;
80
81	cpumask_clear(&info->real_cpu_mask);
82		86
83	/* Start with cpu online mask as the real cpu mask */	87	/* Start with cpu online mask as the real cpu mask */
84	cpumask_copy(&info->real_cpu_mask, cpu_online_mask);	88	cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
85		89
86	/*	90	/*
87	* Remove HT cores from the real cpu mask. Do this in two steps below.	91	* Remove HT cores from the real cpu mask. Do this in two steps below.
88	*/	92	*/
89	possible = cpumask_weight(&info->real_cpu_mask);	93	possible = cpumask_weight(&node_affinity.real_cpu_mask);
90	ht = cpumask_weight(topology_sibling_cpumask(	94	ht = cpumask_weight(topology_sibling_cpumask(
91	cpumask_first(&info->real_cpu_mask)));	95	cpumask_first(&node_affinity.real_cpu_mask)));
92	/*	96	/*
93	* Step 1. Skip over the first N HT siblings and use them as the	97	* Step 1. Skip over the first N HT siblings and use them as the
94	* "real" cores. Assumes that HT cores are not enumerated in	98	* "real" cores. Assumes that HT cores are not enumerated in
95	* succession (except in the single core case).	99	* succession (except in the single core case).
96	*/	100	*/
97	curr_cpu = cpumask_first(&info->real_cpu_mask);	101	curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
98	for (i = 0; i < possible / ht; i++)	102	for (i = 0; i < possible / ht; i++)
99	curr_cpu = cpumask_next(curr_cpu, &info->real_cpu_mask);	103	curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
100	/*	104	/*
101	* Step 2. Remove the remaining HT siblings. Use cpumask_next() to	105	* Step 2. Remove the remaining HT siblings. Use cpumask_next() to
102	* skip any gaps.	106	* skip any gaps.
103	*/	107	*/
104	for (; i < possible; i++) {	108	for (; i < possible; i++) {
105	cpumask_clear_cpu(curr_cpu, &info->real_cpu_mask);	109	cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
106	curr_cpu = cpumask_next(curr_cpu, &info->real_cpu_mask);	110	curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
		111	}
		112	}
		113
		114	int node_affinity_init(void)
		115	{
		116	int node;
		117	struct pci_dev *dev = NULL;
		118	const struct pci_device_id *ids = hfi1_pci_tbl;
		119
		120	cpumask_clear(&node_affinity.proc.used);
		121	cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
		122
		123	node_affinity.proc.gen = 0;
		124	node_affinity.num_core_siblings =
		125	cpumask_weight(topology_sibling_cpumask(
		126	cpumask_first(&node_affinity.proc.mask)
		127	));
		128	node_affinity.num_online_nodes = num_online_nodes();
		129	node_affinity.num_online_cpus = num_online_cpus();
		130
		131	/*
		132	* The real cpu mask is part of the affinity struct but it has to be
		133	* initialized early. It is needed to calculate the number of user
		134	* contexts in set_up_context_variables().
		135	*/
		136	init_real_cpu_mask();
		137
		138	hfi1_per_node_cntr = kcalloc(num_possible_nodes(),
		139	sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
		140	if (!hfi1_per_node_cntr)
		141	return -ENOMEM;
		142
		143	while (ids->vendor) {
		144	dev = NULL;
		145	while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
		146	node = pcibus_to_node(dev->bus);
		147	if (node < 0)
		148	node = numa_node_id();
		149
		150	hfi1_per_node_cntr[node]++;
		151	}
		152	ids++;
107	}	153	}
108		154
109	dd->affinity = info;
110	return 0;	155	return 0;
111	}	156	}
112		157
		158	void node_affinity_destroy(void)
		159	{
		160	struct list_head pos, q;
		161	struct hfi1_affinity_node *entry;
		162
		163	spin_lock(&node_affinity.lock);
		164	list_for_each_safe(pos, q, &node_affinity.list) {
		165	entry = list_entry(pos, struct hfi1_affinity_node,
		166	list);
		167	list_del(pos);
		168	kfree(entry);
		169	}
		170	spin_unlock(&node_affinity.lock);
		171	kfree(hfi1_per_node_cntr);
		172	}
		173
		174	static struct hfi1_affinity_node *node_affinity_allocate(int node)
		175	{
		176	struct hfi1_affinity_node *entry;
		177
		178	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
		179	if (!entry)
		180	return NULL;
		181	entry->node = node;
		182	INIT_LIST_HEAD(&entry->list);
		183
		184	return entry;
		185	}
		186
		187	/*
		188	* It appends an entry to the list.
		189	* It must be called with node_affinity.lock held.
		190	*/
		191	static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
		192	{
		193	list_add_tail(&entry->list, &node_affinity.list);
		194	}
		195
		196	/* It must be called with node_affinity.lock held */
		197	static struct hfi1_affinity_node *node_affinity_lookup(int node)
		198	{
		199	struct list_head *pos;
		200	struct hfi1_affinity_node *entry;
		201
		202	list_for_each(pos, &node_affinity.list) {
		203	entry = list_entry(pos, struct hfi1_affinity_node, list);
		204	if (entry->node == node)
		205	return entry;
		206	}
		207
		208	return NULL;
		209	}
		210
113	/*	211	/*
114	* Interrupt affinity.	212	* Interrupt affinity.
115	*	213	*
@@ -121,10 +219,10 @@ int init_real_cpu_mask(struct hfi1_devdata *dd) · 5 unmodified lines
121	* to the node relative 1 as necessary.	219	* to the node relative 1 as necessary.
122	*	220	*
123	*/	221	*/
124	void hfi1_dev_affinity_init(struct hfi1_devdata *dd)	222	int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
125	{	223	{
126	int node = pcibus_to_node(dd->pcidev->bus);	224	int node = pcibus_to_node(dd->pcidev->bus);
127	struct hfi1_affinity *info = dd->affinity;	225	struct hfi1_affinity_node *entry;
128	const struct cpumask *local_mask;	226	const struct cpumask *local_mask;
129	int curr_cpu, possible, i;	227	int curr_cpu, possible, i;
130		228
@@ -132,56 +230,93 @@ void hfi1_dev_affinity_init(struct hfi1_devdata *dd) · 1 unmodified line
132	node = numa_node_id();	230	node = numa_node_id();
133	dd->node = node;	231	dd->node = node;
134		232
135	spin_lock_init(&info->lock);
136
137	init_cpu_mask_set(&info->def_intr);
138	init_cpu_mask_set(&info->rcv_intr);
139	init_cpu_mask_set(&info->proc);
140
141	local_mask = cpumask_of_node(dd->node);	233	local_mask = cpumask_of_node(dd->node);
142	if (cpumask_first(local_mask) >= nr_cpu_ids)	234	if (cpumask_first(local_mask) >= nr_cpu_ids)
143	local_mask = topology_core_cpumask(0);	235	local_mask = topology_core_cpumask(0);
144	/* Use the "real" cpu mask of this node as the default */	236
145	cpumask_and(&info->def_intr.mask, &info->real_cpu_mask, local_mask);	237	spin_lock(&node_affinity.lock);
146		238	entry = node_affinity_lookup(dd->node);
147	/* fill in the receive list */	239	spin_unlock(&node_affinity.lock);
148	possible = cpumask_weight(&info->def_intr.mask);	240
149	curr_cpu = cpumask_first(&info->def_intr.mask);	241	/*
150	if (possible == 1) {	242	* If this is the first time this NUMA node's affinity is used,
151	/* only one CPU, everyone will use it */	243	* create an entry in the global affinity structure and initialize it.
152	cpumask_set_cpu(curr_cpu, &info->rcv_intr.mask);	244	*/
153	} else {	245	if (!entry) {
154	/*	246	entry = node_affinity_allocate(node);
155	* Retain the first CPU in the default list for the control	247	if (!entry) {
156	* context.	248	dd_dev_err(dd,
157	*/	249	"Unable to allocate global affinity node\n");
158	curr_cpu = cpumask_next(curr_cpu, &info->def_intr.mask);	250	return -ENOMEM;
159	/*
160	* Remove the remaining kernel receive queues from
161	* the default list and add them to the receive list.
162	*/
163	for (i = 0; i < dd->n_krcv_queues - 1; i++) {
164	cpumask_clear_cpu(curr_cpu, &info->def_intr.mask);
165	cpumask_set_cpu(curr_cpu, &info->rcv_intr.mask);
166	curr_cpu = cpumask_next(curr_cpu, &info->def_intr.mask);
167	if (curr_cpu >= nr_cpu_ids)
168	break;
169	}	251	}
170	}	252	init_cpu_mask_set(&entry->def_intr);
		253	init_cpu_mask_set(&entry->rcv_intr);
		254	cpumask_clear(&entry->general_intr_mask);
		255	/* Use the "real" cpu mask of this node as the default */
		256	cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
		257	local_mask);
		258
		259	/* fill in the receive list */
		260	possible = cpumask_weight(&entry->def_intr.mask);
		261	curr_cpu = cpumask_first(&entry->def_intr.mask);
		262
		263	if (possible == 1) {
		264	/* only one CPU, everyone will use it */
		265	cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
		266	cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
		267	} else {
		268	/*
		269	* The general/control context will be the first CPU in
		270	* the default list, so it is removed from the default
		271	* list and added to the general interrupt list.
		272	*/
		273	cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
		274	cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
		275	curr_cpu = cpumask_next(curr_cpu,
		276	&entry->def_intr.mask);
171		277
172	cpumask_copy(&info->proc.mask, cpu_online_mask);	278	/*
173	}	279	* Remove the remaining kernel receive queues from
		280	* the default list and add them to the receive list.
		281	*/
		282	for (i = 0;
		283	i < (dd->n_krcv_queues - 1) *
		284	hfi1_per_node_cntr[dd->node];
		285	i++) {
		286	cpumask_clear_cpu(curr_cpu,
		287	&entry->def_intr.mask);
		288	cpumask_set_cpu(curr_cpu,
		289	&entry->rcv_intr.mask);
		290	curr_cpu = cpumask_next(curr_cpu,
		291	&entry->def_intr.mask);
		292	if (curr_cpu >= nr_cpu_ids)
		293	break;
		294	}
174		295
175	void hfi1_dev_affinity_free(struct hfi1_devdata *dd)	296	/*
176	{	297	* If there ends up being 0 CPU cores leftover for SDMA
177	kfree(dd->affinity);	298	* engines, use the same CPU cores as general/control
		299	* context.
		300	*/
		301	if (cpumask_weight(&entry->def_intr.mask) == 0)
		302	cpumask_copy(&entry->def_intr.mask,
		303	&entry->general_intr_mask);
		304	}
		305
		306	spin_lock(&node_affinity.lock);
		307	node_affinity_add_tail(entry);
		308	spin_unlock(&node_affinity.lock);
		309	}
		310
		311	return 0;
178	}	312	}
179		313
180	int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix)	314	int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix)
181	{	315	{
182	int ret;	316	int ret;
183	cpumask_var_t diff;	317	cpumask_var_t diff;
184	struct cpu_mask_set *set;	318	struct hfi1_affinity_node *entry;
		319	struct cpu_mask_set *set = NULL;
185	struct sdma_engine *sde = NULL;	320	struct sdma_engine *sde = NULL;
186	struct hfi1_ctxtdata *rcd = NULL;	321	struct hfi1_ctxtdata *rcd = NULL;
187	char extra[64];	322	char extra[64];
@@ -194,22 +329,25 @@ int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix) · 6 unmodified lines
194	if (!ret)	329	if (!ret)
195	return -ENOMEM;	330	return -ENOMEM;
196		331
		332	spin_lock(&node_affinity.lock);
		333	entry = node_affinity_lookup(dd->node);
		334	spin_unlock(&node_affinity.lock);
		335
197	switch (msix->type) {	336	switch (msix->type) {
198	case IRQ_SDMA:	337	case IRQ_SDMA:
199	sde = (struct sdma_engine *)msix->arg;	338	sde = (struct sdma_engine *)msix->arg;
200	scnprintf(extra, 64, "engine %u", sde->this_idx);	339	scnprintf(extra, 64, "engine %u", sde->this_idx);
201	/* fall through */	340	set = &entry->def_intr;
		341	break;
202	case IRQ_GENERAL:	342	case IRQ_GENERAL:
203	set = &dd->affinity->def_intr;	343	cpu = cpumask_first(&entry->general_intr_mask);
204	break;	344	break;
205	case IRQ_RCVCTXT:	345	case IRQ_RCVCTXT:
206	rcd = (struct hfi1_ctxtdata *)msix->arg;	346	rcd = (struct hfi1_ctxtdata *)msix->arg;
207	if (rcd->ctxt == HFI1_CTRL_CTXT) {	347	if (rcd->ctxt == HFI1_CTRL_CTXT)
208	set = &dd->affinity->def_intr;	348	cpu = cpumask_first(&entry->general_intr_mask);
209	cpu = cpumask_first(&set->mask);	349	else
210	} else {	350	set = &entry->rcv_intr;
211	set = &dd->affinity->rcv_intr;
212	}
213	scnprintf(extra, 64, "ctxt %u", rcd->ctxt);	351	scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
214	break;	352	break;
215	default:	353	default:
@@ -218,12 +356,12 @@ int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix) · 2 unmodified lines
218	}	356	}
219		357
220	/*	358	/*
221	* The control receive context is placed on a particular CPU, which	359	* The general and control contexts are placed on a particular
222	* is set above. Skip accounting for it. Everything else finds its	360	* CPU, which is set above. Skip accounting for it. Everything else
223	* CPU here.	361	* finds its CPU here.
224	*/	362	*/
225	if (cpu == -1) {	363	if (cpu == -1 && set) {
226	spin_lock(&dd->affinity->lock);	364	spin_lock(&node_affinity.lock);
227	if (cpumask_equal(&set->mask, &set->used)) {	365	if (cpumask_equal(&set->mask, &set->used)) {
228	/*	366	/*
229	* We've used up all the CPUs, bump up the generation	367	* We've used up all the CPUs, bump up the generation
@@ -235,7 +373,7 @@ int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix) · 5 unmodified lines
235	cpumask_andnot(diff, &set->mask, &set->used);	373	cpumask_andnot(diff, &set->mask, &set->used);
236	cpu = cpumask_first(diff);	374	cpu = cpumask_first(diff);
237	cpumask_set_cpu(cpu, &set->used);	375	cpumask_set_cpu(cpu, &set->used);
238	spin_unlock(&dd->affinity->lock);	376	spin_unlock(&node_affinity.lock);
239	}	377	}
240		378
241	switch (msix->type) {	379	switch (msix->type) {
@@ -263,43 +401,84 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd, · 21 unmodified lines
263	{	401	{
264	struct cpu_mask_set *set = NULL;	402	struct cpu_mask_set *set = NULL;
265	struct hfi1_ctxtdata *rcd;	403	struct hfi1_ctxtdata *rcd;
		404	struct hfi1_affinity_node *entry;
		405
		406	spin_lock(&node_affinity.lock);
		407	entry = node_affinity_lookup(dd->node);
		408	spin_unlock(&node_affinity.lock);
266		409
267	switch (msix->type) {	410	switch (msix->type) {
268	case IRQ_SDMA:	411	case IRQ_SDMA:
		412	set = &entry->def_intr;
		413	break;
269	case IRQ_GENERAL:	414	case IRQ_GENERAL:
270	set = &dd->affinity->def_intr;	415	/* Don't do accounting for general contexts */
271	break;	416	break;
272	case IRQ_RCVCTXT:	417	case IRQ_RCVCTXT:
273	rcd = (struct hfi1_ctxtdata *)msix->arg;	418	rcd = (struct hfi1_ctxtdata *)msix->arg;
274	/* only do accounting for non control contexts */	419	/* Don't do accounting for control contexts */
275	if (rcd->ctxt != HFI1_CTRL_CTXT)	420	if (rcd->ctxt != HFI1_CTRL_CTXT)
276	set = &dd->affinity->rcv_intr;	421	set = &entry->rcv_intr;
277	break;	422	break;
278	default:	423	default:
279	return;	424	return;
280	}	425	}
281		426
282	if (set) {	427	if (set) {
283	spin_lock(&dd->affinity->lock);	428	spin_lock(&node_affinity.lock);
284	cpumask_andnot(&set->used, &set->used, &msix->mask);	429	cpumask_andnot(&set->used, &set->used, &msix->mask);
285	if (cpumask_empty(&set->used) && set->gen) {	430	if (cpumask_empty(&set->used) && set->gen) {
286	set->gen--;	431	set->gen--;
287	cpumask_copy(&set->used, &set->mask);	432	cpumask_copy(&set->used, &set->mask);
288	}	433	}
289	spin_unlock(&dd->affinity->lock);	434	spin_unlock(&node_affinity.lock);
290	}	435	}
291		436
292	irq_set_affinity_hint(msix->msix.vector, NULL);	437	irq_set_affinity_hint(msix->msix.vector, NULL);
293	cpumask_clear(&msix->mask);	438	cpumask_clear(&msix->mask);
294	}	439	}
295		440
296	int hfi1_get_proc_affinity(struct hfi1_devdata *dd, int node)	441	/* This should be called with node_affinity.lock held */
		442	static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
		443	struct hfi1_affinity_node_list *affinity)
297	{	444	{
298	int cpu = -1, ret;	445	int possible, curr_cpu, i;
299	cpumask_var_t diff, mask, intrs;	446	uint num_cores_per_socket = node_affinity.num_online_cpus /
		447	affinity->num_core_siblings /
		448	node_affinity.num_online_nodes;
		449
		450	cpumask_copy(hw_thread_mask, &affinity->proc.mask);
		451	if (affinity->num_core_siblings > 0) {
		452	/* Removing other siblings not needed for now */
		453	possible = cpumask_weight(hw_thread_mask);
		454	curr_cpu = cpumask_first(hw_thread_mask);
		455	for (i = 0;
		456	i < num_cores_per_socket * node_affinity.num_online_nodes;
		457	i++)
		458	curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
		459
		460	for (; i < possible; i++) {
		461	cpumask_clear_cpu(curr_cpu, hw_thread_mask);
		462	curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
		463	}
		464
		465	/* Identifying correct HW threads within physical cores */
		466	cpumask_shift_left(hw_thread_mask, hw_thread_mask,
		467	num_cores_per_socket *
		468	node_affinity.num_online_nodes *
		469	hw_thread_no);
		470	}
		471	}
		472
		473	int hfi1_get_proc_affinity(int node)
		474	{
		475	int cpu = -1, ret, i;
		476	struct hfi1_affinity_node *entry;
		477	cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
300	const struct cpumask *node_mask,	478	const struct cpumask *node_mask,
301	*proc_mask = tsk_cpus_allowed(current);	479	*proc_mask = tsk_cpus_allowed(current);
302	struct cpu_mask_set *set = &dd->affinity->proc;	480	struct hfi1_affinity_node_list *affinity = &node_affinity;
		481	struct cpu_mask_set *set = &affinity->proc;
303		482
304	/*	483	/*
305	* check whether process/context affinity has already	484	* check whether process/context affinity has already
@@ -325,22 +504,41 @@ int hfi1_get_proc_affinity(struct hfi1_devdata *dd, int node) · 19 unmodified lines
325		504
326	/*	505	/*
327	* The process does not have a preset CPU affinity so find one to	506	* The process does not have a preset CPU affinity so find one to
328	* recommend. We prefer CPUs on the same NUMA as the device.	507	* recommend using the following algorithm:
		508	*
		509	* For each user process that is opening a context on HFI Y:
		510	* a) If all cores are filled, reinitialize the bitmask
		511	* b) Fill real cores first, then HT cores (First set of HT
		512	* cores on all physical cores, then second set of HT core,
		513	* and, so on) in the following order:
		514	*
		515	* 1. Same NUMA node as HFI Y and not running an IRQ
		516	* handler
		517	* 2. Same NUMA node as HFI Y and running an IRQ handler
		518	* 3. Different NUMA node to HFI Y and not running an IRQ
		519	* handler
		520	* 4. Different NUMA node to HFI Y and running an IRQ
		521	* handler
		522	* c) Mark core as filled in the bitmask. As user processes are
		523	* done, clear cores from the bitmask.
329	*/	524	*/
330		525
331	ret = zalloc_cpumask_var(&diff, GFP_KERNEL);	526	ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
332	if (!ret)	527	if (!ret)
333	goto done;	528	goto done;
334	ret = zalloc_cpumask_var(&mask, GFP_KERNEL);	529	ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
335	if (!ret)	530	if (!ret)
336	goto free_diff;	531	goto free_diff;
337	ret = zalloc_cpumask_var(&intrs, GFP_KERNEL);	532	ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
		533	if (!ret)
		534	goto free_hw_thread_mask;
		535	ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
338	if (!ret)	536	if (!ret)
339	goto free_mask;	537	goto free_available_mask;
340		538
341	spin_lock(&dd->affinity->lock);	539	spin_lock(&affinity->lock);
342	/*	540	/*
343	* If we've used all available CPUs, clear the mask and start	541	* If we've used all available HW threads, clear the mask and start
344	* overloading.	542	* overloading.
345	*/	543	*/
346	if (cpumask_equal(&set->mask, &set->used)) {	544	if (cpumask_equal(&set->mask, &set->used)) {
@@ -348,81 +546,198 @@ int hfi1_get_proc_affinity(struct hfi1_devdata *dd, int node) · 1 unmodified line
348	cpumask_clear(&set->used);	546	cpumask_clear(&set->used);
349	}	547	}
350		548
351	/* CPUs used by interrupt handlers */	549	/*
352	cpumask_copy(intrs, (dd->affinity->def_intr.gen ?	550	* If NUMA node has CPUs used by interrupt handlers, include them in the
353	&dd->affinity->def_intr.mask :	551	* interrupt handler mask.
354	&dd->affinity->def_intr.used));	552	*/
355	cpumask_or(intrs, intrs, (dd->affinity->rcv_intr.gen ?	553	entry = node_affinity_lookup(node);
356	&dd->affinity->rcv_intr.mask :	554	if (entry) {
357	&dd->affinity->rcv_intr.used));	555	cpumask_copy(intrs_mask, (entry->def_intr.gen ?
		556	&entry->def_intr.mask :
		557	&entry->def_intr.used));
		558	cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
		559	&entry->rcv_intr.mask :
		560	&entry->rcv_intr.used));
		561	cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
		562	}
358	hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",	563	hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
359	cpumask_pr_args(intrs));	564	cpumask_pr_args(intrs_mask));
		565
		566	cpumask_copy(hw_thread_mask, &set->mask);
360		567
361	/*	568	/*
362	* If we don't have a NUMA node requested, preference is towards	569	* If HT cores are enabled, identify which HW threads within the
363	* device NUMA node	570	* physical cores should be used.
364	*/	571	*/
365	if (node == -1)	572	if (affinity->num_core_siblings > 0) {
366	node = dd->node;	573	for (i = 0; i < affinity->num_core_siblings; i++) {
		574	find_hw_thread_mask(i, hw_thread_mask, affinity);
		575
		576	/*
		577	* If there's at least one available core for this HW
		578	* thread number, stop looking for a core.
		579	*
		580	* diff will always be not empty at least once in this
		581	* loop as the used mask gets reset when
		582	* (set->mask == set->used) before this loop.
		583	*/
		584	cpumask_andnot(diff, hw_thread_mask, &set->used);
		585	if (!cpumask_empty(diff))
		586	break;
		587	}
		588	}
		589	hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
		590	cpumask_pr_args(hw_thread_mask));
		591
367	node_mask = cpumask_of_node(node);	592	node_mask = cpumask_of_node(node);
368	hfi1_cdbg(PROC, "device on NUMA %u, CPUs %*pbl", node,	593	hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
369	cpumask_pr_args(node_mask));	594	cpumask_pr_args(node_mask));
370		595
371	/* diff will hold all unused cpus */	596	/* Get cpumask of available CPUs on preferred NUMA */
372	cpumask_andnot(diff, &set->mask, &set->used);	597	cpumask_and(available_mask, hw_thread_mask, node_mask);
373	hfi1_cdbg(PROC, "unused CPUs (all) %*pbl", cpumask_pr_args(diff));	598	cpumask_andnot(available_mask, available_mask, &set->used);
374		599	hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
375	/* get cpumask of available CPUs on preferred NUMA */	600	cpumask_pr_args(available_mask));
376	cpumask_and(mask, diff, node_mask);
377	hfi1_cdbg(PROC, "available cpus on NUMA %*pbl", cpumask_pr_args(mask));
378		601
379	/*	602	/*
380	* At first, we don't want to place processes on the same	603	* At first, we don't want to place processes on the same
381	* CPUs as interrupt handlers.	604	* CPUs as interrupt handlers. Then, CPUs running interrupt
		605	* handlers are used.
		606	*
		607	* 1) If diff is not empty, then there are CPUs not running
		608	* non-interrupt handlers available, so diff gets copied
		609	* over to available_mask.
		610	* 2) If diff is empty, then all CPUs not running interrupt
		611	* handlers are taken, so available_mask contains all
		612	* available CPUs running interrupt handlers.
		613	* 3) If available_mask is empty, then all CPUs on the
		614	* preferred NUMA node are taken, so other NUMA nodes are
		615	* used for process assignments using the same method as
		616	* the preferred NUMA node.
382	*/	617	*/
383	cpumask_andnot(diff, mask, intrs);	618	cpumask_andnot(diff, available_mask, intrs_mask);
384	if (!cpumask_empty(diff))	619	if (!cpumask_empty(diff))
385	cpumask_copy(mask, diff);	620	cpumask_copy(available_mask, diff);
386		621
387	/*	622	/* If we don't have CPUs on the preferred node, use other NUMA nodes */
388	* if we don't have a cpu on the preferred NUMA, get	623	if (cpumask_empty(available_mask)) {
389	* the list of the remaining available CPUs	624	cpumask_andnot(available_mask, hw_thread_mask, &set->used);
390	*/	625	/* Excluding preferred NUMA cores */
391	if (cpumask_empty(mask)) {	626	cpumask_andnot(available_mask, available_mask, node_mask);
392	cpumask_andnot(diff, &set->mask, &set->used);	627	hfi1_cdbg(PROC,
393	cpumask_andnot(mask, diff, node_mask);	628	"Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
		629	cpumask_pr_args(available_mask));
		630
		631	/*
		632	* At first, we don't want to place processes on the same
		633	* CPUs as interrupt handlers.
		634	*/
		635	cpumask_andnot(diff, available_mask, intrs_mask);
		636	if (!cpumask_empty(diff))
		637	cpumask_copy(available_mask, diff);
394	}	638	}
395	hfi1_cdbg(PROC, "possible CPUs for process %*pbl",	639	hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
396	cpumask_pr_args(mask));	640	cpumask_pr_args(available_mask));
397		641
398	cpu = cpumask_first(mask);	642	cpu = cpumask_first(available_mask);
399	if (cpu >= nr_cpu_ids) /* empty */	643	if (cpu >= nr_cpu_ids) /* empty */
400	cpu = -1;	644	cpu = -1;
401	else	645	else
402	cpumask_set_cpu(cpu, &set->used);	646	cpumask_set_cpu(cpu, &set->used);
403	spin_unlock(&dd->affinity->lock);	647	spin_unlock(&affinity->lock);
404		648	hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
405	free_cpumask_var(intrs);	649
406	free_mask:	650	free_cpumask_var(intrs_mask);
407	free_cpumask_var(mask);	651	free_available_mask:
		652	free_cpumask_var(available_mask);
		653	free_hw_thread_mask:
		654	free_cpumask_var(hw_thread_mask);
408	free_diff:	655	free_diff:
409	free_cpumask_var(diff);	656	free_cpumask_var(diff);
410	done:	657	done:
411	return cpu;	658	return cpu;
412	}	659	}
413		660
414	void hfi1_put_proc_affinity(struct hfi1_devdata *dd, int cpu)	661	void hfi1_put_proc_affinity(int cpu)
415	{	662	{
416	struct cpu_mask_set *set = &dd->affinity->proc;	663	struct hfi1_affinity_node_list *affinity = &node_affinity;
		664	struct cpu_mask_set *set = &affinity->proc;
417		665
418	if (cpu < 0)	666	if (cpu < 0)
419	return;	667	return;
420	spin_lock(&dd->affinity->lock);	668	spin_lock(&affinity->lock);
421	cpumask_clear_cpu(cpu, &set->used);	669	cpumask_clear_cpu(cpu, &set->used);
		670	hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
422	if (cpumask_empty(&set->used) && set->gen) {	671	if (cpumask_empty(&set->used) && set->gen) {
423	set->gen--;	672	set->gen--;
424	cpumask_copy(&set->used, &set->mask);	673	cpumask_copy(&set->used, &set->mask);
425	}	674	}
426	spin_unlock(&dd->affinity->lock);	675	spin_unlock(&affinity->lock);
427	}	676	}
428		677
		678	/* Prevents concurrent reads and writes of the sdma_affinity attrib */
		679	static DEFINE_MUTEX(sdma_affinity_mutex);
		680
		681	int hfi1_set_sdma_affinity(struct hfi1_devdata dd, const char buf,
		682	size_t count)
		683	{
		684	struct hfi1_affinity_node *entry;
		685	struct cpumask mask;
		686	int ret, i;
		687
		688	spin_lock(&node_affinity.lock);
		689	entry = node_affinity_lookup(dd->node);
		690	spin_unlock(&node_affinity.lock);
		691
		692	if (!entry)
		693	return -EINVAL;
		694
		695	ret = cpulist_parse(buf, &mask);
		696	if (ret)
		697	return ret;
		698
		699	if (!cpumask_subset(&mask, cpu_online_mask) \|\| cpumask_empty(&mask)) {
		700	dd_dev_warn(dd, "Invalid CPU mask\n");
		701	return -EINVAL;
		702	}
		703
		704	mutex_lock(&sdma_affinity_mutex);
		705	/* reset the SDMA interrupt affinity details */
		706	init_cpu_mask_set(&entry->def_intr);
		707	cpumask_copy(&entry->def_intr.mask, &mask);
		708	/*
		709	* Reassign the affinity for each SDMA interrupt.
		710	*/
		711	for (i = 0; i < dd->num_msix_entries; i++) {
		712	struct hfi1_msix_entry *msix;
		713
		714	msix = &dd->msix_entries[i];
		715	if (msix->type != IRQ_SDMA)
		716	continue;
		717
		718	ret = hfi1_get_irq_affinity(dd, msix);
		719
		720	if (ret)
		721	break;
		722	}
		723
		724	mutex_unlock(&sdma_affinity_mutex);
		725	return ret ? ret : strnlen(buf, PAGE_SIZE);
		726	}
		727
		728	int hfi1_get_sdma_affinity(struct hfi1_devdata dd, char buf)
		729	{
		730	struct hfi1_affinity_node *entry;
		731
		732	spin_lock(&node_affinity.lock);
		733	entry = node_affinity_lookup(dd->node);
		734	spin_unlock(&node_affinity.lock);
		735
		736	if (!entry)
		737	return -EINVAL;
		738
		739	mutex_lock(&sdma_affinity_mutex);
		740	cpumap_print_to_pagebuf(true, buf, &entry->def_intr.mask);
		741	mutex_unlock(&sdma_affinity_mutex);
		742	return strnlen(buf, PAGE_SIZE);
		743	}

drivers/infiniband/hw/hfi1/affinity.h+31−7

@@ -73,7 +73,6 @@ struct cpu_mask_set { · 72 unmodified lines
73	struct hfi1_affinity {	73	struct hfi1_affinity {
74	struct cpu_mask_set def_intr;	74	struct cpu_mask_set def_intr;
75	struct cpu_mask_set rcv_intr;	75	struct cpu_mask_set rcv_intr;
76	struct cpu_mask_set proc;
77	struct cpumask real_cpu_mask;	76	struct cpumask real_cpu_mask;
78	/* spin lock to protect affinity struct */	77	/* spin lock to protect affinity struct */
79	spinlock_t lock;	78	spinlock_t lock;
@@ -82,11 +81,9 @@ struct hfi1_affinity { · 2 unmodified lines
82	struct hfi1_msix_entry;	81	struct hfi1_msix_entry;
83		82
84	/* Initialize non-HT cpu cores mask */	83	/* Initialize non-HT cpu cores mask */
85	int init_real_cpu_mask(struct hfi1_devdata *);	84	void init_real_cpu_mask(void);
86	/* Initialize driver affinity data */	85	/* Initialize driver affinity data */
87	void hfi1_dev_affinity_init(struct hfi1_devdata *);	86	int hfi1_dev_affinity_init(struct hfi1_devdata *);
88	/* Free driver affinity data */
89	void hfi1_dev_affinity_free(struct hfi1_devdata *);
90	/*	87	/*
91	* Set IRQ affinity to a CPU. The function will determine the	88	* Set IRQ affinity to a CPU. The function will determine the
92	* CPU and set the affinity to it.	89	* CPU and set the affinity to it.
@@ -101,8 +98,35 @@ void hfi1_put_irq_affinity(struct hfi1_devdata , struct hfi1_msix_entry ); · 8 unmodified lines
101	* Determine a CPU affinity for a user process, if the process does not	98	* Determine a CPU affinity for a user process, if the process does not
102	* have an affinity set yet.	99	* have an affinity set yet.
103	*/	100	*/
104	int hfi1_get_proc_affinity(struct hfi1_devdata *, int);	101	int hfi1_get_proc_affinity(int);
105	/* Release a CPU used by a user process. */	102	/* Release a CPU used by a user process. */
106	void hfi1_put_proc_affinity(struct hfi1_devdata *, int);	103	void hfi1_put_proc_affinity(int);
		104
		105	int hfi1_get_sdma_affinity(struct hfi1_devdata dd, char buf);
		106	int hfi1_set_sdma_affinity(struct hfi1_devdata dd, const char buf,
		107	size_t count);
		108
		109	struct hfi1_affinity_node {
		110	int node;
		111	struct cpu_mask_set def_intr;
		112	struct cpu_mask_set rcv_intr;
		113	struct cpumask general_intr_mask;
		114	struct list_head list;
		115	};
		116
		117	struct hfi1_affinity_node_list {
		118	struct list_head list;
		119	struct cpumask real_cpu_mask;
		120	struct cpu_mask_set proc;
		121	int num_core_siblings;
		122	int num_online_nodes;
		123	int num_online_cpus;
		124	/* protect affinity node list */
		125	spinlock_t lock;
		126	};
		127
		128	int node_affinity_init(void);
		129	void node_affinity_destroy(void);
		130	extern struct hfi1_affinity_node_list node_affinity;
107		131
108	#endif /* _HFI1_AFFINITY_H */	132	#endif /* _HFI1_AFFINITY_H */

drivers/infiniband/hw/hfi1/chip.c+193−103

 #include "efivar.h"
 #include "platform.h"
 #include "aspm.h"
+#include "affinity.h"
 #define NUM_IB_PORTS 1
 #define SEC_SC_HALTED		0x4	/* per-context only */
 #define SEC_SPC_FREEZE		0x8	/* per-HFI only */
+#define DEFAULT_KRCVQS		  2
 #define MIN_KERNEL_KCTXTS         2
 #define FIRST_KERNEL_KCTXT        1
 /* sizes for both the QP and RSM map tables */
 /* all CceStatus sub-block RXE pause bits */
 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
+#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
+#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
 /*
  * CCE Error flags.
  */
 	return dd->sw_send_dma_eng_err_status_cnt[0];
 }
+static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry,
+				 void *context, int vl, int mode,
+				 u64 data)
+{
+	struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
+	u64 val = 0;
+	u64 csr = entry->csr;
+	val = read_write_csr(dd, csr, mode, data);
+	if (mode == CNTR_MODE_R) {
+		val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ?
+			CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors;
+	} else if (mode == CNTR_MODE_W) {
+		dd->sw_rcv_bypass_packet_errors = 0;
+	} else {
+		dd_dev_err(dd, "Invalid cntr register access mode");
+		return 0;
+	}
+	return val;
+}
 #define def_access_sw_cpu(cntr) \
 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry,		      \
 			      void *context, int vl, int mode, u64 data)      \
 			CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
 			      CNTR_SYNTH),
-[C_DC_RCV_ERR] = DC_PERF_CNTR(DcRecvErr, DCC_ERR_PORTRCV_ERR_CNT, CNTR_SYNTH),
+[C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH,
+			    access_dc_rcv_err_cnt),
 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
 				 CNTR_SYNTH),
 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
 	return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
 }
-static void check_fabric_firmware_versions(struct hfi1_devdata *dd)
-{
-	u32 frame, version, prod_id;
-	int ret, lane;
-	/* 4 lanes */
-	for (lane = 0; lane < 4; lane++) {
-		ret = read_8051_config(dd, SPICO_FW_VERSION, lane, &frame);
-		if (ret) {
-			dd_dev_err(dd,
-				   "Unable to read lane %d firmware details\n",
-				   lane);
-			continue;
-		}
-		version = (frame >> SPICO_ROM_VERSION_SHIFT)
-					& SPICO_ROM_VERSION_MASK;
-		prod_id = (frame >> SPICO_ROM_PROD_ID_SHIFT)
-					& SPICO_ROM_PROD_ID_MASK;
-		dd_dev_info(dd,
-			    "Lane %d firmware: version 0x%04x, prod_id 0x%04x\n",
-			    lane, version, prod_id);
-	}
-}
 /*
  * Read an idle LCB message.
  *
 	unsigned long timeout;
 	/*
-	 * Check for QSFP interrupt for t_init (SFF 8679)
+	 * Some QSFP cables have a quirk that asserts the IntN line as a side
+	 * effect of power up on plug-in. We ignore this false positive
+	 * interrupt until the module has finished powering up by waiting for
+	 * a minimum timeout of the module inrush initialization time of
+	 * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the
+	 * module have stabilized.
+	 */
+	msleep(500);
+	/*
+	 * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1)
 	 */
 	timeout = jiffies + msecs_to_jiffies(2000);
 	while (1) {
 		mask = read_csr(dd, dd->hfi1_id ?
 				ASIC_QSFP2_IN : ASIC_QSFP1_IN);
-		if (!(mask & QSFP_HFI0_INT_N)) {
+		if (!(mask & QSFP_HFI0_INT_N))
-			write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR :
-				  ASIC_QSFP1_CLEAR, QSFP_HFI0_INT_N);
 			break;
-		}
 		if (time_after(jiffies, timeout)) {
 			dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
 				    __func__);
 	u64 mask;
 	mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
-	if (enable)
+	if (enable) {
+		/*
+		 * Clear the status register to avoid an immediate interrupt
+		 * when we re-enable the IntN pin
+		 */
+		write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
+			  QSFP_HFI0_INT_N);
 		mask |= (u64)QSFP_HFI0_INT_N;
-	else
+	} else {
 		mask &= ~(u64)QSFP_HFI0_INT_N;
+	}
 	write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
 }
 		hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
 }
-int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
-			struct hfi1_ctxt_info *kinfo)
-{
-	kinfo->runtime_flags = (HFI1_MISC_GET() << HFI1_CAP_USER_SHIFT) |
-		HFI1_CAP_UGET(MASK) | HFI1_CAP_KGET(K2U);
-	return 0;
-}
 struct hfi1_message_header *hfi1_get_msgheader(
 				struct hfi1_devdata *dd, __le32 *rhf_addr)
 {
 	return 0;
 }
+static const char *state_completed_string(u32 completed)
+{
+	static const char * const state_completed[] = {
+		"EstablishComm",
+		"OptimizeEQ",
+		"VerifyCap"
+	};
+	if (completed < ARRAY_SIZE(state_completed))
+		return state_completed[completed];
+	return "unknown";
+}
+static const char all_lanes_dead_timeout_expired[] =
+	"All lanes were inactive – was the interconnect media removed?";
+static const char tx_out_of_policy[] =
+	"Passing lanes on local port do not meet the local link width policy";
+static const char no_state_complete[] =
+	"State timeout occurred before link partner completed the state";
+static const char * const state_complete_reasons[] = {
+	[0x00] = "Reason unknown",
+	[0x01] = "Link was halted by driver, refer to LinkDownReason",
+	[0x02] = "Link partner reported failure",
+	[0x10] = "Unable to achieve frame sync on any lane",
+	[0x11] =
+	  "Unable to find a common bit rate with the link partner",
+	[0x12] =
+	  "Unable to achieve frame sync on sufficient lanes to meet the local link width policy",
+	[0x13] =
+	  "Unable to identify preset equalization on sufficient lanes to meet the local link width policy",
+	[0x14] = no_state_complete,
+	[0x15] =
+	  "State timeout occurred before link partner identified equalization presets",
+	[0x16] =
+	  "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy",
+	[0x17] = tx_out_of_policy,
+	[0x20] = all_lanes_dead_timeout_expired,
+	[0x21] =
+	  "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy",
+	[0x22] = no_state_complete,
+	[0x23] =
+	  "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy",
+	[0x24] = tx_out_of_policy,
+	[0x30] = all_lanes_dead_timeout_expired,
+	[0x31] =
+	  "State timeout occurred waiting for host to process received frames",
+	[0x32] = no_state_complete,
+	[0x33] =
+	  "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy",
+	[0x34] = tx_out_of_policy,
+};
+static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd,
+						     u32 code)
+{
+	const char *str = NULL;
+	if (code < ARRAY_SIZE(state_complete_reasons))
+		str = state_complete_reasons[code];
+	if (str)
+		return str;
+	return "Reserved";
+}
+/* describe the given last state complete frame */
+static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame,
+				  const char *prefix)
+{
+	struct hfi1_devdata *dd = ppd->dd;
+	u32 success;
+	u32 state;
+	u32 reason;
+	u32 lanes;
+	/*
+	 * Decode frame:
+	 *  [ 0: 0] - success
+	 *  [ 3: 1] - state
+	 *  [ 7: 4] - next state timeout
+	 *  [15: 8] - reason code
+	 *  [31:16] - lanes
+	 */
+	success = frame & 0x1;
+	state = (frame >> 1) & 0x7;
+	reason = (frame >> 8) & 0xff;
+	lanes = (frame >> 16) & 0xffff;
+	dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n",
+		   prefix, frame);
+	dd_dev_err(dd, "    last reported state state: %s (0x%x)\n",
+		   state_completed_string(state), state);
+	dd_dev_err(dd, "    state successfully completed: %s\n",
+		   success ? "yes" : "no");
+	dd_dev_err(dd, "    fail reason 0x%x: %s\n",
+		   reason, state_complete_reason_code_string(ppd, reason));
+	dd_dev_err(dd, "    passing lane mask: 0x%x", lanes);
+}
+/*
+ * Read the last state complete frames and explain them.  This routine
+ * expects to be called if the link went down during link negotiation
+ * and initialization (LNI).  That is, anywhere between polling and link up.
+ */
+static void check_lni_states(struct hfi1_pportdata *ppd)
+{
+	u32 last_local_state;
+	u32 last_remote_state;
+	read_last_local_state(ppd->dd, &last_local_state);
+	read_last_remote_state(ppd->dd, &last_remote_state);
+	/*
+	 * Don't report anything if there is nothing to report.  A value of
+	 * 0 means the link was taken down while polling and there was no
+	 * training in-process.
+	 */
+	if (last_local_state == 0 && last_remote_state == 0)
+		return;
+	decode_state_complete(ppd, last_local_state, "transmitted");
+	decode_state_complete(ppd, last_remote_state, "received");
+}
 /*
  * Helper for set_link_state().  Do not call except from that routine.
  * Expects ppd->hls_mutex to be held.
 {
 	struct hfi1_devdata *dd = ppd->dd;
 	u32 pstate, previous_state;
-	u32 last_local_state;
-	u32 last_remote_state;
 	int ret;
 	int do_transition;
 	int do_wait;
 	} else if (previous_state
 			& (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
 		/* went down while attempting link up */
-		/* byte 1 of last_*_state is the failure reason */
+		check_lni_states(ppd);
-		read_last_local_state(dd, &last_local_state);
-		read_last_remote_state(dd, &last_remote_state);
-		dd_dev_err(dd,
-			   "LNI failure last states: local 0x%08x, remote 0x%08x\n",
-			   last_local_state, last_remote_state);
 	}
 	/* the active link width (downgrade) is 0 on link down */
 	dd->cntrnames = NULL;
 }
-#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
-#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
 			      u64 *psval, void *context, int vl)
 {
 	return ib_pstate;
 }
-/*
- * Read/modify/write ASIC_QSFP register bits as selected by mask
- * data: 0 or 1 in the positions depending on what needs to be written
- * dir: 0 for read, 1 for write
- * mask: select by setting
- *      I2CCLK  (bit 0)
- *      I2CDATA (bit 1)
- */
-u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
-		  u32 mask)
-{
-	u64 qsfp_oe, target_oe;
-	target_oe = target ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
-	if (mask) {
-		/* We are writing register bits, so lock access */
-		dir &= mask;
-		data &= mask;
-		qsfp_oe = read_csr(dd, target_oe);
-		qsfp_oe = (qsfp_oe & ~(u64)mask) | (u64)dir;
-		write_csr(dd, target_oe, qsfp_oe);
-	}
-	/* We are exclusively reading bits here, but it is unlikely
-	 * we'll get valid data when we set the direction of the pin
-	 * in the same call, so read should call this function again
-	 * to get valid data
-	 */
-	return read_csr(dd, target ? ASIC_QSFP2_IN : ASIC_QSFP1_IN);
-}
 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
 	/*
 	 * Kernel receive contexts:
-	 * - min of 2 or 1 context/numa (excluding control context)
 	 * - Context 0 - control context (VL15/multicast/error)
 	 * - Context 1 - first kernel context
 	 * - Context 2 - second kernel context
 		 */
 		num_kernel_contexts = n_krcvqs + 1;
 	else
-		num_kernel_contexts = num_online_nodes() + 1;
+		num_kernel_contexts = DEFAULT_KRCVQS + 1;
-	num_kernel_contexts =
-		max_t(int, MIN_KERNEL_KCTXTS, num_kernel_contexts);
 	/*
 	 * Every kernel receive context needs an ACK send context.
 	 * one send context is allocated for each VL{0-7} and VL15
 	 */
 	if (num_user_contexts < 0)
 		num_user_contexts =
-			cpumask_weight(&dd->affinity->real_cpu_mask);
+			cpumask_weight(&node_affinity.real_cpu_mask);
 	total_contexts = num_kernel_contexts + num_user_contexts;
 	}
 	dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
 	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
+	/* first one through - set up i2c devices */
+	if (!peer)
+		ret = set_up_i2c(dd, dd->asic_data);
 	return ret;
 }
 		 (dd->revision >> CCE_REVISION_SW_SHIFT)
 		    & CCE_REVISION_SW_MASK);
-	/*
-	 * The real cpu mask is part of the affinity struct but has to be
-	 * initialized earlier than the rest of the affinity struct because it
-	 * is needed to calculate the number of user contexts in
-	 * set_up_context_variables(). However, hfi1_dev_affinity_init(),
-	 * which initializes the rest of the affinity struct members,
-	 * depends on set_up_context_variables() for the number of kernel
-	 * contexts, so it cannot be called before set_up_context_variables().
-	 */
-	ret = init_real_cpu_mask(dd);
-	if (ret)
-		goto bail_cleanup;
 	ret = set_up_context_variables(dd);
 	if (ret)
 		goto bail_cleanup;
 	/* set up KDETH QP prefix in both RX and TX CSRs */
 	init_kdeth_qp(dd);
-	hfi1_dev_affinity_init(dd);
+	ret = hfi1_dev_affinity_init(dd);
+	if (ret)
+		goto bail_cleanup;
 	/* send contexts must be set up before receive contexts */
 	ret = init_send_contexts(dd);
 	/* set up LCB access - must be after set_up_interrupts() */
 	init_lcb_access(dd);
+	/*
+	 * Serial number is created from the base guid:
+	 * [27:24] = base guid [38:35]
+	 * [23: 0] = base guid [23: 0]
+	 */
 	snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
-		 dd->base_guid & 0xFFFFFF);
+		 (dd->base_guid & 0xFFFFFF) |
+		     ((dd->base_guid >> 11) & 0xF000000));
 	dd->oui1 = dd->base_guid >> 56 & 0xFF;
 	dd->oui2 = dd->base_guid >> 48 & 0xFF;
 	ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
 	if (ret)
 		goto bail_clear_intr;
-	check_fabric_firmware_versions(dd);
 	thermal_init(dd);

drivers/infiniband/hw/hfi1/chip.h+1−4

 /* SBus commands */
 #define RESET_SBUS_RECEIVER 0x20
 #define WRITE_SBUS_RECEIVER 0x21
+#define READ_SBUS_RECEIVER  0x22
 void sbus_request(struct hfi1_devdata *dd,
 		  u8 receiver_addr, u8 data_addr, u8 command, u32 data_in);
 int sbus_request_slow(struct hfi1_devdata *dd,
 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd);
 struct hfi1_message_header *hfi1_get_msgheader(
 				struct hfi1_devdata *dd, __le32 *rhf_addr);
-int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
-			struct hfi1_ctxt_info *kinfo);
-u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
-		  u32 mask);
 int hfi1_init_ctxt(struct send_context *sc);
 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
 		  u32 type, unsigned long pa, u16 order);

drivers/infiniband/hw/hfi1/chip_registers.h+4−0

 #define ASIC_STS_SBUS_RESULT (ASIC + 0x000000000010)
 #define ASIC_STS_SBUS_RESULT_DONE_SMASK 0x1ull
 #define ASIC_STS_SBUS_RESULT_RCV_DATA_VALID_SMASK 0x2ull
+#define ASIC_STS_SBUS_RESULT_RESULT_CODE_SHIFT 2
+#define ASIC_STS_SBUS_RESULT_RESULT_CODE_MASK 0x7ull
+#define ASIC_STS_SBUS_RESULT_DATA_OUT_SHIFT 32
+#define ASIC_STS_SBUS_RESULT_DATA_OUT_MASK 0xFFFFFFFFull
 #define ASIC_STS_THERM (ASIC + 0x000000000058)
 #define ASIC_STS_THERM_CRIT_TEMP_MASK 0x7FFull
 #define ASIC_STS_THERM_CRIT_TEMP_SHIFT 18

drivers/infiniband/hw/hfi1/driver.c+28−24

@@ -392,9 +392,7 @@ static void rcv_hdrerr(struct hfi1_ctxtdata rcd, struct hfi1_pportdata ppd, · 391 unmodified lines
392	u16 rlid;	392	u16 rlid;
393	u8 svc_type, sl, sc5;	393	u8 svc_type, sl, sc5;
394		394
395	sc5 = (be16_to_cpu(rhdr->lrh[0]) >> 12) & 0xf;	395	sc5 = hdr2sc(rhdr, packet->rhf);
396	if (rhf_dc_info(packet->rhf))
397	sc5 \|= 0x10;
398	sl = ibp->sc_to_sl[sc5];	396	sl = ibp->sc_to_sl[sc5];
399		397
400	lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK;	398	lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK;
@@ -450,14 +448,20 @@ static inline void init_packet(struct hfi1_ctxtdata *rcd, · 49 unmodified lines
450	packet->rcv_flags = 0;	448	packet->rcv_flags = 0;
451	}	449	}
452		450
453	static void process_ecn(struct rvt_qp qp, struct hfi1_ib_header hdr,	451	void hfi1_process_ecn_slowpath(struct rvt_qp qp, struct hfi1_packet pkt,
454	struct hfi1_other_headers *ohdr,	452	bool do_cnp)
455	u64 rhf, u32 bth1, struct ib_grh *grh)
456	{	453	{
457	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);	454	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
458	u32 rqpn = 0;	455	struct hfi1_ib_header *hdr = pkt->hdr;
459	u16 rlid;	456	struct hfi1_other_headers *ohdr = pkt->ohdr;
460	u8 sc5, svc_type;	457	struct ib_grh *grh = NULL;
		458	u32 rqpn = 0, bth1;
		459	u16 rlid, dlid = be16_to_cpu(hdr->lrh[1]);
		460	u8 sc, svc_type;
		461	bool is_mcast = false;
		462
		463	if (pkt->rcv_flags & HFI1_HAS_GRH)
		464	grh = &hdr->u.l.grh;
461		465
462	switch (qp->ibqp.qp_type) {	466	switch (qp->ibqp.qp_type) {
463	case IB_QPT_SMI:	467	case IB_QPT_SMI:
@@ -466,6 +470,8 @@ static void process_ecn(struct rvt_qp qp, struct hfi1_ib_header hdr, · 2 unmodified lines
466	rlid = be16_to_cpu(hdr->lrh[3]);	470	rlid = be16_to_cpu(hdr->lrh[3]);
467	rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;	471	rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
468	svc_type = IB_CC_SVCTYPE_UD;	472	svc_type = IB_CC_SVCTYPE_UD;
		473	is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
		474	(dlid != be16_to_cpu(IB_LID_PERMISSIVE));
469	break;	475	break;
470	case IB_QPT_UC:	476	case IB_QPT_UC:
471	rlid = qp->remote_ah_attr.dlid;	477	rlid = qp->remote_ah_attr.dlid;
@@ -481,24 +487,23 @@ static void process_ecn(struct rvt_qp qp, struct hfi1_ib_header hdr, · 9 unmodified lines
481	return;	487	return;
482	}	488	}
483		489
484	sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf;	490	sc = hdr2sc((struct hfi1_message_header *)hdr, pkt->rhf);
485	if (rhf_dc_info(rhf))
486	sc5 \|= 0x10;
487		491
488	if (bth1 & HFI1_FECN_SMASK) {	492	bth1 = be32_to_cpu(ohdr->bth[1]);
		493	if (do_cnp && (bth1 & HFI1_FECN_SMASK)) {
489	u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);	494	u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);
490	u16 dlid = be16_to_cpu(hdr->lrh[1]);
491		495
492	return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc5, grh);	496	return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh);
493	}	497	}
494		498
495	if (bth1 & HFI1_BECN_SMASK) {	499	if (!is_mcast && (bth1 & HFI1_BECN_SMASK)) {
496	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);	500	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
497	u32 lqpn = bth1 & RVT_QPN_MASK;	501	u32 lqpn = bth1 & RVT_QPN_MASK;
498	u8 sl = ibp->sc_to_sl[sc5];	502	u8 sl = ibp->sc_to_sl[sc];
499		503
500	process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);	504	process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
501	}	505	}
		506
502	}	507	}
503		508
504	struct ps_mdata {	509	struct ps_mdata {
@@ -596,7 +601,6 @@ static void __prescan_rxq(struct hfi1_packet *packet) · 91 unmodified lines
596	struct rvt_qp *qp;	601	struct rvt_qp *qp;
597	struct hfi1_ib_header *hdr;	602	struct hfi1_ib_header *hdr;
598	struct hfi1_other_headers *ohdr;	603	struct hfi1_other_headers *ohdr;
599	struct ib_grh *grh = NULL;
600	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;	604	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
601	u64 rhf = rhf_to_cpu(rhf_addr);	605	u64 rhf = rhf_to_cpu(rhf_addr);
602	u32 etype = rhf_rcv_type(rhf), qpn, bth1;	606	u32 etype = rhf_rcv_type(rhf), qpn, bth1;
@@ -616,14 +620,13 @@ static void __prescan_rxq(struct hfi1_packet *packet) · 13 unmodified lines
616	hfi1_get_msgheader(dd, rhf_addr);	620	hfi1_get_msgheader(dd, rhf_addr);
617	lnh = be16_to_cpu(hdr->lrh[0]) & 3;	621	lnh = be16_to_cpu(hdr->lrh[0]) & 3;
618		622
619	if (lnh == HFI1_LRH_BTH) {	623	if (lnh == HFI1_LRH_BTH)
620	ohdr = &hdr->u.oth;	624	ohdr = &hdr->u.oth;
621	} else if (lnh == HFI1_LRH_GRH) {	625	else if (lnh == HFI1_LRH_GRH)
622	ohdr = &hdr->u.l.oth;	626	ohdr = &hdr->u.l.oth;
623	grh = &hdr->u.l.grh;	627	else
624	} else {
625	goto next; /* just in case */	628	goto next; /* just in case */
626	}	629
627	bth1 = be32_to_cpu(ohdr->bth[1]);	630	bth1 = be32_to_cpu(ohdr->bth[1]);
628	is_ecn = !!(bth1 & (HFI1_FECN_SMASK \| HFI1_BECN_SMASK));	631	is_ecn = !!(bth1 & (HFI1_FECN_SMASK \| HFI1_BECN_SMASK));
629		632
@@ -639,7 +642,7 @@ static void __prescan_rxq(struct hfi1_packet *packet) · 9 unmodified lines
639	goto next;	642	goto next;
640	}	643	}
641		644
642	process_ecn(qp, hdr, ohdr, rhf, bth1, grh);	645	process_ecn(qp, packet, true);
643	rcu_read_unlock();	646	rcu_read_unlock();
644		647
645	/* turn off BECN, FECN */	648	/* turn off BECN, FECN */
@@ -1362,6 +1365,7 @@ int process_receive_bypass(struct hfi1_packet *packet) · 716 unmodified lines
1362		1365
1363	dd_dev_err(packet->rcd->dd,	1366	dd_dev_err(packet->rcd->dd,
1364	"Bypass packets are not supported in normal operation. Dropping\n");	1367	"Bypass packets are not supported in normal operation. Dropping\n");
		1368	incr_cntr64(&packet->rcd->dd->sw_rcv_bypass_packet_errors);
1365	return RHF_RCV_CONTINUE;	1369	return RHF_RCV_CONTINUE;
1366	}	1370	}
1367		1371

drivers/infiniband/hw/hfi1/file_ops.c+51−40

@@ -168,6 +168,7 @@ static inline int is_valid_mmap(u64 token) · 167 unmodified lines
168		168
169	static int hfi1_file_open(struct inode inode, struct file fp)	169	static int hfi1_file_open(struct inode inode, struct file fp)
170	{	170	{
		171	struct hfi1_filedata *fd;
171	struct hfi1_devdata *dd = container_of(inode->i_cdev,	172	struct hfi1_devdata *dd = container_of(inode->i_cdev,
172	struct hfi1_devdata,	173	struct hfi1_devdata,
173	user_cdev);	174	user_cdev);
@@ -176,10 +177,17 @@ static int hfi1_file_open(struct inode inode, struct file fp) · 2 unmodified lines
176	kobject_get(&dd->kobj);	177	kobject_get(&dd->kobj);
177		178
178	/* The real work is performed later in assign_ctxt() */	179	/* The real work is performed later in assign_ctxt() */
179	fp->private_data = kzalloc(sizeof(struct hfi1_filedata), GFP_KERNEL);	180
180	if (fp->private_data) /* no cpu affinity by default */	181	fd = kzalloc(sizeof(*fd), GFP_KERNEL);
181	((struct hfi1_filedata *)fp->private_data)->rec_cpu_num = -1;	182
182	return fp->private_data ? 0 : -ENOMEM;	183	if (fd) {
		184	fd->rec_cpu_num = -1; /* no cpu affinity by default */
		185	fd->mm = current->mm;
		186	}
		187
		188	fp->private_data = fd;
		189
		190	return fd ? 0 : -ENOMEM;
183	}	191	}
184		192
185	static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,	193	static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
@@ -392,41 +400,38 @@ static ssize_t hfi1_write_iter(struct kiocb kiocb, struct iov_iter from) · 206 unmodified lines
392	struct hfi1_filedata *fd = kiocb->ki_filp->private_data;	400	struct hfi1_filedata *fd = kiocb->ki_filp->private_data;
393	struct hfi1_user_sdma_pkt_q *pq = fd->pq;	401	struct hfi1_user_sdma_pkt_q *pq = fd->pq;
394	struct hfi1_user_sdma_comp_q *cq = fd->cq;	402	struct hfi1_user_sdma_comp_q *cq = fd->cq;
395	int ret = 0, done = 0, reqs = 0;	403	int done = 0, reqs = 0;
396	unsigned long dim = from->nr_segs;	404	unsigned long dim = from->nr_segs;
397		405
398	if (!cq \|\| !pq) {	406	if (!cq \|\| !pq)
399	ret = -EIO;	407	return -EIO;
400	goto done;
401	}
402		408
403	if (!iter_is_iovec(from) \|\| !dim) {	409	if (!iter_is_iovec(from) \|\| !dim)
404	ret = -EINVAL;	410	return -EINVAL;
405	goto done;
406	}
407		411
408	hfi1_cdbg(SDMA, "SDMA request from %u:%u (%lu)",	412	hfi1_cdbg(SDMA, "SDMA request from %u:%u (%lu)",
409	fd->uctxt->ctxt, fd->subctxt, dim);	413	fd->uctxt->ctxt, fd->subctxt, dim);
410		414
411	if (atomic_read(&pq->n_reqs) == pq->n_max_reqs) {	415	if (atomic_read(&pq->n_reqs) == pq->n_max_reqs)
412	ret = -ENOSPC;	416	return -ENOSPC;
413	goto done;
414	}
415		417
416	while (dim) {	418	while (dim) {
		419	int ret;
417	unsigned long count = 0;	420	unsigned long count = 0;
418		421
419	ret = hfi1_user_sdma_process_request(	422	ret = hfi1_user_sdma_process_request(
420	kiocb->ki_filp, (struct iovec *)(from->iov + done),	423	kiocb->ki_filp, (struct iovec *)(from->iov + done),
421	dim, &count);	424	dim, &count);
422	if (ret)	425	if (ret) {
423	goto done;	426	reqs = ret;
		427	break;
		428	}
424	dim -= count;	429	dim -= count;
425	done += count;	430	done += count;
426	reqs++;	431	reqs++;
427	}	432	}
428	done:	433
429	return ret ? ret : reqs;	434	return reqs;
430	}	435	}
431		436
432	static int hfi1_file_mmap(struct file fp, struct vm_area_struct vma)	437	static int hfi1_file_mmap(struct file fp, struct vm_area_struct vma)
@@ -718,7 +723,7 @@ static int hfi1_file_close(struct inode inode, struct file fp) · 285 unmodified lines
718	hfi1_user_sdma_free_queues(fdata);	723	hfi1_user_sdma_free_queues(fdata);
719		724
720	/* release the cpu */	725	/* release the cpu */
721	hfi1_put_proc_affinity(dd, fdata->rec_cpu_num);	726	hfi1_put_proc_affinity(fdata->rec_cpu_num);
722		727
723	/*	728	/*
724	* Clear any left over, unhandled events so the next process that	729	* Clear any left over, unhandled events so the next process that
@@ -730,7 +735,6 @@ static int hfi1_file_close(struct inode inode, struct file fp) · 5 unmodified lines
730		735
731	if (--uctxt->cnt) {	736	if (--uctxt->cnt) {
732	uctxt->active_slaves &= ~(1 << fdata->subctxt);	737	uctxt->active_slaves &= ~(1 << fdata->subctxt);
733	uctxt->subpid[fdata->subctxt] = 0;
734	mutex_unlock(&hfi1_mutex);	738	mutex_unlock(&hfi1_mutex);
735	goto done;	739	goto done;
736	}	740	}
@@ -756,7 +760,6 @@ static int hfi1_file_close(struct inode inode, struct file fp) · 19 unmodified lines
756	write_kctxt_csr(dd, uctxt->sc->hw_context, SEND_CTXT_CHECK_ENABLE,	760	write_kctxt_csr(dd, uctxt->sc->hw_context, SEND_CTXT_CHECK_ENABLE,
757	hfi1_pkt_default_send_ctxt_mask(dd, uctxt->sc->type));	761	hfi1_pkt_default_send_ctxt_mask(dd, uctxt->sc->type));
758	sc_disable(uctxt->sc);	762	sc_disable(uctxt->sc);
759	uctxt->pid = 0;
760	spin_unlock_irqrestore(&dd->uctxt_lock, flags);	763	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
761		764
762	dd->rcd[uctxt->ctxt] = NULL;	765	dd->rcd[uctxt->ctxt] = NULL;
@@ -818,9 +821,10 @@ static int assign_ctxt(struct file fp, struct hfi1_user_info uinfo) · 55 unmodified lines
818	ret = find_shared_ctxt(fp, uinfo);	821	ret = find_shared_ctxt(fp, uinfo);
819	if (ret < 0)	822	if (ret < 0)
820	goto done_unlock;	823	goto done_unlock;
821	if (ret)	824	if (ret) {
822	fd->rec_cpu_num = hfi1_get_proc_affinity(	825	fd->rec_cpu_num =
823	fd->uctxt->dd, fd->uctxt->numa_id);	826	hfi1_get_proc_affinity(fd->uctxt->numa_id);
		827	}
824	}	828	}
825		829
826	/*	830	/*
@@ -895,7 +899,6 @@ static int find_shared_ctxt(struct file *fp, · 68 unmodified lines
895	}	899	}
896	fd->uctxt = uctxt;	900	fd->uctxt = uctxt;
897	fd->subctxt = uctxt->cnt++;	901	fd->subctxt = uctxt->cnt++;
898	uctxt->subpid[fd->subctxt] = current->pid;
899	uctxt->active_slaves \|= 1 << fd->subctxt;	902	uctxt->active_slaves \|= 1 << fd->subctxt;
900	ret = 1;	903	ret = 1;
901	goto done;	904	goto done;
@@ -932,7 +935,11 @@ static int allocate_ctxt(struct file fp, struct hfi1_devdata dd, · 30 unmodified lines
932	if (ctxt == dd->num_rcv_contexts)	935	if (ctxt == dd->num_rcv_contexts)
933	return -EBUSY;	936	return -EBUSY;
934		937
935	fd->rec_cpu_num = hfi1_get_proc_affinity(dd, -1);	938	/*
		939	* If we don't have a NUMA node requested, preference is towards
		940	* device NUMA node.
		941	*/
		942	fd->rec_cpu_num = hfi1_get_proc_affinity(dd->node);
936	if (fd->rec_cpu_num != -1)	943	if (fd->rec_cpu_num != -1)
937	numa = cpu_to_node(fd->rec_cpu_num);	944	numa = cpu_to_node(fd->rec_cpu_num);
938	else	945	else
@@ -976,8 +983,7 @@ static int allocate_ctxt(struct file fp, struct hfi1_devdata dd, · 37 unmodified lines
976	return ret;	983	return ret;
977	}	984	}
978	uctxt->userversion = uinfo->userversion;	985	uctxt->userversion = uinfo->userversion;
979	uctxt->pid = current->pid;	986	uctxt->flags = hfi1_cap_mask; /* save current flag state */
980	uctxt->flags = HFI1_CAP_UGET(MASK);
981	init_waitqueue_head(&uctxt->wait);	987	init_waitqueue_head(&uctxt->wait);
982	strlcpy(uctxt->comm, current->comm, sizeof(uctxt->comm));	988	strlcpy(uctxt->comm, current->comm, sizeof(uctxt->comm));
983	memcpy(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid));	989	memcpy(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid));
@@ -1080,18 +1086,18 @@ static int user_init(struct file *fp) · 96 unmodified lines
1080	hfi1_set_ctxt_jkey(uctxt->dd, uctxt->ctxt, uctxt->jkey);	1086	hfi1_set_ctxt_jkey(uctxt->dd, uctxt->ctxt, uctxt->jkey);
1081		1087
1082	rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;	1088	rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;
1083	if (HFI1_CAP_KGET_MASK(uctxt->flags, HDRSUPP))	1089	if (HFI1_CAP_UGET_MASK(uctxt->flags, HDRSUPP))
1084	rcvctrl_ops \|= HFI1_RCVCTRL_TIDFLOW_ENB;	1090	rcvctrl_ops \|= HFI1_RCVCTRL_TIDFLOW_ENB;
1085	/*	1091	/*
1086	* Ignore the bit in the flags for now until proper	1092	* Ignore the bit in the flags for now until proper
1087	* support for multiple packet per rcv array entry is	1093	* support for multiple packet per rcv array entry is
1088	* added.	1094	* added.
1089	*/	1095	*/
1090	if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))	1096	if (!HFI1_CAP_UGET_MASK(uctxt->flags, MULTI_PKT_EGR))
1091	rcvctrl_ops \|= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;	1097	rcvctrl_ops \|= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
1092	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))	1098	if (HFI1_CAP_UGET_MASK(uctxt->flags, NODROP_EGR_FULL))
1093	rcvctrl_ops \|= HFI1_RCVCTRL_NO_EGR_DROP_ENB;	1099	rcvctrl_ops \|= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
1094	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))	1100	if (HFI1_CAP_UGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
1095	rcvctrl_ops \|= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;	1101	rcvctrl_ops \|= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
1096	/*	1102	/*
1097	* The RcvCtxtCtrl.TailUpd bit has to be explicitly written.	1103	* The RcvCtxtCtrl.TailUpd bit has to be explicitly written.
@@ -1099,7 +1105,7 @@ static int user_init(struct file *fp) · 1 unmodified line
1099	* uses of the chip or ctxt. Therefore, add the rcvctrl op	1105	* uses of the chip or ctxt. Therefore, add the rcvctrl op
1100	* for both cases.	1106	* for both cases.
1101	*/	1107	*/
1102	if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))	1108	if (HFI1_CAP_UGET_MASK(uctxt->flags, DMA_RTAIL))
1103	rcvctrl_ops \|= HFI1_RCVCTRL_TAILUPD_ENB;	1109	rcvctrl_ops \|= HFI1_RCVCTRL_TAILUPD_ENB;
1104	else	1110	else
1105	rcvctrl_ops \|= HFI1_RCVCTRL_TAILUPD_DIS;	1111	rcvctrl_ops \|= HFI1_RCVCTRL_TAILUPD_DIS;
@@ -1122,9 +1128,14 @@ static int get_ctxt_info(struct file fp, void __user ubase, __u32 len) · 16 unmodified lines
1122	int ret = 0;	1128	int ret = 0;
1123		1129
1124	memset(&cinfo, 0, sizeof(cinfo));	1130	memset(&cinfo, 0, sizeof(cinfo));
1125	ret = hfi1_get_base_kinfo(uctxt, &cinfo);	1131	cinfo.runtime_flags = (((uctxt->flags >> HFI1_CAP_MISC_SHIFT) &
1126	if (ret < 0)	1132	HFI1_CAP_MISC_MASK) << HFI1_CAP_USER_SHIFT) \|
1127	goto done;	1133	HFI1_CAP_UGET_MASK(uctxt->flags, MASK) \|
		1134	HFI1_CAP_KGET_MASK(uctxt->flags, K2U);
		1135	/* adjust flag if this fd is not able to cache */
		1136	if (!fd->handler)
		1137	cinfo.runtime_flags \|= HFI1_CAP_TID_UNMAP; /* no caching */
		1138
1128	cinfo.num_active = hfi1_count_active_units();	1139	cinfo.num_active = hfi1_count_active_units();
1129	cinfo.unit = uctxt->dd->unit;	1140	cinfo.unit = uctxt->dd->unit;
1130	cinfo.ctxt = uctxt->ctxt;	1141	cinfo.ctxt = uctxt->ctxt;
@@ -1146,7 +1157,7 @@ static int get_ctxt_info(struct file fp, void __user ubase, __u32 len) · 15 unmodified lines
1146	trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, cinfo);	1157	trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, cinfo);
1147	if (copy_to_user(ubase, &cinfo, sizeof(cinfo)))	1158	if (copy_to_user(ubase, &cinfo, sizeof(cinfo)))
1148	ret = -EFAULT;	1159	ret = -EFAULT;
1149	done:	1160
1150	return ret;	1161	return ret;
1151	}	1162	}
1152		1163

drivers/infiniband/hw/hfi1/firmware.c+125−0

@@ -206,6 +206,9 @@ static const struct firmware *platform_config; · 205 unmodified lines
206	/* the number of fabric SerDes on the SBus */	206	/* the number of fabric SerDes on the SBus */
207	#define NUM_FABRIC_SERDES 4	207	#define NUM_FABRIC_SERDES 4
208		208
		209	/* ASIC_STS_SBUS_RESULT.RESULT_CODE value */
		210	#define SBUS_READ_COMPLETE 0x4
		211
209	/* SBus fabric SerDes addresses, one set per HFI */	212	/* SBus fabric SerDes addresses, one set per HFI */
210	static const u8 fabric_serdes_addrs[2][NUM_FABRIC_SERDES] = {	213	static const u8 fabric_serdes_addrs[2][NUM_FABRIC_SERDES] = {
211	{ 0x01, 0x02, 0x03, 0x04 },	214	{ 0x01, 0x02, 0x03, 0x04 },
@@ -240,6 +243,7 @@ static const u8 all_pcie_serdes_broadcast = 0xe0; · 28 unmodified lines
240	static void dispose_one_firmware(struct firmware_details *fdet);	243	static void dispose_one_firmware(struct firmware_details *fdet);
241	static int load_fabric_serdes_firmware(struct hfi1_devdata *dd,	244	static int load_fabric_serdes_firmware(struct hfi1_devdata *dd,
242	struct firmware_details *fdet);	245	struct firmware_details *fdet);
		246	static void dump_fw_version(struct hfi1_devdata *dd);
243		247
244	/*	248	/*
245	* Read a single 64-bit value from 8051 data memory.	249	* Read a single 64-bit value from 8051 data memory.
@@ -1078,6 +1082,44 @@ void sbus_request(struct hfi1_devdata *dd, · 832 unmodified lines
1078	ASIC_CFG_SBUS_REQUEST_RECEIVER_ADDR_SHIFT));	1082	ASIC_CFG_SBUS_REQUEST_RECEIVER_ADDR_SHIFT));
1079	}	1083	}
1080		1084
		1085	/*
		1086	* Read a value from the SBus.
		1087	*
		1088	* Requires the caller to be in fast mode
		1089	*/
		1090	static u32 sbus_read(struct hfi1_devdata *dd, u8 receiver_addr, u8 data_addr,
		1091	u32 data_in)
		1092	{
		1093	u64 reg;
		1094	int retries;
		1095	int success = 0;
		1096	u32 result = 0;
		1097	u32 result_code = 0;
		1098
		1099	sbus_request(dd, receiver_addr, data_addr, READ_SBUS_RECEIVER, data_in);
		1100
		1101	for (retries = 0; retries < 100; retries++) {
		1102	usleep_range(1000, 1200); /* arbitrary */
		1103	reg = read_csr(dd, ASIC_STS_SBUS_RESULT);
		1104	result_code = (reg >> ASIC_STS_SBUS_RESULT_RESULT_CODE_SHIFT)
		1105	& ASIC_STS_SBUS_RESULT_RESULT_CODE_MASK;
		1106	if (result_code != SBUS_READ_COMPLETE)
		1107	continue;
		1108
		1109	success = 1;
		1110	result = (reg >> ASIC_STS_SBUS_RESULT_DATA_OUT_SHIFT)
		1111	& ASIC_STS_SBUS_RESULT_DATA_OUT_MASK;
		1112	break;
		1113	}
		1114
		1115	if (!success) {
		1116	dd_dev_err(dd, "%s: read failed, result code 0x%x\n", __func__,
		1117	result_code);
		1118	}
		1119
		1120	return result;
		1121	}
		1122
1081	/*	1123	/*
1082	* Turn off the SBus and fabric serdes spicos.	1124	* Turn off the SBus and fabric serdes spicos.
1083	*	1125	*
@@ -1636,6 +1678,7 @@ int load_firmware(struct hfi1_devdata *dd) · 552 unmodified lines
1636	return ret;	1678	return ret;
1637	}	1679	}
1638		1680
		1681	dump_fw_version(dd);
1639	return 0;	1682	return 0;
1640	}	1683	}
1641		1684
@@ -2054,3 +2097,85 @@ void read_guid(struct hfi1_devdata *dd) · 412 unmodified lines
2054	dd_dev_info(dd, "GUID %llx",	2097	dd_dev_info(dd, "GUID %llx",
2055	(unsigned long long)dd->base_guid);	2098	(unsigned long long)dd->base_guid);
2056	}	2099	}
		2100
		2101	/* read and display firmware version info */
		2102	static void dump_fw_version(struct hfi1_devdata *dd)
		2103	{
		2104	u32 pcie_vers[NUM_PCIE_SERDES];
		2105	u32 fabric_vers[NUM_FABRIC_SERDES];
		2106	u32 sbus_vers;
		2107	int i;
		2108	int all_same;
		2109	int ret;
		2110	u8 rcv_addr;
		2111
		2112	ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
		2113	if (ret) {
		2114	dd_dev_err(dd, "Unable to acquire SBus to read firmware versions\n");
		2115	return;
		2116	}
		2117
		2118	/* set fast mode */
		2119	set_sbus_fast_mode(dd);
		2120
		2121	/* read version for SBus Master */
		2122	sbus_request(dd, SBUS_MASTER_BROADCAST, 0x02, WRITE_SBUS_RECEIVER, 0);
		2123	sbus_request(dd, SBUS_MASTER_BROADCAST, 0x07, WRITE_SBUS_RECEIVER, 0x1);
		2124	/* wait for interrupt to be processed */
		2125	usleep_range(10000, 11000);
		2126	sbus_vers = sbus_read(dd, SBUS_MASTER_BROADCAST, 0x08, 0x1);
		2127	dd_dev_info(dd, "SBus Master firmware version 0x%08x\n", sbus_vers);
		2128
		2129	/* read version for PCIe SerDes */
		2130	all_same = 1;
		2131	pcie_vers[0] = 0;
		2132	for (i = 0; i < NUM_PCIE_SERDES; i++) {
		2133	rcv_addr = pcie_serdes_addrs[dd->hfi1_id][i];
		2134	sbus_request(dd, rcv_addr, 0x03, WRITE_SBUS_RECEIVER, 0);
		2135	/* wait for interrupt to be processed */
		2136	usleep_range(10000, 11000);
		2137	pcie_vers[i] = sbus_read(dd, rcv_addr, 0x04, 0x0);
		2138	if (i > 0 && pcie_vers[0] != pcie_vers[i])
		2139	all_same = 0;
		2140	}
		2141
		2142	if (all_same) {
		2143	dd_dev_info(dd, "PCIe SerDes firmware version 0x%x\n",
		2144	pcie_vers[0]);
		2145	} else {
		2146	dd_dev_warn(dd, "PCIe SerDes do not have the same firmware version\n");
		2147	for (i = 0; i < NUM_PCIE_SERDES; i++) {
		2148	dd_dev_info(dd,
		2149	"PCIe SerDes lane %d firmware version 0x%x\n",
		2150	i, pcie_vers[i]);
		2151	}
		2152	}
		2153
		2154	/* read version for fabric SerDes */
		2155	all_same = 1;
		2156	fabric_vers[0] = 0;
		2157	for (i = 0; i < NUM_FABRIC_SERDES; i++) {
		2158	rcv_addr = fabric_serdes_addrs[dd->hfi1_id][i];
		2159	sbus_request(dd, rcv_addr, 0x03, WRITE_SBUS_RECEIVER, 0);
		2160	/* wait for interrupt to be processed */
		2161	usleep_range(10000, 11000);
		2162	fabric_vers[i] = sbus_read(dd, rcv_addr, 0x04, 0x0);
		2163	if (i > 0 && fabric_vers[0] != fabric_vers[i])
		2164	all_same = 0;
		2165	}
		2166
		2167	if (all_same) {
		2168	dd_dev_info(dd, "Fabric SerDes firmware version 0x%x\n",
		2169	fabric_vers[0]);
		2170	} else {
		2171	dd_dev_warn(dd, "Fabric SerDes do not have the same firmware version\n");
		2172	for (i = 0; i < NUM_FABRIC_SERDES; i++) {
		2173	dd_dev_info(dd,
		2174	"Fabric SerDes lane %d firmware version 0x%x\n",
		2175	i, fabric_vers[i]);
		2176	}
		2177	}
		2178
		2179	clear_sbus_fast_mode(dd);
		2180	release_chip_resource(dd, CR_SBUS);
		2181	}

drivers/infiniband/hw/hfi1/hfi.h+104−17

@@ -62,6 +62,8 @@ · 61 unmodified lines
62	#include <linux/cdev.h>	62	#include <linux/cdev.h>
63	#include <linux/delay.h>	63	#include <linux/delay.h>
64	#include <linux/kthread.h>	64	#include <linux/kthread.h>
		65	#include <linux/i2c.h>
		66	#include <linux/i2c-algo-bit.h>
65	#include <rdma/rdma_vt.h>	67	#include <rdma/rdma_vt.h>
66		68
67	#include "chip_registers.h"	69	#include "chip_registers.h"
@@ -253,7 +255,7 @@ struct hfi1_ctxtdata { · 185 unmodified lines
253	/* chip offset of PIO buffers for this ctxt */	255	/* chip offset of PIO buffers for this ctxt */
254	u32 piobufs;	256	u32 piobufs;
255	/* per-context configuration flags */	257	/* per-context configuration flags */
256	u32 flags;	258	unsigned long flags;
257	/* per-context event flags for fileops/intr communication */	259	/* per-context event flags for fileops/intr communication */
258	unsigned long event_flags;	260	unsigned long event_flags;
259	/* WAIT_RCV that timed out, no interrupt */	261	/* WAIT_RCV that timed out, no interrupt */
@@ -268,9 +270,6 @@ struct hfi1_ctxtdata { · 8 unmodified lines
268	u32 urgent;	270	u32 urgent;
269	/* saved total number of polled urgent packets for poll edge trigger */	271	/* saved total number of polled urgent packets for poll edge trigger */
270	u32 urgent_poll;	272	u32 urgent_poll;
271	/* pid of process using this ctxt */
272	pid_t pid;
273	pid_t subpid[HFI1_MAX_SHARED_CTXTS];
274	/* same size as task_struct .comm[], command that opened context */	273	/* same size as task_struct .comm[], command that opened context */
275	char comm[TASK_COMM_LEN];	274	char comm[TASK_COMM_LEN];
276	/* so file ops can get at unit */	275	/* so file ops can get at unit */
@@ -366,11 +365,6 @@ struct hfi1_packet { · 89 unmodified lines
366	u8 etype;	365	u8 etype;
367	};	366	};
368		367
369	static inline bool has_sc4_bit(struct hfi1_packet *p)
370	{
371	return !!rhf_dc_info(p->rhf);
372	}
373
374	/*	368	/*
375	* Private data for snoop/capture support.	369	* Private data for snoop/capture support.
376	*/	370	*/
@@ -805,10 +799,19 @@ struct hfi1_temp { · 428 unmodified lines
805	u8 triggers; /* temperature triggers */	799	u8 triggers; /* temperature triggers */
806	};	800	};
807		801
		802	struct hfi1_i2c_bus {
		803	struct hfi1_devdata controlling_dd; / current controlling device */
		804	struct i2c_adapter adapter; /* bus details */
		805	struct i2c_algo_bit_data algo; /* bus algorithm details */
		806	int num; /* bus number, 0 or 1 */
		807	};
		808
808	/* common data between shared ASIC HFIs */	809	/* common data between shared ASIC HFIs */
809	struct hfi1_asic_data {	810	struct hfi1_asic_data {
810	struct hfi1_devdata dds[2]; / back pointers */	811	struct hfi1_devdata dds[2]; / back pointers */
811	struct mutex asic_resource_mutex;	812	struct mutex asic_resource_mutex;
		813	struct hfi1_i2c_bus *i2c_bus0;
		814	struct hfi1_i2c_bus *i2c_bus1;
812	};	815	};
813		816
814	/* device data struct now contains only "general per-device" info.	817	/* device data struct now contains only "general per-device" info.
@@ -1128,7 +1131,8 @@ struct hfi1_devdata { · 313 unmodified lines
1128	NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS];	1131	NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS];
1129	/* Software counter that aggregates all cce_err_status errors */	1132	/* Software counter that aggregates all cce_err_status errors */
1130	u64 sw_cce_err_status_aggregate;	1133	u64 sw_cce_err_status_aggregate;
1131		1134	/* Software counter that aggregates all bypass packet rcv errors */
		1135	u64 sw_rcv_bypass_packet_errors;
1132	/* receive interrupt functions */	1136	/* receive interrupt functions */
1133	rhf_rcv_function_ptr *rhf_rcv_function_map;	1137	rhf_rcv_function_ptr *rhf_rcv_function_map;
1134	rhf_rcv_function_ptr normal_rhf_rcv_functions[8];	1138	rhf_rcv_function_ptr normal_rhf_rcv_functions[8];
@@ -1184,6 +1188,7 @@ struct hfi1_devdata { · 49 unmodified lines
1184		1188
1185	struct tid_rb_node;	1189	struct tid_rb_node;
1186	struct mmu_rb_node;	1190	struct mmu_rb_node;
		1191	struct mmu_rb_handler;
1187		1192
1188	/* Private data for file operations */	1193	/* Private data for file operations */
1189	struct hfi1_filedata {	1194	struct hfi1_filedata {
@@ -1194,7 +1199,7 @@ struct hfi1_filedata { · 4 unmodified lines
1194	/* for cpu affinity; -1 if none */	1199	/* for cpu affinity; -1 if none */
1195	int rec_cpu_num;	1200	int rec_cpu_num;
1196	u32 tid_n_pinned;	1201	u32 tid_n_pinned;
1197	struct rb_root tid_rb_root;	1202	struct mmu_rb_handler *handler;
1198	struct tid_rb_node **entry_to_rb;	1203	struct tid_rb_node **entry_to_rb;
1199	spinlock_t tid_lock; /* protect tid_[limit,used] counters */	1204	spinlock_t tid_lock; /* protect tid_[limit,used] counters */
1200	u32 tid_limit;	1205	u32 tid_limit;
@@ -1203,6 +1208,7 @@ struct hfi1_filedata { · 2 unmodified lines
1203	u32 invalid_tid_idx;	1208	u32 invalid_tid_idx;
1204	/* protect invalid_tids array and invalid_tid_idx */	1209	/* protect invalid_tids array and invalid_tid_idx */
1205	spinlock_t invalid_lock;	1210	spinlock_t invalid_lock;
		1211	struct mm_struct *mm;
1206	};	1212	};
1207		1213
1208	extern struct list_head hfi1_dev_list;	1214	extern struct list_head hfi1_dev_list;
@@ -1236,6 +1242,8 @@ int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *, int); · 27 unmodified lines
1236	int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *, int);	1242	int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *, int);
1237	void set_all_slowpath(struct hfi1_devdata *dd);	1243	void set_all_slowpath(struct hfi1_devdata *dd);
1238		1244
		1245	extern const struct pci_device_id hfi1_pci_tbl[];
		1246
1239	/* receive packet handler dispositions */	1247	/* receive packet handler dispositions */
1240	#define RCV_PKT_OK 0x0 /* keep going */	1248	#define RCV_PKT_OK 0x0 /* keep going */
1241	#define RCV_PKT_LIMIT 0x1 /* stop, hit limit, start thread */	1249	#define RCV_PKT_LIMIT 0x1 /* stop, hit limit, start thread */
@@ -1261,7 +1269,7 @@ void receive_interrupt_work(struct work_struct *work); · 19 unmodified lines
1261	static inline int hdr2sc(struct hfi1_message_header *hdr, u64 rhf)	1269	static inline int hdr2sc(struct hfi1_message_header *hdr, u64 rhf)
1262	{	1270	{
1263	return ((be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf) \|	1271	return ((be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf) \|
1264	((!!(rhf & RHF_DC_INFO_SMASK)) << 4);	1272	((!!(rhf_dc_info(rhf))) << 4);
1265	}	1273	}
1266		1274
1267	static inline u16 generate_jkey(kuid_t uid)	1275	static inline u16 generate_jkey(kuid_t uid)
@@ -1571,6 +1579,22 @@ static inline struct hfi1_ibport to_iport(struct ib_device ibdev, u8 port) · 303 unmodified lines
1571	return &dd->pport[pidx].ibport_data;	1579	return &dd->pport[pidx].ibport_data;
1572	}	1580	}
1573		1581
		1582	void hfi1_process_ecn_slowpath(struct rvt_qp qp, struct hfi1_packet pkt,
		1583	bool do_cnp);
		1584	static inline bool process_ecn(struct rvt_qp qp, struct hfi1_packet pkt,
		1585	bool do_cnp)
		1586	{
		1587	struct hfi1_other_headers *ohdr = pkt->ohdr;
		1588	u32 bth1;
		1589
		1590	bth1 = be32_to_cpu(ohdr->bth[1]);
		1591	if (unlikely(bth1 & (HFI1_BECN_SMASK \| HFI1_FECN_SMASK))) {
		1592	hfi1_process_ecn_slowpath(qp, pkt, do_cnp);
		1593	return bth1 & HFI1_FECN_SMASK;
		1594	}
		1595	return false;
		1596	}
		1597
1574	/*	1598	/*
1575	* Return the indexed PKEY from the port PKEY table.	1599	* Return the indexed PKEY from the port PKEY table.
1576	*/	1600	*/
@@ -1588,14 +1612,23 @@ static inline u16 hfi1_get_pkey(struct hfi1_ibport *ibp, unsigned index) · 11 unmodified lines
1588	}	1612	}
1589		1613
1590	/*	1614	/*
1591	* Readers of cc_state must call get_cc_state() under rcu_read_lock().	1615	* Called by readers of cc_state only, must call under rcu_read_lock().
1592	* Writers of cc_state must call get_cc_state() under cc_state_lock.
1593	*/	1616	*/
1594	static inline struct cc_state get_cc_state(struct hfi1_pportdata ppd)	1617	static inline struct cc_state get_cc_state(struct hfi1_pportdata ppd)
1595	{	1618	{
1596	return rcu_dereference(ppd->cc_state);	1619	return rcu_dereference(ppd->cc_state);
1597	}	1620	}
1598		1621
		1622	/*
		1623	* Called by writers of cc_state only, must call under cc_state_lock.
		1624	*/
		1625	static inline
		1626	struct cc_state get_cc_state_protected(struct hfi1_pportdata ppd)
		1627	{
		1628	return rcu_dereference_protected(ppd->cc_state,
		1629	lockdep_is_held(&ppd->cc_state_lock));
		1630	}
		1631
1599	/*	1632	/*
1600	* values for dd->flags (_device_ related flags)	1633	* values for dd->flags (_device_ related flags)
1601	*/	1634	*/
@@ -1671,9 +1704,12 @@ void shutdown_led_override(struct hfi1_pportdata *ppd); · 69 unmodified lines
1671	*/	1704	*/
1672	#define DEFAULT_RCVHDR_ENTSIZE 32	1705	#define DEFAULT_RCVHDR_ENTSIZE 32
1673		1706
1674	bool hfi1_can_pin_pages(struct hfi1_devdata *, u32, u32);	1707	bool hfi1_can_pin_pages(struct hfi1_devdata dd, struct mm_struct mm,
1675	int hfi1_acquire_user_pages(unsigned long, size_t, bool, struct page **);	1708	u32 nlocked, u32 npages);
1676	void hfi1_release_user_pages(struct mm_struct , struct page *, size_t, bool);	1709	int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr,
		1710	size_t npages, bool writable, struct page **pages);
		1711	void hfi1_release_user_pages(struct mm_struct mm, struct page *p,
		1712	size_t npages, bool dirty);
1677		1713
1678	static inline void clear_rcvhdrtail(const struct hfi1_ctxtdata *rcd)	1714	static inline void clear_rcvhdrtail(const struct hfi1_ctxtdata *rcd)
1679	{	1715	{
@@ -1949,4 +1985,55 @@ static inline u32 qsfp_resource(struct hfi1_devdata *dd) · 269 unmodified lines
1949		1985
1950	int hfi1_tempsense_rd(struct hfi1_devdata dd, struct hfi1_temp temp);	1986	int hfi1_tempsense_rd(struct hfi1_devdata dd, struct hfi1_temp temp);
1951		1987
		1988	#define DD_DEV_ENTRY(dd) __string(dev, dev_name(&(dd)->pcidev->dev))
		1989	#define DD_DEV_ASSIGN(dd) __assign_str(dev, dev_name(&(dd)->pcidev->dev))
		1990
		1991	#define packettype_name(etype) { RHF_RCV_TYPE_##etype, #etype }
		1992	#define show_packettype(etype) \
		1993	__print_symbolic(etype, \
		1994	packettype_name(EXPECTED), \
		1995	packettype_name(EAGER), \
		1996	packettype_name(IB), \
		1997	packettype_name(ERROR), \
		1998	packettype_name(BYPASS))
		1999
		2000	#define ib_opcode_name(opcode) { IB_OPCODE_##opcode, #opcode }
		2001	#define show_ib_opcode(opcode) \
		2002	__print_symbolic(opcode, \
		2003	ib_opcode_name(RC_SEND_FIRST), \
		2004	ib_opcode_name(RC_SEND_MIDDLE), \
		2005	ib_opcode_name(RC_SEND_LAST), \
		2006	ib_opcode_name(RC_SEND_LAST_WITH_IMMEDIATE), \
		2007	ib_opcode_name(RC_SEND_ONLY), \
		2008	ib_opcode_name(RC_SEND_ONLY_WITH_IMMEDIATE), \
		2009	ib_opcode_name(RC_RDMA_WRITE_FIRST), \
		2010	ib_opcode_name(RC_RDMA_WRITE_MIDDLE), \
		2011	ib_opcode_name(RC_RDMA_WRITE_LAST), \
		2012	ib_opcode_name(RC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
		2013	ib_opcode_name(RC_RDMA_WRITE_ONLY), \
		2014	ib_opcode_name(RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
		2015	ib_opcode_name(RC_RDMA_READ_REQUEST), \
		2016	ib_opcode_name(RC_RDMA_READ_RESPONSE_FIRST), \
		2017	ib_opcode_name(RC_RDMA_READ_RESPONSE_MIDDLE), \
		2018	ib_opcode_name(RC_RDMA_READ_RESPONSE_LAST), \
		2019	ib_opcode_name(RC_RDMA_READ_RESPONSE_ONLY), \
		2020	ib_opcode_name(RC_ACKNOWLEDGE), \
		2021	ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE), \
		2022	ib_opcode_name(RC_COMPARE_SWAP), \
		2023	ib_opcode_name(RC_FETCH_ADD), \
		2024	ib_opcode_name(UC_SEND_FIRST), \
		2025	ib_opcode_name(UC_SEND_MIDDLE), \
		2026	ib_opcode_name(UC_SEND_LAST), \
		2027	ib_opcode_name(UC_SEND_LAST_WITH_IMMEDIATE), \
		2028	ib_opcode_name(UC_SEND_ONLY), \
		2029	ib_opcode_name(UC_SEND_ONLY_WITH_IMMEDIATE), \
		2030	ib_opcode_name(UC_RDMA_WRITE_FIRST), \
		2031	ib_opcode_name(UC_RDMA_WRITE_MIDDLE), \
		2032	ib_opcode_name(UC_RDMA_WRITE_LAST), \
		2033	ib_opcode_name(UC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
		2034	ib_opcode_name(UC_RDMA_WRITE_ONLY), \
		2035	ib_opcode_name(UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
		2036	ib_opcode_name(UD_SEND_ONLY), \
		2037	ib_opcode_name(UD_SEND_ONLY_WITH_IMMEDIATE), \
		2038	ib_opcode_name(CNP))
1952	#endif /* _HFI1_KERNEL_H */	2039	#endif /* _HFI1_KERNEL_H */

drivers/infiniband/hw/hfi1/init.c+34−17

 #include "debugfs.h"
 #include "verbs.h"
 #include "aspm.h"
+#include "affinity.h"
 #undef pr_fmt
 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
 void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
 			 struct hfi1_devdata *dd, u8 hw_pidx, u8 port)
 {
-	int i, size;
+	int i;
 	uint default_pkey_idx;
+	struct cc_state *cc_state;
 	ppd->dd = dd;
 	ppd->hw_pidx = hw_pidx;
 	spin_lock_init(&ppd->cc_state_lock);
 	spin_lock_init(&ppd->cc_log_lock);
-	size = sizeof(struct cc_state);
+	cc_state = kzalloc(sizeof(*cc_state), GFP_KERNEL);
-	RCU_INIT_POINTER(ppd->cc_state, kzalloc(size, GFP_KERNEL));
+	RCU_INIT_POINTER(ppd->cc_state, cc_state);
-	if (!rcu_dereference(ppd->cc_state))
+	if (!cc_state)
 		goto bail;
 	return;
 /*
  * Release our hold on the shared asic data.  If we are the last one,
- * free the structure.  Must be holding hfi1_devs_lock.
+ * return the structure to be finalized outside the lock.  Must be
+ * holding hfi1_devs_lock.
  */
-static void release_asic_data(struct hfi1_devdata *dd)
+static struct hfi1_asic_data *release_asic_data(struct hfi1_devdata *dd)
 {
+	struct hfi1_asic_data *ad;
 	int other;
 	if (!dd->asic_data)
-		return;
+		return NULL;
 	dd->asic_data->dds[dd->hfi1_id] = NULL;
 	other = dd->hfi1_id ? 0 : 1;
-	if (!dd->asic_data->dds[other]) {
+	ad = dd->asic_data;
-		/* we are the last holder, free it */
-		kfree(dd->asic_data);
-	}
 	dd->asic_data = NULL;
+	/* return NULL if the other dd still has a link */
+	return ad->dds[other] ? NULL : ad;
+}
+static void finalize_asic_data(struct hfi1_devdata *dd,
+			       struct hfi1_asic_data *ad)
+{
+	clean_up_i2c(dd, ad);
+	kfree(ad);
 }
 static void __hfi1_free_devdata(struct kobject *kobj)
 {
 	struct hfi1_devdata *dd =
 		container_of(kobj, struct hfi1_devdata, kobj);
+	struct hfi1_asic_data *ad;
 	unsigned long flags;
 	spin_lock_irqsave(&hfi1_devs_lock, flags);
 	idr_remove(&hfi1_unit_table, dd->unit);
 	list_del(&dd->list);
-	release_asic_data(dd);
+	ad = release_asic_data(dd);
 	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
+	if (ad)
+		finalize_asic_data(dd, ad);
 	free_platform_config(dd);
 	rcu_barrier(); /* wait for rcu callbacks to complete */
 	free_percpu(dd->int_counter);
 	free_percpu(dd->rcv_limit);
-	hfi1_dev_affinity_free(dd);
 	free_percpu(dd->send_schedule);
 	rvt_dealloc_device(&dd->verbs_dev.rdi);
 }
 #define DRIVER_LOAD_MSG "Intel " DRIVER_NAME " loaded: "
 #define PFX DRIVER_NAME ": "
-static const struct pci_device_id hfi1_pci_tbl[] = {
+const struct pci_device_id hfi1_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL0) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL1) },
 	{ 0, }
 	if (ret)
 		goto bail;
+	ret = node_affinity_init();
+	if (ret)
+		goto bail;
 	/* validate max MTU before any devices start */
 	if (!valid_opa_max_mtu(hfi1_max_mtu)) {
 		pr_err("Invalid max_mtu 0x%x, using 0x%x instead\n",
 static void __exit hfi1_mod_cleanup(void)
 {
 	pci_unregister_driver(&hfi1_pci_driver);
+	node_affinity_destroy();
 	hfi1_wss_exit();
 	hfi1_dbg_exit();
 	hfi1_cpulist_count = 0;
 			hrtimer_cancel(&ppd->cca_timer[i].hrtimer);
 		spin_lock(&ppd->cc_state_lock);
-		cc_state = get_cc_state(ppd);
+		cc_state = get_cc_state_protected(ppd);
 		RCU_INIT_POINTER(ppd->cc_state, NULL);
 		spin_unlock(&ppd->cc_state_lock);
 	hfi1_cdbg(PROC,
 		  "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %zuKB\n",
-		  rcd->ctxt, rcd->egrbufs.alloced, rcd->egrbufs.rcvtid_size,
+		  rcd->ctxt, rcd->egrbufs.alloced,
-		  rcd->egrbufs.size);
+		  rcd->egrbufs.rcvtid_size / 1024, rcd->egrbufs.size / 1024);
 	/*
 	 * Set the contexts rcv array head update threshold to the closest

drivers/infiniband/hw/hfi1/mad.c+21−39

@@ -588,7 +588,6 @@ static int __subn_get_opa_portinfo(struct opa_smp smp, u32 am, u8 data, · 587 unmodified lines
588		588
589	pi->port_phys_conf = (ppd->port_type & 0xf);	589	pi->port_phys_conf = (ppd->port_type & 0xf);
590		590
591	#if PI_LED_ENABLE_SUP
592	pi->port_states.ledenable_offlinereason = ppd->neighbor_normal << 4;	591	pi->port_states.ledenable_offlinereason = ppd->neighbor_normal << 4;
593	pi->port_states.ledenable_offlinereason \|=	592	pi->port_states.ledenable_offlinereason \|=
594	ppd->is_sm_config_started << 5;	593	ppd->is_sm_config_started << 5;
@@ -602,11 +601,6 @@ static int __subn_get_opa_portinfo(struct opa_smp smp, u32 am, u8 data, · 7 unmodified lines
602	pi->port_states.ledenable_offlinereason \|= is_beaconing_active << 6;	601	pi->port_states.ledenable_offlinereason \|= is_beaconing_active << 6;
603	pi->port_states.ledenable_offlinereason \|=	602	pi->port_states.ledenable_offlinereason \|=
604	ppd->offline_disabled_reason;	603	ppd->offline_disabled_reason;
605	#else
606	pi->port_states.offline_reason = ppd->neighbor_normal << 4;
607	pi->port_states.offline_reason \|= ppd->is_sm_config_started << 5;
608	pi->port_states.offline_reason \|= ppd->offline_disabled_reason;
609	#endif /* PI_LED_ENABLE_SUP */
610		604
611	pi->port_states.portphysstate_portstate =	605	pi->port_states.portphysstate_portstate =
612	(hfi1_ibphys_portstate(ppd) << 4) \| state;	606	(hfi1_ibphys_portstate(ppd) << 4) \| state;
@@ -1752,17 +1746,11 @@ static int __subn_get_opa_psi(struct opa_smp smp, u32 am, u8 data, · 1139 unmodified lines
1752	if (start_of_sm_config && (lstate == IB_PORT_INIT))	1746	if (start_of_sm_config && (lstate == IB_PORT_INIT))
1753	ppd->is_sm_config_started = 1;	1747	ppd->is_sm_config_started = 1;
1754		1748
1755	#if PI_LED_ENABLE_SUP
1756	psi->port_states.ledenable_offlinereason = ppd->neighbor_normal << 4;	1749	psi->port_states.ledenable_offlinereason = ppd->neighbor_normal << 4;
1757	psi->port_states.ledenable_offlinereason \|=	1750	psi->port_states.ledenable_offlinereason \|=
1758	ppd->is_sm_config_started << 5;	1751	ppd->is_sm_config_started << 5;
1759	psi->port_states.ledenable_offlinereason \|=	1752	psi->port_states.ledenable_offlinereason \|=
1760	ppd->offline_disabled_reason;	1753	ppd->offline_disabled_reason;
1761	#else
1762	psi->port_states.offline_reason = ppd->neighbor_normal << 4;
1763	psi->port_states.offline_reason \|= ppd->is_sm_config_started << 5;
1764	psi->port_states.offline_reason \|= ppd->offline_disabled_reason;
1765	#endif /* PI_LED_ENABLE_SUP */
1766		1754
1767	psi->port_states.portphysstate_portstate =	1755	psi->port_states.portphysstate_portstate =
1768	(hfi1_ibphys_portstate(ppd) << 4) \| (lstate & 0xf);	1756	(hfi1_ibphys_portstate(ppd) << 4) \| (lstate & 0xf);
@@ -2430,14 +2418,9 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, · 661 unmodified lines
2430	rsp->port_rcv_remote_physical_errors =	2418	rsp->port_rcv_remote_physical_errors =
2431	cpu_to_be64(read_dev_cntr(dd, C_DC_RMT_PHY_ERR,	2419	cpu_to_be64(read_dev_cntr(dd, C_DC_RMT_PHY_ERR,
2432	CNTR_INVALID_VL));	2420	CNTR_INVALID_VL));
2433	tmp = read_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL);	2421	rsp->local_link_integrity_errors =
2434	tmp2 = tmp + read_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL);	2422	cpu_to_be64(read_dev_cntr(dd, C_DC_RX_REPLAY,
2435	if (tmp2 < tmp) {	2423	CNTR_INVALID_VL));
2436	/* overflow/wrapped */
2437	rsp->local_link_integrity_errors = cpu_to_be64(~0);
2438	} else {
2439	rsp->local_link_integrity_errors = cpu_to_be64(tmp2);
2440	}
2441	tmp = read_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL);	2424	tmp = read_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL);
2442	tmp2 = tmp + read_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT,	2425	tmp2 = tmp + read_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT,
2443	CNTR_INVALID_VL);	2426	CNTR_INVALID_VL);
@@ -2499,6 +2482,9 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, · 55 unmodified lines
2499	cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_BCN_VL,	2482	cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_BCN_VL,
2500	idx_from_vl(vl)));	2483	idx_from_vl(vl)));
2501		2484
		2485	rsp->vls[vfi].port_vl_xmit_discards =
		2486	cpu_to_be64(read_port_cntr(ppd, C_SW_XMIT_DSCD_VL,
		2487	idx_from_vl(vl)));
2502	vlinfo++;	2488	vlinfo++;
2503	vfi++;	2489	vfi++;
2504	}	2490	}
@@ -2529,9 +2515,8 @@ static u64 get_error_counter_summary(struct ib_device *ibdev, u8 port, · 24 unmodified lines
2529	error_counter_summary += read_dev_cntr(dd, C_DC_RMT_PHY_ERR,	2515	error_counter_summary += read_dev_cntr(dd, C_DC_RMT_PHY_ERR,
2530	CNTR_INVALID_VL);	2516	CNTR_INVALID_VL);
2531	/* local link integrity must be right-shifted by the lli resolution */	2517	/* local link integrity must be right-shifted by the lli resolution */
2532	tmp = read_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL);	2518	error_counter_summary += (read_dev_cntr(dd, C_DC_RX_REPLAY,
2533	tmp += read_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL);	2519	CNTR_INVALID_VL) >> res_lli);
2534	error_counter_summary += (tmp >> res_lli);
2535	/* link error recovery must b right-shifted by the ler resolution */	2520	/* link error recovery must b right-shifted by the ler resolution */
2536	tmp = read_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL);	2521	tmp = read_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL);
2537	tmp += read_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT, CNTR_INVALID_VL);	2522	tmp += read_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT, CNTR_INVALID_VL);
@@ -2800,14 +2785,9 @@ static void pma_get_opa_port_ectrs(struct ib_device *ibdev, · 262 unmodified lines
2800	rsp->port_rcv_constraint_errors =	2785	rsp->port_rcv_constraint_errors =
2801	cpu_to_be64(read_port_cntr(ppd, C_SW_RCV_CSTR_ERR,	2786	cpu_to_be64(read_port_cntr(ppd, C_SW_RCV_CSTR_ERR,
2802	CNTR_INVALID_VL));	2787	CNTR_INVALID_VL));
2803	tmp = read_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL);	2788	rsp->local_link_integrity_errors =
2804	tmp2 = tmp + read_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL);	2789	cpu_to_be64(read_dev_cntr(dd, C_DC_RX_REPLAY,
2805	if (tmp2 < tmp) {	2790	CNTR_INVALID_VL));
2806	/* overflow/wrapped */
2807	rsp->local_link_integrity_errors = cpu_to_be64(~0);
2808	} else {
2809	rsp->local_link_integrity_errors = cpu_to_be64(tmp2);
2810	}
2811	rsp->excessive_buffer_overruns =	2791	rsp->excessive_buffer_overruns =
2812	cpu_to_be64(read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL));	2792	cpu_to_be64(read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL));
2813	}	2793	}
@@ -2883,14 +2863,17 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, · 69 unmodified lines
2883	tmp = read_dev_cntr(dd, C_DC_UNC_ERR, CNTR_INVALID_VL);	2863	tmp = read_dev_cntr(dd, C_DC_UNC_ERR, CNTR_INVALID_VL);
2884		2864
2885	rsp->uncorrectable_errors = tmp < 0x100 ? (tmp & 0xff) : 0xff;	2865	rsp->uncorrectable_errors = tmp < 0x100 ? (tmp & 0xff) : 0xff;
2886		2866	rsp->port_rcv_errors =
		2867	cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_ERR, CNTR_INVALID_VL));
2887	vlinfo = &rsp->vls[0];	2868	vlinfo = &rsp->vls[0];
2888	vfi = 0;	2869	vfi = 0;
2889	vl_select_mask = be32_to_cpu(req->vl_select_mask);	2870	vl_select_mask = be32_to_cpu(req->vl_select_mask);
2890	for_each_set_bit(vl, (unsigned long *)&(vl_select_mask),	2871	for_each_set_bit(vl, (unsigned long *)&(vl_select_mask),
2891	8 * sizeof(req->vl_select_mask)) {	2872	8 * sizeof(req->vl_select_mask)) {
2892	memset(vlinfo, 0, sizeof(*vlinfo));	2873	memset(vlinfo, 0, sizeof(*vlinfo));
2893	/* vlinfo->vls[vfi].port_vl_xmit_discards ??? */	2874	rsp->vls[vfi].port_vl_xmit_discards =
		2875	cpu_to_be64(read_port_cntr(ppd, C_SW_XMIT_DSCD_VL,
		2876	idx_from_vl(vl)));
2894	vlinfo += 1;	2877	vlinfo += 1;
2895	vfi++;	2878	vfi++;
2896	}	2879	}
@@ -3162,10 +3145,8 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, · 265 unmodified lines
3162	if (counter_select & CS_PORT_RCV_REMOTE_PHYSICAL_ERRORS)	3145	if (counter_select & CS_PORT_RCV_REMOTE_PHYSICAL_ERRORS)
3163	write_dev_cntr(dd, C_DC_RMT_PHY_ERR, CNTR_INVALID_VL, 0);	3146	write_dev_cntr(dd, C_DC_RMT_PHY_ERR, CNTR_INVALID_VL, 0);
3164		3147
3165	if (counter_select & CS_LOCAL_LINK_INTEGRITY_ERRORS) {	3148	if (counter_select & CS_LOCAL_LINK_INTEGRITY_ERRORS)
3166	write_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL, 0);
3167	write_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL, 0);	3149	write_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL, 0);
3168	}
3169		3150
3170	if (counter_select & CS_LINK_ERROR_RECOVERY) {	3151	if (counter_select & CS_LINK_ERROR_RECOVERY) {
3171	write_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL, 0);	3152	write_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL, 0);
@@ -3223,7 +3204,9 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, · 51 unmodified lines
3223	/* if (counter_select & CS_PORT_MARK_FECN)	3204	/* if (counter_select & CS_PORT_MARK_FECN)
3224	* write_csr(dd, DCC_PRF_PORT_VL_MARK_FECN_CNT + offset, 0);	3205	* write_csr(dd, DCC_PRF_PORT_VL_MARK_FECN_CNT + offset, 0);
3225	*/	3206	*/
3226	/* port_vl_xmit_discards ??? */	3207	if (counter_select & C_SW_XMIT_DSCD_VL)
		3208	write_port_cntr(ppd, C_SW_XMIT_DSCD_VL,
		3209	idx_from_vl(vl), 0);
3227	}	3210	}
3228		3211
3229	if (resp_len)	3212	if (resp_len)
@@ -3392,7 +3375,7 @@ static void apply_cc_state(struct hfi1_pportdata *ppd) · 162 unmodified lines
3392	*/	3375	*/
3393	spin_lock(&ppd->cc_state_lock);	3376	spin_lock(&ppd->cc_state_lock);
3394		3377
3395	old_cc_state = get_cc_state(ppd);	3378	old_cc_state = get_cc_state_protected(ppd);
3396	if (!old_cc_state) {	3379	if (!old_cc_state) {
3397	/* never active, or shutting down */	3380	/* never active, or shutting down */
3398	spin_unlock(&ppd->cc_state_lock);	3381	spin_unlock(&ppd->cc_state_lock);
@@ -3960,7 +3943,6 @@ void clear_linkup_counters(struct hfi1_devdata *dd) · 561 unmodified lines
3960	write_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL, 0);	3943	write_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL, 0);
3961	write_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT, CNTR_INVALID_VL, 0);	3944	write_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT, CNTR_INVALID_VL, 0);
3962	/* LocalLinkIntegrityErrors */	3945	/* LocalLinkIntegrityErrors */
3963	write_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL, 0);
3964	write_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL, 0);	3946	write_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL, 0);
3965	/* ExcessiveBufferOverruns */	3947	/* ExcessiveBufferOverruns */
3966	write_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL, 0);	3948	write_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL, 0);

drivers/infiniband/hw/hfi1/mad.h+0−7

 #define _HFI1_MAD_H
 #include <rdma/ib_pma.h>
-#define USE_PI_LED_ENABLE	1 /*
-				   * use led enabled bit in struct
-				   * opa_port_states, if available
-				   */
 #include <rdma/opa_smi.h>
 #include <rdma/opa_port_info.h>
-#ifndef PI_LED_ENABLE_SUP
-#define PI_LED_ENABLE_SUP 0
-#endif
 #include "opa_compat.h"
 /*

drivers/infiniband/hw/hfi1/mmu_rb.c+149−105

@@ -53,19 +53,20 @@ · 52 unmodified lines
53	#include "trace.h"	53	#include "trace.h"
54		54
55	struct mmu_rb_handler {	55	struct mmu_rb_handler {
56	struct list_head list;
57	struct mmu_notifier mn;	56	struct mmu_notifier mn;
58	struct rb_root *root;	57	struct rb_root root;
		58	void *ops_arg;
59	spinlock_t lock; /* protect the RB tree */	59	spinlock_t lock; /* protect the RB tree */
60	struct mmu_rb_ops *ops;	60	struct mmu_rb_ops *ops;
		61	struct mm_struct *mm;
		62	struct list_head lru_list;
		63	struct work_struct del_work;
		64	struct list_head del_list;
		65	struct workqueue_struct *wq;
61	};	66	};
62		67
63	static LIST_HEAD(mmu_rb_handlers);
64	static DEFINE_SPINLOCK(mmu_rb_lock); /* protect mmu_rb_handlers list */
65
66	static unsigned long mmu_node_start(struct mmu_rb_node *);	68	static unsigned long mmu_node_start(struct mmu_rb_node *);
67	static unsigned long mmu_node_last(struct mmu_rb_node *);	69	static unsigned long mmu_node_last(struct mmu_rb_node *);
68	static struct mmu_rb_handler find_mmu_handler(struct rb_root );
69	static inline void mmu_notifier_page(struct mmu_notifier , struct mm_struct ,	70	static inline void mmu_notifier_page(struct mmu_notifier , struct mm_struct ,
70	unsigned long);	71	unsigned long);
71	static inline void mmu_notifier_range_start(struct mmu_notifier *,	72	static inline void mmu_notifier_range_start(struct mmu_notifier *,
@@ -76,6 +77,9 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *, · 4 unmodified lines
76	unsigned long, unsigned long);	77	unsigned long, unsigned long);
77	static struct mmu_rb_node __mmu_rb_search(struct mmu_rb_handler ,	78	static struct mmu_rb_node __mmu_rb_search(struct mmu_rb_handler ,
78	unsigned long, unsigned long);	79	unsigned long, unsigned long);
		80	static void do_remove(struct mmu_rb_handler *handler,
		81	struct list_head *del_list);
		82	static void handle_remove(struct work_struct *work);
79		83
80	static struct mmu_notifier_ops mn_opts = {	84	static struct mmu_notifier_ops mn_opts = {
81	.invalidate_page = mmu_notifier_page,	85	.invalidate_page = mmu_notifier_page,
@@ -95,73 +99,79 @@ static unsigned long mmu_node_last(struct mmu_rb_node *node) · 13 unmodified lines
95	return PAGE_ALIGN(node->addr + node->len) - 1;	99	return PAGE_ALIGN(node->addr + node->len) - 1;
96	}	100	}
97		101
98	int hfi1_mmu_rb_register(struct rb_root root, struct mmu_rb_ops ops)	102	int hfi1_mmu_rb_register(void ops_arg, struct mm_struct mm,
		103	struct mmu_rb_ops *ops,
		104	struct workqueue_struct *wq,
		105	struct mmu_rb_handler **handler)
99	{	106	{
100	struct mmu_rb_handler *handlr;	107	struct mmu_rb_handler *handlr;
101		108	int ret;
102	if (!ops->invalidate)
103	return -EINVAL;
104		109
105	handlr = kmalloc(sizeof(*handlr), GFP_KERNEL);	110	handlr = kmalloc(sizeof(*handlr), GFP_KERNEL);
106	if (!handlr)	111	if (!handlr)
107	return -ENOMEM;	112	return -ENOMEM;
108		113
109	handlr->root = root;	114	handlr->root = RB_ROOT;
110	handlr->ops = ops;	115	handlr->ops = ops;
		116	handlr->ops_arg = ops_arg;
111	INIT_HLIST_NODE(&handlr->mn.hlist);	117	INIT_HLIST_NODE(&handlr->mn.hlist);
112	spin_lock_init(&handlr->lock);	118	spin_lock_init(&handlr->lock);
113	handlr->mn.ops = &mn_opts;	119	handlr->mn.ops = &mn_opts;
114	spin_lock(&mmu_rb_lock);	120	handlr->mm = mm;
115	list_add_tail_rcu(&handlr->list, &mmu_rb_handlers);	121	INIT_WORK(&handlr->del_work, handle_remove);
116	spin_unlock(&mmu_rb_lock);	122	INIT_LIST_HEAD(&handlr->del_list);
		123	INIT_LIST_HEAD(&handlr->lru_list);
		124	handlr->wq = wq;
		125
		126	ret = mmu_notifier_register(&handlr->mn, handlr->mm);
		127	if (ret) {
		128	kfree(handlr);
		129	return ret;
		130	}
117		131
118	return mmu_notifier_register(&handlr->mn, current->mm);	132	*handler = handlr;
		133	return 0;
119	}	134	}
120		135
121	void hfi1_mmu_rb_unregister(struct rb_root *root)	136	void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)
122	{	137	{
123	struct mmu_rb_handler *handler = find_mmu_handler(root);	138	struct mmu_rb_node *rbnode;
		139	struct rb_node *node;
124	unsigned long flags;	140	unsigned long flags;
125		141	struct list_head del_list;
126	if (!handler)
127	return;
128		142
129	/* Unregister first so we don't get any more notifications. */	143	/* Unregister first so we don't get any more notifications. */
130	if (current->mm)	144	mmu_notifier_unregister(&handler->mn, handler->mm);
131	mmu_notifier_unregister(&handler->mn, current->mm);
132		145
133	spin_lock(&mmu_rb_lock);	146	/*
134	list_del_rcu(&handler->list);	147	* Make sure the wq delete handler is finished running. It will not
135	spin_unlock(&mmu_rb_lock);	148	* be triggered once the mmu notifiers are unregistered above.
136	synchronize_rcu();	149	*/
		150	flush_work(&handler->del_work);
		151
		152	INIT_LIST_HEAD(&del_list);
137		153
138	spin_lock_irqsave(&handler->lock, flags);	154	spin_lock_irqsave(&handler->lock, flags);
139	if (!RB_EMPTY_ROOT(root)) {	155	while ((node = rb_first(&handler->root))) {
140	struct rb_node *node;	156	rbnode = rb_entry(node, struct mmu_rb_node, node);
141	struct mmu_rb_node *rbnode;	157	rb_erase(node, &handler->root);
142		158	/* move from LRU list to delete list */
143	while ((node = rb_first(root))) {	159	list_move(&rbnode->list, &del_list);
144	rbnode = rb_entry(node, struct mmu_rb_node, node);
145	rb_erase(node, root);
146	if (handler->ops->remove)
147	handler->ops->remove(root, rbnode, NULL);
148	}
149	}	160	}
150	spin_unlock_irqrestore(&handler->lock, flags);	161	spin_unlock_irqrestore(&handler->lock, flags);
151		162
		163	do_remove(handler, &del_list);
		164
152	kfree(handler);	165	kfree(handler);
153	}	166	}
154		167
155	int hfi1_mmu_rb_insert(struct rb_root root, struct mmu_rb_node mnode)	168	int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
		169	struct mmu_rb_node *mnode)
156	{	170	{
157	struct mmu_rb_handler *handler = find_mmu_handler(root);
158	struct mmu_rb_node *node;	171	struct mmu_rb_node *node;
159	unsigned long flags;	172	unsigned long flags;
160	int ret = 0;	173	int ret = 0;
161		174
162	if (!handler)
163	return -EINVAL;
164
165	spin_lock_irqsave(&handler->lock, flags);	175	spin_lock_irqsave(&handler->lock, flags);
166	hfi1_cdbg(MMU, "Inserting node addr 0x%llx, len %u", mnode->addr,	176	hfi1_cdbg(MMU, "Inserting node addr 0x%llx, len %u", mnode->addr,
167	mnode->len);	177	mnode->len);
@@ -170,12 +180,13 @@ int hfi1_mmu_rb_insert(struct rb_root root, struct mmu_rb_node mnode) · 2 unmodified lines
170	ret = -EINVAL;	180	ret = -EINVAL;
171	goto unlock;	181	goto unlock;
172	}	182	}
173	__mmu_int_rb_insert(mnode, root);	183	__mmu_int_rb_insert(mnode, &handler->root);
		184	list_add(&mnode->list, &handler->lru_list);
174		185
175	if (handler->ops->insert) {	186	ret = handler->ops->insert(handler->ops_arg, mnode);
176	ret = handler->ops->insert(root, mnode);	187	if (ret) {
177	if (ret)	188	__mmu_int_rb_remove(mnode, &handler->root);
178	__mmu_int_rb_remove(mnode, root);	189	list_del(&mnode->list); /* remove from LRU list */
179	}	190	}
180	unlock:	191	unlock:
181	spin_unlock_irqrestore(&handler->lock, flags);	192	spin_unlock_irqrestore(&handler->lock, flags);
@@ -191,10 +202,10 @@ static struct mmu_rb_node __mmu_rb_search(struct mmu_rb_handler handler, · 9 unmodified lines
191		202
192	hfi1_cdbg(MMU, "Searching for addr 0x%llx, len %u", addr, len);	203	hfi1_cdbg(MMU, "Searching for addr 0x%llx, len %u", addr, len);
193	if (!handler->ops->filter) {	204	if (!handler->ops->filter) {
194	node = __mmu_int_rb_iter_first(handler->root, addr,	205	node = __mmu_int_rb_iter_first(&handler->root, addr,
195	(addr + len) - 1);	206	(addr + len) - 1);
196	} else {	207	} else {
197	for (node = __mmu_int_rb_iter_first(handler->root, addr,	208	for (node = __mmu_int_rb_iter_first(&handler->root, addr,
198	(addr + len) - 1);	209	(addr + len) - 1);
199	node;	210	node;
200	node = __mmu_int_rb_iter_next(node, addr,	211	node = __mmu_int_rb_iter_next(node, addr,
@@ -206,82 +217,72 @@ static struct mmu_rb_node __mmu_rb_search(struct mmu_rb_handler handler, · 5 unmodified lines
206	return node;	217	return node;
207	}	218	}
208		219
209	/* Caller must not hold handler lock. */	220	struct mmu_rb_node hfi1_mmu_rb_extract(struct mmu_rb_handler handler,
210	static void __mmu_rb_remove(struct mmu_rb_handler *handler,	221	unsigned long addr, unsigned long len)
211	struct mmu_rb_node node, struct mm_struct mm)
212	{
213	unsigned long flags;
214
215	/* Validity of handler and node pointers has been checked by caller. */
216	hfi1_cdbg(MMU, "Removing node addr 0x%llx, len %u", node->addr,
217	node->len);
218	spin_lock_irqsave(&handler->lock, flags);
219	__mmu_int_rb_remove(node, handler->root);
220	spin_unlock_irqrestore(&handler->lock, flags);
221
222	if (handler->ops->remove)
223	handler->ops->remove(handler->root, node, mm);
224	}
225
226	struct mmu_rb_node hfi1_mmu_rb_search(struct rb_root root, unsigned long addr,
227	unsigned long len)
228	{	222	{
229	struct mmu_rb_handler *handler = find_mmu_handler(root);
230	struct mmu_rb_node *node;	223	struct mmu_rb_node *node;
231	unsigned long flags;	224	unsigned long flags;
232		225
233	if (!handler)
234	return ERR_PTR(-EINVAL);
235
236	spin_lock_irqsave(&handler->lock, flags);	226	spin_lock_irqsave(&handler->lock, flags);
237	node = __mmu_rb_search(handler, addr, len);	227	node = __mmu_rb_search(handler, addr, len);
		228	if (node) {
		229	__mmu_int_rb_remove(node, &handler->root);
		230	list_del(&node->list); /* remove from LRU list */
		231	}
238	spin_unlock_irqrestore(&handler->lock, flags);	232	spin_unlock_irqrestore(&handler->lock, flags);
239		233
240	return node;	234	return node;
241	}	235	}
242		236
243	struct mmu_rb_node hfi1_mmu_rb_extract(struct rb_root root,	237	void hfi1_mmu_rb_evict(struct mmu_rb_handler handler, void evict_arg)
244	unsigned long addr, unsigned long len)
245	{	238	{
246	struct mmu_rb_handler *handler = find_mmu_handler(root);	239	struct mmu_rb_node rbnode, ptr;
247	struct mmu_rb_node *node;	240	struct list_head del_list;
248	unsigned long flags;	241	unsigned long flags;
		242	bool stop = false;
249		243
250	if (!handler)	244	INIT_LIST_HEAD(&del_list);
251	return ERR_PTR(-EINVAL);
252		245
253	spin_lock_irqsave(&handler->lock, flags);	246	spin_lock_irqsave(&handler->lock, flags);
254	node = __mmu_rb_search(handler, addr, len);	247	list_for_each_entry_safe_reverse(rbnode, ptr, &handler->lru_list,
255	if (node)	248	list) {
256	__mmu_int_rb_remove(node, handler->root);	249	if (handler->ops->evict(handler->ops_arg, rbnode, evict_arg,
		250	&stop)) {
		251	__mmu_int_rb_remove(rbnode, &handler->root);
		252	/* move from LRU list to delete list */
		253	list_move(&rbnode->list, &del_list);
		254	}
		255	if (stop)
		256	break;
		257	}
257	spin_unlock_irqrestore(&handler->lock, flags);	258	spin_unlock_irqrestore(&handler->lock, flags);
258		259
259	return node;	260	while (!list_empty(&del_list)) {
		261	rbnode = list_first_entry(&del_list, struct mmu_rb_node, list);
		262	list_del(&rbnode->list);
		263	handler->ops->remove(handler->ops_arg, rbnode);
		264	}
260	}	265	}
261		266
262	void hfi1_mmu_rb_remove(struct rb_root root, struct mmu_rb_node node)	267	/*
		268	* It is up to the caller to ensure that this function does not race with the
		269	* mmu invalidate notifier which may be calling the users remove callback on
		270	* 'node'.
		271	*/
		272	void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
		273	struct mmu_rb_node *node)
263	{	274	{
264	struct mmu_rb_handler *handler = find_mmu_handler(root);	275	unsigned long flags;
265
266	if (!handler \|\| !node)
267	return;
268
269	__mmu_rb_remove(handler, node, NULL);
270	}
271		276
272	static struct mmu_rb_handler find_mmu_handler(struct rb_root root)	277	/* Validity of handler and node pointers has been checked by caller. */
273	{	278	hfi1_cdbg(MMU, "Removing node addr 0x%llx, len %u", node->addr,
274	struct mmu_rb_handler *handler;	279	node->len);
		280	spin_lock_irqsave(&handler->lock, flags);
		281	__mmu_int_rb_remove(node, &handler->root);
		282	list_del(&node->list); /* remove from LRU list */
		283	spin_unlock_irqrestore(&handler->lock, flags);
275		284
276	rcu_read_lock();	285	handler->ops->remove(handler->ops_arg, node);
277	list_for_each_entry_rcu(handler, &mmu_rb_handlers, list) {
278	if (handler->root == root)
279	goto unlock;
280	}
281	handler = NULL;
282	unlock:
283	rcu_read_unlock();
284	return handler;
285	}	286	}
286		287
287	static inline void mmu_notifier_page(struct mmu_notifier *mn,	288	static inline void mmu_notifier_page(struct mmu_notifier *mn,
@@ -304,9 +305,10 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn, · 16 unmodified lines
304	{	305	{
305	struct mmu_rb_handler *handler =	306	struct mmu_rb_handler *handler =
306	container_of(mn, struct mmu_rb_handler, mn);	307	container_of(mn, struct mmu_rb_handler, mn);
307	struct rb_root *root = handler->root;	308	struct rb_root *root = &handler->root;
308	struct mmu_rb_node node, ptr = NULL;	309	struct mmu_rb_node node, ptr = NULL;
309	unsigned long flags;	310	unsigned long flags;
		311	bool added = false;
310		312
311	spin_lock_irqsave(&handler->lock, flags);	313	spin_lock_irqsave(&handler->lock, flags);
312	for (node = __mmu_int_rb_iter_first(root, start, end - 1);	314	for (node = __mmu_int_rb_iter_first(root, start, end - 1);
@@ -315,11 +317,53 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn, · 2 unmodified lines
315	ptr = __mmu_int_rb_iter_next(node, start, end - 1);	317	ptr = __mmu_int_rb_iter_next(node, start, end - 1);
316	hfi1_cdbg(MMU, "Invalidating node addr 0x%llx, len %u",	318	hfi1_cdbg(MMU, "Invalidating node addr 0x%llx, len %u",
317	node->addr, node->len);	319	node->addr, node->len);
318	if (handler->ops->invalidate(root, node)) {	320	if (handler->ops->invalidate(handler->ops_arg, node)) {
319	__mmu_int_rb_remove(node, root);	321	__mmu_int_rb_remove(node, root);
320	if (handler->ops->remove)	322	/* move from LRU list to delete list */
321	handler->ops->remove(root, node, mm);	323	list_move(&node->list, &handler->del_list);
		324	added = true;
322	}	325	}
323	}	326	}
324	spin_unlock_irqrestore(&handler->lock, flags);	327	spin_unlock_irqrestore(&handler->lock, flags);
		328
		329	if (added)
		330	queue_work(handler->wq, &handler->del_work);
		331	}
		332
		333	/*
		334	* Call the remove function for the given handler and the list. This
		335	* is expected to be called with a delete list extracted from handler.
		336	* The caller should not be holding the handler lock.
		337	*/
		338	static void do_remove(struct mmu_rb_handler *handler,
		339	struct list_head *del_list)
		340	{
		341	struct mmu_rb_node *node;
		342
		343	while (!list_empty(del_list)) {
		344	node = list_first_entry(del_list, struct mmu_rb_node, list);
		345	list_del(&node->list);
		346	handler->ops->remove(handler->ops_arg, node);
		347	}
		348	}
		349
		350	/*
		351	* Work queue function to remove all nodes that have been queued up to
		352	* be removed. The key feature is that mm->mmap_sem is not being held
		353	* and the remove callback can sleep while taking it, if needed.
		354	*/
		355	static void handle_remove(struct work_struct *work)
		356	{
		357	struct mmu_rb_handler *handler = container_of(work,
		358	struct mmu_rb_handler,
		359	del_work);
		360	struct list_head del_list;
		361	unsigned long flags;
		362
		363	/* remove anything that is queued to get removed */
		364	spin_lock_irqsave(&handler->lock, flags);
		365	list_replace_init(&handler->del_list, &del_list);
		366	spin_unlock_irqrestore(&handler->lock, flags);
		367
		368	do_remove(handler, &del_list);
325	}	369	}

drivers/infiniband/hw/hfi1/mmu_rb.h+24−13

@@ -54,23 +54,34 @@ struct mmu_rb_node { · 53 unmodified lines
54	unsigned long len;	54	unsigned long len;
55	unsigned long __last;	55	unsigned long __last;
56	struct rb_node node;	56	struct rb_node node;
		57	struct list_head list;
57	};	58	};
58		59
		60	/*
		61	* NOTE: filter, insert, invalidate, and evict must not sleep. Only remove is
		62	* allowed to sleep.
		63	*/
59	struct mmu_rb_ops {	64	struct mmu_rb_ops {
60	bool (filter)(struct mmu_rb_node , unsigned long, unsigned long);	65	bool (filter)(struct mmu_rb_node node, unsigned long addr,
61	int (insert)(struct rb_root , struct mmu_rb_node *);	66	unsigned long len);
62	void (remove)(struct rb_root , struct mmu_rb_node *,	67	int (insert)(void ops_arg, struct mmu_rb_node *mnode);
63	struct mm_struct *);	68	void (remove)(void ops_arg, struct mmu_rb_node *mnode);
64	int (invalidate)(struct rb_root , struct mmu_rb_node *);	69	int (invalidate)(void ops_arg, struct mmu_rb_node *node);
		70	int (evict)(void ops_arg, struct mmu_rb_node *mnode,
		71	void evict_arg, bool stop);
65	};	72	};
66		73
67	int hfi1_mmu_rb_register(struct rb_root root, struct mmu_rb_ops ops);	74	int hfi1_mmu_rb_register(void ops_arg, struct mm_struct mm,
68	void hfi1_mmu_rb_unregister(struct rb_root *);	75	struct mmu_rb_ops *ops,
69	int hfi1_mmu_rb_insert(struct rb_root , struct mmu_rb_node );	76	struct workqueue_struct *wq,
70	void hfi1_mmu_rb_remove(struct rb_root , struct mmu_rb_node );	77	struct mmu_rb_handler **handler);
71	struct mmu_rb_node hfi1_mmu_rb_search(struct rb_root , unsigned long,	78	void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler);
72	unsigned long);	79	int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
73	struct mmu_rb_node hfi1_mmu_rb_extract(struct rb_root , unsigned long,	80	struct mmu_rb_node *mnode);
74	unsigned long);	81	void hfi1_mmu_rb_evict(struct mmu_rb_handler handler, void evict_arg);
		82	void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
		83	struct mmu_rb_node *mnode);
		84	struct mmu_rb_node hfi1_mmu_rb_extract(struct mmu_rb_handler handler,
		85	unsigned long addr, unsigned long len);
75		86
76	#endif /* _HFI1_MMU_RB_H */	87	#endif /* _HFI1_MMU_RB_H */

drivers/infiniband/hw/hfi1/pcie.c+64−4

@@ -679,6 +679,10 @@ static uint pcie_pset = UNSET_PSET; · 678 unmodified lines
679	module_param(pcie_pset, uint, S_IRUGO);	679	module_param(pcie_pset, uint, S_IRUGO);
680	MODULE_PARM_DESC(pcie_pset, "PCIe Eq Pset value to use, range is 0-10");	680	MODULE_PARM_DESC(pcie_pset, "PCIe Eq Pset value to use, range is 0-10");
681		681
		682	static uint pcie_ctle = 1; /* discrete on, integrated off */
		683	module_param(pcie_ctle, uint, S_IRUGO);
		684	MODULE_PARM_DESC(pcie_ctle, "PCIe static CTLE mode, bit 0 - discrete on/off, bit 1 - integrated on/off");
		685
682	/* equalization columns */	686	/* equalization columns */
683	#define PREC 0	687	#define PREC 0
684	#define ATTN 1	688	#define ATTN 1
@@ -716,6 +720,36 @@ static const u8 integrated_preliminary_eq[11][3] = { · 31 unmodified lines
716	{ 0x00, 0x1e, 0x0a }, /* p10 */	720	{ 0x00, 0x1e, 0x0a }, /* p10 */
717	};	721	};
718		722
		723	static const u8 discrete_ctle_tunings[11][4] = {
		724	/* DC LF HF BW */
		725	{ 0x48, 0x0b, 0x04, 0x04 }, /* p0 */
		726	{ 0x60, 0x05, 0x0f, 0x0a }, /* p1 */
		727	{ 0x50, 0x09, 0x06, 0x06 }, /* p2 */
		728	{ 0x68, 0x05, 0x0f, 0x0a }, /* p3 */
		729	{ 0x80, 0x05, 0x0f, 0x0a }, /* p4 */
		730	{ 0x70, 0x05, 0x0f, 0x0a }, /* p5 */
		731	{ 0x68, 0x05, 0x0f, 0x0a }, /* p6 */
		732	{ 0x38, 0x0f, 0x00, 0x00 }, /* p7 */
		733	{ 0x48, 0x09, 0x06, 0x06 }, /* p8 */
		734	{ 0x60, 0x05, 0x0f, 0x0a }, /* p9 */
		735	{ 0x38, 0x0f, 0x00, 0x00 }, /* p10 */
		736	};
		737
		738	static const u8 integrated_ctle_tunings[11][4] = {
		739	/* DC LF HF BW */
		740	{ 0x38, 0x0f, 0x00, 0x00 }, /* p0 */
		741	{ 0x38, 0x0f, 0x00, 0x00 }, /* p1 */
		742	{ 0x38, 0x0f, 0x00, 0x00 }, /* p2 */
		743	{ 0x38, 0x0f, 0x00, 0x00 }, /* p3 */
		744	{ 0x58, 0x0a, 0x05, 0x05 }, /* p4 */
		745	{ 0x48, 0x0a, 0x05, 0x05 }, /* p5 */
		746	{ 0x40, 0x0a, 0x05, 0x05 }, /* p6 */
		747	{ 0x38, 0x0f, 0x00, 0x00 }, /* p7 */
		748	{ 0x38, 0x0f, 0x00, 0x00 }, /* p8 */
		749	{ 0x38, 0x09, 0x06, 0x06 }, /* p9 */
		750	{ 0x38, 0x0e, 0x01, 0x01 }, /* p10 */
		751	};
		752
719	/* helper to format the value to write to hardware */	753	/* helper to format the value to write to hardware */
720	#define eq_value(pre, curr, post) \	754	#define eq_value(pre, curr, post) \
721	((((u32)(pre)) << \	755	((((u32)(pre)) << \
@@ -951,11 +985,14 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd) · 229 unmodified lines
951	u32 status, err;	985	u32 status, err;
952	int ret;	986	int ret;
953	int do_retry, retry_count = 0;	987	int do_retry, retry_count = 0;
		988	int intnum = 0;
954	uint default_pset;	989	uint default_pset;
955	u16 target_vector, target_speed;	990	u16 target_vector, target_speed;
956	u16 lnkctl2, vendor;	991	u16 lnkctl2, vendor;
957	u8 div;	992	u8 div;
958	const u8 (*eq)[3];	993	const u8 (*eq)[3];
		994	const u8 (*ctle_tunings)[4];
		995	uint static_ctle_mode;
959	int return_error = 0;	996	int return_error = 0;
960		997
961	/* PCIe Gen3 is for the ASIC only */	998	/* PCIe Gen3 is for the ASIC only */
@@ -1089,6 +1126,9 @@ retry: · 127 unmodified lines
1089	div = 3;	1126	div = 3;
1090	eq = discrete_preliminary_eq;	1127	eq = discrete_preliminary_eq;
1091	default_pset = DEFAULT_DISCRETE_PSET;	1128	default_pset = DEFAULT_DISCRETE_PSET;
		1129	ctle_tunings = discrete_ctle_tunings;
		1130	/* bit 0 - discrete on/off */
		1131	static_ctle_mode = pcie_ctle & 0x1;
1092	} else {	1132	} else {
1093	/* 400mV, FS=29, LF = 9 */	1133	/* 400mV, FS=29, LF = 9 */
1094	fs = 29;	1134	fs = 29;
@@ -1096,6 +1136,9 @@ retry: · 1 unmodified line
1096	div = 1;	1136	div = 1;
1097	eq = integrated_preliminary_eq;	1137	eq = integrated_preliminary_eq;
1098	default_pset = DEFAULT_MCP_PSET;	1138	default_pset = DEFAULT_MCP_PSET;
		1139	ctle_tunings = integrated_ctle_tunings;
		1140	/* bit 1 - integrated on/off */
		1141	static_ctle_mode = (pcie_ctle >> 1) & 0x1;
1099	}	1142	}
1100	pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL101,	1143	pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL101,
1101	(fs <<	1144	(fs <<
@@ -1135,16 +1178,33 @@ retry: · 33 unmodified lines
1135	* step 5c: Program gasket interrupts	1178	* step 5c: Program gasket interrupts
1136	*/	1179	*/
1137	/* set the Rx Bit Rate to REFCLK ratio */	1180	/* set the Rx Bit Rate to REFCLK ratio */
1138	write_gasket_interrupt(dd, 0, 0x0006, 0x0050);	1181	write_gasket_interrupt(dd, intnum++, 0x0006, 0x0050);
1139	/* disable pCal for PCIe Gen3 RX equalization */	1182	/* disable pCal for PCIe Gen3 RX equalization */
1140	write_gasket_interrupt(dd, 1, 0x0026, 0x5b01);	1183	/* select adaptive or static CTLE */
		1184	write_gasket_interrupt(dd, intnum++, 0x0026,
		1185	0x5b01 \| (static_ctle_mode << 3));
1141	/*	1186	/*
1142	* Enable iCal for PCIe Gen3 RX equalization, and set which	1187	* Enable iCal for PCIe Gen3 RX equalization, and set which
1143	* evaluation of RX_EQ_EVAL will launch the iCal procedure.	1188	* evaluation of RX_EQ_EVAL will launch the iCal procedure.
1144	*/	1189	*/
1145	write_gasket_interrupt(dd, 2, 0x0026, 0x5202);	1190	write_gasket_interrupt(dd, intnum++, 0x0026, 0x5202);
		1191
		1192	if (static_ctle_mode) {
		1193	/* apply static CTLE tunings */
		1194	u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw;
		1195
		1196	pcie_dc = ctle_tunings[pcie_pset][0];
		1197	pcie_lf = ctle_tunings[pcie_pset][1];
		1198	pcie_hf = ctle_tunings[pcie_pset][2];
		1199	pcie_bw = ctle_tunings[pcie_pset][3];
		1200	write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 \| pcie_dc);
		1201	write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 \| pcie_lf);
		1202	write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 \| pcie_hf);
		1203	write_gasket_interrupt(dd, intnum++, 0x0026, 0x5500 \| pcie_bw);
		1204	}
		1205
1146	/* terminate list */	1206	/* terminate list */
1147	write_gasket_interrupt(dd, 3, 0x0000, 0x0000);	1207	write_gasket_interrupt(dd, intnum++, 0x0000, 0x0000);
1148		1208
1149	/*	1209	/*
1150	* step 5d: program XMT margin	1210	* step 5d: program XMT margin

drivers/infiniband/hw/hfi1/pio.c+17−4

 	dd->vld[15].sc = sc_alloc(dd, SC_VL15,
 				  dd->rcd[0]->rcvhdrqentsize, dd->node);
 	if (!dd->vld[15].sc)
-		goto nomem;
+		return -ENOMEM;
 	hfi1_init_ctxt(dd->vld[15].sc);
 	dd->vld[15].mtu = enum_to_mtu(OPA_MTU_2048);
-	dd->kernel_send_context = kmalloc_node(dd->num_send_contexts *
+	dd->kernel_send_context = kzalloc_node(dd->num_send_contexts *
 					sizeof(struct send_context *),
 					GFP_KERNEL, dd->node);
+	if (!dd->kernel_send_context)
+		goto freesc15;
 	dd->kernel_send_context[0] = dd->vld[15].sc;
 	for (i = 0; i < num_vls; i++) {
 	if (pio_map_init(dd, ppd->port - 1, num_vls, NULL))
 		goto nomem;
 	return 0;
 nomem:
-	sc_free(dd->vld[15].sc);
+	for (i = 0; i < num_vls; i++) {
-	for (i = 0; i < num_vls; i++)
 		sc_free(dd->vld[i].sc);
+		dd->vld[i].sc = NULL;
+	}
 	for (i = num_vls; i < INIT_SC_PER_VL * num_vls; i++)
 		sc_free(dd->kernel_send_context[i + 1]);
+	kfree(dd->kernel_send_context);
+	dd->kernel_send_context = NULL;
+freesc15:
+	sc_free(dd->vld[15].sc);
 	return -ENOMEM;
 }

drivers/infiniband/hw/hfi1/platform.c+5−15

 	u8 precur = 0, attn = 0, postcur = 0, external_device_config = 0;
 	u8 *cache = ppd->qsfp_info.cache;
-	/* Enable external device config if channel is limiting active */
-	read_8051_config(ppd->dd, LINK_OPTIMIZATION_SETTINGS,
-			 GENERAL_CONFIG, &config_data);
-	config_data &= ~(0xff << ENABLE_EXT_DEV_CONFIG_SHIFT);
-	config_data |= ((u32)limiting_active << ENABLE_EXT_DEV_CONFIG_SHIFT);
-	ret = load_8051_config(ppd->dd, LINK_OPTIMIZATION_SETTINGS,
-			       GENERAL_CONFIG, config_data);
-	if (ret != HCMD_SUCCESS)
-		dd_dev_err(
-			ppd->dd,
-			"%s: Failed to set enable external device config\n",
-			__func__);
-	config_data = 0; /* re-init  */
 	/* Pass tuning method to 8051 */
 	read_8051_config(ppd->dd, LINK_TUNING_PARAMETERS, GENERAL_CONFIG,
 			 &config_data);
 	if (ret)
 		return ret;
+	/*
+	 * We'll change the QSFP memory contents from here on out, thus we set a
+	 * flag here to remind ourselves to reset the QSFP module. This prevents
+	 * reuse of stale settings established in our previous pass through.
+	 */
 	if (ppd->qsfp_info.reset_needed) {
 		reset_qsfp(ppd);
-		ppd->qsfp_info.reset_needed = 0;
 		refresh_qsfp_cache(ppd, &ppd->qsfp_info);
 	} else {
 		ppd->qsfp_info.reset_needed = 1;

drivers/infiniband/hw/hfi1/qp.c+65−3

@@ -52,6 +52,7 @@ · 51 unmodified lines
52	#include <linux/seq_file.h>	52	#include <linux/seq_file.h>
53	#include <rdma/rdma_vt.h>	53	#include <rdma/rdma_vt.h>
54	#include <rdma/rdmavt_qp.h>	54	#include <rdma/rdmavt_qp.h>
		55	#include <rdma/ib_verbs.h>
55		56
56	#include "hfi.h"	57	#include "hfi.h"
57	#include "qp.h"	58	#include "qp.h"
@@ -115,6 +116,66 @@ static const u16 credit_table[31] = { · 57 unmodified lines
115	32768 /* 1E */	116	32768 /* 1E */
116	};	117	};
117		118
		119	const struct rvt_operation_params hfi1_post_parms[RVT_OPERATION_MAX] = {
		120	[IB_WR_RDMA_WRITE] = {
		121	.length = sizeof(struct ib_rdma_wr),
		122	.qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		123	},
		124
		125	[IB_WR_RDMA_READ] = {
		126	.length = sizeof(struct ib_rdma_wr),
		127	.qpt_support = BIT(IB_QPT_RC),
		128	.flags = RVT_OPERATION_ATOMIC,
		129	},
		130
		131	[IB_WR_ATOMIC_CMP_AND_SWP] = {
		132	.length = sizeof(struct ib_atomic_wr),
		133	.qpt_support = BIT(IB_QPT_RC),
		134	.flags = RVT_OPERATION_ATOMIC \| RVT_OPERATION_ATOMIC_SGE,
		135	},
		136
		137	[IB_WR_ATOMIC_FETCH_AND_ADD] = {
		138	.length = sizeof(struct ib_atomic_wr),
		139	.qpt_support = BIT(IB_QPT_RC),
		140	.flags = RVT_OPERATION_ATOMIC \| RVT_OPERATION_ATOMIC_SGE,
		141	},
		142
		143	[IB_WR_RDMA_WRITE_WITH_IMM] = {
		144	.length = sizeof(struct ib_rdma_wr),
		145	.qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		146	},
		147
		148	[IB_WR_SEND] = {
		149	.length = sizeof(struct ib_send_wr),
		150	.qpt_support = BIT(IB_QPT_UD) \| BIT(IB_QPT_SMI) \| BIT(IB_QPT_GSI) \|
		151	BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		152	},
		153
		154	[IB_WR_SEND_WITH_IMM] = {
		155	.length = sizeof(struct ib_send_wr),
		156	.qpt_support = BIT(IB_QPT_UD) \| BIT(IB_QPT_SMI) \| BIT(IB_QPT_GSI) \|
		157	BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		158	},
		159
		160	[IB_WR_REG_MR] = {
		161	.length = sizeof(struct ib_reg_wr),
		162	.qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		163	.flags = RVT_OPERATION_LOCAL,
		164	},
		165
		166	[IB_WR_LOCAL_INV] = {
		167	.length = sizeof(struct ib_send_wr),
		168	.qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		169	.flags = RVT_OPERATION_LOCAL,
		170	},
		171
		172	[IB_WR_SEND_WITH_INV] = {
		173	.length = sizeof(struct ib_send_wr),
		174	.qpt_support = BIT(IB_QPT_RC),
		175	},
		176
		177	};
		178
118	static void flush_tx_list(struct rvt_qp *qp)	179	static void flush_tx_list(struct rvt_qp *qp)
119	{	180	{
120	struct hfi1_qp_priv *priv = qp->priv;	181	struct hfi1_qp_priv *priv = qp->priv;
@@ -745,8 +806,9 @@ void qp_priv_alloc(struct rvt_dev_info rdi, struct rvt_qp *qp, · 624 unmodified lines
745		806
746	priv->owner = qp;	807	priv->owner = qp;
747		808
748	priv->s_hdr = kzalloc_node(sizeof(*priv->s_hdr), gfp, rdi->dparms.node);	809	priv->s_ahg = kzalloc_node(sizeof(*priv->s_ahg), gfp,
749	if (!priv->s_hdr) {	810	rdi->dparms.node);
		811	if (!priv->s_ahg) {
750	kfree(priv);	812	kfree(priv);
751	return ERR_PTR(-ENOMEM);	813	return ERR_PTR(-ENOMEM);
752	}	814	}
@@ -759,7 +821,7 @@ void qp_priv_free(struct rvt_dev_info rdi, struct rvt_qp qp) · 6 unmodified lines
759	{	821	{
760	struct hfi1_qp_priv *priv = qp->priv;	822	struct hfi1_qp_priv *priv = qp->priv;
761		823
762	kfree(priv->s_hdr);	824	kfree(priv->s_ahg);
763	kfree(priv);	825	kfree(priv);
764	}	826	}
765		827

drivers/infiniband/hw/hfi1/qp.h+3−1

 extern unsigned int hfi1_qp_table_size;
+extern const struct rvt_operation_params hfi1_post_parms[];
 /*
  * free_ahg - clear ahg from QP
  */
 {
 	struct hfi1_qp_priv *priv = qp->priv;
-	priv->s_hdr->ahgcount = 0;
+	priv->s_ahg->ahgcount = 0;
 	qp->s_flags &= ~(RVT_S_AHG_VALID | RVT_S_AHG_CLEAR);
 	if (priv->s_sde && qp->s_ahgidx >= 0)
 		sdma_ahg_free(priv->s_sde, qp->s_ahgidx);

drivers/infiniband/hw/hfi1/qsfp.c+302−107

@@ -50,46 +50,285 @@ · 49 unmodified lines
50	#include <linux/vmalloc.h>	50	#include <linux/vmalloc.h>
51		51
52	#include "hfi.h"	52	#include "hfi.h"
53	#include "twsi.h"	53
		54	/* for the given bus number, return the CSR for reading an i2c line */
		55	static inline u32 i2c_in_csr(u32 bus_num)
		56	{
		57	return bus_num ? ASIC_QSFP2_IN : ASIC_QSFP1_IN;
		58	}
		59
		60	/* for the given bus number, return the CSR for writing an i2c line */
		61	static inline u32 i2c_oe_csr(u32 bus_num)
		62	{
		63	return bus_num ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
		64	}
		65
		66	static void hfi1_setsda(void *data, int state)
		67	{
		68	struct hfi1_i2c_bus bus = (struct hfi1_i2c_bus )data;
		69	struct hfi1_devdata *dd = bus->controlling_dd;
		70	u64 reg;
		71	u32 target_oe;
		72
		73	target_oe = i2c_oe_csr(bus->num);
		74	reg = read_csr(dd, target_oe);
		75	/*
		76	* The OE bit value is inverted and connected to the pin. When
		77	* OE is 0 the pin is left to be pulled up, when the OE is 1
		78	* the pin is driven low. This matches the "open drain" or "open
		79	* collector" convention.
		80	*/
		81	if (state)
		82	reg &= ~QSFP_HFI0_I2CDAT;
		83	else
		84	reg \|= QSFP_HFI0_I2CDAT;
		85	write_csr(dd, target_oe, reg);
		86	/* do a read to force the write into the chip */
		87	(void)read_csr(dd, target_oe);
		88	}
		89
		90	static void hfi1_setscl(void *data, int state)
		91	{
		92	struct hfi1_i2c_bus bus = (struct hfi1_i2c_bus )data;
		93	struct hfi1_devdata *dd = bus->controlling_dd;
		94	u64 reg;
		95	u32 target_oe;
		96
		97	target_oe = i2c_oe_csr(bus->num);
		98	reg = read_csr(dd, target_oe);
		99	/*
		100	* The OE bit value is inverted and connected to the pin. When
		101	* OE is 0 the pin is left to be pulled up, when the OE is 1
		102	* the pin is driven low. This matches the "open drain" or "open
		103	* collector" convention.
		104	*/
		105	if (state)
		106	reg &= ~QSFP_HFI0_I2CCLK;
		107	else
		108	reg \|= QSFP_HFI0_I2CCLK;
		109	write_csr(dd, target_oe, reg);
		110	/* do a read to force the write into the chip */
		111	(void)read_csr(dd, target_oe);
		112	}
		113
		114	static int hfi1_getsda(void *data)
		115	{
		116	struct hfi1_i2c_bus bus = (struct hfi1_i2c_bus )data;
		117	u64 reg;
		118	u32 target_in;
		119
		120	hfi1_setsda(data, 1); /* clear OE so we do not pull line down */
		121	udelay(2); /* 1us pull up + 250ns hold */
		122
		123	target_in = i2c_in_csr(bus->num);
		124	reg = read_csr(bus->controlling_dd, target_in);
		125	return !!(reg & QSFP_HFI0_I2CDAT);
		126	}
		127
		128	static int hfi1_getscl(void *data)
		129	{
		130	struct hfi1_i2c_bus bus = (struct hfi1_i2c_bus )data;
		131	u64 reg;
		132	u32 target_in;
		133
		134	hfi1_setscl(data, 1); /* clear OE so we do not pull line down */
		135	udelay(2); /* 1us pull up + 250ns hold */
		136
		137	target_in = i2c_in_csr(bus->num);
		138	reg = read_csr(bus->controlling_dd, target_in);
		139	return !!(reg & QSFP_HFI0_I2CCLK);
		140	}
54		141
55	/*	142	/*
56	* QSFP support for hfi driver, using "Two Wire Serial Interface" driver	143	* Allocate and initialize the given i2c bus number.
57	* in twsi.c	144	* Returns NULL on failure.
58	*/	145	*/
59	#define I2C_MAX_RETRY 4	146	static struct hfi1_i2c_bus init_i2c_bus(struct hfi1_devdata dd,
		147	struct hfi1_asic_data *ad, int num)
		148	{
		149	struct hfi1_i2c_bus *bus;
		150	int ret;
		151
		152	bus = kzalloc(sizeof(*bus), GFP_KERNEL);
		153	if (!bus)
		154	return NULL;
		155
		156	bus->controlling_dd = dd;
		157	bus->num = num; /* our bus number */
		158
		159	bus->algo.setsda = hfi1_setsda;
		160	bus->algo.setscl = hfi1_setscl;
		161	bus->algo.getsda = hfi1_getsda;
		162	bus->algo.getscl = hfi1_getscl;
		163	bus->algo.udelay = 5;
		164	bus->algo.timeout = usecs_to_jiffies(50);
		165	bus->algo.data = bus;
		166
		167	bus->adapter.owner = THIS_MODULE;
		168	bus->adapter.algo_data = &bus->algo;
		169	bus->adapter.dev.parent = &dd->pcidev->dev;
		170	snprintf(bus->adapter.name, sizeof(bus->adapter.name),
		171	"hfi1_i2c%d", num);
		172
		173	ret = i2c_bit_add_bus(&bus->adapter);
		174	if (ret) {
		175	dd_dev_info(dd, "%s: unable to add i2c bus %d, err %d\n",
		176	__func__, num, ret);
		177	kfree(bus);
		178	return NULL;
		179	}
		180
		181	return bus;
		182	}
60		183
61	/*	184	/*
62	* Raw i2c write. No set-up or lock checking.	185	* Initialize i2c buses.
		186	* Return 0 on success, -errno on error.
63	*/	187	*/
64	static int __i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr,	188	int set_up_i2c(struct hfi1_devdata dd, struct hfi1_asic_data ad)
65	int offset, void *bp, int len)
66	{	189	{
67	struct hfi1_devdata *dd = ppd->dd;	190	ad->i2c_bus0 = init_i2c_bus(dd, ad, 0);
68	int ret, cnt;	191	ad->i2c_bus1 = init_i2c_bus(dd, ad, 1);
69	u8 *buff = bp;	192	if (!ad->i2c_bus0 \|\| !ad->i2c_bus1)
		193	return -ENOMEM;
		194	return 0;
		195	};
70		196
71	cnt = 0;	197	static void clean_i2c_bus(struct hfi1_i2c_bus *bus)
72	while (cnt < len) {	198	{
73	int wlen = len - cnt;	199	if (bus) {
		200	i2c_del_adapter(&bus->adapter);
		201	kfree(bus);
		202	}
		203	}
74		204
75	ret = hfi1_twsi_blk_wr(dd, target, i2c_addr, offset,	205	void clean_up_i2c(struct hfi1_devdata dd, struct hfi1_asic_data ad)
76	buff + cnt, wlen);	206	{
77	if (ret) {	207	clean_i2c_bus(ad->i2c_bus0);
78	/* hfi1_twsi_blk_wr() 1 for error, else 0 */	208	ad->i2c_bus0 = NULL;
79	return -EIO;	209	clean_i2c_bus(ad->i2c_bus1);
80	}	210	ad->i2c_bus1 = NULL;
81	offset += wlen;	211	}
82	cnt += wlen;	212
		213	static int i2c_bus_write(struct hfi1_devdata dd, struct hfi1_i2c_bus i2c,
		214	u8 slave_addr, int offset, int offset_size,
		215	u8 *data, u16 len)
		216	{
		217	int ret;
		218	int num_msgs;
		219	u8 offset_bytes[2];
		220	struct i2c_msg msgs[2];
		221
		222	switch (offset_size) {
		223	case 0:
		224	num_msgs = 1;
		225	msgs[0].addr = slave_addr;
		226	msgs[0].flags = 0;
		227	msgs[0].len = len;
		228	msgs[0].buf = data;
		229	break;
		230	case 2:
		231	offset_bytes[1] = (offset >> 8) & 0xff;
		232	/* fall through */
		233	case 1:
		234	num_msgs = 2;
		235	offset_bytes[0] = offset & 0xff;
		236
		237	msgs[0].addr = slave_addr;
		238	msgs[0].flags = 0;
		239	msgs[0].len = offset_size;
		240	msgs[0].buf = offset_bytes;
		241
		242	msgs[1].addr = slave_addr;
		243	msgs[1].flags = I2C_M_NOSTART,
		244	msgs[1].len = len;
		245	msgs[1].buf = data;
		246	break;
		247	default:
		248	return -EINVAL;
83	}	249	}
84		250
85	/* Must wait min 20us between qsfp i2c transactions */	251	i2c->controlling_dd = dd;
86	udelay(20);	252	ret = i2c_transfer(&i2c->adapter, msgs, num_msgs);
		253	if (ret != num_msgs) {
		254	dd_dev_err(dd, "%s: bus %d, i2c slave 0x%x, offset 0x%x, len 0x%x; write failed, ret %d\n",
		255	__func__, i2c->num, slave_addr, offset, len, ret);
		256	return ret < 0 ? ret : -EIO;
		257	}
		258	return 0;
		259	}
		260
		261	static int i2c_bus_read(struct hfi1_devdata dd, struct hfi1_i2c_bus bus,
		262	u8 slave_addr, int offset, int offset_size,
		263	u8 *data, u16 len)
		264	{
		265	int ret;
		266	int num_msgs;
		267	u8 offset_bytes[2];
		268	struct i2c_msg msgs[2];
		269
		270	switch (offset_size) {
		271	case 0:
		272	num_msgs = 1;
		273	msgs[0].addr = slave_addr;
		274	msgs[0].flags = I2C_M_RD;
		275	msgs[0].len = len;
		276	msgs[0].buf = data;
		277	break;
		278	case 2:
		279	offset_bytes[1] = (offset >> 8) & 0xff;
		280	/* fall through */
		281	case 1:
		282	num_msgs = 2;
		283	offset_bytes[0] = offset & 0xff;
		284
		285	msgs[0].addr = slave_addr;
		286	msgs[0].flags = 0;
		287	msgs[0].len = offset_size;
		288	msgs[0].buf = offset_bytes;
		289
		290	msgs[1].addr = slave_addr;
		291	msgs[1].flags = I2C_M_RD,
		292	msgs[1].len = len;
		293	msgs[1].buf = data;
		294	break;
		295	default:
		296	return -EINVAL;
		297	}
87		298
88	return cnt;	299	bus->controlling_dd = dd;
		300	ret = i2c_transfer(&bus->adapter, msgs, num_msgs);
		301	if (ret != num_msgs) {
		302	dd_dev_err(dd, "%s: bus %d, i2c slave 0x%x, offset 0x%x, len 0x%x; read failed, ret %d\n",
		303	__func__, bus->num, slave_addr, offset, len, ret);
		304	return ret < 0 ? ret : -EIO;
		305	}
		306	return 0;
		307	}
		308
		309	/*
		310	* Raw i2c write. No set-up or lock checking.
		311	*
		312	* Return 0 on success, -errno on error.
		313	*/
		314	static int __i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr,
		315	int offset, void *bp, int len)
		316	{
		317	struct hfi1_devdata *dd = ppd->dd;
		318	struct hfi1_i2c_bus *bus;
		319	u8 slave_addr;
		320	int offset_size;
		321
		322	bus = target ? dd->asic_data->i2c_bus1 : dd->asic_data->i2c_bus0;
		323	slave_addr = (i2c_addr & 0xff) >> 1; /* convert to 7-bit addr */
		324	offset_size = (i2c_addr >> 8) & 0x3;
		325	return i2c_bus_write(dd, bus, slave_addr, offset, offset_size, bp, len);
89	}	326	}
90		327
91	/*	328	/*
92	* Caller must hold the i2c chain resource.	329	* Caller must hold the i2c chain resource.
		330	*
		331	* Return number of bytes written, or -errno.
93	*/	332	*/
94	int i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset,	333	int i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset,
95	void *bp, int len)	334	void *bp, int len)
@@ -99,63 +338,36 @@ int i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset, · 3 unmodified lines
99	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))	338	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))
100	return -EACCES;	339	return -EACCES;
101		340
102	/* make sure the TWSI bus is in a sane state */	341	ret = __i2c_write(ppd, target, i2c_addr, offset, bp, len);
103	ret = hfi1_twsi_reset(ppd->dd, target);	342	if (ret)
104	if (ret) {
105	hfi1_dev_porterr(ppd->dd, ppd->port,
106	"I2C chain %d write interface reset failed\n",
107	target);
108	return ret;	343	return ret;
109	}
110		344
111	return __i2c_write(ppd, target, i2c_addr, offset, bp, len);	345	return len;
112	}	346	}
113		347
114	/*	348	/*
115	* Raw i2c read. No set-up or lock checking.	349	* Raw i2c read. No set-up or lock checking.
		350	*
		351	* Return 0 on success, -errno on error.
116	*/	352	*/
117	static int __i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr,	353	static int __i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr,
118	int offset, void *bp, int len)	354	int offset, void *bp, int len)
119	{	355	{
120	struct hfi1_devdata *dd = ppd->dd;	356	struct hfi1_devdata *dd = ppd->dd;
121	int ret, cnt, pass = 0;	357	struct hfi1_i2c_bus *bus;
122	int orig_offset = offset;	358	u8 slave_addr;
123		359	int offset_size;
124	cnt = 0;	360
125	while (cnt < len) {	361	bus = target ? dd->asic_data->i2c_bus1 : dd->asic_data->i2c_bus0;
126	int rlen = len - cnt;	362	slave_addr = (i2c_addr & 0xff) >> 1; /* convert to 7-bit addr */
127		363	offset_size = (i2c_addr >> 8) & 0x3;
128	ret = hfi1_twsi_blk_rd(dd, target, i2c_addr, offset,	364	return i2c_bus_read(dd, bus, slave_addr, offset, offset_size, bp, len);
129	bp + cnt, rlen);
130	/* Some QSFP's fail first try. Retry as experiment */
131	if (ret && cnt == 0 && ++pass < I2C_MAX_RETRY)
132	continue;
133	if (ret) {
134	/* hfi1_twsi_blk_rd() 1 for error, else 0 */
135	ret = -EIO;
136	goto exit;
137	}
138	offset += rlen;
139	cnt += rlen;
140	}
141
142	ret = cnt;
143
144	exit:
145	if (ret < 0) {
146	hfi1_dev_porterr(dd, ppd->port,
147	"I2C chain %d read failed, addr 0x%x, offset 0x%x, len %d\n",
148	target, i2c_addr, orig_offset, len);
149	}
150
151	/* Must wait min 20us between qsfp i2c transactions */
152	udelay(20);
153
154	return ret;
155	}	365	}
156		366
157	/*	367	/*
158	* Caller must hold the i2c chain resource.	368	* Caller must hold the i2c chain resource.
		369	*
		370	* Return number of bytes read, or -errno.
159	*/	371	*/
160	int i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset,	372	int i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset,
161	void *bp, int len)	373	void *bp, int len)
@@ -165,16 +377,11 @@ int i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset, · 3 unmodified lines
165	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))	377	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))
166	return -EACCES;	378	return -EACCES;
167		379
168	/* make sure the TWSI bus is in a sane state */	380	ret = __i2c_read(ppd, target, i2c_addr, offset, bp, len);
169	ret = hfi1_twsi_reset(ppd->dd, target);	381	if (ret)
170	if (ret) {
171	hfi1_dev_porterr(ppd->dd, ppd->port,
172	"I2C chain %d read interface reset failed\n",
173	target);
174	return ret;	382	return ret;
175	}
176		383
177	return __i2c_read(ppd, target, i2c_addr, offset, bp, len);	384	return len;
178	}	385	}
179		386
180	/*	387	/*
@@ -182,6 +389,8 @@ int i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset, · 1 unmodified line
182	* by writing @addr = ((256 * n) + m)	389	* by writing @addr = ((256 * n) + m)
183	*	390	*
184	* Caller must hold the i2c chain resource.	391	* Caller must hold the i2c chain resource.
		392	*
		393	* Return number of bytes written or -errno.
185	*/	394	*/
186	int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp,	395	int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp,
187	int len)	396	int len)
@@ -189,21 +398,12 @@ int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 1 unmodified line
189	int count = 0;	398	int count = 0;
190	int offset;	399	int offset;
191	int nwrite;	400	int nwrite;
192	int ret;	401	int ret = 0;
193	u8 page;	402	u8 page;
194		403
195	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))	404	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))
196	return -EACCES;	405	return -EACCES;
197		406
198	/* make sure the TWSI bus is in a sane state */
199	ret = hfi1_twsi_reset(ppd->dd, target);
200	if (ret) {
201	hfi1_dev_porterr(ppd->dd, ppd->port,
202	"QSFP chain %d write interface reset failed\n",
203	target);
204	return ret;
205	}
206
207	while (count < len) {	407	while (count < len) {
208	/*	408	/*
209	* Set the qsfp page based on a zero-based address	409	* Set the qsfp page based on a zero-based address
@@ -213,11 +413,12 @@ int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 3 unmodified lines
213		413
214	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,	414	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,
215	QSFP_PAGE_SELECT_BYTE_OFFS, &page, 1);	415	QSFP_PAGE_SELECT_BYTE_OFFS, &page, 1);
216	if (ret != 1) {	416	/* QSFPs require a 5-10msec delay after write operations */
		417	mdelay(5);
		418	if (ret) {
217	hfi1_dev_porterr(ppd->dd, ppd->port,	419	hfi1_dev_porterr(ppd->dd, ppd->port,
218	"QSFP chain %d can't write QSFP_PAGE_SELECT_BYTE: %d\n",	420	"QSFP chain %d can't write QSFP_PAGE_SELECT_BYTE: %d\n",
219	target, ret);	421	target, ret);
220	ret = -EIO;
221	break;	422	break;
222	}	423	}
223		424
@@ -229,11 +430,13 @@ int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 5 unmodified lines
229		430
230	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,	431	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,
231	offset, bp + count, nwrite);	432	offset, bp + count, nwrite);
232	if (ret <= 0) /* stop on error or nothing written */	433	/* QSFPs require a 5-10msec delay after write operations */
		434	mdelay(5);
		435	if (ret) /* stop on error */
233	break;	436	break;
234		437
235	count += ret;	438	count += nwrite;
236	addr += ret;	439	addr += nwrite;
237	}	440	}
238		441
239	if (ret < 0)	442	if (ret < 0)
@@ -243,7 +446,7 @@ int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 3 unmodified lines
243		446
244	/*	447	/*
245	* Perform a stand-alone single QSFP write. Acquire the resource, do the	448	* Perform a stand-alone single QSFP write. Acquire the resource, do the
246	* read, then release the resource.	449	* write, then release the resource.
247	*/	450	*/
248	int one_qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp,	451	int one_qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp,
249	int len)	452	int len)
@@ -266,6 +469,8 @@ int one_qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 16 unmodified lines
266	* by reading @addr = ((256 * n) + m)	469	* by reading @addr = ((256 * n) + m)
267	*	470	*
268	* Caller must hold the i2c chain resource.	471	* Caller must hold the i2c chain resource.
		472	*
		473	* Return the number of bytes read or -errno.
269	*/	474	*/
270	int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp,	475	int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp,
271	int len)	476	int len)
@@ -273,21 +478,12 @@ int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 1 unmodified line
273	int count = 0;	478	int count = 0;
274	int offset;	479	int offset;
275	int nread;	480	int nread;
276	int ret;	481	int ret = 0;
277	u8 page;	482	u8 page;
278		483
279	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))	484	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))
280	return -EACCES;	485	return -EACCES;
281		486
282	/* make sure the TWSI bus is in a sane state */
283	ret = hfi1_twsi_reset(ppd->dd, target);
284	if (ret) {
285	hfi1_dev_porterr(ppd->dd, ppd->port,
286	"QSFP chain %d read interface reset failed\n",
287	target);
288	return ret;
289	}
290
291	while (count < len) {	487	while (count < len) {
292	/*	488	/*
293	* Set the qsfp page based on a zero-based address	489	* Set the qsfp page based on a zero-based address
@@ -296,11 +492,12 @@ int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 2 unmodified lines
296	page = (u8)(addr / QSFP_PAGESIZE);	492	page = (u8)(addr / QSFP_PAGESIZE);
297	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,	493	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,
298	QSFP_PAGE_SELECT_BYTE_OFFS, &page, 1);	494	QSFP_PAGE_SELECT_BYTE_OFFS, &page, 1);
299	if (ret != 1) {	495	/* QSFPs require a 5-10msec delay after write operations */
		496	mdelay(5);
		497	if (ret) {
300	hfi1_dev_porterr(ppd->dd, ppd->port,	498	hfi1_dev_porterr(ppd->dd, ppd->port,
301	"QSFP chain %d can't write QSFP_PAGE_SELECT_BYTE: %d\n",	499	"QSFP chain %d can't write QSFP_PAGE_SELECT_BYTE: %d\n",
302	target, ret);	500	target, ret);
303	ret = -EIO;
304	break;	501	break;
305	}	502	}
306		503
@@ -310,15 +507,13 @@ int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 3 unmodified lines
310	if (((addr % QSFP_RW_BOUNDARY) + nread) > QSFP_RW_BOUNDARY)	507	if (((addr % QSFP_RW_BOUNDARY) + nread) > QSFP_RW_BOUNDARY)
311	nread = QSFP_RW_BOUNDARY - (addr % QSFP_RW_BOUNDARY);	508	nread = QSFP_RW_BOUNDARY - (addr % QSFP_RW_BOUNDARY);
312		509
313	/* QSFPs require a 5-10msec delay after write operations */
314	mdelay(5);
315	ret = __i2c_read(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,	510	ret = __i2c_read(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,
316	offset, bp + count, nread);	511	offset, bp + count, nread);
317	if (ret <= 0) /* stop on error or nothing read */	512	if (ret) /* stop on error */
318	break;	513	break;
319		514
320	count += ret;	515	count += nread;
321	addr += ret;	516	addr += nread;
322	}	517	}
323		518
324	if (ret < 0)	519	if (ret < 0)

drivers/infiniband/hw/hfi1/qsfp.h+3−0

@@ -238,3 +238,6 @@ int one_qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 237 unmodified lines
238	int len);	238	int len);
239	int one_qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp,	239	int one_qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp,
240	int len);	240	int len);
		241	struct hfi1_asic_data;
		242	int set_up_i2c(struct hfi1_devdata dd, struct hfi1_asic_data ad);
		243	void clean_up_i2c(struct hfi1_devdata dd, struct hfi1_asic_data ad);

drivers/infiniband/hw/hfi1/rc.c+65−25

 				qp->s_flags |= RVT_S_WAIT_FENCE;
 				goto bail;
 			}
+			/*
+			 * Local operations are processed immediately
+			 * after all prior requests have completed
+			 */
+			if (wqe->wr.opcode == IB_WR_REG_MR ||
+			    wqe->wr.opcode == IB_WR_LOCAL_INV) {
+				int local_ops = 0;
+				int err = 0;
+				if (qp->s_last != qp->s_cur)
+					goto bail;
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				if (++qp->s_tail == qp->s_size)
+					qp->s_tail = 0;
+				if (!(wqe->wr.send_flags &
+				      RVT_SEND_COMPLETION_ONLY)) {
+					err = rvt_invalidate_rkey(
+						qp,
+						wqe->wr.ex.invalidate_rkey);
+					local_ops = 1;
+				}
+				hfi1_send_complete(qp, wqe,
+						   err ? IB_WC_LOC_PROT_ERR
+						       : IB_WC_SUCCESS);
+				if (local_ops)
+					atomic_dec(&qp->local_ops_pending);
+				qp->s_hdrwords = 0;
+				goto done_free_tx;
+			}
 			newreq = 1;
 			qp->s_psn = wqe->psn;
 		}
 		switch (wqe->wr.opcode) {
 		case IB_WR_SEND:
 		case IB_WR_SEND_WITH_IMM:
+		case IB_WR_SEND_WITH_INV:
 			/* If no credit, return. */
 			if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
 			    cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
 			}
 			if (wqe->wr.opcode == IB_WR_SEND) {
 				qp->s_state = OP(SEND_ONLY);
-			} else {
+			} else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
 				qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
 				/* Immediate data comes after the BTH */
 				ohdr->u.imm_data = wqe->wr.ex.imm_data;
 				hwords += 1;
+			} else {
+				qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE);
+				/* Invalidate rkey comes after the BTH */
+				ohdr->u.ieth = cpu_to_be32(
+						wqe->wr.ex.invalidate_rkey);
+				hwords += 1;
 			}
 			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
 				bth0 |= IB_BTH_SOLICITED;
 		}
 		if (wqe->wr.opcode == IB_WR_SEND) {
 			qp->s_state = OP(SEND_LAST);
-		} else {
+		} else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
 			qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
 			/* Immediate data comes after the BTH */
 			ohdr->u.imm_data = wqe->wr.ex.imm_data;
 			hwords += 1;
+		} else {
+			qp->s_state = OP(SEND_LAST_WITH_INVALIDATE);
+			/* invalidate data comes after the BTH */
+			ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey);
+			hwords += 1;
 		}
 		if (wqe->wr.send_flags & IB_SEND_SOLICITED)
 			bth0 |= IB_BTH_SOLICITED;
 		ibp->rvp.n_rc_timeouts++;
 		qp->s_flags &= ~RVT_S_TIMER;
 		del_timer(&qp->s_timer);
-		trace_hfi1_rc_timeout(qp, qp->s_last_psn + 1);
+		trace_hfi1_timeout(qp, qp->s_last_psn + 1);
 		restart_rc(qp, qp->s_last_psn + 1, 1);
 		hfi1_schedule_send(qp);
 	}
 	 * If we were waiting for sends to complete before re-sending,
 	 * and they are now complete, restart sending.
 	 */
-	trace_hfi1_rc_sendcomplete(qp, psn);
+	trace_hfi1_sendcomplete(qp, psn);
 	if (qp->s_flags & RVT_S_WAIT_PSN &&
 	    cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
 		qp->s_flags &= ~RVT_S_WAIT_PSN;
 	spin_lock_irqsave(&qp->s_lock, flags);
-	trace_hfi1_rc_ack(qp, psn);
+	trace_hfi1_ack(qp, psn);
 	/* Ignore invalid responses. */
 	smp_read_barrier_depends(); /* see post_one_send */
 	u8 i, prev;
 	int old_req;
-	trace_hfi1_rc_rcv_error(qp, psn);
+	trace_hfi1_rcv_error(qp, psn);
 	if (diff > 0) {
 		/*
 		 * Packet sequence error.
 	u32 tlen = packet->tlen;
 	struct rvt_qp *qp = packet->qp;
 	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
-	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 	struct hfi1_other_headers *ohdr = packet->ohdr;
 	u32 bth0, opcode;
 	u32 hdrsize = packet->hlen;
 	int diff;
 	struct ib_reth *reth;
 	unsigned long flags;
-	u32 bth1;
 	int ret, is_fecn = 0;
 	int copy_last = 0;
+	u32 rkey;
 	bth0 = be32_to_cpu(ohdr->bth[0]);
 	if (hfi1_ruc_check_hdr(ibp, hdr, rcv_flags & HFI1_HAS_GRH, qp, bth0))
 		return;
-	bth1 = be32_to_cpu(ohdr->bth[1]);
+	is_fecn = process_ecn(qp, packet, false);
-	if (unlikely(bth1 & (HFI1_BECN_SMASK | HFI1_FECN_SMASK))) {
-		if (bth1 & HFI1_BECN_SMASK) {
-			u16 rlid = qp->remote_ah_attr.dlid;
-			u32 lqpn, rqpn;
-			lqpn = qp->ibqp.qp_num;
-			rqpn = qp->remote_qpn;
-			process_becn(
-				ppd,
-				qp->remote_ah_attr.sl,
-				rlid, lqpn, rqpn,
-				IB_CC_SVCTYPE_RC);
-		}
-		is_fecn = bth1 & HFI1_FECN_SMASK;
-	}
 	psn = be32_to_cpu(ohdr->bth[2]);
 	opcode = (bth0 >> 24) & 0xff;
 	case OP(SEND_MIDDLE):
 		if (opcode == OP(SEND_MIDDLE) ||
 		    opcode == OP(SEND_LAST) ||
-		    opcode == OP(SEND_LAST_WITH_IMMEDIATE))
+		    opcode == OP(SEND_LAST_WITH_IMMEDIATE) ||
+		    opcode == OP(SEND_LAST_WITH_INVALIDATE))
 			break;
 		goto nack_inv;
 		if (opcode == OP(SEND_MIDDLE) ||
 		    opcode == OP(SEND_LAST) ||
 		    opcode == OP(SEND_LAST_WITH_IMMEDIATE) ||
+		    opcode == OP(SEND_LAST_WITH_INVALIDATE) ||
 		    opcode == OP(RDMA_WRITE_MIDDLE) ||
 		    opcode == OP(RDMA_WRITE_LAST) ||
 		    opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
 	case OP(SEND_ONLY):
 	case OP(SEND_ONLY_WITH_IMMEDIATE):
+	case OP(SEND_ONLY_WITH_INVALIDATE):
 		ret = hfi1_rvt_get_rwqe(qp, 0);
 		if (ret < 0)
 			goto nack_op_err;
 		qp->r_rcv_len = 0;
 		if (opcode == OP(SEND_ONLY))
 			goto no_immediate_data;
+		if (opcode == OP(SEND_ONLY_WITH_INVALIDATE))
+			goto send_last_inv;
 		/* FALLTHROUGH for SEND_ONLY_WITH_IMMEDIATE */
 	case OP(SEND_LAST_WITH_IMMEDIATE):
 send_last_imm:
 		wc.ex.imm_data = ohdr->u.imm_data;
 		wc.wc_flags = IB_WC_WITH_IMM;
 		goto send_last;
+	case OP(SEND_LAST_WITH_INVALIDATE):
+send_last_inv:
+		rkey = be32_to_cpu(ohdr->u.ieth);
+		if (rvt_invalidate_rkey(qp, rkey))
+			goto no_immediate_data;
+		wc.ex.invalidate_rkey = rkey;
+		wc.wc_flags = IB_WC_WITH_INVALIDATE;
+		goto send_last;
 	case OP(RDMA_WRITE_LAST):
 		copy_last = ibpd_to_rvtpd(qp->ibqp.pd)->user;
 		/* fall through */

drivers/infiniband/hw/hfi1/ruc.c+40−17

 	int ret;
 	int copy_last = 0;
 	u32 to;
+	int local_ops = 0;
 	rcu_read_lock();
 	sqp->s_sge.num_sge = wqe->wr.num_sge;
 	sqp->s_len = wqe->length;
 	switch (wqe->wr.opcode) {
+	case IB_WR_REG_MR:
+		goto send_comp;
+	case IB_WR_LOCAL_INV:
+		if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
+			if (rvt_invalidate_rkey(sqp,
+						wqe->wr.ex.invalidate_rkey))
+				send_status = IB_WC_LOC_PROT_ERR;
+			local_ops = 1;
+		}
+		goto send_comp;
+	case IB_WR_SEND_WITH_INV:
+		if (!rvt_invalidate_rkey(qp, wqe->wr.ex.invalidate_rkey)) {
+			wc.wc_flags = IB_WC_WITH_INVALIDATE;
+			wc.ex.invalidate_rkey = wqe->wr.ex.invalidate_rkey;
+		}
+		goto send;
 	case IB_WR_SEND_WITH_IMM:
 		wc.wc_flags = IB_WC_WITH_IMM;
 		wc.ex.imm_data = wqe->wr.ex.imm_data;
 		/* FALLTHROUGH */
 	case IB_WR_SEND:
+send:
 		ret = hfi1_rvt_get_rwqe(qp, 0);
 		if (ret < 0)
 			goto op_err;
 flush_send:
 	sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
 	hfi1_send_complete(sqp, wqe, send_status);
+	if (local_ops) {
+		atomic_dec(&sqp->local_ops_pending);
+		local_ops = 0;
+	}
 	goto again;
 rnr_nak:
 	return sizeof(struct ib_grh) / sizeof(u32);
 }
-#define BTH2_OFFSET (offsetof(struct hfi1_pio_header, hdr.u.oth.bth[2]) / 4)
+#define BTH2_OFFSET (offsetof(struct hfi1_sdma_header, hdr.u.oth.bth[2]) / 4)
 /**
- * build_ahg - create ahg in s_hdr
+ * build_ahg - create ahg in s_ahg
  * @qp: a pointer to QP
  * @npsn: the next PSN for the request/response
  *
 			qp->s_ahgidx = sdma_ahg_alloc(priv->s_sde);
 		if (qp->s_ahgidx >= 0) {
 			qp->s_ahgpsn = npsn;
-			priv->s_hdr->tx_flags |= SDMA_TXREQ_F_AHG_COPY;
+			priv->s_ahg->tx_flags |= SDMA_TXREQ_F_AHG_COPY;
 			/* save to protect a change in another thread */
-			priv->s_hdr->sde = priv->s_sde;
+			priv->s_ahg->ahgidx = qp->s_ahgidx;
-			priv->s_hdr->ahgidx = qp->s_ahgidx;
 			qp->s_flags |= RVT_S_AHG_VALID;
 		}
 	} else {
 		/* subsequent middle after valid */
 		if (qp->s_ahgidx >= 0) {
-			priv->s_hdr->tx_flags |= SDMA_TXREQ_F_USE_AHG;
+			priv->s_ahg->tx_flags |= SDMA_TXREQ_F_USE_AHG;
-			priv->s_hdr->ahgidx = qp->s_ahgidx;
+			priv->s_ahg->ahgidx = qp->s_ahgidx;
-			priv->s_hdr->ahgcount++;
+			priv->s_ahg->ahgcount++;
-			priv->s_hdr->ahgdesc[0] =
+			priv->s_ahg->ahgdesc[0] =
 				sdma_build_ahg_descriptor(
 					(__force u16)cpu_to_be16((u16)npsn),
 					BTH2_OFFSET,
 					16);
 			if ((npsn & 0xffff0000) !=
 					(qp->s_ahgpsn & 0xffff0000)) {
-				priv->s_hdr->ahgcount++;
+				priv->s_ahg->ahgcount++;
-				priv->s_hdr->ahgdesc[1] =
+				priv->s_ahg->ahgdesc[1] =
 					sdma_build_ahg_descriptor(
 						(__force u16)cpu_to_be16(
 							(u16)(npsn >> 16)),
 	}
 	lrh0 |= (priv->s_sc & 0xf) << 12 | (qp->remote_ah_attr.sl & 0xf) << 4;
 	/*
-	 * reset s_hdr/AHG fields
+	 * reset s_ahg/AHG fields
 	 *
 	 * This insures that the ahgentry/ahgcount
 	 * are at a non-AHG default to protect
 	 * build_ahg() will modify as appropriate
 	 * to use the AHG feature.
 	 */
-	priv->s_hdr->tx_flags = 0;
+	priv->s_ahg->tx_flags = 0;
-	priv->s_hdr->ahgcount = 0;
+	priv->s_ahg->ahgcount = 0;
-	priv->s_hdr->ahgidx = 0;
+	priv->s_ahg->ahgidx = 0;
-	priv->s_hdr->sde = NULL;
 	if (qp->s_mig_state == IB_MIG_MIGRATED)
 		bth0 |= IB_BTH_MIG_REQ;
 	else
 			 */
 			if (hfi1_verbs_send(qp, &ps))
 				return;
-			/* Record that s_hdr is empty. */
+			/* Record that s_ahg is empty. */
 			qp->s_hdrwords = 0;
 			/* allow other tasks to run */
 			if (unlikely(time_after(jiffies, timeout))) {

drivers/infiniband/hw/hfi1/sysfs.c+25−0

@@ -49,6 +49,7 @@ · 48 unmodified lines
49	#include "hfi.h"	49	#include "hfi.h"
50	#include "mad.h"	50	#include "mad.h"
51	#include "trace.h"	51	#include "trace.h"
		52	#include "affinity.h"
52		53
53	/*	54	/*
54	* Start of per-port congestion control structures and support code	55	* Start of per-port congestion control structures and support code
@@ -622,6 +623,27 @@ static ssize_t show_tempsense(struct device *device, · 567 unmodified lines
622	return ret;	623	return ret;
623	}	624	}
624		625
		626	static ssize_t show_sdma_affinity(struct device *device,
		627	struct device_attribute attr, char buf)
		628	{
		629	struct hfi1_ibdev *dev =
		630	container_of(device, struct hfi1_ibdev, rdi.ibdev.dev);
		631	struct hfi1_devdata *dd = dd_from_dev(dev);
		632
		633	return hfi1_get_sdma_affinity(dd, buf);
		634	}
		635
		636	static ssize_t store_sdma_affinity(struct device *device,
		637	struct device_attribute *attr,
		638	const char *buf, size_t count)
		639	{
		640	struct hfi1_ibdev *dev =
		641	container_of(device, struct hfi1_ibdev, rdi.ibdev.dev);
		642	struct hfi1_devdata *dd = dd_from_dev(dev);
		643
		644	return hfi1_set_sdma_affinity(dd, buf, count);
		645	}
		646
625	/*	647	/*
626	* end of per-unit (or driver, in some cases, but replicated	648	* end of per-unit (or driver, in some cases, but replicated
627	* per unit) functions	649	* per unit) functions
@@ -636,6 +658,8 @@ static DEVICE_ATTR(serial, S_IRUGO, show_serial, NULL); · 8 unmodified lines
636	static DEVICE_ATTR(boardversion, S_IRUGO, show_boardversion, NULL);	658	static DEVICE_ATTR(boardversion, S_IRUGO, show_boardversion, NULL);
637	static DEVICE_ATTR(tempsense, S_IRUGO, show_tempsense, NULL);	659	static DEVICE_ATTR(tempsense, S_IRUGO, show_tempsense, NULL);
638	static DEVICE_ATTR(chip_reset, S_IWUSR, NULL, store_chip_reset);	660	static DEVICE_ATTR(chip_reset, S_IWUSR, NULL, store_chip_reset);
		661	static DEVICE_ATTR(sdma_affinity, S_IWUSR \| S_IRUGO, show_sdma_affinity,
		662	store_sdma_affinity);
639		663
640	static struct device_attribute *hfi1_attributes[] = {	664	static struct device_attribute *hfi1_attributes[] = {
641	&dev_attr_hw_rev,	665	&dev_attr_hw_rev,
@@ -646,6 +670,7 @@ static struct device_attribute *hfi1_attributes[] = { · 4 unmodified lines
646	&dev_attr_boardversion,	670	&dev_attr_boardversion,
647	&dev_attr_tempsense,	671	&dev_attr_tempsense,
648	&dev_attr_chip_reset,	672	&dev_attr_chip_reset,
		673	&dev_attr_sdma_affinity,
649	};	674	};
650		675
651	int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,	676	int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,

drivers/infiniband/hw/hfi1/trace.h+7−1326

@@ -44,1329 +44,10 @@ · 43 unmodified lines
44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45	*	45	*
46	*/	46	*/
47	#undef TRACE_SYSTEM_VAR	47	#include "trace_dbg.h"
48	#define TRACE_SYSTEM_VAR hfi1	48	#include "trace_misc.h"
49		49	#include "trace_ctxts.h"
50	#if !defined(__HFI1_TRACE_H) \|\| defined(TRACE_HEADER_MULTI_READ)	50	#include "trace_ibhdrs.h"
51	#define __HFI1_TRACE_H	51	#include "trace_rc.h"
52		52	#include "trace_rx.h"
53	#include <linux/tracepoint.h>	53	#include "trace_tx.h"
54	#include <linux/trace_seq.h>
55
56	#include "hfi.h"
57	#include "mad.h"
58	#include "sdma.h"
59
60	#define DD_DEV_ENTRY(dd) __string(dev, dev_name(&(dd)->pcidev->dev))
61	#define DD_DEV_ASSIGN(dd) __assign_str(dev, dev_name(&(dd)->pcidev->dev))
62
63	#define packettype_name(etype) { RHF_RCV_TYPE_##etype, #etype }
64	#define show_packettype(etype) \
65	__print_symbolic(etype, \
66	packettype_name(EXPECTED), \
67	packettype_name(EAGER), \
68	packettype_name(IB), \
69	packettype_name(ERROR), \
70	packettype_name(BYPASS))
71
72	#undef TRACE_SYSTEM
73	#define TRACE_SYSTEM hfi1_rx
74
75	TRACE_EVENT(hfi1_rcvhdr,
76	TP_PROTO(struct hfi1_devdata *dd,
77	u32 ctxt,
78	u64 eflags,
79	u32 etype,
80	u32 hlen,
81	u32 tlen,
82	u32 updegr,
83	u32 etail
84	),
85	TP_ARGS(dd, ctxt, eflags, etype, hlen, tlen, updegr, etail),
86	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
87	__field(u64, eflags)
88	__field(u32, ctxt)
89	__field(u32, etype)
90	__field(u32, hlen)
91	__field(u32, tlen)
92	__field(u32, updegr)
93	__field(u32, etail)
94	),
95	TP_fast_assign(DD_DEV_ASSIGN(dd);
96	__entry->eflags = eflags;
97	__entry->ctxt = ctxt;
98	__entry->etype = etype;
99	__entry->hlen = hlen;
100	__entry->tlen = tlen;
101	__entry->updegr = updegr;
102	__entry->etail = etail;
103	),
104	TP_printk(
105	"[%s] ctxt %d eflags 0x%llx etype %d,%s hlen %d tlen %d updegr %d etail %d",
106	__get_str(dev),
107	__entry->ctxt,
108	__entry->eflags,
109	__entry->etype, show_packettype(__entry->etype),
110	__entry->hlen,
111	__entry->tlen,
112	__entry->updegr,
113	__entry->etail
114	)
115	);
116
117	TRACE_EVENT(hfi1_receive_interrupt,
118	TP_PROTO(struct hfi1_devdata *dd, u32 ctxt),
119	TP_ARGS(dd, ctxt),
120	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
121	__field(u32, ctxt)
122	__field(u8, slow_path)
123	__field(u8, dma_rtail)
124	),
125	TP_fast_assign(DD_DEV_ASSIGN(dd);
126	__entry->ctxt = ctxt;
127	if (dd->rcd[ctxt]->do_interrupt ==
128	&handle_receive_interrupt) {
129	__entry->slow_path = 1;
130	__entry->dma_rtail = 0xFF;
131	} else if (dd->rcd[ctxt]->do_interrupt ==
132	&handle_receive_interrupt_dma_rtail){
133	__entry->dma_rtail = 1;
134	__entry->slow_path = 0;
135	} else if (dd->rcd[ctxt]->do_interrupt ==
136	&handle_receive_interrupt_nodma_rtail) {
137	__entry->dma_rtail = 0;
138	__entry->slow_path = 0;
139	}
140	),
141	TP_printk("[%s] ctxt %d SlowPath: %d DmaRtail: %d",
142	__get_str(dev),
143	__entry->ctxt,
144	__entry->slow_path,
145	__entry->dma_rtail
146	)
147	);
148
149	TRACE_EVENT(hfi1_exp_tid_reg,
150	TP_PROTO(unsigned ctxt, u16 subctxt, u32 rarr,
151	u32 npages, unsigned long va, unsigned long pa,
152	dma_addr_t dma),
153	TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
154	TP_STRUCT__entry(
155	__field(unsigned, ctxt)
156	__field(u16, subctxt)
157	__field(u32, rarr)
158	__field(u32, npages)
159	__field(unsigned long, va)
160	__field(unsigned long, pa)
161	__field(dma_addr_t, dma)
162	),
163	TP_fast_assign(
164	__entry->ctxt = ctxt;
165	__entry->subctxt = subctxt;
166	__entry->rarr = rarr;
167	__entry->npages = npages;
168	__entry->va = va;
169	__entry->pa = pa;
170	__entry->dma = dma;
171	),
172	TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
173	__entry->ctxt,
174	__entry->subctxt,
175	__entry->rarr,
176	__entry->npages,
177	__entry->pa,
178	__entry->va,
179	__entry->dma
180	)
181	);
182
183	TRACE_EVENT(hfi1_exp_tid_unreg,
184	TP_PROTO(unsigned ctxt, u16 subctxt, u32 rarr, u32 npages,
185	unsigned long va, unsigned long pa, dma_addr_t dma),
186	TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
187	TP_STRUCT__entry(
188	__field(unsigned, ctxt)
189	__field(u16, subctxt)
190	__field(u32, rarr)
191	__field(u32, npages)
192	__field(unsigned long, va)
193	__field(unsigned long, pa)
194	__field(dma_addr_t, dma)
195	),
196	TP_fast_assign(
197	__entry->ctxt = ctxt;
198	__entry->subctxt = subctxt;
199	__entry->rarr = rarr;
200	__entry->npages = npages;
201	__entry->va = va;
202	__entry->pa = pa;
203	__entry->dma = dma;
204	),
205	TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
206	__entry->ctxt,
207	__entry->subctxt,
208	__entry->rarr,
209	__entry->npages,
210	__entry->pa,
211	__entry->va,
212	__entry->dma
213	)
214	);
215
216	TRACE_EVENT(hfi1_exp_tid_inval,
217	TP_PROTO(unsigned ctxt, u16 subctxt, unsigned long va, u32 rarr,
218	u32 npages, dma_addr_t dma),
219	TP_ARGS(ctxt, subctxt, va, rarr, npages, dma),
220	TP_STRUCT__entry(
221	__field(unsigned, ctxt)
222	__field(u16, subctxt)
223	__field(unsigned long, va)
224	__field(u32, rarr)
225	__field(u32, npages)
226	__field(dma_addr_t, dma)
227	),
228	TP_fast_assign(
229	__entry->ctxt = ctxt;
230	__entry->subctxt = subctxt;
231	__entry->va = va;
232	__entry->rarr = rarr;
233	__entry->npages = npages;
234	__entry->dma = dma;
235	),
236	TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx dma: 0x%llx",
237	__entry->ctxt,
238	__entry->subctxt,
239	__entry->rarr,
240	__entry->npages,
241	__entry->va,
242	__entry->dma
243	)
244	);
245
246	TRACE_EVENT(hfi1_mmu_invalidate,
247	TP_PROTO(unsigned ctxt, u16 subctxt, const char *type,
248	unsigned long start, unsigned long end),
249	TP_ARGS(ctxt, subctxt, type, start, end),
250	TP_STRUCT__entry(
251	__field(unsigned, ctxt)
252	__field(u16, subctxt)
253	__string(type, type)
254	__field(unsigned long, start)
255	__field(unsigned long, end)
256	),
257	TP_fast_assign(
258	__entry->ctxt = ctxt;
259	__entry->subctxt = subctxt;
260	__assign_str(type, type);
261	__entry->start = start;
262	__entry->end = end;
263	),
264	TP_printk("[%3u:%02u] MMU Invalidate (%s) 0x%lx - 0x%lx",
265	__entry->ctxt,
266	__entry->subctxt,
267	__get_str(type),
268	__entry->start,
269	__entry->end
270	)
271	);
272
273	#undef TRACE_SYSTEM
274	#define TRACE_SYSTEM hfi1_tx
275
276	TRACE_EVENT(hfi1_piofree,
277	TP_PROTO(struct send_context *sc, int extra),
278	TP_ARGS(sc, extra),
279	TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
280	__field(u32, sw_index)
281	__field(u32, hw_context)
282	__field(int, extra)
283	),
284	TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
285	__entry->sw_index = sc->sw_index;
286	__entry->hw_context = sc->hw_context;
287	__entry->extra = extra;
288	),
289	TP_printk("[%s] ctxt %u(%u) extra %d",
290	__get_str(dev),
291	__entry->sw_index,
292	__entry->hw_context,
293	__entry->extra
294	)
295	);
296
297	TRACE_EVENT(hfi1_wantpiointr,
298	TP_PROTO(struct send_context *sc, u32 needint, u64 credit_ctrl),
299	TP_ARGS(sc, needint, credit_ctrl),
300	TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
301	__field(u32, sw_index)
302	__field(u32, hw_context)
303	__field(u32, needint)
304	__field(u64, credit_ctrl)
305	),
306	TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
307	__entry->sw_index = sc->sw_index;
308	__entry->hw_context = sc->hw_context;
309	__entry->needint = needint;
310	__entry->credit_ctrl = credit_ctrl;
311	),
312	TP_printk("[%s] ctxt %u(%u) on %d credit_ctrl 0x%llx",
313	__get_str(dev),
314	__entry->sw_index,
315	__entry->hw_context,
316	__entry->needint,
317	(unsigned long long)__entry->credit_ctrl
318	)
319	);
320
321	DECLARE_EVENT_CLASS(hfi1_qpsleepwakeup_template,
322	TP_PROTO(struct rvt_qp *qp, u32 flags),
323	TP_ARGS(qp, flags),
324	TP_STRUCT__entry(
325	DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
326	__field(u32, qpn)
327	__field(u32, flags)
328	__field(u32, s_flags)
329	),
330	TP_fast_assign(
331	DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device))
332	__entry->flags = flags;
333	__entry->qpn = qp->ibqp.qp_num;
334	__entry->s_flags = qp->s_flags;
335	),
336	TP_printk(
337	"[%s] qpn 0x%x flags 0x%x s_flags 0x%x",
338	__get_str(dev),
339	__entry->qpn,
340	__entry->flags,
341	__entry->s_flags
342	)
343	);
344
345	DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpwakeup,
346	TP_PROTO(struct rvt_qp *qp, u32 flags),
347	TP_ARGS(qp, flags));
348
349	DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpsleep,
350	TP_PROTO(struct rvt_qp *qp, u32 flags),
351	TP_ARGS(qp, flags));
352
353	#undef TRACE_SYSTEM
354	#define TRACE_SYSTEM hfi1_ibhdrs
355
356	u8 ibhdr_exhdr_len(struct hfi1_ib_header *hdr);
357	const char parse_everbs_hdrs(struct trace_seq p, u8 opcode, void *ehdrs);
358
359	#define __parse_ib_ehdrs(op, ehdrs) parse_everbs_hdrs(p, op, ehdrs)
360
361	const char parse_sdma_flags(struct trace_seq p, u64 desc0, u64 desc1);
362
363	#define __parse_sdma_flags(desc0, desc1) parse_sdma_flags(p, desc0, desc1)
364
365	#define lrh_name(lrh) { HFI1_##lrh, #lrh }
366	#define show_lnh(lrh) \
367	__print_symbolic(lrh, \
368	lrh_name(LRH_BTH), \
369	lrh_name(LRH_GRH))
370
371	#define ib_opcode_name(opcode) { IB_OPCODE_##opcode, #opcode }
372	#define show_ib_opcode(opcode) \
373	__print_symbolic(opcode, \
374	ib_opcode_name(RC_SEND_FIRST), \
375	ib_opcode_name(RC_SEND_MIDDLE), \
376	ib_opcode_name(RC_SEND_LAST), \
377	ib_opcode_name(RC_SEND_LAST_WITH_IMMEDIATE), \
378	ib_opcode_name(RC_SEND_ONLY), \
379	ib_opcode_name(RC_SEND_ONLY_WITH_IMMEDIATE), \
380	ib_opcode_name(RC_RDMA_WRITE_FIRST), \
381	ib_opcode_name(RC_RDMA_WRITE_MIDDLE), \
382	ib_opcode_name(RC_RDMA_WRITE_LAST), \
383	ib_opcode_name(RC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
384	ib_opcode_name(RC_RDMA_WRITE_ONLY), \
385	ib_opcode_name(RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
386	ib_opcode_name(RC_RDMA_READ_REQUEST), \
387	ib_opcode_name(RC_RDMA_READ_RESPONSE_FIRST), \
388	ib_opcode_name(RC_RDMA_READ_RESPONSE_MIDDLE), \
389	ib_opcode_name(RC_RDMA_READ_RESPONSE_LAST), \
390	ib_opcode_name(RC_RDMA_READ_RESPONSE_ONLY), \
391	ib_opcode_name(RC_ACKNOWLEDGE), \
392	ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE), \
393	ib_opcode_name(RC_COMPARE_SWAP), \
394	ib_opcode_name(RC_FETCH_ADD), \
395	ib_opcode_name(RC_SEND_LAST_WITH_INVALIDATE), \
396	ib_opcode_name(RC_SEND_ONLY_WITH_INVALIDATE), \
397	ib_opcode_name(UC_SEND_FIRST), \
398	ib_opcode_name(UC_SEND_MIDDLE), \
399	ib_opcode_name(UC_SEND_LAST), \
400	ib_opcode_name(UC_SEND_LAST_WITH_IMMEDIATE), \
401	ib_opcode_name(UC_SEND_ONLY), \
402	ib_opcode_name(UC_SEND_ONLY_WITH_IMMEDIATE), \
403	ib_opcode_name(UC_RDMA_WRITE_FIRST), \
404	ib_opcode_name(UC_RDMA_WRITE_MIDDLE), \
405	ib_opcode_name(UC_RDMA_WRITE_LAST), \
406	ib_opcode_name(UC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
407	ib_opcode_name(UC_RDMA_WRITE_ONLY), \
408	ib_opcode_name(UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
409	ib_opcode_name(UD_SEND_ONLY), \
410	ib_opcode_name(UD_SEND_ONLY_WITH_IMMEDIATE), \
411	ib_opcode_name(CNP))
412
413	#define LRH_PRN "vl %d lver %d sl %d lnh %d,%s dlid %.4x len %d slid %.4x"
414	#define BTH_PRN \
415	"op 0x%.2x,%s se %d m %d pad %d tver %d pkey 0x%.4x " \
416	"f %d b %d qpn 0x%.6x a %d psn 0x%.8x"
417	#define EHDR_PRN "%s"
418
419	DECLARE_EVENT_CLASS(hfi1_ibhdr_template,
420	TP_PROTO(struct hfi1_devdata *dd,
421	struct hfi1_ib_header *hdr),
422	TP_ARGS(dd, hdr),
423	TP_STRUCT__entry(
424	DD_DEV_ENTRY(dd)
425	/* LRH */
426	__field(u8, vl)
427	__field(u8, lver)
428	__field(u8, sl)
429	__field(u8, lnh)
430	__field(u16, dlid)
431	__field(u16, len)
432	__field(u16, slid)
433	/* BTH */
434	__field(u8, opcode)
435	__field(u8, se)
436	__field(u8, m)
437	__field(u8, pad)
438	__field(u8, tver)
439	__field(u16, pkey)
440	__field(u8, f)
441	__field(u8, b)
442	__field(u32, qpn)
443	__field(u8, a)
444	__field(u32, psn)
445	/* extended headers */
446	__dynamic_array(u8, ehdrs, ibhdr_exhdr_len(hdr))
447	),
448	TP_fast_assign(
449	struct hfi1_other_headers *ohdr;
450
451	DD_DEV_ASSIGN(dd);
452	/* LRH */
453	__entry->vl =
454	(u8)(be16_to_cpu(hdr->lrh[0]) >> 12);
455	__entry->lver =
456	(u8)(be16_to_cpu(hdr->lrh[0]) >> 8) & 0xf;
457	__entry->sl =
458	(u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
459	__entry->lnh =
460	(u8)(be16_to_cpu(hdr->lrh[0]) & 3);
461	__entry->dlid =
462	be16_to_cpu(hdr->lrh[1]);
463	/* allow for larger len */
464	__entry->len =
465	be16_to_cpu(hdr->lrh[2]);
466	__entry->slid =
467	be16_to_cpu(hdr->lrh[3]);
468	/* BTH */
469	if (__entry->lnh == HFI1_LRH_BTH)
470	ohdr = &hdr->u.oth;
471	else
472	ohdr = &hdr->u.l.oth;
473	__entry->opcode =
474	(be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
475	__entry->se =
476	(be32_to_cpu(ohdr->bth[0]) >> 23) & 1;
477	__entry->m =
478	(be32_to_cpu(ohdr->bth[0]) >> 22) & 1;
479	__entry->pad =
480	(be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
481	__entry->tver =
482	(be32_to_cpu(ohdr->bth[0]) >> 16) & 0xf;
483	__entry->pkey =
484	be32_to_cpu(ohdr->bth[0]) & 0xffff;
485	__entry->f =
486	(be32_to_cpu(ohdr->bth[1]) >> HFI1_FECN_SHIFT) &
487	HFI1_FECN_MASK;
488	__entry->b =
489	(be32_to_cpu(ohdr->bth[1]) >> HFI1_BECN_SHIFT) &
490	HFI1_BECN_MASK;
491	__entry->qpn =
492	be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
493	__entry->a =
494	(be32_to_cpu(ohdr->bth[2]) >> 31) & 1;
495	/* allow for larger PSN */
496	__entry->psn =
497	be32_to_cpu(ohdr->bth[2]) & 0x7fffffff;
498	/* extended headers */
499	memcpy(__get_dynamic_array(ehdrs), &ohdr->u,
500	ibhdr_exhdr_len(hdr));
501	),
502	TP_printk("[%s] " LRH_PRN " " BTH_PRN " " EHDR_PRN,
503	__get_str(dev),
504	/* LRH */
505	__entry->vl,
506	__entry->lver,
507	__entry->sl,
508	__entry->lnh, show_lnh(__entry->lnh),
509	__entry->dlid,
510	__entry->len,
511	__entry->slid,
512	/* BTH */
513	__entry->opcode, show_ib_opcode(__entry->opcode),
514	__entry->se,
515	__entry->m,
516	__entry->pad,
517	__entry->tver,
518	__entry->pkey,
519	__entry->f,
520	__entry->b,
521	__entry->qpn,
522	__entry->a,
523	__entry->psn,
524	/* extended headers */
525	__parse_ib_ehdrs(
526	__entry->opcode,
527	(void *)__get_dynamic_array(ehdrs))
528	)
529	);
530
531	DEFINE_EVENT(hfi1_ibhdr_template, input_ibhdr,
532	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
533	TP_ARGS(dd, hdr));
534
535	DEFINE_EVENT(hfi1_ibhdr_template, pio_output_ibhdr,
536	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
537	TP_ARGS(dd, hdr));
538
539	DEFINE_EVENT(hfi1_ibhdr_template, ack_output_ibhdr,
540	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
541	TP_ARGS(dd, hdr));
542
543	DEFINE_EVENT(hfi1_ibhdr_template, sdma_output_ibhdr,
544	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
545	TP_ARGS(dd, hdr));
546
547	#define SNOOP_PRN \
548	"slid %.4x dlid %.4x qpn 0x%.6x opcode 0x%.2x,%s " \
549	"svc lvl %d pkey 0x%.4x [header = %d bytes] [data = %d bytes]"
550
551	#undef TRACE_SYSTEM
552	#define TRACE_SYSTEM hfi1_snoop
553
554	TRACE_EVENT(snoop_capture,
555	TP_PROTO(struct hfi1_devdata *dd,
556	int hdr_len,
557	struct hfi1_ib_header *hdr,
558	int data_len,
559	void *data),
560	TP_ARGS(dd, hdr_len, hdr, data_len, data),
561	TP_STRUCT__entry(
562	DD_DEV_ENTRY(dd)
563	__field(u16, slid)
564	__field(u16, dlid)
565	__field(u32, qpn)
566	__field(u8, opcode)
567	__field(u8, sl)
568	__field(u16, pkey)
569	__field(u32, hdr_len)
570	__field(u32, data_len)
571	__field(u8, lnh)
572	__dynamic_array(u8, raw_hdr, hdr_len)
573	__dynamic_array(u8, raw_pkt, data_len)
574	),
575	TP_fast_assign(
576	struct hfi1_other_headers *ohdr;
577
578	__entry->lnh = (u8)(be16_to_cpu(hdr->lrh[0]) & 3);
579	if (__entry->lnh == HFI1_LRH_BTH)
580	ohdr = &hdr->u.oth;
581	else
582	ohdr = &hdr->u.l.oth;
583	DD_DEV_ASSIGN(dd);
584	__entry->slid = be16_to_cpu(hdr->lrh[3]);
585	__entry->dlid = be16_to_cpu(hdr->lrh[1]);
586	__entry->qpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
587	__entry->opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
588	__entry->sl = (u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
589	__entry->pkey = be32_to_cpu(ohdr->bth[0]) & 0xffff;
590	__entry->hdr_len = hdr_len;
591	__entry->data_len = data_len;
592	memcpy(__get_dynamic_array(raw_hdr), hdr, hdr_len);
593	memcpy(__get_dynamic_array(raw_pkt), data, data_len);
594	),
595	TP_printk(
596	"[%s] " SNOOP_PRN,
597	__get_str(dev),
598	__entry->slid,
599	__entry->dlid,
600	__entry->qpn,
601	__entry->opcode,
602	show_ib_opcode(__entry->opcode),
603	__entry->sl,
604	__entry->pkey,
605	__entry->hdr_len,
606	__entry->data_len
607	)
608	);
609
610	#undef TRACE_SYSTEM
611	#define TRACE_SYSTEM hfi1_ctxts
612
613	#define UCTXT_FMT \
614	"cred:%u, credaddr:0x%llx, piobase:0x%llx, rcvhdr_cnt:%u, " \
615	"rcvbase:0x%llx, rcvegrc:%u, rcvegrb:0x%llx"
616	TRACE_EVENT(hfi1_uctxtdata,
617	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ctxtdata uctxt),
618	TP_ARGS(dd, uctxt),
619	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
620	__field(unsigned, ctxt)
621	__field(u32, credits)
622	__field(u64, hw_free)
623	__field(u64, piobase)
624	__field(u16, rcvhdrq_cnt)
625	__field(u64, rcvhdrq_phys)
626	__field(u32, eager_cnt)
627	__field(u64, rcvegr_phys)
628	),
629	TP_fast_assign(DD_DEV_ASSIGN(dd);
630	__entry->ctxt = uctxt->ctxt;
631	__entry->credits = uctxt->sc->credits;
632	__entry->hw_free = (u64)uctxt->sc->hw_free;
633	__entry->piobase = (u64)uctxt->sc->base_addr;
634	__entry->rcvhdrq_cnt = uctxt->rcvhdrq_cnt;
635	__entry->rcvhdrq_phys = uctxt->rcvhdrq_phys;
636	__entry->eager_cnt = uctxt->egrbufs.alloced;
637	__entry->rcvegr_phys =
638	uctxt->egrbufs.rcvtids[0].phys;
639	),
640	TP_printk("[%s] ctxt %u " UCTXT_FMT,
641	__get_str(dev),
642	__entry->ctxt,
643	__entry->credits,
644	__entry->hw_free,
645	__entry->piobase,
646	__entry->rcvhdrq_cnt,
647	__entry->rcvhdrq_phys,
648	__entry->eager_cnt,
649	__entry->rcvegr_phys
650	)
651	);
652
653	#define CINFO_FMT \
654	"egrtids:%u, egr_size:%u, hdrq_cnt:%u, hdrq_size:%u, sdma_ring_size:%u"
655	TRACE_EVENT(hfi1_ctxt_info,
656	TP_PROTO(struct hfi1_devdata *dd, unsigned ctxt, unsigned subctxt,
657	struct hfi1_ctxt_info cinfo),
658	TP_ARGS(dd, ctxt, subctxt, cinfo),
659	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
660	__field(unsigned, ctxt)
661	__field(unsigned, subctxt)
662	__field(u16, egrtids)
663	__field(u16, rcvhdrq_cnt)
664	__field(u16, rcvhdrq_size)
665	__field(u16, sdma_ring_size)
666	__field(u32, rcvegr_size)
667	),
668	TP_fast_assign(DD_DEV_ASSIGN(dd);
669	__entry->ctxt = ctxt;
670	__entry->subctxt = subctxt;
671	__entry->egrtids = cinfo.egrtids;
672	__entry->rcvhdrq_cnt = cinfo.rcvhdrq_cnt;
673	__entry->rcvhdrq_size = cinfo.rcvhdrq_entsize;
674	__entry->sdma_ring_size = cinfo.sdma_ring_size;
675	__entry->rcvegr_size = cinfo.rcvegr_size;
676	),
677	TP_printk("[%s] ctxt %u:%u " CINFO_FMT,
678	__get_str(dev),
679	__entry->ctxt,
680	__entry->subctxt,
681	__entry->egrtids,
682	__entry->rcvegr_size,
683	__entry->rcvhdrq_cnt,
684	__entry->rcvhdrq_size,
685	__entry->sdma_ring_size
686	)
687	);
688
689	#undef TRACE_SYSTEM
690	#define TRACE_SYSTEM hfi1_sma
691
692	#define BCT_FORMAT \
693	"shared_limit %x vls 0-7 [%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x] 15 [%x,%x]"
694
695	#define BCT(field) \
696	be16_to_cpu( \
697	((struct buffer_control *)__get_dynamic_array(bct))->field \
698	)
699
700	DECLARE_EVENT_CLASS(hfi1_bct_template,
701	TP_PROTO(struct hfi1_devdata *dd,
702	struct buffer_control *bc),
703	TP_ARGS(dd, bc),
704	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
705	__dynamic_array(u8, bct, sizeof(*bc))
706	),
707	TP_fast_assign(DD_DEV_ASSIGN(dd);
708	memcpy(__get_dynamic_array(bct), bc,
709	sizeof(*bc));
710	),
711	TP_printk(BCT_FORMAT,
712	BCT(overall_shared_limit),
713
714	BCT(vl[0].dedicated),
715	BCT(vl[0].shared),
716
717	BCT(vl[1].dedicated),
718	BCT(vl[1].shared),
719
720	BCT(vl[2].dedicated),
721	BCT(vl[2].shared),
722
723	BCT(vl[3].dedicated),
724	BCT(vl[3].shared),
725
726	BCT(vl[4].dedicated),
727	BCT(vl[4].shared),
728
729	BCT(vl[5].dedicated),
730	BCT(vl[5].shared),
731
732	BCT(vl[6].dedicated),
733	BCT(vl[6].shared),
734
735	BCT(vl[7].dedicated),
736	BCT(vl[7].shared),
737
738	BCT(vl[15].dedicated),
739	BCT(vl[15].shared)
740	)
741	);
742
743	DEFINE_EVENT(hfi1_bct_template, bct_set,
744	TP_PROTO(struct hfi1_devdata dd, struct buffer_control bc),
745	TP_ARGS(dd, bc));
746
747	DEFINE_EVENT(hfi1_bct_template, bct_get,
748	TP_PROTO(struct hfi1_devdata dd, struct buffer_control bc),
749	TP_ARGS(dd, bc));
750
751	#undef TRACE_SYSTEM
752	#define TRACE_SYSTEM hfi1_sdma
753
754	TRACE_EVENT(hfi1_sdma_descriptor,
755	TP_PROTO(struct sdma_engine *sde,
756	u64 desc0,
757	u64 desc1,
758	u16 e,
759	void *descp),
760	TP_ARGS(sde, desc0, desc1, e, descp),
761	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
762	__field(void *, descp)
763	__field(u64, desc0)
764	__field(u64, desc1)
765	__field(u16, e)
766	__field(u8, idx)
767	),
768	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
769	__entry->desc0 = desc0;
770	__entry->desc1 = desc1;
771	__entry->idx = sde->this_idx;
772	__entry->descp = descp;
773	__entry->e = e;
774	),
775	TP_printk(
776	"[%s] SDE(%u) flags:%s addr:0x%016llx gen:%u len:%u d0:%016llx d1:%016llx to %p,%u",
777	__get_str(dev),
778	__entry->idx,
779	__parse_sdma_flags(__entry->desc0, __entry->desc1),
780	(__entry->desc0 >> SDMA_DESC0_PHY_ADDR_SHIFT) &
781	SDMA_DESC0_PHY_ADDR_MASK,
782	(u8)((__entry->desc1 >> SDMA_DESC1_GENERATION_SHIFT) &
783	SDMA_DESC1_GENERATION_MASK),
784	(u16)((__entry->desc0 >> SDMA_DESC0_BYTE_COUNT_SHIFT) &
785	SDMA_DESC0_BYTE_COUNT_MASK),
786	__entry->desc0,
787	__entry->desc1,
788	__entry->descp,
789	__entry->e
790	)
791	);
792
793	TRACE_EVENT(hfi1_sdma_engine_select,
794	TP_PROTO(struct hfi1_devdata *dd, u32 sel, u8 vl, u8 idx),
795	TP_ARGS(dd, sel, vl, idx),
796	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
797	__field(u32, sel)
798	__field(u8, vl)
799	__field(u8, idx)
800	),
801	TP_fast_assign(DD_DEV_ASSIGN(dd);
802	__entry->sel = sel;
803	__entry->vl = vl;
804	__entry->idx = idx;
805	),
806	TP_printk("[%s] selecting SDE %u sel 0x%x vl %u",
807	__get_str(dev),
808	__entry->idx,
809	__entry->sel,
810	__entry->vl
811	)
812	);
813
814	DECLARE_EVENT_CLASS(hfi1_sdma_engine_class,
815	TP_PROTO(struct sdma_engine *sde, u64 status),
816	TP_ARGS(sde, status),
817	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
818	__field(u64, status)
819	__field(u8, idx)
820	),
821	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
822	__entry->status = status;
823	__entry->idx = sde->this_idx;
824	),
825	TP_printk("[%s] SDE(%u) status %llx",
826	__get_str(dev),
827	__entry->idx,
828	(unsigned long long)__entry->status
829	)
830	);
831
832	DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_interrupt,
833	TP_PROTO(struct sdma_engine *sde, u64 status),
834	TP_ARGS(sde, status)
835	);
836
837	DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_progress,
838	TP_PROTO(struct sdma_engine *sde, u64 status),
839	TP_ARGS(sde, status)
840	);
841
842	DECLARE_EVENT_CLASS(hfi1_sdma_ahg_ad,
843	TP_PROTO(struct sdma_engine *sde, int aidx),
844	TP_ARGS(sde, aidx),
845	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
846	__field(int, aidx)
847	__field(u8, idx)
848	),
849	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
850	__entry->idx = sde->this_idx;
851	__entry->aidx = aidx;
852	),
853	TP_printk("[%s] SDE(%u) aidx %d",
854	__get_str(dev),
855	__entry->idx,
856	__entry->aidx
857	)
858	);
859
860	DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_allocate,
861	TP_PROTO(struct sdma_engine *sde, int aidx),
862	TP_ARGS(sde, aidx));
863
864	DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_deallocate,
865	TP_PROTO(struct sdma_engine *sde, int aidx),
866	TP_ARGS(sde, aidx));
867
868	#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
869	TRACE_EVENT(hfi1_sdma_progress,
870	TP_PROTO(struct sdma_engine *sde,
871	u16 hwhead,
872	u16 swhead,
873	struct sdma_txreq *txp
874	),
875	TP_ARGS(sde, hwhead, swhead, txp),
876	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
877	__field(u64, sn)
878	__field(u16, hwhead)
879	__field(u16, swhead)
880	__field(u16, txnext)
881	__field(u16, tx_tail)
882	__field(u16, tx_head)
883	__field(u8, idx)
884	),
885	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
886	__entry->hwhead = hwhead;
887	__entry->swhead = swhead;
888	__entry->tx_tail = sde->tx_tail;
889	__entry->tx_head = sde->tx_head;
890	__entry->txnext = txp ? txp->next_descq_idx : ~0;
891	__entry->idx = sde->this_idx;
892	__entry->sn = txp ? txp->sn : ~0;
893	),
894	TP_printk(
895	"[%s] SDE(%u) sn %llu hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
896	__get_str(dev),
897	__entry->idx,
898	__entry->sn,
899	__entry->hwhead,
900	__entry->swhead,
901	__entry->txnext,
902	__entry->tx_head,
903	__entry->tx_tail
904	)
905	);
906	#else
907	TRACE_EVENT(hfi1_sdma_progress,
908	TP_PROTO(struct sdma_engine *sde,
909	u16 hwhead, u16 swhead,
910	struct sdma_txreq *txp
911	),
912	TP_ARGS(sde, hwhead, swhead, txp),
913	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
914	__field(u16, hwhead)
915	__field(u16, swhead)
916	__field(u16, txnext)
917	__field(u16, tx_tail)
918	__field(u16, tx_head)
919	__field(u8, idx)
920	),
921	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
922	__entry->hwhead = hwhead;
923	__entry->swhead = swhead;
924	__entry->tx_tail = sde->tx_tail;
925	__entry->tx_head = sde->tx_head;
926	__entry->txnext = txp ? txp->next_descq_idx : ~0;
927	__entry->idx = sde->this_idx;
928	),
929	TP_printk(
930	"[%s] SDE(%u) hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
931	__get_str(dev),
932	__entry->idx,
933	__entry->hwhead,
934	__entry->swhead,
935	__entry->txnext,
936	__entry->tx_head,
937	__entry->tx_tail
938	)
939	);
940	#endif
941
942	DECLARE_EVENT_CLASS(hfi1_sdma_sn,
943	TP_PROTO(struct sdma_engine *sde, u64 sn),
944	TP_ARGS(sde, sn),
945	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
946	__field(u64, sn)
947	__field(u8, idx)
948	),
949	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
950	__entry->sn = sn;
951	__entry->idx = sde->this_idx;
952	),
953	TP_printk("[%s] SDE(%u) sn %llu",
954	__get_str(dev),
955	__entry->idx,
956	__entry->sn
957	)
958	);
959
960	DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_out_sn,
961	TP_PROTO(
962	struct sdma_engine *sde,
963	u64 sn
964	),
965	TP_ARGS(sde, sn)
966	);
967
968	DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_in_sn,
969	TP_PROTO(struct sdma_engine *sde, u64 sn),
970	TP_ARGS(sde, sn)
971	);
972
973	#define USDMA_HDR_FORMAT \
974	"[%s:%u:%u:%u] PBC=(0x%x 0x%x) LRH=(0x%x 0x%x) BTH=(0x%x 0x%x 0x%x) KDETH=(0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x) TIDVal=0x%x"
975
976	TRACE_EVENT(hfi1_sdma_user_header,
977	TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
978	struct hfi1_pkt_header *hdr, u32 tidval),
979	TP_ARGS(dd, ctxt, subctxt, req, hdr, tidval),
980	TP_STRUCT__entry(
981	DD_DEV_ENTRY(dd)
982	__field(u16, ctxt)
983	__field(u8, subctxt)
984	__field(u16, req)
985	__field(__le32, pbc0)
986	__field(__le32, pbc1)
987	__field(__be32, lrh0)
988	__field(__be32, lrh1)
989	__field(__be32, bth0)
990	__field(__be32, bth1)
991	__field(__be32, bth2)
992	__field(__le32, kdeth0)
993	__field(__le32, kdeth1)
994	__field(__le32, kdeth2)
995	__field(__le32, kdeth3)
996	__field(__le32, kdeth4)
997	__field(__le32, kdeth5)
998	__field(__le32, kdeth6)
999	__field(__le32, kdeth7)
1000	__field(__le32, kdeth8)
1001	__field(u32, tidval)
1002	),
1003	TP_fast_assign(
1004	__le32 pbc = (__le32 )hdr->pbc;
1005	__be32 lrh = (__be32 )hdr->lrh;
1006	__be32 bth = (__be32 )hdr->bth;
1007	__le32 kdeth = (__le32 )&hdr->kdeth;
1008
1009	DD_DEV_ASSIGN(dd);
1010	__entry->ctxt = ctxt;
1011	__entry->subctxt = subctxt;
1012	__entry->req = req;
1013	__entry->pbc0 = pbc[0];
1014	__entry->pbc1 = pbc[1];
1015	__entry->lrh0 = be32_to_cpu(lrh[0]);
1016	__entry->lrh1 = be32_to_cpu(lrh[1]);
1017	__entry->bth0 = be32_to_cpu(bth[0]);
1018	__entry->bth1 = be32_to_cpu(bth[1]);
1019	__entry->bth2 = be32_to_cpu(bth[2]);
1020	__entry->kdeth0 = kdeth[0];
1021	__entry->kdeth1 = kdeth[1];
1022	__entry->kdeth2 = kdeth[2];
1023	__entry->kdeth3 = kdeth[3];
1024	__entry->kdeth4 = kdeth[4];
1025	__entry->kdeth5 = kdeth[5];
1026	__entry->kdeth6 = kdeth[6];
1027	__entry->kdeth7 = kdeth[7];
1028	__entry->kdeth8 = kdeth[8];
1029	__entry->tidval = tidval;
1030	),
1031	TP_printk(USDMA_HDR_FORMAT,
1032	__get_str(dev),
1033	__entry->ctxt,
1034	__entry->subctxt,
1035	__entry->req,
1036	__entry->pbc1,
1037	__entry->pbc0,
1038	__entry->lrh0,
1039	__entry->lrh1,
1040	__entry->bth0,
1041	__entry->bth1,
1042	__entry->bth2,
1043	__entry->kdeth0,
1044	__entry->kdeth1,
1045	__entry->kdeth2,
1046	__entry->kdeth3,
1047	__entry->kdeth4,
1048	__entry->kdeth5,
1049	__entry->kdeth6,
1050	__entry->kdeth7,
1051	__entry->kdeth8,
1052	__entry->tidval
1053	)
1054	);
1055
1056	#define SDMA_UREQ_FMT \
1057	"[%s:%u:%u] ver/op=0x%x, iovcnt=%u, npkts=%u, frag=%u, idx=%u"
1058	TRACE_EVENT(hfi1_sdma_user_reqinfo,
1059	TP_PROTO(struct hfi1_devdata dd, u16 ctxt, u8 subctxt, u16 i),
1060	TP_ARGS(dd, ctxt, subctxt, i),
1061	TP_STRUCT__entry(
1062	DD_DEV_ENTRY(dd);
1063	__field(u16, ctxt)
1064	__field(u8, subctxt)
1065	__field(u8, ver_opcode)
1066	__field(u8, iovcnt)
1067	__field(u16, npkts)
1068	__field(u16, fragsize)
1069	__field(u16, comp_idx)
1070	),
1071	TP_fast_assign(
1072	DD_DEV_ASSIGN(dd);
1073	__entry->ctxt = ctxt;
1074	__entry->subctxt = subctxt;
1075	__entry->ver_opcode = i[0] & 0xff;
1076	__entry->iovcnt = (i[0] >> 8) & 0xff;
1077	__entry->npkts = i[1];
1078	__entry->fragsize = i[2];
1079	__entry->comp_idx = i[3];
1080	),
1081	TP_printk(SDMA_UREQ_FMT,
1082	__get_str(dev),
1083	__entry->ctxt,
1084	__entry->subctxt,
1085	__entry->ver_opcode,
1086	__entry->iovcnt,
1087	__entry->npkts,
1088	__entry->fragsize,
1089	__entry->comp_idx
1090	)
1091	);
1092
1093	#define usdma_complete_name(st) { st, #st }
1094	#define show_usdma_complete_state(st) \
1095	__print_symbolic(st, \
1096	usdma_complete_name(FREE), \
1097	usdma_complete_name(QUEUED), \
1098	usdma_complete_name(COMPLETE), \
1099	usdma_complete_name(ERROR))
1100
1101	TRACE_EVENT(hfi1_sdma_user_completion,
1102	TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 idx,
1103	u8 state, int code),
1104	TP_ARGS(dd, ctxt, subctxt, idx, state, code),
1105	TP_STRUCT__entry(
1106	DD_DEV_ENTRY(dd)
1107	__field(u16, ctxt)
1108	__field(u8, subctxt)
1109	__field(u16, idx)
1110	__field(u8, state)
1111	__field(int, code)
1112	),
1113	TP_fast_assign(
1114	DD_DEV_ASSIGN(dd);
1115	__entry->ctxt = ctxt;
1116	__entry->subctxt = subctxt;
1117	__entry->idx = idx;
1118	__entry->state = state;
1119	__entry->code = code;
1120	),
1121	TP_printk("[%s:%u:%u:%u] SDMA completion state %s (%d)",
1122	__get_str(dev), __entry->ctxt, __entry->subctxt,
1123	__entry->idx, show_usdma_complete_state(__entry->state),
1124	__entry->code)
1125	);
1126
1127	const char print_u32_array(struct trace_seq , u32 *, int);
1128	#define __print_u32_hex(arr, len) print_u32_array(p, arr, len)
1129
1130	TRACE_EVENT(hfi1_sdma_user_header_ahg,
1131	TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
1132	u8 sde, u8 ahgidx, u32 *ahg, int len, u32 tidval),
1133	TP_ARGS(dd, ctxt, subctxt, req, sde, ahgidx, ahg, len, tidval),
1134	TP_STRUCT__entry(
1135	DD_DEV_ENTRY(dd)
1136	__field(u16, ctxt)
1137	__field(u8, subctxt)
1138	__field(u16, req)
1139	__field(u8, sde)
1140	__field(u8, idx)
1141	__field(int, len)
1142	__field(u32, tidval)
1143	__array(u32, ahg, 10)
1144	),
1145	TP_fast_assign(
1146	DD_DEV_ASSIGN(dd);
1147	__entry->ctxt = ctxt;
1148	__entry->subctxt = subctxt;
1149	__entry->req = req;
1150	__entry->sde = sde;
1151	__entry->idx = ahgidx;
1152	__entry->len = len;
1153	__entry->tidval = tidval;
1154	memcpy(__entry->ahg, ahg, len * sizeof(u32));
1155	),
1156	TP_printk("[%s:%u:%u:%u] (SDE%u/AHG%u) ahg[0-%d]=(%s) TIDVal=0x%x",
1157	__get_str(dev),
1158	__entry->ctxt,
1159	__entry->subctxt,
1160	__entry->req,
1161	__entry->sde,
1162	__entry->idx,
1163	__entry->len - 1,
1164	__print_u32_hex(__entry->ahg, __entry->len),
1165	__entry->tidval
1166	)
1167	);
1168
1169	TRACE_EVENT(hfi1_sdma_state,
1170	TP_PROTO(struct sdma_engine *sde,
1171	const char *cstate,
1172	const char *nstate
1173	),
1174	TP_ARGS(sde, cstate, nstate),
1175	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
1176	__string(curstate, cstate)
1177	__string(newstate, nstate)
1178	),
1179	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
1180	__assign_str(curstate, cstate);
1181	__assign_str(newstate, nstate);
1182	),
1183	TP_printk("[%s] current state %s new state %s",
1184	__get_str(dev),
1185	__get_str(curstate),
1186	__get_str(newstate)
1187	)
1188	);
1189
1190	#undef TRACE_SYSTEM
1191	#define TRACE_SYSTEM hfi1_rc
1192
1193	DECLARE_EVENT_CLASS(hfi1_rc_template,
1194	TP_PROTO(struct rvt_qp *qp, u32 psn),
1195	TP_ARGS(qp, psn),
1196	TP_STRUCT__entry(
1197	DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
1198	__field(u32, qpn)
1199	__field(u32, s_flags)
1200	__field(u32, psn)
1201	__field(u32, s_psn)
1202	__field(u32, s_next_psn)
1203	__field(u32, s_sending_psn)
1204	__field(u32, s_sending_hpsn)
1205	__field(u32, r_psn)
1206	),
1207	TP_fast_assign(
1208	DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device))
1209	__entry->qpn = qp->ibqp.qp_num;
1210	__entry->s_flags = qp->s_flags;
1211	__entry->psn = psn;
1212	__entry->s_psn = qp->s_psn;
1213	__entry->s_next_psn = qp->s_next_psn;
1214	__entry->s_sending_psn = qp->s_sending_psn;
1215	__entry->s_sending_hpsn = qp->s_sending_hpsn;
1216	__entry->r_psn = qp->r_psn;
1217	),
1218	TP_printk(
1219	"[%s] qpn 0x%x s_flags 0x%x psn 0x%x s_psn 0x%x s_next_psn 0x%x s_sending_psn 0x%x sending_hpsn 0x%x r_psn 0x%x",
1220	__get_str(dev),
1221	__entry->qpn,
1222	__entry->s_flags,
1223	__entry->psn,
1224	__entry->s_psn,
1225	__entry->s_next_psn,
1226	__entry->s_sending_psn,
1227	__entry->s_sending_hpsn,
1228	__entry->r_psn
1229	)
1230	);
1231
1232	DEFINE_EVENT(hfi1_rc_template, hfi1_rc_sendcomplete,
1233	TP_PROTO(struct rvt_qp *qp, u32 psn),
1234	TP_ARGS(qp, psn)
1235	);
1236
1237	DEFINE_EVENT(hfi1_rc_template, hfi1_rc_ack,
1238	TP_PROTO(struct rvt_qp *qp, u32 psn),
1239	TP_ARGS(qp, psn)
1240	);
1241
1242	DEFINE_EVENT(hfi1_rc_template, hfi1_rc_timeout,
1243	TP_PROTO(struct rvt_qp *qp, u32 psn),
1244	TP_ARGS(qp, psn)
1245	);
1246
1247	DEFINE_EVENT(hfi1_rc_template, hfi1_rc_rcv_error,
1248	TP_PROTO(struct rvt_qp *qp, u32 psn),
1249	TP_ARGS(qp, psn)
1250	);
1251
1252	#undef TRACE_SYSTEM
1253	#define TRACE_SYSTEM hfi1_misc
1254
1255	TRACE_EVENT(hfi1_interrupt,
1256	TP_PROTO(struct hfi1_devdata dd, const struct is_table is_entry,
1257	int src),
1258	TP_ARGS(dd, is_entry, src),
1259	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
1260	__array(char, buf, 64)
1261	__field(int, src)
1262	),
1263	TP_fast_assign(DD_DEV_ASSIGN(dd)
1264	is_entry->is_name(__entry->buf, 64,
1265	src - is_entry->start);
1266	__entry->src = src;
1267	),
1268	TP_printk("[%s] source: %s [%d]", __get_str(dev), __entry->buf,
1269	__entry->src)
1270	);
1271
1272	/*
1273	* Note:
1274	* This produces a REALLY ugly trace in the console output when the string is
1275	* too long.
1276	*/
1277
1278	#undef TRACE_SYSTEM
1279	#define TRACE_SYSTEM hfi1_trace
1280
1281	#define MAX_MSG_LEN 512
1282
1283	DECLARE_EVENT_CLASS(hfi1_trace_template,
1284	TP_PROTO(const char function, struct va_format vaf),
1285	TP_ARGS(function, vaf),
1286	TP_STRUCT__entry(__string(function, function)
1287	__dynamic_array(char, msg, MAX_MSG_LEN)
1288	),
1289	TP_fast_assign(__assign_str(function, function);
1290	WARN_ON_ONCE(vsnprintf
1291	(__get_dynamic_array(msg),
1292	MAX_MSG_LEN, vaf->fmt,
1293	*vaf->va) >=
1294	MAX_MSG_LEN);
1295	),
1296	TP_printk("(%s) %s",
1297	__get_str(function),
1298	__get_str(msg))
1299	);
1300
1301	/*
1302	* It may be nice to macroize the __hfi1_trace but the va_* stuff requires an
1303	* actual function to work and can not be in a macro.
1304	*/
1305	#define __hfi1_trace_def(lvl) \
1306	void __hfi1_trace_##lvl(const char funct, char fmt, ...); \
1307	\
1308	DEFINE_EVENT(hfi1_trace_template, hfi1_ ##lvl, \
1309	TP_PROTO(const char function, struct va_format vaf), \
1310	TP_ARGS(function, vaf))
1311
1312	#define __hfi1_trace_fn(lvl) \
1313	void __hfi1_trace_##lvl(const char func, char fmt, ...) \
1314	{ \
1315	struct va_format vaf = { \
1316	.fmt = fmt, \
1317	}; \
1318	va_list args; \
1319	\
1320	va_start(args, fmt); \
1321	vaf.va = &args; \
1322	trace_hfi1_ ##lvl(func, &vaf); \
1323	va_end(args); \
1324	return; \
1325	}
1326
1327	/*
1328	* To create a new trace level simply define it below and as a __hfi1_trace_fn
1329	* in trace.c. This will create all the hooks for calling
1330	* hfi1_cdbg(LVL, fmt, ...); as well as take care of all
1331	* the debugfs stuff.
1332	*/
1333	__hfi1_trace_def(PKT);
1334	__hfi1_trace_def(PROC);
1335	__hfi1_trace_def(SDMA);
1336	__hfi1_trace_def(LINKVERB);
1337	__hfi1_trace_def(DEBUG);
1338	__hfi1_trace_def(SNOOP);
1339	__hfi1_trace_def(CNTR);
1340	__hfi1_trace_def(PIO);
1341	__hfi1_trace_def(DC8051);
1342	__hfi1_trace_def(FIRMWARE);
1343	__hfi1_trace_def(RCVCTRL);
1344	__hfi1_trace_def(TID);
1345	__hfi1_trace_def(MMU);
1346	__hfi1_trace_def(IOCTL);
1347
1348	#define hfi1_cdbg(which, fmt, ...) \
1349	__hfi1_trace_##which(__func__, fmt, ##__VA_ARGS__)
1350
1351	#define hfi1_dbg(fmt, ...) \
1352	hfi1_cdbg(DEBUG, fmt, ##__VA_ARGS__)
1353
1354	/*
1355	* Define HFI1_EARLY_DBG at compile time or here to enable early trace
1356	* messages. Do not check in an enablement for this.
1357	*/
1358
1359	#ifdef HFI1_EARLY_DBG
1360	#define hfi1_dbg_early(fmt, ...) \
1361	trace_printk(fmt, ##__VA_ARGS__)
1362	#else
1363	#define hfi1_dbg_early(fmt, ...)
1364	#endif
1365
1366	#endif /* __HFI1_TRACE_H */
1367
1368	#undef TRACE_INCLUDE_PATH
1369	#undef TRACE_INCLUDE_FILE
1370	#define TRACE_INCLUDE_PATH .
1371	#define TRACE_INCLUDE_FILE trace
1372	#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_ctxts.h+141−0

@@ -0,0 +1,141 @@
	1	/*
	2	* Copyright(c) 2015, 2016 Intel Corporation.
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of version 2 of the GNU General Public License as
	11	* published by the Free Software Foundation.
	12	*
	13	* This program is distributed in the hope that it will be useful, but
	14	* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* General Public License for more details.
	17	*
	18	* BSD LICENSE
	19	*
	20	* Redistribution and use in source and binary forms, with or without
	21	* modification, are permitted provided that the following conditions
	22	* are met:
	23	*
	24	* - Redistributions of source code must retain the above copyright
	25	* notice, this list of conditions and the following disclaimer.
	26	* - Redistributions in binary form must reproduce the above copyright
	27	* notice, this list of conditions and the following disclaimer in
	28	* the documentation and/or other materials provided with the
	29	* distribution.
	30	* - Neither the name of Intel Corporation nor the names of its
	31	* contributors may be used to endorse or promote products derived
	32	* from this software without specific prior written permission.
	33	*
	34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	*
	46	*/
	47	#if !defined(__HFI1_TRACE_CTXTS_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	#define __HFI1_TRACE_CTXTS_H
	49
	50	#include <linux/tracepoint.h>
	51	#include <linux/trace_seq.h>
	52
	53	#include "hfi.h"
	54
	55	#undef TRACE_SYSTEM
	56	#define TRACE_SYSTEM hfi1_ctxts
	57
	58	#define UCTXT_FMT \
	59	"cred:%u, credaddr:0x%llx, piobase:0x%p, rcvhdr_cnt:%u, " \
	60	"rcvbase:0x%llx, rcvegrc:%u, rcvegrb:0x%llx"
	61	TRACE_EVENT(hfi1_uctxtdata,
	62	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ctxtdata uctxt),
	63	TP_ARGS(dd, uctxt),
	64	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	65	__field(unsigned int, ctxt)
	66	__field(u32, credits)
	67	__field(u64, hw_free)
	68	__field(void __iomem *, piobase)
	69	__field(u16, rcvhdrq_cnt)
	70	__field(u64, rcvhdrq_phys)
	71	__field(u32, eager_cnt)
	72	__field(u64, rcvegr_phys)
	73	),
	74	TP_fast_assign(DD_DEV_ASSIGN(dd);
	75	__entry->ctxt = uctxt->ctxt;
	76	__entry->credits = uctxt->sc->credits;
	77	__entry->hw_free = le64_to_cpu(*uctxt->sc->hw_free);
	78	__entry->piobase = uctxt->sc->base_addr;
	79	__entry->rcvhdrq_cnt = uctxt->rcvhdrq_cnt;
	80	__entry->rcvhdrq_phys = uctxt->rcvhdrq_phys;
	81	__entry->eager_cnt = uctxt->egrbufs.alloced;
	82	__entry->rcvegr_phys =
	83	uctxt->egrbufs.rcvtids[0].phys;
	84	),
	85	TP_printk("[%s] ctxt %u " UCTXT_FMT,
	86	__get_str(dev),
	87	__entry->ctxt,
	88	__entry->credits,
	89	__entry->hw_free,
	90	__entry->piobase,
	91	__entry->rcvhdrq_cnt,
	92	__entry->rcvhdrq_phys,
	93	__entry->eager_cnt,
	94	__entry->rcvegr_phys
	95	)
	96	);
	97
	98	#define CINFO_FMT \
	99	"egrtids:%u, egr_size:%u, hdrq_cnt:%u, hdrq_size:%u, sdma_ring_size:%u"
	100	TRACE_EVENT(hfi1_ctxt_info,
	101	TP_PROTO(struct hfi1_devdata *dd, unsigned int ctxt,
	102	unsigned int subctxt,
	103	struct hfi1_ctxt_info cinfo),
	104	TP_ARGS(dd, ctxt, subctxt, cinfo),
	105	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	106	__field(unsigned int, ctxt)
	107	__field(unsigned int, subctxt)
	108	__field(u16, egrtids)
	109	__field(u16, rcvhdrq_cnt)
	110	__field(u16, rcvhdrq_size)
	111	__field(u16, sdma_ring_size)
	112	__field(u32, rcvegr_size)
	113	),
	114	TP_fast_assign(DD_DEV_ASSIGN(dd);
	115	__entry->ctxt = ctxt;
	116	__entry->subctxt = subctxt;
	117	__entry->egrtids = cinfo.egrtids;
	118	__entry->rcvhdrq_cnt = cinfo.rcvhdrq_cnt;
	119	__entry->rcvhdrq_size = cinfo.rcvhdrq_entsize;
	120	__entry->sdma_ring_size = cinfo.sdma_ring_size;
	121	__entry->rcvegr_size = cinfo.rcvegr_size;
	122	),
	123	TP_printk("[%s] ctxt %u:%u " CINFO_FMT,
	124	__get_str(dev),
	125	__entry->ctxt,
	126	__entry->subctxt,
	127	__entry->egrtids,
	128	__entry->rcvegr_size,
	129	__entry->rcvhdrq_cnt,
	130	__entry->rcvhdrq_size,
	131	__entry->sdma_ring_size
	132	)
	133	);
	134
	135	#endif /* __HFI1_TRACE_CTXTS_H */
	136
	137	#undef TRACE_INCLUDE_PATH
	138	#undef TRACE_INCLUDE_FILE
	139	#define TRACE_INCLUDE_PATH .
	140	#define TRACE_INCLUDE_FILE trace_ctxts
	141	#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_dbg.h+155−0

@@ -0,0 +1,155 @@
	1	/*
	2	* Copyright(c) 2015, 2016 Intel Corporation.
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of version 2 of the GNU General Public License as
	11	* published by the Free Software Foundation.
	12	*
	13	* This program is distributed in the hope that it will be useful, but
	14	* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* General Public License for more details.
	17	*
	18	* BSD LICENSE
	19	*
	20	* Redistribution and use in source and binary forms, with or without
	21	* modification, are permitted provided that the following conditions
	22	* are met:
	23	*
	24	* - Redistributions of source code must retain the above copyright
	25	* notice, this list of conditions and the following disclaimer.
	26	* - Redistributions in binary form must reproduce the above copyright
	27	* notice, this list of conditions and the following disclaimer in
	28	* the documentation and/or other materials provided with the
	29	* distribution.
	30	* - Neither the name of Intel Corporation nor the names of its
	31	* contributors may be used to endorse or promote products derived
	32	* from this software without specific prior written permission.
	33	*
	34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	*
	46	*/
	47	#if !defined(__HFI1_TRACE_EXTRA_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	#define __HFI1_TRACE_EXTRA_H
	49
	50	#include <linux/tracepoint.h>
	51	#include <linux/trace_seq.h>
	52
	53	#include "hfi.h"
	54
	55	/*
	56	* Note:
	57	* This produces a REALLY ugly trace in the console output when the string is
	58	* too long.
	59	*/
	60
	61	#undef TRACE_SYSTEM
	62	#define TRACE_SYSTEM hfi1_dbg
	63
	64	#define MAX_MSG_LEN 512
	65
	66	DECLARE_EVENT_CLASS(hfi1_trace_template,
	67	TP_PROTO(const char function, struct va_format vaf),
	68	TP_ARGS(function, vaf),
	69	TP_STRUCT__entry(__string(function, function)
	70	__dynamic_array(char, msg, MAX_MSG_LEN)
	71	),
	72	TP_fast_assign(__assign_str(function, function);
	73	WARN_ON_ONCE(vsnprintf
	74	(__get_dynamic_array(msg),
	75	MAX_MSG_LEN, vaf->fmt,
	76	*vaf->va) >=
	77	MAX_MSG_LEN);
	78	),
	79	TP_printk("(%s) %s",
	80	__get_str(function),
	81	__get_str(msg))
	82	);
	83
	84	/*
	85	* It may be nice to macroize the __hfi1_trace but the va_* stuff requires an
	86	* actual function to work and can not be in a macro.
	87	*/
	88	#define __hfi1_trace_def(lvl) \
	89	void __hfi1_trace_##lvl(const char funct, char fmt, ...); \
	90	\
	91	DEFINE_EVENT(hfi1_trace_template, hfi1_ ##lvl, \
	92	TP_PROTO(const char function, struct va_format vaf), \
	93	TP_ARGS(function, vaf))
	94
	95	#define __hfi1_trace_fn(lvl) \
	96	void __hfi1_trace_##lvl(const char func, char fmt, ...) \
	97	{ \
	98	struct va_format vaf = { \
	99	.fmt = fmt, \
	100	}; \
	101	va_list args; \
	102	\
	103	va_start(args, fmt); \
	104	vaf.va = &args; \
	105	trace_hfi1_ ##lvl(func, &vaf); \
	106	va_end(args); \
	107	return; \
	108	}
	109
	110	/*
	111	* To create a new trace level simply define it below and as a __hfi1_trace_fn
	112	* in trace.c. This will create all the hooks for calling
	113	* hfi1_cdbg(LVL, fmt, ...); as well as take care of all
	114	* the debugfs stuff.
	115	*/
	116	__hfi1_trace_def(PKT);
	117	__hfi1_trace_def(PROC);
	118	__hfi1_trace_def(SDMA);
	119	__hfi1_trace_def(LINKVERB);
	120	__hfi1_trace_def(DEBUG);
	121	__hfi1_trace_def(SNOOP);
	122	__hfi1_trace_def(CNTR);
	123	__hfi1_trace_def(PIO);
	124	__hfi1_trace_def(DC8051);
	125	__hfi1_trace_def(FIRMWARE);
	126	__hfi1_trace_def(RCVCTRL);
	127	__hfi1_trace_def(TID);
	128	__hfi1_trace_def(MMU);
	129	__hfi1_trace_def(IOCTL);
	130
	131	#define hfi1_cdbg(which, fmt, ...) \
	132	__hfi1_trace_##which(__func__, fmt, ##__VA_ARGS__)
	133
	134	#define hfi1_dbg(fmt, ...) \
	135	hfi1_cdbg(DEBUG, fmt, ##__VA_ARGS__)
	136
	137	/*
	138	* Define HFI1_EARLY_DBG at compile time or here to enable early trace
	139	* messages. Do not check in an enablement for this.
	140	*/
	141
	142	#ifdef HFI1_EARLY_DBG
	143	#define hfi1_dbg_early(fmt, ...) \
	144	trace_printk(fmt, ##__VA_ARGS__)
	145	#else
	146	#define hfi1_dbg_early(fmt, ...)
	147	#endif
	148
	149	#endif /* __HFI1_TRACE_EXTRA_H */
	150
	151	#undef TRACE_INCLUDE_PATH
	152	#undef TRACE_INCLUDE_FILE
	153	#define TRACE_INCLUDE_PATH .
	154	#define TRACE_INCLUDE_FILE trace_dbg
	155	#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_ibhdrs.h+209−0

@@ -0,0 +1,209 @@
	1	/*
	2	* Copyright(c) 2015, 2016 Intel Corporation.
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of version 2 of the GNU General Public License as
	11	* published by the Free Software Foundation.
	12	*
	13	* This program is distributed in the hope that it will be useful, but
	14	* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* General Public License for more details.
	17	*
	18	* BSD LICENSE
	19	*
	20	* Redistribution and use in source and binary forms, with or without
	21	* modification, are permitted provided that the following conditions
	22	* are met:
	23	*
	24	* - Redistributions of source code must retain the above copyright
	25	* notice, this list of conditions and the following disclaimer.
	26	* - Redistributions in binary form must reproduce the above copyright
	27	* notice, this list of conditions and the following disclaimer in
	28	* the documentation and/or other materials provided with the
	29	* distribution.
	30	* - Neither the name of Intel Corporation nor the names of its
	31	* contributors may be used to endorse or promote products derived
	32	* from this software without specific prior written permission.
	33	*
	34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	*
	46	*/
	47	#if !defined(__HFI1_TRACE_IBHDRS_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	#define __HFI1_TRACE_IBHDRS_H
	49
	50	#include <linux/tracepoint.h>
	51	#include <linux/trace_seq.h>
	52
	53	#include "hfi.h"
	54
	55	#undef TRACE_SYSTEM
	56	#define TRACE_SYSTEM hfi1_ibhdrs
	57
	58	u8 ibhdr_exhdr_len(struct hfi1_ib_header *hdr);
	59	const char parse_everbs_hdrs(struct trace_seq p, u8 opcode, void *ehdrs);
	60
	61	#define __parse_ib_ehdrs(op, ehdrs) parse_everbs_hdrs(p, op, ehdrs)
	62
	63	#define lrh_name(lrh) { HFI1_##lrh, #lrh }
	64	#define show_lnh(lrh) \
	65	__print_symbolic(lrh, \
	66	lrh_name(LRH_BTH), \
	67	lrh_name(LRH_GRH))
	68
	69	#define LRH_PRN "vl %d lver %d sl %d lnh %d,%s dlid %.4x len %d slid %.4x"
	70	#define BTH_PRN \
	71	"op 0x%.2x,%s se %d m %d pad %d tver %d pkey 0x%.4x " \
	72	"f %d b %d qpn 0x%.6x a %d psn 0x%.8x"
	73	#define EHDR_PRN "%s"
	74
	75	DECLARE_EVENT_CLASS(hfi1_ibhdr_template,
	76	TP_PROTO(struct hfi1_devdata *dd,
	77	struct hfi1_ib_header *hdr),
	78	TP_ARGS(dd, hdr),
	79	TP_STRUCT__entry(
	80	DD_DEV_ENTRY(dd)
	81	/* LRH */
	82	__field(u8, vl)
	83	__field(u8, lver)
	84	__field(u8, sl)
	85	__field(u8, lnh)
	86	__field(u16, dlid)
	87	__field(u16, len)
	88	__field(u16, slid)
	89	/* BTH */
	90	__field(u8, opcode)
	91	__field(u8, se)
	92	__field(u8, m)
	93	__field(u8, pad)
	94	__field(u8, tver)
	95	__field(u16, pkey)
	96	__field(u8, f)
	97	__field(u8, b)
	98	__field(u32, qpn)
	99	__field(u8, a)
	100	__field(u32, psn)
	101	/* extended headers */
	102	__dynamic_array(u8, ehdrs, ibhdr_exhdr_len(hdr))
	103	),
	104	TP_fast_assign(
	105	struct hfi1_other_headers *ohdr;
	106
	107	DD_DEV_ASSIGN(dd);
	108	/* LRH */
	109	__entry->vl =
	110	(u8)(be16_to_cpu(hdr->lrh[0]) >> 12);
	111	__entry->lver =
	112	(u8)(be16_to_cpu(hdr->lrh[0]) >> 8) & 0xf;
	113	__entry->sl =
	114	(u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
	115	__entry->lnh =
	116	(u8)(be16_to_cpu(hdr->lrh[0]) & 3);
	117	__entry->dlid =
	118	be16_to_cpu(hdr->lrh[1]);
	119	/* allow for larger len */
	120	__entry->len =
	121	be16_to_cpu(hdr->lrh[2]);
	122	__entry->slid =
	123	be16_to_cpu(hdr->lrh[3]);
	124	/* BTH */
	125	if (__entry->lnh == HFI1_LRH_BTH)
	126	ohdr = &hdr->u.oth;
	127	else
	128	ohdr = &hdr->u.l.oth;
	129	__entry->opcode =
	130	(be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
	131	__entry->se =
	132	(be32_to_cpu(ohdr->bth[0]) >> 23) & 1;
	133	__entry->m =
	134	(be32_to_cpu(ohdr->bth[0]) >> 22) & 1;
	135	__entry->pad =
	136	(be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
	137	__entry->tver =
	138	(be32_to_cpu(ohdr->bth[0]) >> 16) & 0xf;
	139	__entry->pkey =
	140	be32_to_cpu(ohdr->bth[0]) & 0xffff;
	141	__entry->f =
	142	(be32_to_cpu(ohdr->bth[1]) >> HFI1_FECN_SHIFT) &
	143	HFI1_FECN_MASK;
	144	__entry->b =
	145	(be32_to_cpu(ohdr->bth[1]) >> HFI1_BECN_SHIFT) &
	146	HFI1_BECN_MASK;
	147	__entry->qpn =
	148	be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
	149	__entry->a =
	150	(be32_to_cpu(ohdr->bth[2]) >> 31) & 1;
	151	/* allow for larger PSN */
	152	__entry->psn =
	153	be32_to_cpu(ohdr->bth[2]) & 0x7fffffff;
	154	/* extended headers */
	155	memcpy(__get_dynamic_array(ehdrs), &ohdr->u,
	156	ibhdr_exhdr_len(hdr));
	157	),
	158	TP_printk("[%s] " LRH_PRN " " BTH_PRN " " EHDR_PRN,
	159	__get_str(dev),
	160	/* LRH */
	161	__entry->vl,
	162	__entry->lver,
	163	__entry->sl,
	164	__entry->lnh, show_lnh(__entry->lnh),
	165	__entry->dlid,
	166	__entry->len,
	167	__entry->slid,
	168	/* BTH */
	169	__entry->opcode, show_ib_opcode(__entry->opcode),
	170	__entry->se,
	171	__entry->m,
	172	__entry->pad,
	173	__entry->tver,
	174	__entry->pkey,
	175	__entry->f,
	176	__entry->b,
	177	__entry->qpn,
	178	__entry->a,
	179	__entry->psn,
	180	/* extended headers */
	181	__parse_ib_ehdrs(
	182	__entry->opcode,
	183	(void *)__get_dynamic_array(ehdrs))
	184	)
	185	);
	186
	187	DEFINE_EVENT(hfi1_ibhdr_template, input_ibhdr,
	188	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
	189	TP_ARGS(dd, hdr));
	190
	191	DEFINE_EVENT(hfi1_ibhdr_template, pio_output_ibhdr,
	192	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
	193	TP_ARGS(dd, hdr));
	194
	195	DEFINE_EVENT(hfi1_ibhdr_template, ack_output_ibhdr,
	196	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
	197	TP_ARGS(dd, hdr));
	198
	199	DEFINE_EVENT(hfi1_ibhdr_template, sdma_output_ibhdr,
	200	TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
	201	TP_ARGS(dd, hdr));
	202
	203	#endif /* __HFI1_TRACE_IBHDRS_H */
	204
	205	#undef TRACE_INCLUDE_PATH
	206	#undef TRACE_INCLUDE_FILE
	207	#define TRACE_INCLUDE_PATH .
	208	#define TRACE_INCLUDE_FILE trace_ibhdrs
	209	#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_misc.h+81−0

@@ -0,0 +1,81 @@
	1	/*
	2	* Copyright(c) 2015, 2016 Intel Corporation.
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of version 2 of the GNU General Public License as
	11	* published by the Free Software Foundation.
	12	*
	13	* This program is distributed in the hope that it will be useful, but
	14	* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* General Public License for more details.
	17	*
	18	* BSD LICENSE
	19	*
	20	* Redistribution and use in source and binary forms, with or without
	21	* modification, are permitted provided that the following conditions
	22	* are met:
	23	*
	24	* - Redistributions of source code must retain the above copyright
	25	* notice, this list of conditions and the following disclaimer.
	26	* - Redistributions in binary form must reproduce the above copyright
	27	* notice, this list of conditions and the following disclaimer in
	28	* the documentation and/or other materials provided with the
	29	* distribution.
	30	* - Neither the name of Intel Corporation nor the names of its
	31	* contributors may be used to endorse or promote products derived
	32	* from this software without specific prior written permission.
	33	*
	34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	*
	46	*/
	47	#if !defined(__HFI1_TRACE_MISC_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	#define __HFI1_TRACE_MISC_H
	49
	50	#include <linux/tracepoint.h>
	51	#include <linux/trace_seq.h>
	52
	53	#include "hfi.h"
	54
	55	#undef TRACE_SYSTEM
	56	#define TRACE_SYSTEM hfi1_misc
	57
	58	TRACE_EVENT(hfi1_interrupt,
	59	TP_PROTO(struct hfi1_devdata dd, const struct is_table is_entry,
	60	int src),
	61	TP_ARGS(dd, is_entry, src),
	62	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	63	__array(char, buf, 64)
	64	__field(int, src)
	65	),
	66	TP_fast_assign(DD_DEV_ASSIGN(dd)
	67	is_entry->is_name(__entry->buf, 64,
	68	src - is_entry->start);
	69	__entry->src = src;
	70	),
	71	TP_printk("[%s] source: %s [%d]", __get_str(dev), __entry->buf,
	72	__entry->src)
	73	);
	74
	75	#endif /* __HFI1_TRACE_MISC_H */
	76
	77	#undef TRACE_INCLUDE_PATH
	78	#undef TRACE_INCLUDE_FILE
	79	#define TRACE_INCLUDE_PATH .
	80	#define TRACE_INCLUDE_FILE trace_misc
	81	#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_rc.h+123−0

@@ -0,0 +1,123 @@
	1	/*
	2	* Copyright(c) 2015, 2016 Intel Corporation.
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of version 2 of the GNU General Public License as
	11	* published by the Free Software Foundation.
	12	*
	13	* This program is distributed in the hope that it will be useful, but
	14	* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* General Public License for more details.
	17	*
	18	* BSD LICENSE
	19	*
	20	* Redistribution and use in source and binary forms, with or without
	21	* modification, are permitted provided that the following conditions
	22	* are met:
	23	*
	24	* - Redistributions of source code must retain the above copyright
	25	* notice, this list of conditions and the following disclaimer.
	26	* - Redistributions in binary form must reproduce the above copyright
	27	* notice, this list of conditions and the following disclaimer in
	28	* the documentation and/or other materials provided with the
	29	* distribution.
	30	* - Neither the name of Intel Corporation nor the names of its
	31	* contributors may be used to endorse or promote products derived
	32	* from this software without specific prior written permission.
	33	*
	34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	*
	46	*/
	47	#if !defined(__HFI1_TRACE_RC_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	#define __HFI1_TRACE_RC_H
	49
	50	#include <linux/tracepoint.h>
	51	#include <linux/trace_seq.h>
	52
	53	#include "hfi.h"
	54
	55	#undef TRACE_SYSTEM
	56	#define TRACE_SYSTEM hfi1_rc
	57
	58	DECLARE_EVENT_CLASS(hfi1_rc_template,
	59	TP_PROTO(struct rvt_qp *qp, u32 psn),
	60	TP_ARGS(qp, psn),
	61	TP_STRUCT__entry(
	62	DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
	63	__field(u32, qpn)
	64	__field(u32, s_flags)
	65	__field(u32, psn)
	66	__field(u32, s_psn)
	67	__field(u32, s_next_psn)
	68	__field(u32, s_sending_psn)
	69	__field(u32, s_sending_hpsn)
	70	__field(u32, r_psn)
	71	),
	72	TP_fast_assign(
	73	DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device))
	74	__entry->qpn = qp->ibqp.qp_num;
	75	__entry->s_flags = qp->s_flags;
	76	__entry->psn = psn;
	77	__entry->s_psn = qp->s_psn;
	78	__entry->s_next_psn = qp->s_next_psn;
	79	__entry->s_sending_psn = qp->s_sending_psn;
	80	__entry->s_sending_hpsn = qp->s_sending_hpsn;
	81	__entry->r_psn = qp->r_psn;
	82	),
	83	TP_printk(
	84	"[%s] qpn 0x%x s_flags 0x%x psn 0x%x s_psn 0x%x s_next_psn 0x%x s_sending_psn 0x%x sending_hpsn 0x%x r_psn 0x%x",
	85	__get_str(dev),
	86	__entry->qpn,
	87	__entry->s_flags,
	88	__entry->psn,
	89	__entry->s_psn,
	90	__entry->s_next_psn,
	91	__entry->s_sending_psn,
	92	__entry->s_sending_hpsn,
	93	__entry->r_psn
	94	)
	95	);
	96
	97	DEFINE_EVENT(hfi1_rc_template, hfi1_sendcomplete,
	98	TP_PROTO(struct rvt_qp *qp, u32 psn),
	99	TP_ARGS(qp, psn)
	100	);
	101
	102	DEFINE_EVENT(hfi1_rc_template, hfi1_ack,
	103	TP_PROTO(struct rvt_qp *qp, u32 psn),
	104	TP_ARGS(qp, psn)
	105	);
	106
	107	DEFINE_EVENT(hfi1_rc_template, hfi1_timeout,
	108	TP_PROTO(struct rvt_qp *qp, u32 psn),
	109	TP_ARGS(qp, psn)
	110	);
	111
	112	DEFINE_EVENT(hfi1_rc_template, hfi1_rcv_error,
	113	TP_PROTO(struct rvt_qp *qp, u32 psn),
	114	TP_ARGS(qp, psn)
	115	);
	116
	117	#endif /* __HFI1_TRACE_RC_H */
	118
	119	#undef TRACE_INCLUDE_PATH
	120	#undef TRACE_INCLUDE_FILE
	121	#define TRACE_INCLUDE_PATH .
	122	#define TRACE_INCLUDE_FILE trace_rc
	123	#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_rx.h+322−0

@@ -0,0 +1,322 @@
	1	/*
	2	* Copyright(c) 2015, 2016 Intel Corporation.
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of version 2 of the GNU General Public License as
	11	* published by the Free Software Foundation.
	12	*
	13	* This program is distributed in the hope that it will be useful, but
	14	* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* General Public License for more details.
	17	*
	18	* BSD LICENSE
	19	*
	20	* Redistribution and use in source and binary forms, with or without
	21	* modification, are permitted provided that the following conditions
	22	* are met:
	23	*
	24	* - Redistributions of source code must retain the above copyright
	25	* notice, this list of conditions and the following disclaimer.
	26	* - Redistributions in binary form must reproduce the above copyright
	27	* notice, this list of conditions and the following disclaimer in
	28	* the documentation and/or other materials provided with the
	29	* distribution.
	30	* - Neither the name of Intel Corporation nor the names of its
	31	* contributors may be used to endorse or promote products derived
	32	* from this software without specific prior written permission.
	33	*
	34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	*
	46	*/
	47	#if !defined(__HFI1_TRACE_RX_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	#define __HFI1_TRACE_RX_H
	49
	50	#include <linux/tracepoint.h>
	51	#include <linux/trace_seq.h>
	52
	53	#include "hfi.h"
	54
	55	#undef TRACE_SYSTEM
	56	#define TRACE_SYSTEM hfi1_rx
	57
	58	TRACE_EVENT(hfi1_rcvhdr,
	59	TP_PROTO(struct hfi1_devdata *dd,
	60	u32 ctxt,
	61	u64 eflags,
	62	u32 etype,
	63	u32 hlen,
	64	u32 tlen,
	65	u32 updegr,
	66	u32 etail
	67	),
	68	TP_ARGS(dd, ctxt, eflags, etype, hlen, tlen, updegr, etail),
	69	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	70	__field(u64, eflags)
	71	__field(u32, ctxt)
	72	__field(u32, etype)
	73	__field(u32, hlen)
	74	__field(u32, tlen)
	75	__field(u32, updegr)
	76	__field(u32, etail)
	77	),
	78	TP_fast_assign(DD_DEV_ASSIGN(dd);
	79	__entry->eflags = eflags;
	80	__entry->ctxt = ctxt;
	81	__entry->etype = etype;
	82	__entry->hlen = hlen;
	83	__entry->tlen = tlen;
	84	__entry->updegr = updegr;
	85	__entry->etail = etail;
	86	),
	87	TP_printk(
	88	"[%s] ctxt %d eflags 0x%llx etype %d,%s hlen %d tlen %d updegr %d etail %d",
	89	__get_str(dev),
	90	__entry->ctxt,
	91	__entry->eflags,
	92	__entry->etype, show_packettype(__entry->etype),
	93	__entry->hlen,
	94	__entry->tlen,
	95	__entry->updegr,
	96	__entry->etail
	97	)
	98	);
	99
	100	TRACE_EVENT(hfi1_receive_interrupt,
	101	TP_PROTO(struct hfi1_devdata *dd, u32 ctxt),
	102	TP_ARGS(dd, ctxt),
	103	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	104	__field(u32, ctxt)
	105	__field(u8, slow_path)
	106	__field(u8, dma_rtail)
	107	),
	108	TP_fast_assign(DD_DEV_ASSIGN(dd);
	109	__entry->ctxt = ctxt;
	110	if (dd->rcd[ctxt]->do_interrupt ==
	111	&handle_receive_interrupt) {
	112	__entry->slow_path = 1;
	113	__entry->dma_rtail = 0xFF;
	114	} else if (dd->rcd[ctxt]->do_interrupt ==
	115	&handle_receive_interrupt_dma_rtail){
	116	__entry->dma_rtail = 1;
	117	__entry->slow_path = 0;
	118	} else if (dd->rcd[ctxt]->do_interrupt ==
	119	&handle_receive_interrupt_nodma_rtail) {
	120	__entry->dma_rtail = 0;
	121	__entry->slow_path = 0;
	122	}
	123	),
	124	TP_printk("[%s] ctxt %d SlowPath: %d DmaRtail: %d",
	125	__get_str(dev),
	126	__entry->ctxt,
	127	__entry->slow_path,
	128	__entry->dma_rtail
	129	)
	130	);
	131
	132	TRACE_EVENT(hfi1_exp_tid_reg,
	133	TP_PROTO(unsigned int ctxt, u16 subctxt, u32 rarr,
	134	u32 npages, unsigned long va, unsigned long pa,
	135	dma_addr_t dma),
	136	TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
	137	TP_STRUCT__entry(
	138	__field(unsigned int, ctxt)
	139	__field(u16, subctxt)
	140	__field(u32, rarr)
	141	__field(u32, npages)
	142	__field(unsigned long, va)
	143	__field(unsigned long, pa)
	144	__field(dma_addr_t, dma)
	145	),
	146	TP_fast_assign(
	147	__entry->ctxt = ctxt;
	148	__entry->subctxt = subctxt;
	149	__entry->rarr = rarr;
	150	__entry->npages = npages;
	151	__entry->va = va;
	152	__entry->pa = pa;
	153	__entry->dma = dma;
	154	),
	155	TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
	156	__entry->ctxt,
	157	__entry->subctxt,
	158	__entry->rarr,
	159	__entry->npages,
	160	__entry->pa,
	161	__entry->va,
	162	__entry->dma
	163	)
	164	);
	165
	166	TRACE_EVENT(hfi1_exp_tid_unreg,
	167	TP_PROTO(unsigned int ctxt, u16 subctxt, u32 rarr, u32 npages,
	168	unsigned long va, unsigned long pa, dma_addr_t dma),
	169	TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
	170	TP_STRUCT__entry(
	171	__field(unsigned int, ctxt)
	172	__field(u16, subctxt)
	173	__field(u32, rarr)
	174	__field(u32, npages)
	175	__field(unsigned long, va)
	176	__field(unsigned long, pa)
	177	__field(dma_addr_t, dma)
	178	),
	179	TP_fast_assign(
	180	__entry->ctxt = ctxt;
	181	__entry->subctxt = subctxt;
	182	__entry->rarr = rarr;
	183	__entry->npages = npages;
	184	__entry->va = va;
	185	__entry->pa = pa;
	186	__entry->dma = dma;
	187	),
	188	TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
	189	__entry->ctxt,
	190	__entry->subctxt,
	191	__entry->rarr,
	192	__entry->npages,
	193	__entry->pa,
	194	__entry->va,
	195	__entry->dma
	196	)
	197	);
	198
	199	TRACE_EVENT(hfi1_exp_tid_inval,
	200	TP_PROTO(unsigned int ctxt, u16 subctxt, unsigned long va, u32 rarr,
	201	u32 npages, dma_addr_t dma),
	202	TP_ARGS(ctxt, subctxt, va, rarr, npages, dma),
	203	TP_STRUCT__entry(
	204	__field(unsigned int, ctxt)
	205	__field(u16, subctxt)
	206	__field(unsigned long, va)
	207	__field(u32, rarr)
	208	__field(u32, npages)
	209	__field(dma_addr_t, dma)
	210	),
	211	TP_fast_assign(
	212	__entry->ctxt = ctxt;
	213	__entry->subctxt = subctxt;
	214	__entry->va = va;
	215	__entry->rarr = rarr;
	216	__entry->npages = npages;
	217	__entry->dma = dma;
	218	),
	219	TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx dma: 0x%llx",
	220	__entry->ctxt,
	221	__entry->subctxt,
	222	__entry->rarr,
	223	__entry->npages,
	224	__entry->va,
	225	__entry->dma
	226	)
	227	);
	228
	229	TRACE_EVENT(hfi1_mmu_invalidate,
	230	TP_PROTO(unsigned int ctxt, u16 subctxt, const char *type,
	231	unsigned long start, unsigned long end),
	232	TP_ARGS(ctxt, subctxt, type, start, end),
	233	TP_STRUCT__entry(
	234	__field(unsigned int, ctxt)
	235	__field(u16, subctxt)
	236	__string(type, type)
	237	__field(unsigned long, start)
	238	__field(unsigned long, end)
	239	),
	240	TP_fast_assign(
	241	__entry->ctxt = ctxt;
	242	__entry->subctxt = subctxt;
	243	__assign_str(type, type);
	244	__entry->start = start;
	245	__entry->end = end;
	246	),
	247	TP_printk("[%3u:%02u] MMU Invalidate (%s) 0x%lx - 0x%lx",
	248	__entry->ctxt,
	249	__entry->subctxt,
	250	__get_str(type),
	251	__entry->start,
	252	__entry->end
	253	)
	254	);
	255
	256	#define SNOOP_PRN \
	257	"slid %.4x dlid %.4x qpn 0x%.6x opcode 0x%.2x,%s " \
	258	"svc lvl %d pkey 0x%.4x [header = %d bytes] [data = %d bytes]"
	259
	260	TRACE_EVENT(snoop_capture,
	261	TP_PROTO(struct hfi1_devdata *dd,
	262	int hdr_len,
	263	struct hfi1_ib_header *hdr,
	264	int data_len,
	265	void *data),
	266	TP_ARGS(dd, hdr_len, hdr, data_len, data),
	267	TP_STRUCT__entry(
	268	DD_DEV_ENTRY(dd)
	269	__field(u16, slid)
	270	__field(u16, dlid)
	271	__field(u32, qpn)
	272	__field(u8, opcode)
	273	__field(u8, sl)
	274	__field(u16, pkey)
	275	__field(u32, hdr_len)
	276	__field(u32, data_len)
	277	__field(u8, lnh)
	278	__dynamic_array(u8, raw_hdr, hdr_len)
	279	__dynamic_array(u8, raw_pkt, data_len)
	280	),
	281	TP_fast_assign(
	282	struct hfi1_other_headers *ohdr;
	283
	284	__entry->lnh = (u8)(be16_to_cpu(hdr->lrh[0]) & 3);
	285	if (__entry->lnh == HFI1_LRH_BTH)
	286	ohdr = &hdr->u.oth;
	287	else
	288	ohdr = &hdr->u.l.oth;
	289	DD_DEV_ASSIGN(dd);
	290	__entry->slid = be16_to_cpu(hdr->lrh[3]);
	291	__entry->dlid = be16_to_cpu(hdr->lrh[1]);
	292	__entry->qpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
	293	__entry->opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
	294	__entry->sl = (u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
	295	__entry->pkey = be32_to_cpu(ohdr->bth[0]) & 0xffff;
	296	__entry->hdr_len = hdr_len;
	297	__entry->data_len = data_len;
	298	memcpy(__get_dynamic_array(raw_hdr), hdr, hdr_len);
	299	memcpy(__get_dynamic_array(raw_pkt), data, data_len);
	300	),
	301	TP_printk(
	302	"[%s] " SNOOP_PRN,
	303	__get_str(dev),
	304	__entry->slid,
	305	__entry->dlid,
	306	__entry->qpn,
	307	__entry->opcode,
	308	show_ib_opcode(__entry->opcode),
	309	__entry->sl,
	310	__entry->pkey,
	311	__entry->hdr_len,
	312	__entry->data_len
	313	)
	314	);
	315
	316	#endif /* __HFI1_TRACE_RX_H */
	317
	318	#undef TRACE_INCLUDE_PATH
	319	#undef TRACE_INCLUDE_FILE
	320	#define TRACE_INCLUDE_PATH .
	321	#define TRACE_INCLUDE_FILE trace_rx
	322	#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_tx.h+642−0

@@ -0,0 +1,642 @@
	1	/*
	2	* Copyright(c) 2015, 2016 Intel Corporation.
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of version 2 of the GNU General Public License as
	11	* published by the Free Software Foundation.
	12	*
	13	* This program is distributed in the hope that it will be useful, but
	14	* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* General Public License for more details.
	17	*
	18	* BSD LICENSE
	19	*
	20	* Redistribution and use in source and binary forms, with or without
	21	* modification, are permitted provided that the following conditions
	22	* are met:
	23	*
	24	* - Redistributions of source code must retain the above copyright
	25	* notice, this list of conditions and the following disclaimer.
	26	* - Redistributions in binary form must reproduce the above copyright
	27	* notice, this list of conditions and the following disclaimer in
	28	* the documentation and/or other materials provided with the
	29	* distribution.
	30	* - Neither the name of Intel Corporation nor the names of its
	31	* contributors may be used to endorse or promote products derived
	32	* from this software without specific prior written permission.
	33	*
	34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	*
	46	*/
	47	#if !defined(__HFI1_TRACE_TX_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	#define __HFI1_TRACE_TX_H
	49
	50	#include <linux/tracepoint.h>
	51	#include <linux/trace_seq.h>
	52
	53	#include "hfi.h"
	54	#include "mad.h"
	55	#include "sdma.h"
	56
	57	const char parse_sdma_flags(struct trace_seq p, u64 desc0, u64 desc1);
	58
	59	#define __parse_sdma_flags(desc0, desc1) parse_sdma_flags(p, desc0, desc1)
	60
	61	#undef TRACE_SYSTEM
	62	#define TRACE_SYSTEM hfi1_tx
	63
	64	TRACE_EVENT(hfi1_piofree,
	65	TP_PROTO(struct send_context *sc, int extra),
	66	TP_ARGS(sc, extra),
	67	TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
	68	__field(u32, sw_index)
	69	__field(u32, hw_context)
	70	__field(int, extra)
	71	),
	72	TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
	73	__entry->sw_index = sc->sw_index;
	74	__entry->hw_context = sc->hw_context;
	75	__entry->extra = extra;
	76	),
	77	TP_printk("[%s] ctxt %u(%u) extra %d",
	78	__get_str(dev),
	79	__entry->sw_index,
	80	__entry->hw_context,
	81	__entry->extra
	82	)
	83	);
	84
	85	TRACE_EVENT(hfi1_wantpiointr,
	86	TP_PROTO(struct send_context *sc, u32 needint, u64 credit_ctrl),
	87	TP_ARGS(sc, needint, credit_ctrl),
	88	TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
	89	__field(u32, sw_index)
	90	__field(u32, hw_context)
	91	__field(u32, needint)
	92	__field(u64, credit_ctrl)
	93	),
	94	TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
	95	__entry->sw_index = sc->sw_index;
	96	__entry->hw_context = sc->hw_context;
	97	__entry->needint = needint;
	98	__entry->credit_ctrl = credit_ctrl;
	99	),
	100	TP_printk("[%s] ctxt %u(%u) on %d credit_ctrl 0x%llx",
	101	__get_str(dev),
	102	__entry->sw_index,
	103	__entry->hw_context,
	104	__entry->needint,
	105	(unsigned long long)__entry->credit_ctrl
	106	)
	107	);
	108
	109	DECLARE_EVENT_CLASS(hfi1_qpsleepwakeup_template,
	110	TP_PROTO(struct rvt_qp *qp, u32 flags),
	111	TP_ARGS(qp, flags),
	112	TP_STRUCT__entry(
	113	DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
	114	__field(u32, qpn)
	115	__field(u32, flags)
	116	__field(u32, s_flags)
	117	),
	118	TP_fast_assign(
	119	DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device))
	120	__entry->flags = flags;
	121	__entry->qpn = qp->ibqp.qp_num;
	122	__entry->s_flags = qp->s_flags;
	123	),
	124	TP_printk(
	125	"[%s] qpn 0x%x flags 0x%x s_flags 0x%x",
	126	__get_str(dev),
	127	__entry->qpn,
	128	__entry->flags,
	129	__entry->s_flags
	130	)
	131	);
	132
	133	DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpwakeup,
	134	TP_PROTO(struct rvt_qp *qp, u32 flags),
	135	TP_ARGS(qp, flags));
	136
	137	DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpsleep,
	138	TP_PROTO(struct rvt_qp *qp, u32 flags),
	139	TP_ARGS(qp, flags));
	140
	141	TRACE_EVENT(hfi1_sdma_descriptor,
	142	TP_PROTO(struct sdma_engine *sde,
	143	u64 desc0,
	144	u64 desc1,
	145	u16 e,
	146	void *descp),
	147	TP_ARGS(sde, desc0, desc1, e, descp),
	148	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	149	__field(void *, descp)
	150	__field(u64, desc0)
	151	__field(u64, desc1)
	152	__field(u16, e)
	153	__field(u8, idx)
	154	),
	155	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	156	__entry->desc0 = desc0;
	157	__entry->desc1 = desc1;
	158	__entry->idx = sde->this_idx;
	159	__entry->descp = descp;
	160	__entry->e = e;
	161	),
	162	TP_printk(
	163	"[%s] SDE(%u) flags:%s addr:0x%016llx gen:%u len:%u d0:%016llx d1:%016llx to %p,%u",
	164	__get_str(dev),
	165	__entry->idx,
	166	__parse_sdma_flags(__entry->desc0, __entry->desc1),
	167	(__entry->desc0 >> SDMA_DESC0_PHY_ADDR_SHIFT) &
	168	SDMA_DESC0_PHY_ADDR_MASK,
	169	(u8)((__entry->desc1 >> SDMA_DESC1_GENERATION_SHIFT) &
	170	SDMA_DESC1_GENERATION_MASK),
	171	(u16)((__entry->desc0 >> SDMA_DESC0_BYTE_COUNT_SHIFT) &
	172	SDMA_DESC0_BYTE_COUNT_MASK),
	173	__entry->desc0,
	174	__entry->desc1,
	175	__entry->descp,
	176	__entry->e
	177	)
	178	);
	179
	180	TRACE_EVENT(hfi1_sdma_engine_select,
	181	TP_PROTO(struct hfi1_devdata *dd, u32 sel, u8 vl, u8 idx),
	182	TP_ARGS(dd, sel, vl, idx),
	183	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	184	__field(u32, sel)
	185	__field(u8, vl)
	186	__field(u8, idx)
	187	),
	188	TP_fast_assign(DD_DEV_ASSIGN(dd);
	189	__entry->sel = sel;
	190	__entry->vl = vl;
	191	__entry->idx = idx;
	192	),
	193	TP_printk("[%s] selecting SDE %u sel 0x%x vl %u",
	194	__get_str(dev),
	195	__entry->idx,
	196	__entry->sel,
	197	__entry->vl
	198	)
	199	);
	200
	201	DECLARE_EVENT_CLASS(hfi1_sdma_engine_class,
	202	TP_PROTO(struct sdma_engine *sde, u64 status),
	203	TP_ARGS(sde, status),
	204	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	205	__field(u64, status)
	206	__field(u8, idx)
	207	),
	208	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	209	__entry->status = status;
	210	__entry->idx = sde->this_idx;
	211	),
	212	TP_printk("[%s] SDE(%u) status %llx",
	213	__get_str(dev),
	214	__entry->idx,
	215	(unsigned long long)__entry->status
	216	)
	217	);
	218
	219	DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_interrupt,
	220	TP_PROTO(struct sdma_engine *sde, u64 status),
	221	TP_ARGS(sde, status)
	222	);
	223
	224	DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_progress,
	225	TP_PROTO(struct sdma_engine *sde, u64 status),
	226	TP_ARGS(sde, status)
	227	);
	228
	229	DECLARE_EVENT_CLASS(hfi1_sdma_ahg_ad,
	230	TP_PROTO(struct sdma_engine *sde, int aidx),
	231	TP_ARGS(sde, aidx),
	232	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	233	__field(int, aidx)
	234	__field(u8, idx)
	235	),
	236	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	237	__entry->idx = sde->this_idx;
	238	__entry->aidx = aidx;
	239	),
	240	TP_printk("[%s] SDE(%u) aidx %d",
	241	__get_str(dev),
	242	__entry->idx,
	243	__entry->aidx
	244	)
	245	);
	246
	247	DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_allocate,
	248	TP_PROTO(struct sdma_engine *sde, int aidx),
	249	TP_ARGS(sde, aidx));
	250
	251	DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_deallocate,
	252	TP_PROTO(struct sdma_engine *sde, int aidx),
	253	TP_ARGS(sde, aidx));
	254
	255	#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
	256	TRACE_EVENT(hfi1_sdma_progress,
	257	TP_PROTO(struct sdma_engine *sde,
	258	u16 hwhead,
	259	u16 swhead,
	260	struct sdma_txreq *txp
	261	),
	262	TP_ARGS(sde, hwhead, swhead, txp),
	263	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	264	__field(u64, sn)
	265	__field(u16, hwhead)
	266	__field(u16, swhead)
	267	__field(u16, txnext)
	268	__field(u16, tx_tail)
	269	__field(u16, tx_head)
	270	__field(u8, idx)
	271	),
	272	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	273	__entry->hwhead = hwhead;
	274	__entry->swhead = swhead;
	275	__entry->tx_tail = sde->tx_tail;
	276	__entry->tx_head = sde->tx_head;
	277	__entry->txnext = txp ? txp->next_descq_idx : ~0;
	278	__entry->idx = sde->this_idx;
	279	__entry->sn = txp ? txp->sn : ~0;
	280	),
	281	TP_printk(
	282	"[%s] SDE(%u) sn %llu hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
	283	__get_str(dev),
	284	__entry->idx,
	285	__entry->sn,
	286	__entry->hwhead,
	287	__entry->swhead,
	288	__entry->txnext,
	289	__entry->tx_head,
	290	__entry->tx_tail
	291	)
	292	);
	293	#else
	294	TRACE_EVENT(hfi1_sdma_progress,
	295	TP_PROTO(struct sdma_engine *sde,
	296	u16 hwhead, u16 swhead,
	297	struct sdma_txreq *txp
	298	),
	299	TP_ARGS(sde, hwhead, swhead, txp),
	300	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	301	__field(u16, hwhead)
	302	__field(u16, swhead)
	303	__field(u16, txnext)
	304	__field(u16, tx_tail)
	305	__field(u16, tx_head)
	306	__field(u8, idx)
	307	),
	308	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	309	__entry->hwhead = hwhead;
	310	__entry->swhead = swhead;
	311	__entry->tx_tail = sde->tx_tail;
	312	__entry->tx_head = sde->tx_head;
	313	__entry->txnext = txp ? txp->next_descq_idx : ~0;
	314	__entry->idx = sde->this_idx;
	315	),
	316	TP_printk(
	317	"[%s] SDE(%u) hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
	318	__get_str(dev),
	319	__entry->idx,
	320	__entry->hwhead,
	321	__entry->swhead,
	322	__entry->txnext,
	323	__entry->tx_head,
	324	__entry->tx_tail
	325	)
	326	);
	327	#endif
	328
	329	DECLARE_EVENT_CLASS(hfi1_sdma_sn,
	330	TP_PROTO(struct sdma_engine *sde, u64 sn),
	331	TP_ARGS(sde, sn),
	332	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	333	__field(u64, sn)
	334	__field(u8, idx)
	335	),
	336	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	337	__entry->sn = sn;
	338	__entry->idx = sde->this_idx;
	339	),
	340	TP_printk("[%s] SDE(%u) sn %llu",
	341	__get_str(dev),
	342	__entry->idx,
	343	__entry->sn
	344	)
	345	);
	346
	347	DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_out_sn,
	348	TP_PROTO(
	349	struct sdma_engine *sde,
	350	u64 sn
	351	),
	352	TP_ARGS(sde, sn)
	353	);
	354
	355	DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_in_sn,
	356	TP_PROTO(struct sdma_engine *sde, u64 sn),
	357	TP_ARGS(sde, sn)
	358	);
	359
	360	#define USDMA_HDR_FORMAT \
	361	"[%s:%u:%u:%u] PBC=(0x%x 0x%x) LRH=(0x%x 0x%x) BTH=(0x%x 0x%x 0x%x) KDETH=(0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x) TIDVal=0x%x"
	362
	363	TRACE_EVENT(hfi1_sdma_user_header,
	364	TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
	365	struct hfi1_pkt_header *hdr, u32 tidval),
	366	TP_ARGS(dd, ctxt, subctxt, req, hdr, tidval),
	367	TP_STRUCT__entry(
	368	DD_DEV_ENTRY(dd)
	369	__field(u16, ctxt)
	370	__field(u8, subctxt)
	371	__field(u16, req)
	372	__field(u32, pbc0)
	373	__field(u32, pbc1)
	374	__field(u32, lrh0)
	375	__field(u32, lrh1)
	376	__field(u32, bth0)
	377	__field(u32, bth1)
	378	__field(u32, bth2)
	379	__field(u32, kdeth0)
	380	__field(u32, kdeth1)
	381	__field(u32, kdeth2)
	382	__field(u32, kdeth3)
	383	__field(u32, kdeth4)
	384	__field(u32, kdeth5)
	385	__field(u32, kdeth6)
	386	__field(u32, kdeth7)
	387	__field(u32, kdeth8)
	388	__field(u32, tidval)
	389	),
	390	TP_fast_assign(
	391	__le32 pbc = (__le32 )hdr->pbc;
	392	__be32 lrh = (__be32 )hdr->lrh;
	393	__be32 bth = (__be32 )hdr->bth;
	394	__le32 kdeth = (__le32 )&hdr->kdeth;
	395
	396	DD_DEV_ASSIGN(dd);
	397	__entry->ctxt = ctxt;
	398	__entry->subctxt = subctxt;
	399	__entry->req = req;
	400	__entry->pbc0 = le32_to_cpu(pbc[0]);
	401	__entry->pbc1 = le32_to_cpu(pbc[1]);
	402	__entry->lrh0 = be32_to_cpu(lrh[0]);
	403	__entry->lrh1 = be32_to_cpu(lrh[1]);
	404	__entry->bth0 = be32_to_cpu(bth[0]);
	405	__entry->bth1 = be32_to_cpu(bth[1]);
	406	__entry->bth2 = be32_to_cpu(bth[2]);
	407	__entry->kdeth0 = le32_to_cpu(kdeth[0]);
	408	__entry->kdeth1 = le32_to_cpu(kdeth[1]);
	409	__entry->kdeth2 = le32_to_cpu(kdeth[2]);
	410	__entry->kdeth3 = le32_to_cpu(kdeth[3]);
	411	__entry->kdeth4 = le32_to_cpu(kdeth[4]);
	412	__entry->kdeth5 = le32_to_cpu(kdeth[5]);
	413	__entry->kdeth6 = le32_to_cpu(kdeth[6]);
	414	__entry->kdeth7 = le32_to_cpu(kdeth[7]);
	415	__entry->kdeth8 = le32_to_cpu(kdeth[8]);
	416	__entry->tidval = tidval;
	417	),
	418	TP_printk(USDMA_HDR_FORMAT,
	419	__get_str(dev),
	420	__entry->ctxt,
	421	__entry->subctxt,
	422	__entry->req,
	423	__entry->pbc1,
	424	__entry->pbc0,
	425	__entry->lrh0,
	426	__entry->lrh1,
	427	__entry->bth0,
	428	__entry->bth1,
	429	__entry->bth2,
	430	__entry->kdeth0,
	431	__entry->kdeth1,
	432	__entry->kdeth2,
	433	__entry->kdeth3,
	434	__entry->kdeth4,
	435	__entry->kdeth5,
	436	__entry->kdeth6,
	437	__entry->kdeth7,
	438	__entry->kdeth8,
	439	__entry->tidval
	440	)
	441	);
	442
	443	#define SDMA_UREQ_FMT \
	444	"[%s:%u:%u] ver/op=0x%x, iovcnt=%u, npkts=%u, frag=%u, idx=%u"
	445	TRACE_EVENT(hfi1_sdma_user_reqinfo,
	446	TP_PROTO(struct hfi1_devdata dd, u16 ctxt, u8 subctxt, u16 i),
	447	TP_ARGS(dd, ctxt, subctxt, i),
	448	TP_STRUCT__entry(
	449	DD_DEV_ENTRY(dd);
	450	__field(u16, ctxt)
	451	__field(u8, subctxt)
	452	__field(u8, ver_opcode)
	453	__field(u8, iovcnt)
	454	__field(u16, npkts)
	455	__field(u16, fragsize)
	456	__field(u16, comp_idx)
	457	),
	458	TP_fast_assign(
	459	DD_DEV_ASSIGN(dd);
	460	__entry->ctxt = ctxt;
	461	__entry->subctxt = subctxt;
	462	__entry->ver_opcode = i[0] & 0xff;
	463	__entry->iovcnt = (i[0] >> 8) & 0xff;
	464	__entry->npkts = i[1];
	465	__entry->fragsize = i[2];
	466	__entry->comp_idx = i[3];
	467	),
	468	TP_printk(SDMA_UREQ_FMT,
	469	__get_str(dev),
	470	__entry->ctxt,
	471	__entry->subctxt,
	472	__entry->ver_opcode,
	473	__entry->iovcnt,
	474	__entry->npkts,
	475	__entry->fragsize,
	476	__entry->comp_idx
	477	)
	478	);
	479
	480	#define usdma_complete_name(st) { st, #st }
	481	#define show_usdma_complete_state(st) \
	482	__print_symbolic(st, \
	483	usdma_complete_name(FREE), \
	484	usdma_complete_name(QUEUED), \
	485	usdma_complete_name(COMPLETE), \
	486	usdma_complete_name(ERROR))
	487
	488	TRACE_EVENT(hfi1_sdma_user_completion,
	489	TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 idx,
	490	u8 state, int code),
	491	TP_ARGS(dd, ctxt, subctxt, idx, state, code),
	492	TP_STRUCT__entry(
	493	DD_DEV_ENTRY(dd)
	494	__field(u16, ctxt)
	495	__field(u8, subctxt)
	496	__field(u16, idx)
	497	__field(u8, state)
	498	__field(int, code)
	499	),
	500	TP_fast_assign(
	501	DD_DEV_ASSIGN(dd);
	502	__entry->ctxt = ctxt;
	503	__entry->subctxt = subctxt;
	504	__entry->idx = idx;
	505	__entry->state = state;
	506	__entry->code = code;
	507	),
	508	TP_printk("[%s:%u:%u:%u] SDMA completion state %s (%d)",
	509	__get_str(dev), __entry->ctxt, __entry->subctxt,
	510	__entry->idx, show_usdma_complete_state(__entry->state),
	511	__entry->code)
	512	);
	513
	514	const char print_u32_array(struct trace_seq , u32 *, int);
	515	#define __print_u32_hex(arr, len) print_u32_array(p, arr, len)
	516
	517	TRACE_EVENT(hfi1_sdma_user_header_ahg,
	518	TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
	519	u8 sde, u8 ahgidx, u32 *ahg, int len, u32 tidval),
	520	TP_ARGS(dd, ctxt, subctxt, req, sde, ahgidx, ahg, len, tidval),
	521	TP_STRUCT__entry(
	522	DD_DEV_ENTRY(dd)
	523	__field(u16, ctxt)
	524	__field(u8, subctxt)
	525	__field(u16, req)
	526	__field(u8, sde)
	527	__field(u8, idx)
	528	__field(int, len)
	529	__field(u32, tidval)
	530	__array(u32, ahg, 10)
	531	),
	532	TP_fast_assign(
	533	DD_DEV_ASSIGN(dd);
	534	__entry->ctxt = ctxt;
	535	__entry->subctxt = subctxt;
	536	__entry->req = req;
	537	__entry->sde = sde;
	538	__entry->idx = ahgidx;
	539	__entry->len = len;
	540	__entry->tidval = tidval;
	541	memcpy(__entry->ahg, ahg, len * sizeof(u32));
	542	),
	543	TP_printk("[%s:%u:%u:%u] (SDE%u/AHG%u) ahg[0-%d]=(%s) TIDVal=0x%x",
	544	__get_str(dev),
	545	__entry->ctxt,
	546	__entry->subctxt,
	547	__entry->req,
	548	__entry->sde,
	549	__entry->idx,
	550	__entry->len - 1,
	551	__print_u32_hex(__entry->ahg, __entry->len),
	552	__entry->tidval
	553	)
	554	);
	555
	556	TRACE_EVENT(hfi1_sdma_state,
	557	TP_PROTO(struct sdma_engine *sde,
	558	const char *cstate,
	559	const char *nstate
	560	),
	561	TP_ARGS(sde, cstate, nstate),
	562	TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	563	__string(curstate, cstate)
	564	__string(newstate, nstate)
	565	),
	566	TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	567	__assign_str(curstate, cstate);
	568	__assign_str(newstate, nstate);
	569	),
	570	TP_printk("[%s] current state %s new state %s",
	571	__get_str(dev),
	572	__get_str(curstate),
	573	__get_str(newstate)
	574	)
	575	);
	576
	577	#define BCT_FORMAT \
	578	"shared_limit %x vls 0-7 [%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x] 15 [%x,%x]"
	579
	580	#define BCT(field) \
	581	be16_to_cpu( \
	582	((struct buffer_control *)__get_dynamic_array(bct))->field \
	583	)
	584
	585	DECLARE_EVENT_CLASS(hfi1_bct_template,
	586	TP_PROTO(struct hfi1_devdata *dd,
	587	struct buffer_control *bc),
	588	TP_ARGS(dd, bc),
	589	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	590	__dynamic_array(u8, bct, sizeof(*bc))
	591	),
	592	TP_fast_assign(DD_DEV_ASSIGN(dd);
	593	memcpy(__get_dynamic_array(bct), bc,
	594	sizeof(*bc));
	595	),
	596	TP_printk(BCT_FORMAT,
	597	BCT(overall_shared_limit),
	598
	599	BCT(vl[0].dedicated),
	600	BCT(vl[0].shared),
	601
	602	BCT(vl[1].dedicated),
	603	BCT(vl[1].shared),
	604
	605	BCT(vl[2].dedicated),
	606	BCT(vl[2].shared),
	607
	608	BCT(vl[3].dedicated),
	609	BCT(vl[3].shared),
	610
	611	BCT(vl[4].dedicated),
	612	BCT(vl[4].shared),
	613
	614	BCT(vl[5].dedicated),
	615	BCT(vl[5].shared),
	616
	617	BCT(vl[6].dedicated),
	618	BCT(vl[6].shared),
	619
	620	BCT(vl[7].dedicated),
	621	BCT(vl[7].shared),
	622
	623	BCT(vl[15].dedicated),
	624	BCT(vl[15].shared)
	625	)
	626	);
	627
	628	DEFINE_EVENT(hfi1_bct_template, bct_set,
	629	TP_PROTO(struct hfi1_devdata dd, struct buffer_control bc),
	630	TP_ARGS(dd, bc));
	631
	632	DEFINE_EVENT(hfi1_bct_template, bct_get,
	633	TP_PROTO(struct hfi1_devdata dd, struct buffer_control bc),
	634	TP_ARGS(dd, bc));
	635
	636	#endif /* __HFI1_TRACE_TX_H */
	637
	638	#undef TRACE_INCLUDE_PATH
	639	#undef TRACE_INCLUDE_FILE
	640	#define TRACE_INCLUDE_PATH .
	641	#define TRACE_INCLUDE_FILE trace_tx
	642	#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/twsi.c+0−489

@@ -1,489 +0,0 @@
1	/*
2	* Copyright(c) 2015, 2016 Intel Corporation.
3	*
4	* This file is provided under a dual BSD/GPLv2 license. When using or
5	* redistributing this file, you may do so under either license.
6	*
7	* GPL LICENSE SUMMARY
8	*
9	* This program is free software; you can redistribute it and/or modify
10	* it under the terms of version 2 of the GNU General Public License as
11	* published by the Free Software Foundation.
12	*
13	* This program is distributed in the hope that it will be useful, but
14	* WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16	* General Public License for more details.
17	*
18	* BSD LICENSE
19	*
20	* Redistribution and use in source and binary forms, with or without
21	* modification, are permitted provided that the following conditions
22	* are met:
23	*
24	* - Redistributions of source code must retain the above copyright
25	* notice, this list of conditions and the following disclaimer.
26	* - Redistributions in binary form must reproduce the above copyright
27	* notice, this list of conditions and the following disclaimer in
28	* the documentation and/or other materials provided with the
29	* distribution.
30	* - Neither the name of Intel Corporation nor the names of its
31	* contributors may be used to endorse or promote products derived
32	* from this software without specific prior written permission.
33	*
34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45	*
46	*/
47
48	#include <linux/delay.h>
49	#include <linux/pci.h>
50	#include <linux/vmalloc.h>
51
52	#include "hfi.h"
53	#include "twsi.h"
54
55	/*
56	* "Two Wire Serial Interface" support.
57	*
58	* Originally written for a not-quite-i2c serial eeprom, which is
59	* still used on some supported boards. Later boards have added a
60	* variety of other uses, most board-specific, so the bit-boffing
61	* part has been split off to this file, while the other parts
62	* have been moved to chip-specific files.
63	*
64	* We have also dropped all pretense of fully generic (e.g. pretend
65	* we don't know whether '1' is the higher voltage) interface, as
66	* the restrictions of the generic i2c interface (e.g. no access from
67	* driver itself) make it unsuitable for this use.
68	*/
69
70	#define READ_CMD 1
71	#define WRITE_CMD 0
72
73	/**
74	* i2c_wait_for_writes - wait for a write
75	* @dd: the hfi1_ib device
76	*
77	* We use this instead of udelay directly, so we can make sure
78	* that previous register writes have been flushed all the way
79	* to the chip. Since we are delaying anyway, the cost doesn't
80	* hurt, and makes the bit twiddling more regular
81	*/
82	static void i2c_wait_for_writes(struct hfi1_devdata *dd, u32 target)
83	{
84	/*
85	* implicit read of EXTStatus is as good as explicit
86	* read of scratch, if all we want to do is flush
87	* writes.
88	*/
89	hfi1_gpio_mod(dd, target, 0, 0, 0);
90	rmb(); /* inlined, so prevent compiler reordering */
91	}
92
93	/*
94	* QSFP modules are allowed to hold SCL low for 500uSec. Allow twice that
95	* for "almost compliant" modules
96	*/
97	#define SCL_WAIT_USEC 1000
98
99	/* BUF_WAIT is time bus must be free between STOP or ACK and to next START.
100	* Should be 20, but some chips need more.
101	*/
102	#define TWSI_BUF_WAIT_USEC 60
103
104	static void scl_out(struct hfi1_devdata *dd, u32 target, u8 bit)
105	{
106	u32 mask;
107
108	udelay(1);
109
110	mask = QSFP_HFI0_I2CCLK;
111
112	/* SCL is meant to be bare-drain, so never set "OUT", just DIR */
113	hfi1_gpio_mod(dd, target, 0, bit ? 0 : mask, mask);
114
115	/*
116	* Allow for slow slaves by simple
117	* delay for falling edge, sampling on rise.
118	*/
119	if (!bit) {
120	udelay(2);
121	} else {
122	int rise_usec;
123
124	for (rise_usec = SCL_WAIT_USEC; rise_usec > 0; rise_usec -= 2) {
125	if (mask & hfi1_gpio_mod(dd, target, 0, 0, 0))
126	break;
127	udelay(2);
128	}
129	if (rise_usec <= 0)
130	dd_dev_err(dd, "SCL interface stuck low > %d uSec\n",
131	SCL_WAIT_USEC);
132	}
133	i2c_wait_for_writes(dd, target);
134	}
135
136	static u8 scl_in(struct hfi1_devdata *dd, u32 target, int wait)
137	{
138	u32 read_val, mask;
139
140	mask = QSFP_HFI0_I2CCLK;
141	/* SCL is meant to be bare-drain, so never set "OUT", just DIR */
142	hfi1_gpio_mod(dd, target, 0, 0, mask);
143	read_val = hfi1_gpio_mod(dd, target, 0, 0, 0);
144	if (wait)
145	i2c_wait_for_writes(dd, target);
146	return (read_val & mask) >> GPIO_SCL_NUM;
147	}
148
149	static void sda_out(struct hfi1_devdata *dd, u32 target, u8 bit)
150	{
151	u32 mask;
152
153	mask = QSFP_HFI0_I2CDAT;
154
155	/* SDA is meant to be bare-drain, so never set "OUT", just DIR */
156	hfi1_gpio_mod(dd, target, 0, bit ? 0 : mask, mask);
157
158	i2c_wait_for_writes(dd, target);
159	udelay(2);
160	}
161
162	static u8 sda_in(struct hfi1_devdata *dd, u32 target, int wait)
163	{
164	u32 read_val, mask;
165
166	mask = QSFP_HFI0_I2CDAT;
167	/* SDA is meant to be bare-drain, so never set "OUT", just DIR */
168	hfi1_gpio_mod(dd, target, 0, 0, mask);
169	read_val = hfi1_gpio_mod(dd, target, 0, 0, 0);
170	if (wait)
171	i2c_wait_for_writes(dd, target);
172	return (read_val & mask) >> GPIO_SDA_NUM;
173	}
174
175	/**
176	* i2c_ackrcv - see if ack following write is true
177	* @dd: the hfi1_ib device
178	*/
179	static int i2c_ackrcv(struct hfi1_devdata *dd, u32 target)
180	{
181	u8 ack_received;
182
183	/* AT ENTRY SCL = LOW */
184	/* change direction, ignore data */
185	ack_received = sda_in(dd, target, 1);
186	scl_out(dd, target, 1);
187	ack_received = sda_in(dd, target, 1) == 0;
188	scl_out(dd, target, 0);
189	return ack_received;
190	}
191
192	static void stop_cmd(struct hfi1_devdata *dd, u32 target);
193
194	/**
195	* rd_byte - read a byte, sending STOP on last, else ACK
196	* @dd: the hfi1_ib device
197	*
198	* Returns byte shifted out of device
199	*/
200	static int rd_byte(struct hfi1_devdata *dd, u32 target, int last)
201	{
202	int bit_cntr, data;
203
204	data = 0;
205
206	for (bit_cntr = 7; bit_cntr >= 0; --bit_cntr) {
207	data <<= 1;
208	scl_out(dd, target, 1);
209	data \|= sda_in(dd, target, 0);
210	scl_out(dd, target, 0);
211	}
212	if (last) {
213	scl_out(dd, target, 1);
214	stop_cmd(dd, target);
215	} else {
216	sda_out(dd, target, 0);
217	scl_out(dd, target, 1);
218	scl_out(dd, target, 0);
219	sda_out(dd, target, 1);
220	}
221	return data;
222	}
223
224	/**
225	* wr_byte - write a byte, one bit at a time
226	* @dd: the hfi1_ib device
227	* @data: the byte to write
228	*
229	* Returns 0 if we got the following ack, otherwise 1
230	*/
231	static int wr_byte(struct hfi1_devdata *dd, u32 target, u8 data)
232	{
233	int bit_cntr;
234	u8 bit;
235
236	for (bit_cntr = 7; bit_cntr >= 0; bit_cntr--) {
237	bit = (data >> bit_cntr) & 1;
238	sda_out(dd, target, bit);
239	scl_out(dd, target, 1);
240	scl_out(dd, target, 0);
241	}
242	return (!i2c_ackrcv(dd, target)) ? 1 : 0;
243	}
244
245	/*
246	* issue TWSI start sequence:
247	* (both clock/data high, clock high, data low while clock is high)
248	*/
249	static void start_seq(struct hfi1_devdata *dd, u32 target)
250	{
251	sda_out(dd, target, 1);
252	scl_out(dd, target, 1);
253	sda_out(dd, target, 0);
254	udelay(1);
255	scl_out(dd, target, 0);
256	}
257
258	/**
259	* stop_seq - transmit the stop sequence
260	* @dd: the hfi1_ib device
261	*
262	* (both clock/data low, clock high, data high while clock is high)
263	*/
264	static void stop_seq(struct hfi1_devdata *dd, u32 target)
265	{
266	scl_out(dd, target, 0);
267	sda_out(dd, target, 0);
268	scl_out(dd, target, 1);
269	sda_out(dd, target, 1);
270	}
271
272	/**
273	* stop_cmd - transmit the stop condition
274	* @dd: the hfi1_ib device
275	*
276	* (both clock/data low, clock high, data high while clock is high)
277	*/
278	static void stop_cmd(struct hfi1_devdata *dd, u32 target)
279	{
280	stop_seq(dd, target);
281	udelay(TWSI_BUF_WAIT_USEC);
282	}
283
284	/**
285	* hfi1_twsi_reset - reset I2C communication
286	* @dd: the hfi1_ib device
287	* returns 0 if ok, -EIO on error
288	*/
289	int hfi1_twsi_reset(struct hfi1_devdata *dd, u32 target)
290	{
291	int clock_cycles_left = 9;
292	u32 mask;
293
294	/* Both SCL and SDA should be high. If not, there
295	* is something wrong.
296	*/
297	mask = QSFP_HFI0_I2CCLK \| QSFP_HFI0_I2CDAT;
298
299	/*
300	* Force pins to desired innocuous state.
301	* This is the default power-on state with out=0 and dir=0,
302	* So tri-stated and should be floating high (barring HW problems)
303	*/
304	hfi1_gpio_mod(dd, target, 0, 0, mask);
305
306	/* Check if SCL is low, if it is low then we have a slave device
307	* misbehaving and there is not much we can do.
308	*/
309	if (!scl_in(dd, target, 0))
310	return -EIO;
311
312	/* Check if SDA is low, if it is low then we have to clock SDA
313	* up to 9 times for the device to release the bus
314	*/
315	while (clock_cycles_left--) {
316	if (sda_in(dd, target, 0))
317	return 0;
318	scl_out(dd, target, 0);
319	scl_out(dd, target, 1);
320	}
321
322	return -EIO;
323	}
324
325	#define HFI1_TWSI_START 0x100
326	#define HFI1_TWSI_STOP 0x200
327
328	/* Write byte to TWSI, optionally prefixed with START or suffixed with
329	* STOP.
330	* returns 0 if OK (ACK received), else != 0
331	*/
332	static int twsi_wr(struct hfi1_devdata *dd, u32 target, int data, int flags)
333	{
334	int ret = 1;
335
336	if (flags & HFI1_TWSI_START)
337	start_seq(dd, target);
338
339	/* Leaves SCL low (from i2c_ackrcv()) */
340	ret = wr_byte(dd, target, data);
341
342	if (flags & HFI1_TWSI_STOP)
343	stop_cmd(dd, target);
344	return ret;
345	}
346
347	/* Added functionality for IBA7220-based cards */
348	#define HFI1_TEMP_DEV 0x98
349
350	/*
351	* hfi1_twsi_blk_rd
352	* General interface for data transfer from twsi devices.
353	* One vestige of its former role is that it recognizes a device
354	* HFI1_TWSI_NO_DEV and does the correct operation for the legacy part,
355	* which responded to all TWSI device codes, interpreting them as
356	* address within device. On all other devices found on board handled by
357	* this driver, the device is followed by a N-byte "address" which selects
358	* the "register" or "offset" within the device from which data should
359	* be read.
360	*/
361	int hfi1_twsi_blk_rd(struct hfi1_devdata *dd, u32 target, int dev, int addr,
362	void *buffer, int len)
363	{
364	u8 *bp = buffer;
365	int ret = 1;
366	int i;
367	int offset_size;
368
369	/* obtain the offset size, strip it from the device address */
370	offset_size = (dev >> 8) & 0xff;
371	dev &= 0xff;
372
373	/* allow at most a 2 byte offset */
374	if (offset_size > 2)
375	goto bail;
376
377	if (dev == HFI1_TWSI_NO_DEV) {
378	/* legacy not-really-I2C */
379	addr = (addr << 1) \| READ_CMD;
380	ret = twsi_wr(dd, target, addr, HFI1_TWSI_START);
381	} else {
382	/* Actual I2C */
383	if (offset_size) {
384	ret = twsi_wr(dd, target,
385	dev \| WRITE_CMD, HFI1_TWSI_START);
386	if (ret) {
387	stop_cmd(dd, target);
388	goto bail;
389	}
390
391	for (i = 0; i < offset_size; i++) {
392	ret = twsi_wr(dd, target,
393	(addr >> (i * 8)) & 0xff, 0);
394	udelay(TWSI_BUF_WAIT_USEC);
395	if (ret) {
396	dd_dev_err(dd, "Failed to write byte %d of offset 0x%04X\n",
397	i, addr);
398	goto bail;
399	}
400	}
401	}
402	ret = twsi_wr(dd, target, dev \| READ_CMD, HFI1_TWSI_START);
403	}
404	if (ret) {
405	stop_cmd(dd, target);
406	goto bail;
407	}
408
409	/*
410	* block devices keeps clocking data out as long as we ack,
411	* automatically incrementing the address. Some have "pages"
412	* whose boundaries will not be crossed, but the handling
413	* of these is left to the caller, who is in a better
414	* position to know.
415	*/
416	while (len-- > 0) {
417	/*
418	* Get and store data, sending ACK if length remaining,
419	* else STOP
420	*/
421	*bp++ = rd_byte(dd, target, !len);
422	}
423
424	ret = 0;
425
426	bail:
427	return ret;
428	}
429
430	/*
431	* hfi1_twsi_blk_wr
432	* General interface for data transfer to twsi devices.
433	* One vestige of its former role is that it recognizes a device
434	* HFI1_TWSI_NO_DEV and does the correct operation for the legacy part,
435	* which responded to all TWSI device codes, interpreting them as
436	* address within device. On all other devices found on board handled by
437	* this driver, the device is followed by a N-byte "address" which selects
438	* the "register" or "offset" within the device to which data should
439	* be written.
440	*/
441	int hfi1_twsi_blk_wr(struct hfi1_devdata *dd, u32 target, int dev, int addr,
442	const void *buffer, int len)
443	{
444	const u8 *bp = buffer;
445	int ret = 1;
446	int i;
447	int offset_size;
448
449	/* obtain the offset size, strip it from the device address */
450	offset_size = (dev >> 8) & 0xff;
451	dev &= 0xff;
452
453	/* allow at most a 2 byte offset */
454	if (offset_size > 2)
455	goto bail;
456
457	if (dev == HFI1_TWSI_NO_DEV) {
458	if (twsi_wr(dd, target, (addr << 1) \| WRITE_CMD,
459	HFI1_TWSI_START)) {
460	goto failed_write;
461	}
462	} else {
463	/* Real I2C */
464	if (twsi_wr(dd, target, dev \| WRITE_CMD, HFI1_TWSI_START))
465	goto failed_write;
466	}
467
468	for (i = 0; i < offset_size; i++) {
469	ret = twsi_wr(dd, target, (addr >> (i * 8)) & 0xff, 0);
470	udelay(TWSI_BUF_WAIT_USEC);
471	if (ret) {
472	dd_dev_err(dd, "Failed to write byte %d of offset 0x%04X\n",
473	i, addr);
474	goto bail;
475	}
476	}
477
478	for (i = 0; i < len; i++)
479	if (twsi_wr(dd, target, *bp++, 0))
480	goto failed_write;
481
482	ret = 0;
483
484	failed_write:
485	stop_cmd(dd, target);
486
487	bail:
488	return ret;
489	}

drivers/infiniband/hw/hfi1/twsi.h+0−65

@@ -1,65 +0,0 @@
1	#ifndef _TWSI_H
2	#define _TWSI_H
3	/*
4	* Copyright(c) 2015, 2016 Intel Corporation.
5	*
6	* This file is provided under a dual BSD/GPLv2 license. When using or
7	* redistributing this file, you may do so under either license.
8	*
9	* GPL LICENSE SUMMARY
10	*
11	* This program is free software; you can redistribute it and/or modify
12	* it under the terms of version 2 of the GNU General Public License as
13	* published by the Free Software Foundation.
14	*
15	* This program is distributed in the hope that it will be useful, but
16	* WITHOUT ANY WARRANTY; without even the implied warranty of
17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18	* General Public License for more details.
19	*
20	* BSD LICENSE
21	*
22	* Redistribution and use in source and binary forms, with or without
23	* modification, are permitted provided that the following conditions
24	* are met:
25	*
26	* - Redistributions of source code must retain the above copyright
27	* notice, this list of conditions and the following disclaimer.
28	* - Redistributions in binary form must reproduce the above copyright
29	* notice, this list of conditions and the following disclaimer in
30	* the documentation and/or other materials provided with the
31	* distribution.
32	* - Neither the name of Intel Corporation nor the names of its
33	* contributors may be used to endorse or promote products derived
34	* from this software without specific prior written permission.
35	*
36	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
39	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
40	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
41	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
42	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
43	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
44	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
45	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
46	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47	*
48	*/
49
50	#define HFI1_TWSI_NO_DEV 0xFF
51
52	struct hfi1_devdata;
53
54	/* Bit position of SDA/SCL pins in ASIC_QSFP* registers */
55	#define GPIO_SDA_NUM 1
56	#define GPIO_SCL_NUM 0
57
58	/* these functions must be called with qsfp_lock held */
59	int hfi1_twsi_reset(struct hfi1_devdata *dd, u32 target);
60	int hfi1_twsi_blk_rd(struct hfi1_devdata *dd, u32 target, int dev, int addr,
61	void *buffer, int len);
62	int hfi1_twsi_blk_wr(struct hfi1_devdata *dd, u32 target, int dev, int addr,
63	const void *buffer, int len);
64
65	#endif /* _TWSI_H */

drivers/infiniband/hw/hfi1/uc.c+26−35

 			clear_ahg(qp);
 			goto bail;
 		}
+		/*
+		 * Local operations are processed immediately
+		 * after all prior requests have completed.
+		 */
+		if (wqe->wr.opcode == IB_WR_REG_MR ||
+		    wqe->wr.opcode == IB_WR_LOCAL_INV) {
+			int local_ops = 0;
+			int err = 0;
+			if (qp->s_last != qp->s_cur)
+				goto bail;
+			if (++qp->s_cur == qp->s_size)
+				qp->s_cur = 0;
+			if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
+				err = rvt_invalidate_rkey(
+					qp, wqe->wr.ex.invalidate_rkey);
+				local_ops = 1;
+			}
+			hfi1_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR
+							: IB_WC_SUCCESS);
+			if (local_ops)
+				atomic_dec(&qp->local_ops_pending);
+			qp->s_hdrwords = 0;
+			goto done_free_tx;
+		}
 		/*
 		 * Start a new request.
 		 */
 	struct ib_reth *reth;
 	int has_grh = rcv_flags & HFI1_HAS_GRH;
 	int ret;
-	u32 bth1;
 	bth0 = be32_to_cpu(ohdr->bth[0]);
 	if (hfi1_ruc_check_hdr(ibp, hdr, has_grh, qp, bth0))
 		return;
-	bth1 = be32_to_cpu(ohdr->bth[1]);
+	process_ecn(qp, packet, true);
-	if (unlikely(bth1 & (HFI1_BECN_SMASK | HFI1_FECN_SMASK))) {
-		if (bth1 & HFI1_BECN_SMASK) {
-			struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
-			u32 rqpn, lqpn;
-			u16 rlid = be16_to_cpu(hdr->lrh[3]);
-			u8 sl, sc5;
-			lqpn = bth1 & RVT_QPN_MASK;
-			rqpn = qp->remote_qpn;
-			sc5 = ibp->sl_to_sc[qp->remote_ah_attr.sl];
-			sl = ibp->sc_to_sl[sc5];
-			process_becn(ppd, sl, rlid, lqpn, rqpn,
-				     IB_CC_SVCTYPE_UC);
-		}
-		if (bth1 & HFI1_FECN_SMASK) {
-			struct ib_grh *grh = NULL;
-			u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);
-			u16 slid = be16_to_cpu(hdr->lrh[3]);
-			u16 dlid = be16_to_cpu(hdr->lrh[1]);
-			u32 src_qp = qp->remote_qpn;
-			u8 sc5;
-			sc5 = ibp->sl_to_sc[qp->remote_ah_attr.sl];
-			if (has_grh)
-				grh = &hdr->u.l.grh;
-			return_cnp(ibp, qp, src_qp, pkey, dlid, slid, sc5,
-				   grh);
-		}
-	}
 	psn = be32_to_cpu(ohdr->bth[2]);
 	opcode = (bth0 >> 24) & 0xff;

drivers/infiniband/hw/hfi1/ud.c+29−57

 	}
 	if (ah_attr->ah_flags & IB_AH_GRH) {
-		hfi1_copy_sge(&qp->r_sge, &ah_attr->grh,
+		struct ib_grh grh;
-			      sizeof(struct ib_grh), 1, 0);
+		struct ib_global_route grd = ah_attr->grh;
+		hfi1_make_grh(ibp, &grh, &grd, 0, 0);
+		hfi1_copy_sge(&qp->r_sge, &grh,
+			      sizeof(grh), 1, 0);
 		wc.wc_flags |= IB_WC_GRH;
 	} else {
 		hfi1_skip_sge(&qp->r_sge, sizeof(struct ib_grh), 1);
 					 qp->qkey : wqe->ud_wr.remote_qkey);
 	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
 	/* disarm any ahg */
-	priv->s_hdr->ahgcount = 0;
+	priv->s_ahg->ahgcount = 0;
-	priv->s_hdr->ahgidx = 0;
+	priv->s_ahg->ahgidx = 0;
-	priv->s_hdr->tx_flags = 0;
+	priv->s_ahg->tx_flags = 0;
-	priv->s_hdr->sde = NULL;
 	/* pbc */
 	ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
 	struct hfi1_other_headers *ohdr = packet->ohdr;
 	int opcode;
 	u32 hdrsize = packet->hlen;
-	u32 pad;
 	struct ib_wc wc;
 	u32 qkey;
 	u32 src_qp;
 	u16 dlid, pkey;
 	int mgmt_pkey_idx = -1;
 	struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 	struct hfi1_ib_header *hdr = packet->hdr;
 	u32 rcv_flags = packet->rcv_flags;
 	void *data = packet->ebuf;
 	bool has_grh = rcv_flags & HFI1_HAS_GRH;
 	u8 sc5 = hdr2sc((struct hfi1_message_header *)hdr, packet->rhf);
 	u32 bth1;
-	int is_mcast;
+	u8 sl_from_sc, sl;
-	struct ib_grh *grh = NULL;
+	u16 slid;
+	u8 extra_bytes;
 	qkey = be32_to_cpu(ohdr->u.ud.deth[0]);
 	src_qp = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
 	dlid = be16_to_cpu(hdr->lrh[1]);
-	is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
-			(dlid != be16_to_cpu(IB_LID_PERMISSIVE));
 	bth1 = be32_to_cpu(ohdr->bth[1]);
-	if (unlikely(bth1 & HFI1_BECN_SMASK)) {
+	slid = be16_to_cpu(hdr->lrh[3]);
-		/*
+	pkey = (u16)be32_to_cpu(ohdr->bth[0]);
-		 * In pre-B0 h/w the CNP_OPCODE is handled via an
+	sl = (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
-		 * error path.
+	extra_bytes = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
-		 */
+	extra_bytes += (SIZE_OF_CRC << 2);
-		struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	sl_from_sc = ibp->sc_to_sl[sc5];
-		u32 lqpn =  be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
-		u8 sl;
-		sl = ibp->sc_to_sl[sc5];
-		process_becn(ppd, sl, 0, lqpn, 0, IB_CC_SVCTYPE_UD);
-	}
-	/*
-	 * The opcode is in the low byte when its in network order
-	 * (top byte when in host order).
-	 */
 	opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
 	opcode &= 0xff;
-	pkey = (u16)be32_to_cpu(ohdr->bth[0]);
+	process_ecn(qp, packet, (opcode != IB_OPCODE_CNP));
-	if (!is_mcast && (opcode != IB_OPCODE_CNP) && bth1 & HFI1_FECN_SMASK) {
-		u16 slid = be16_to_cpu(hdr->lrh[3]);
-		return_cnp(ibp, qp, src_qp, pkey, dlid, slid, sc5, grh);
-	}
 	/*
 	 * Get the number of bytes the message was padded by
 	 * and drop incomplete packets.
 	 */
-	pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
+	if (unlikely(tlen < (hdrsize + extra_bytes)))
-	if (unlikely(tlen < (hdrsize + pad + 4)))
 		goto drop;
-	tlen -= hdrsize + pad + 4;
+	tlen -= hdrsize + extra_bytes;
 	/*
 	 * Check that the permissive LID is only used on QP0
 			     hdr->lrh[3] == IB_LID_PERMISSIVE))
 			goto drop;
 		if (qp->ibqp.qp_num > 1) {
-			struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
-			u16 slid;
-			slid = be16_to_cpu(hdr->lrh[3]);
 			if (unlikely(rcv_pkey_check(ppd, pkey, sc5, slid))) {
 				/*
 				 * Traps will not be sent for packets dropped
 				 * IB spec (release 1.3, section 10.9.4)
 				 */
 				hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_P_KEY,
-					       pkey,
+					       pkey, sl,
-					       (be16_to_cpu(hdr->lrh[0]) >> 4) &
-						0xF,
 					       src_qp, qp->ibqp.qp_num,
-					       be16_to_cpu(hdr->lrh[3]),
+					       slid, dlid);
-					       be16_to_cpu(hdr->lrh[1]));
 				return;
 			}
 		} else {
 				goto drop;
 		}
 		if (unlikely(qkey != qp->qkey)) {
-			hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_Q_KEY, qkey,
+			hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_Q_KEY, qkey, sl,
-				       (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF,
 				       src_qp, qp->ibqp.qp_num,
-				       be16_to_cpu(hdr->lrh[3]),
+				       slid, dlid);
-				       be16_to_cpu(hdr->lrh[1]));
 			return;
 		}
 		/* Drop invalid MAD packets (see 13.5.3.1). */
 		if (unlikely(qp->ibqp.qp_num == 1 &&
-			     (tlen > 2048 ||
+			     (tlen > 2048 || (sc5 == 0xF))))
-			      (be16_to_cpu(hdr->lrh[0]) >> 12) == 15)))
 			goto drop;
 	} else {
 		/* Received on QP0, and so by definition, this is an SMP */
 		struct opa_smp *smp = (struct opa_smp *)data;
-		u16 slid = be16_to_cpu(hdr->lrh[3]);
 		if (opa_smp_check(ibp, pkey, sc5, qp, slid, smp))
 			goto drop;
 	    qp->ibqp.qp_type == IB_QPT_SMI) {
 		if (mgmt_pkey_idx < 0) {
 			if (net_ratelimit()) {
-				struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 				struct hfi1_devdata *dd = ppd->dd;
 				dd_dev_err(dd, "QP type %d mgmt_pkey_idx < 0 and packet not dropped???\n",
 		wc.pkey_index = 0;
 	}
-	wc.slid = be16_to_cpu(hdr->lrh[3]);
+	wc.slid = slid;
-	wc.sl = ibp->sc_to_sl[sc5];
+	wc.sl = sl_from_sc;
 	/*
 	 * Save the LMC lower bits if the destination LID is a unicast LID.

drivers/infiniband/hw/hfi1/user_exp_rcv.c+65−59

@@ -82,24 +82,25 @@ struct tid_pageset { · 81 unmodified lines
82	((unsigned long)vaddr & PAGE_MASK)) >> PAGE_SHIFT))	82	((unsigned long)vaddr & PAGE_MASK)) >> PAGE_SHIFT))
83		83
84	static void unlock_exp_tids(struct hfi1_ctxtdata , struct exp_tid_set ,	84	static void unlock_exp_tids(struct hfi1_ctxtdata , struct exp_tid_set ,
85	struct rb_root *);	85	struct hfi1_filedata *);
86	static u32 find_phys_blocks(struct page *, unsigned, struct tid_pageset );	86	static u32 find_phys_blocks(struct page *, unsigned, struct tid_pageset );
87	static int set_rcvarray_entry(struct file *, unsigned long, u32,	87	static int set_rcvarray_entry(struct file *, unsigned long, u32,
88	struct tid_group , struct page *, unsigned);	88	struct tid_group , struct page *, unsigned);
89	static int mmu_rb_insert(struct rb_root , struct mmu_rb_node );	89	static int tid_rb_insert(void , struct mmu_rb_node );
90	static void mmu_rb_remove(struct rb_root , struct mmu_rb_node ,	90	static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,
91	struct mm_struct *);	91	struct tid_rb_node *tnode);
92	static int mmu_rb_invalidate(struct rb_root , struct mmu_rb_node );	92	static void tid_rb_remove(void , struct mmu_rb_node );
		93	static int tid_rb_invalidate(void , struct mmu_rb_node );
93	static int program_rcvarray(struct file , unsigned long, struct tid_group ,	94	static int program_rcvarray(struct file , unsigned long, struct tid_group ,
94	struct tid_pageset , unsigned, u16, struct page *,	95	struct tid_pageset , unsigned, u16, struct page *,
95	u32 , unsigned , unsigned *);	96	u32 , unsigned , unsigned *);
96	static int unprogram_rcvarray(struct file , u32, struct tid_group *);	97	static int unprogram_rcvarray(struct file , u32, struct tid_group *);
97	static void clear_tid_node(struct hfi1_filedata , u16, struct tid_rb_node );	98	static void clear_tid_node(struct hfi1_filedata fd, struct tid_rb_node node);
98		99
99	static struct mmu_rb_ops tid_rb_ops = {	100	static struct mmu_rb_ops tid_rb_ops = {
100	.insert = mmu_rb_insert,	101	.insert = tid_rb_insert,
101	.remove = mmu_rb_remove,	102	.remove = tid_rb_remove,
102	.invalidate = mmu_rb_invalidate	103	.invalidate = tid_rb_invalidate
103	};	104	};
104		105
105	static inline u32 rcventry2tidinfo(u32 rcventry)	106	static inline u32 rcventry2tidinfo(u32 rcventry)
@@ -162,7 +163,6 @@ int hfi1_user_exp_rcv_init(struct file *fp) · 56 unmodified lines
162		163
163	spin_lock_init(&fd->tid_lock);	164	spin_lock_init(&fd->tid_lock);
164	spin_lock_init(&fd->invalid_lock);	165	spin_lock_init(&fd->invalid_lock);
165	fd->tid_rb_root = RB_ROOT;
166		166
167	if (!uctxt->subctxt_cnt \|\| !fd->subctxt) {	167	if (!uctxt->subctxt_cnt \|\| !fd->subctxt) {
168	exp_tid_group_init(&uctxt->tid_group_list);	168	exp_tid_group_init(&uctxt->tid_group_list);
@@ -197,7 +197,7 @@ int hfi1_user_exp_rcv_init(struct file *fp) · 28 unmodified lines
197	if (!fd->entry_to_rb)	197	if (!fd->entry_to_rb)
198	return -ENOMEM;	198	return -ENOMEM;
199		199
200	if (!HFI1_CAP_IS_USET(TID_UNMAP)) {	200	if (!HFI1_CAP_UGET_MASK(uctxt->flags, TID_UNMAP)) {
201	fd->invalid_tid_idx = 0;	201	fd->invalid_tid_idx = 0;
202	fd->invalid_tids = kzalloc(uctxt->expected_count *	202	fd->invalid_tids = kzalloc(uctxt->expected_count *
203	sizeof(u32), GFP_KERNEL);	203	sizeof(u32), GFP_KERNEL);
@@ -208,15 +208,15 @@ int hfi1_user_exp_rcv_init(struct file *fp) · 4 unmodified lines
208		208
209	/*	209	/*
210	* Register MMU notifier callbacks. If the registration	210	* Register MMU notifier callbacks. If the registration
211	* fails, continue but turn off the TID caching for	211	* fails, continue without TID caching for this context.
212	* all user contexts.
213	*/	212	*/
214	ret = hfi1_mmu_rb_register(&fd->tid_rb_root, &tid_rb_ops);	213	ret = hfi1_mmu_rb_register(fd, fd->mm, &tid_rb_ops,
		214	dd->pport->hfi1_wq,
		215	&fd->handler);
215	if (ret) {	216	if (ret) {
216	dd_dev_info(dd,	217	dd_dev_info(dd,
217	"Failed MMU notifier registration %d\n",	218	"Failed MMU notifier registration %d\n",
218	ret);	219	ret);
219	HFI1_CAP_USET(TID_UNMAP);
220	ret = 0;	220	ret = 0;
221	}	221	}
222	}	222	}
@@ -235,7 +235,7 @@ int hfi1_user_exp_rcv_init(struct file *fp) · 12 unmodified lines
235	* init.	235	* init.
236	*/	236	*/
237	spin_lock(&fd->tid_lock);	237	spin_lock(&fd->tid_lock);
238	if (uctxt->subctxt_cnt && !HFI1_CAP_IS_USET(TID_UNMAP)) {	238	if (uctxt->subctxt_cnt && fd->handler) {
239	u16 remainder;	239	u16 remainder;
240		240
241	fd->tid_limit = uctxt->expected_count / uctxt->subctxt_cnt;	241	fd->tid_limit = uctxt->expected_count / uctxt->subctxt_cnt;
@@ -261,18 +261,16 @@ int hfi1_user_exp_rcv_free(struct hfi1_filedata *fd) · 19 unmodified lines
261	* The notifier would have been removed when the process'es mm	261	* The notifier would have been removed when the process'es mm
262	* was freed.	262	* was freed.
263	*/	263	*/
264	if (!HFI1_CAP_IS_USET(TID_UNMAP))	264	if (fd->handler)
265	hfi1_mmu_rb_unregister(&fd->tid_rb_root);	265	hfi1_mmu_rb_unregister(fd->handler);
266		266
267	kfree(fd->invalid_tids);	267	kfree(fd->invalid_tids);
268		268
269	if (!uctxt->cnt) {	269	if (!uctxt->cnt) {
270	if (!EXP_TID_SET_EMPTY(uctxt->tid_full_list))	270	if (!EXP_TID_SET_EMPTY(uctxt->tid_full_list))
271	unlock_exp_tids(uctxt, &uctxt->tid_full_list,	271	unlock_exp_tids(uctxt, &uctxt->tid_full_list, fd);
272	&fd->tid_rb_root);
273	if (!EXP_TID_SET_EMPTY(uctxt->tid_used_list))	272	if (!EXP_TID_SET_EMPTY(uctxt->tid_used_list))
274	unlock_exp_tids(uctxt, &uctxt->tid_used_list,	273	unlock_exp_tids(uctxt, &uctxt->tid_used_list, fd);
275	&fd->tid_rb_root);
276	list_for_each_entry_safe(grp, gptr, &uctxt->tid_group_list.list,	274	list_for_each_entry_safe(grp, gptr, &uctxt->tid_group_list.list,
277	list) {	275	list) {
278	list_del_init(&grp->list);	276	list_del_init(&grp->list);
@@ -399,12 +397,12 @@ int hfi1_user_exp_rcv_setup(struct file fp, struct hfi1_tid_info tinfo) · 120 unmodified lines
399	* pages, accept the amount pinned so far and program only that.	397	* pages, accept the amount pinned so far and program only that.
400	* User space knows how to deal with partially programmed buffers.	398	* User space knows how to deal with partially programmed buffers.
401	*/	399	*/
402	if (!hfi1_can_pin_pages(dd, fd->tid_n_pinned, npages)) {	400	if (!hfi1_can_pin_pages(dd, fd->mm, fd->tid_n_pinned, npages)) {
403	ret = -ENOMEM;	401	ret = -ENOMEM;
404	goto bail;	402	goto bail;
405	}	403	}
406		404
407	pinned = hfi1_acquire_user_pages(vaddr, npages, true, pages);	405	pinned = hfi1_acquire_user_pages(fd->mm, vaddr, npages, true, pages);
408	if (pinned <= 0) {	406	if (pinned <= 0) {
409	ret = pinned;	407	ret = pinned;
410	goto bail;	408	goto bail;
@@ -559,7 +557,7 @@ nomem: · 148 unmodified lines
559	* for example), unpin all unmapped pages so we can pin them nex time.	557	* for example), unpin all unmapped pages so we can pin them nex time.
560	*/	558	*/
561	if (mapped_pages != pinned) {	559	if (mapped_pages != pinned) {
562	hfi1_release_user_pages(current->mm, &pages[mapped_pages],	560	hfi1_release_user_pages(fd->mm, &pages[mapped_pages],
563	pinned - mapped_pages,	561	pinned - mapped_pages,
564	false);	562	false);
565	fd->tid_n_pinned -= pinned - mapped_pages;	563	fd->tid_n_pinned -= pinned - mapped_pages;
@@ -829,7 +827,6 @@ static int set_rcvarray_entry(struct file *fp, unsigned long vaddr, · 263 unmodified lines
829	struct hfi1_ctxtdata *uctxt = fd->uctxt;	827	struct hfi1_ctxtdata *uctxt = fd->uctxt;
830	struct tid_rb_node *node;	828	struct tid_rb_node *node;
831	struct hfi1_devdata *dd = uctxt->dd;	829	struct hfi1_devdata *dd = uctxt->dd;
832	struct rb_root *root = &fd->tid_rb_root;
833	dma_addr_t phys;	830	dma_addr_t phys;
834		831
835	/*	832	/*
@@ -861,10 +858,10 @@ static int set_rcvarray_entry(struct file *fp, unsigned long vaddr, · 25 unmodified lines
861	node->freed = false;	858	node->freed = false;
862	memcpy(node->pages, pages, sizeof(struct page ) npages);	859	memcpy(node->pages, pages, sizeof(struct page ) npages);
863		860
864	if (HFI1_CAP_IS_USET(TID_UNMAP))	861	if (!fd->handler)
865	ret = mmu_rb_insert(root, &node->mmu);	862	ret = tid_rb_insert(fd, &node->mmu);
866	else	863	else
867	ret = hfi1_mmu_rb_insert(root, &node->mmu);	864	ret = hfi1_mmu_rb_insert(fd->handler, &node->mmu);
868		865
869	if (ret) {	866	if (ret) {
870	hfi1_cdbg(TID, "Failed to insert RB node %u 0x%lx, 0x%lx %d",	867	hfi1_cdbg(TID, "Failed to insert RB node %u 0x%lx, 0x%lx %d",
@@ -904,19 +901,19 @@ static int unprogram_rcvarray(struct file *fp, u32 tidinfo, · 33 unmodified lines
904	node = fd->entry_to_rb[rcventry];	901	node = fd->entry_to_rb[rcventry];
905	if (!node \|\| node->rcventry != (uctxt->expected_base + rcventry))	902	if (!node \|\| node->rcventry != (uctxt->expected_base + rcventry))
906	return -EBADF;	903	return -EBADF;
907	if (HFI1_CAP_IS_USET(TID_UNMAP))
908	mmu_rb_remove(&fd->tid_rb_root, &node->mmu, NULL);
909	else
910	hfi1_mmu_rb_remove(&fd->tid_rb_root, &node->mmu);
911		904
912	if (grp)	905	if (grp)
913	*grp = node->grp;	906	*grp = node->grp;
914	clear_tid_node(fd, fd->subctxt, node);	907
		908	if (!fd->handler)
		909	cacheless_tid_rb_remove(fd, node);
		910	else
		911	hfi1_mmu_rb_remove(fd->handler, &node->mmu);
		912
915	return 0;	913	return 0;
916	}	914	}
917		915
918	static void clear_tid_node(struct hfi1_filedata *fd, u16 subctxt,	916	static void clear_tid_node(struct hfi1_filedata fd, struct tid_rb_node node)
919	struct tid_rb_node *node)
920	{	917	{
921	struct hfi1_ctxtdata *uctxt = fd->uctxt;	918	struct hfi1_ctxtdata *uctxt = fd->uctxt;
922	struct hfi1_devdata *dd = uctxt->dd;	919	struct hfi1_devdata *dd = uctxt->dd;
@@ -934,7 +931,7 @@ static void clear_tid_node(struct hfi1_filedata *fd, u16 subctxt, · 11 unmodified lines
934		931
935	pci_unmap_single(dd->pcidev, node->dma_addr, node->mmu.len,	932	pci_unmap_single(dd->pcidev, node->dma_addr, node->mmu.len,
936	PCI_DMA_FROMDEVICE);	933	PCI_DMA_FROMDEVICE);
937	hfi1_release_user_pages(current->mm, node->pages, node->npages, true);	934	hfi1_release_user_pages(fd->mm, node->pages, node->npages, true);
938	fd->tid_n_pinned -= node->npages;	935	fd->tid_n_pinned -= node->npages;
939		936
940	node->grp->used--;	937	node->grp->used--;
@@ -949,12 +946,15 @@ static void clear_tid_node(struct hfi1_filedata *fd, u16 subctxt, · 8 unmodified lines
949	kfree(node);	946	kfree(node);
950	}	947	}
951		948
		949	/*
		950	* As a simple helper for hfi1_user_exp_rcv_free, this function deals with
		951	* clearing nodes in the non-cached case.
		952	*/
952	static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,	953	static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,
953	struct exp_tid_set set, struct rb_root root)	954	struct exp_tid_set *set,
		955	struct hfi1_filedata *fd)
954	{	956	{
955	struct tid_group grp, ptr;	957	struct tid_group grp, ptr;
956	struct hfi1_filedata *fd = container_of(root, struct hfi1_filedata,
957	tid_rb_root);
958	int i;	958	int i;
959		959
960	list_for_each_entry_safe(grp, ptr, &set->list, list) {	960	list_for_each_entry_safe(grp, ptr, &set->list, list) {
@@ -969,22 +969,23 @@ static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt, · 8 unmodified lines
969	uctxt->expected_base];	969	uctxt->expected_base];
970	if (!node \|\| node->rcventry != rcventry)	970	if (!node \|\| node->rcventry != rcventry)
971	continue;	971	continue;
972	if (HFI1_CAP_IS_USET(TID_UNMAP))	972
973	mmu_rb_remove(&fd->tid_rb_root,	973	cacheless_tid_rb_remove(fd, node);
974	&node->mmu, NULL);
975	else
976	hfi1_mmu_rb_remove(&fd->tid_rb_root,
977	&node->mmu);
978	clear_tid_node(fd, -1, node);
979	}	974	}
980	}	975	}
981	}	976	}
982	}	977	}
983		978
984	static int mmu_rb_invalidate(struct rb_root root, struct mmu_rb_node mnode)	979	/*
		980	* Always return 0 from this function. A non-zero return indicates that the
		981	* remove operation will be called and that memory should be unpinned.
		982	* However, the driver cannot unpin out from under PSM. Instead, retain the
		983	* memory (by returning 0) and inform PSM that the memory is going away. PSM
		984	* will call back later when it has removed the memory from its list.
		985	*/
		986	static int tid_rb_invalidate(void arg, struct mmu_rb_node mnode)
985	{	987	{
986	struct hfi1_filedata *fdata =	988	struct hfi1_filedata *fdata = arg;
987	container_of(root, struct hfi1_filedata, tid_rb_root);
988	struct hfi1_ctxtdata *uctxt = fdata->uctxt;	989	struct hfi1_ctxtdata *uctxt = fdata->uctxt;
989	struct tid_rb_node *node =	990	struct tid_rb_node *node =
990	container_of(mnode, struct tid_rb_node, mmu);	991	container_of(mnode, struct tid_rb_node, mmu);
@@ -1025,10 +1026,9 @@ static int mmu_rb_invalidate(struct rb_root root, struct mmu_rb_node mnode) · 34 unmodified lines
1025	return 0;	1026	return 0;
1026	}	1027	}
1027		1028
1028	static int mmu_rb_insert(struct rb_root root, struct mmu_rb_node node)	1029	static int tid_rb_insert(void arg, struct mmu_rb_node node)
1029	{	1030	{
1030	struct hfi1_filedata *fdata =	1031	struct hfi1_filedata *fdata = arg;
1031	container_of(root, struct hfi1_filedata, tid_rb_root);
1032	struct tid_rb_node *tnode =	1032	struct tid_rb_node *tnode =
1033	container_of(node, struct tid_rb_node, mmu);	1033	container_of(node, struct tid_rb_node, mmu);
1034	u32 base = fdata->uctxt->expected_base;	1034	u32 base = fdata->uctxt->expected_base;
@@ -1037,14 +1037,20 @@ static int mmu_rb_insert(struct rb_root root, struct mmu_rb_node node) · 2 unmodified lines
1037	return 0;	1037	return 0;
1038	}	1038	}
1039		1039
1040	static void mmu_rb_remove(struct rb_root root, struct mmu_rb_node node,	1040	static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,
1041	struct mm_struct *mm)	1041	struct tid_rb_node *tnode)
1042	{	1042	{
1043	struct hfi1_filedata *fdata =
1044	container_of(root, struct hfi1_filedata, tid_rb_root);
1045	struct tid_rb_node *tnode =
1046	container_of(node, struct tid_rb_node, mmu);
1047	u32 base = fdata->uctxt->expected_base;	1043	u32 base = fdata->uctxt->expected_base;
1048		1044
1049	fdata->entry_to_rb[tnode->rcventry - base] = NULL;	1045	fdata->entry_to_rb[tnode->rcventry - base] = NULL;
		1046	clear_tid_node(fdata, tnode);
		1047	}
		1048
		1049	static void tid_rb_remove(void arg, struct mmu_rb_node node)
		1050	{
		1051	struct hfi1_filedata *fdata = arg;
		1052	struct tid_rb_node *tnode =
		1053	container_of(node, struct tid_rb_node, mmu);
		1054
		1055	cacheless_tid_rb_remove(fdata, tnode);
1050	}	1056	}

drivers/infiniband/hw/hfi1/user_pages.c+10−9

  * could keeping caching buffers.
  *
  */
-bool hfi1_can_pin_pages(struct hfi1_devdata *dd, u32 nlocked, u32 npages)
+bool hfi1_can_pin_pages(struct hfi1_devdata *dd, struct mm_struct *mm,
+			u32 nlocked, u32 npages)
 {
 	unsigned long ulimit = rlimit(RLIMIT_MEMLOCK), pinned, cache_limit,
 		size = (cache_size * (1UL << 20)); /* convert to bytes */
 	/* Convert to number of pages */
 	size = DIV_ROUND_UP(size, PAGE_SIZE);
-	down_read(&current->mm->mmap_sem);
+	down_read(&mm->mmap_sem);
-	pinned = current->mm->pinned_vm;
+	pinned = mm->pinned_vm;
-	up_read(&current->mm->mmap_sem);
+	up_read(&mm->mmap_sem);
 	/* First, check the absolute limit against all pinned pages. */
 	if (pinned + npages >= ulimit && !can_lock)
 	return ((nlocked + npages) <= size) || can_lock;
 }
-int hfi1_acquire_user_pages(unsigned long vaddr, size_t npages, bool writable,
+int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t npages,
-			    struct page **pages)
+			    bool writable, struct page **pages)
 {
 	int ret;
 	if (ret < 0)
 		return ret;
-	down_write(&current->mm->mmap_sem);
+	down_write(&mm->mmap_sem);
-	current->mm->pinned_vm += ret;
+	mm->pinned_vm += ret;
-	up_write(&current->mm->mmap_sem);
+	up_write(&mm->mmap_sem);
 	return ret;
 }

drivers/infiniband/hw/hfi1/user_sdma.c+171−130

@@ -145,7 +145,7 @@ MODULE_PARM_DESC(sdma_comp_size, "Size of User SDMA completion ring. Default: 12 · 144 unmodified lines
145	/* Last packet in the request */	145	/* Last packet in the request */
146	#define TXREQ_FLAGS_REQ_LAST_PKT BIT(0)	146	#define TXREQ_FLAGS_REQ_LAST_PKT BIT(0)
147		147
148	#define SDMA_REQ_IN_USE 0	148	/* SDMA request flag bits */
149	#define SDMA_REQ_FOR_THREAD 1	149	#define SDMA_REQ_FOR_THREAD 1
150	#define SDMA_REQ_SEND_DONE 2	150	#define SDMA_REQ_SEND_DONE 2
151	#define SDMA_REQ_HAVE_AHG 3	151	#define SDMA_REQ_HAVE_AHG 3
@@ -183,16 +183,18 @@ struct user_sdma_iovec { · 31 unmodified lines
183	struct sdma_mmu_node *node;	183	struct sdma_mmu_node *node;
184	};	184	};
185		185
186	#define SDMA_CACHE_NODE_EVICT 0
187
188	struct sdma_mmu_node {	186	struct sdma_mmu_node {
189	struct mmu_rb_node rb;	187	struct mmu_rb_node rb;
190	struct list_head list;
191	struct hfi1_user_sdma_pkt_q *pq;	188	struct hfi1_user_sdma_pkt_q *pq;
192	atomic_t refcount;	189	atomic_t refcount;
193	struct page **pages;	190	struct page **pages;
194	unsigned npages;	191	unsigned npages;
195	unsigned long flags;	192	};
		193
		194	/* evict operation argument */
		195	struct evict_data {
		196	u32 cleared; /* count evicted so far */
		197	u32 target; /* target count to evict */
196	};	198	};
197		199
198	struct user_sdma_request {	200	struct user_sdma_request {
@@ -305,14 +307,16 @@ static int defer_packet_queue( · 106 unmodified lines
305	unsigned seq);	307	unsigned seq);
306	static void activate_packet_queue(struct iowait *, int);	308	static void activate_packet_queue(struct iowait *, int);
307	static bool sdma_rb_filter(struct mmu_rb_node *, unsigned long, unsigned long);	309	static bool sdma_rb_filter(struct mmu_rb_node *, unsigned long, unsigned long);
308	static int sdma_rb_insert(struct rb_root , struct mmu_rb_node );	310	static int sdma_rb_insert(void , struct mmu_rb_node );
309	static void sdma_rb_remove(struct rb_root , struct mmu_rb_node ,	311	static int sdma_rb_evict(void arg, struct mmu_rb_node mnode,
310	struct mm_struct *);	312	void arg2, bool stop);
311	static int sdma_rb_invalidate(struct rb_root , struct mmu_rb_node );	313	static void sdma_rb_remove(void , struct mmu_rb_node );
		314	static int sdma_rb_invalidate(void , struct mmu_rb_node );
312		315
313	static struct mmu_rb_ops sdma_rb_ops = {	316	static struct mmu_rb_ops sdma_rb_ops = {
314	.filter = sdma_rb_filter,	317	.filter = sdma_rb_filter,
315	.insert = sdma_rb_insert,	318	.insert = sdma_rb_insert,
		319	.evict = sdma_rb_evict,
316	.remove = sdma_rb_remove,	320	.remove = sdma_rb_remove,
317	.invalidate = sdma_rb_invalidate	321	.invalidate = sdma_rb_invalidate
318	};	322	};
@@ -397,6 +401,11 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata uctxt, struct file fp) · 78 unmodified lines
397	if (!pq->reqs)	401	if (!pq->reqs)
398	goto pq_reqs_nomem;	402	goto pq_reqs_nomem;
399		403
		404	memsize = BITS_TO_LONGS(hfi1_sdma_comp_ring_size) * sizeof(long);
		405	pq->req_in_use = kzalloc(memsize, GFP_KERNEL);
		406	if (!pq->req_in_use)
		407	goto pq_reqs_no_in_use;
		408
400	INIT_LIST_HEAD(&pq->list);	409	INIT_LIST_HEAD(&pq->list);
401	pq->dd = dd;	410	pq->dd = dd;
402	pq->ctxt = uctxt->ctxt;	411	pq->ctxt = uctxt->ctxt;
@@ -405,9 +414,8 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata uctxt, struct file fp) · 2 unmodified lines
405	pq->state = SDMA_PKT_Q_INACTIVE;	414	pq->state = SDMA_PKT_Q_INACTIVE;
406	atomic_set(&pq->n_reqs, 0);	415	atomic_set(&pq->n_reqs, 0);
407	init_waitqueue_head(&pq->wait);	416	init_waitqueue_head(&pq->wait);
408	pq->sdma_rb_root = RB_ROOT;	417	atomic_set(&pq->n_locked, 0);
409	INIT_LIST_HEAD(&pq->evict);	418	pq->mm = fd->mm;
410	spin_lock_init(&pq->evict_lock);
411		419
412	iowait_init(&pq->busy, 0, NULL, defer_packet_queue,	420	iowait_init(&pq->busy, 0, NULL, defer_packet_queue,
413	activate_packet_queue, NULL);	421	activate_packet_queue, NULL);
@@ -437,7 +445,8 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata uctxt, struct file fp) · 23 unmodified lines
437	cq->nentries = hfi1_sdma_comp_ring_size;	445	cq->nentries = hfi1_sdma_comp_ring_size;
438	fd->cq = cq;	446	fd->cq = cq;
439		447
440	ret = hfi1_mmu_rb_register(&pq->sdma_rb_root, &sdma_rb_ops);	448	ret = hfi1_mmu_rb_register(pq, pq->mm, &sdma_rb_ops, dd->pport->hfi1_wq,
		449	&pq->handler);
441	if (ret) {	450	if (ret) {
442	dd_dev_err(dd, "Failed to register with MMU %d", ret);	451	dd_dev_err(dd, "Failed to register with MMU %d", ret);
443	goto done;	452	goto done;
@@ -453,6 +462,8 @@ cq_comps_nomem: · 9 unmodified lines
453	cq_nomem:	462	cq_nomem:
454	kmem_cache_destroy(pq->txreq_cache);	463	kmem_cache_destroy(pq->txreq_cache);
455	pq_txreq_nomem:	464	pq_txreq_nomem:
		465	kfree(pq->req_in_use);
		466	pq_reqs_no_in_use:
456	kfree(pq->reqs);	467	kfree(pq->reqs);
457	pq_reqs_nomem:	468	pq_reqs_nomem:
458	kfree(pq);	469	kfree(pq);
@@ -472,8 +483,9 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd) · 13 unmodified lines
472	hfi1_cdbg(SDMA, "[%u:%u:%u] Freeing user SDMA queues", uctxt->dd->unit,	483	hfi1_cdbg(SDMA, "[%u:%u:%u] Freeing user SDMA queues", uctxt->dd->unit,
473	uctxt->ctxt, fd->subctxt);	484	uctxt->ctxt, fd->subctxt);
474	pq = fd->pq;	485	pq = fd->pq;
475	hfi1_mmu_rb_unregister(&pq->sdma_rb_root);
476	if (pq) {	486	if (pq) {
		487	if (pq->handler)
		488	hfi1_mmu_rb_unregister(pq->handler);
477	spin_lock_irqsave(&uctxt->sdma_qlock, flags);	489	spin_lock_irqsave(&uctxt->sdma_qlock, flags);
478	if (!list_empty(&pq->list))	490	if (!list_empty(&pq->list))
479	list_del_init(&pq->list);	491	list_del_init(&pq->list);
@@ -484,6 +496,7 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd) · 4 unmodified lines
484	pq->wait,	496	pq->wait,
485	(ACCESS_ONCE(pq->state) == SDMA_PKT_Q_INACTIVE));	497	(ACCESS_ONCE(pq->state) == SDMA_PKT_Q_INACTIVE));
486	kfree(pq->reqs);	498	kfree(pq->reqs);
		499	kfree(pq->req_in_use);
487	kmem_cache_destroy(pq->txreq_cache);	500	kmem_cache_destroy(pq->txreq_cache);
488	kfree(pq);	501	kfree(pq);
489	fd->pq = NULL;	502	fd->pq = NULL;
@@ -496,10 +509,31 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd) · 6 unmodified lines
496	return 0;	509	return 0;
497	}	510	}
498		511
		512	static u8 dlid_to_selector(u16 dlid)
		513	{
		514	static u8 mapping[256];
		515	static int initialized;
		516	static u8 next;
		517	int hash;
		518
		519	if (!initialized) {
		520	memset(mapping, 0xFF, 256);
		521	initialized = 1;
		522	}
		523
		524	hash = ((dlid >> 8) ^ dlid) & 0xFF;
		525	if (mapping[hash] == 0xFF) {
		526	mapping[hash] = next;
		527	next = (next + 1) & 0x7F;
		528	}
		529
		530	return mapping[hash];
		531	}
		532
499	int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec,	533	int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec,
500	unsigned long dim, unsigned long *count)	534	unsigned long dim, unsigned long *count)
501	{	535	{
502	int ret = 0, i = 0;	536	int ret = 0, i;
503	struct hfi1_filedata *fd = fp->private_data;	537	struct hfi1_filedata *fd = fp->private_data;
504	struct hfi1_ctxtdata *uctxt = fd->uctxt;	538	struct hfi1_ctxtdata *uctxt = fd->uctxt;
505	struct hfi1_user_sdma_pkt_q *pq = fd->pq;	539	struct hfi1_user_sdma_pkt_q *pq = fd->pq;
@@ -511,6 +545,8 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 5 unmodified lines
511	struct user_sdma_request *req;	545	struct user_sdma_request *req;
512	u8 opcode, sc, vl;	546	u8 opcode, sc, vl;
513	int req_queued = 0;	547	int req_queued = 0;
		548	u16 dlid;
		549	u8 selector;
514		550
515	if (iovec[idx].iov_len < sizeof(info) + sizeof(req->hdr)) {	551	if (iovec[idx].iov_len < sizeof(info) + sizeof(req->hdr)) {
516	hfi1_cdbg(	552	hfi1_cdbg(
@@ -529,30 +565,48 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 12 unmodified lines
529		565
530	trace_hfi1_sdma_user_reqinfo(dd, uctxt->ctxt, fd->subctxt,	566	trace_hfi1_sdma_user_reqinfo(dd, uctxt->ctxt, fd->subctxt,
531	(u16 *)&info);	567	(u16 *)&info);
532	if (cq->comps[info.comp_idx].status == QUEUED \|\|	568
533	test_bit(SDMA_REQ_IN_USE, &pq->reqs[info.comp_idx].flags)) {	569	if (info.comp_idx >= hfi1_sdma_comp_ring_size) {
534	hfi1_cdbg(SDMA, "[%u:%u:%u] Entry %u is in QUEUED state",	570	hfi1_cdbg(SDMA,
535	dd->unit, uctxt->ctxt, fd->subctxt,	571	"[%u:%u:%u:%u] Invalid comp index",
536	info.comp_idx);	572	dd->unit, uctxt->ctxt, fd->subctxt, info.comp_idx);
537	return -EBADSLT;	573	return -EINVAL;
538	}	574	}
		575
		576	/*
		577	* Sanity check the header io vector count. Need at least 1 vector
		578	* (header) and cannot be larger than the actual io vector count.
		579	*/
		580	if (req_iovcnt(info.ctrl) < 1 \|\| req_iovcnt(info.ctrl) > dim) {
		581	hfi1_cdbg(SDMA,
		582	"[%u:%u:%u:%u] Invalid iov count %d, dim %ld",
		583	dd->unit, uctxt->ctxt, fd->subctxt, info.comp_idx,
		584	req_iovcnt(info.ctrl), dim);
		585	return -EINVAL;
		586	}
		587
539	if (!info.fragsize) {	588	if (!info.fragsize) {
540	hfi1_cdbg(SDMA,	589	hfi1_cdbg(SDMA,
541	"[%u:%u:%u:%u] Request does not specify fragsize",	590	"[%u:%u:%u:%u] Request does not specify fragsize",
542	dd->unit, uctxt->ctxt, fd->subctxt, info.comp_idx);	591	dd->unit, uctxt->ctxt, fd->subctxt, info.comp_idx);
543	return -EINVAL;	592	return -EINVAL;
544	}	593	}
		594
		595	/* Try to claim the request. */
		596	if (test_and_set_bit(info.comp_idx, pq->req_in_use)) {
		597	hfi1_cdbg(SDMA, "[%u:%u:%u] Entry %u is in use",
		598	dd->unit, uctxt->ctxt, fd->subctxt,
		599	info.comp_idx);
		600	return -EBADSLT;
		601	}
545	/*	602	/*
546	* We've done all the safety checks that we can up to this point,	603	* All safety checks have been done and this request has been claimed.
547	* "allocate" the request entry.
548	*/	604	*/
549	hfi1_cdbg(SDMA, "[%u:%u:%u] Using req/comp entry %u\n", dd->unit,	605	hfi1_cdbg(SDMA, "[%u:%u:%u] Using req/comp entry %u\n", dd->unit,
550	uctxt->ctxt, fd->subctxt, info.comp_idx);	606	uctxt->ctxt, fd->subctxt, info.comp_idx);
551	req = pq->reqs + info.comp_idx;	607	req = pq->reqs + info.comp_idx;
552	memset(req, 0, sizeof(*req));	608	memset(req, 0, sizeof(*req));
553	/* Mark the request as IN_USE before we start filling it in. */	609	req->data_iovs = req_iovcnt(info.ctrl) - 1; /* subtract header vector */
554	set_bit(SDMA_REQ_IN_USE, &req->flags);
555	req->data_iovs = req_iovcnt(info.ctrl) - 1;
556	req->pq = pq;	610	req->pq = pq;
557	req->cq = cq;	611	req->cq = cq;
558	req->status = -1;	612	req->status = -1;
@@ -560,13 +614,22 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 1 unmodified line
560		614
561	memcpy(&req->info, &info, sizeof(info));	615	memcpy(&req->info, &info, sizeof(info));
562		616
563	if (req_opcode(info.ctrl) == EXPECTED)	617	if (req_opcode(info.ctrl) == EXPECTED) {
		618	/* expected must have a TID info and at least one data vector */
		619	if (req->data_iovs < 2) {
		620	SDMA_DBG(req,
		621	"Not enough vectors for expected request");
		622	ret = -EINVAL;
		623	goto free_req;
		624	}
564	req->data_iovs--;	625	req->data_iovs--;
		626	}
565		627
566	if (!info.npkts \|\| req->data_iovs > MAX_VECTORS_PER_REQ) {	628	if (!info.npkts \|\| req->data_iovs > MAX_VECTORS_PER_REQ) {
567	SDMA_DBG(req, "Too many vectors (%u/%u)", req->data_iovs,	629	SDMA_DBG(req, "Too many vectors (%u/%u)", req->data_iovs,
568	MAX_VECTORS_PER_REQ);	630	MAX_VECTORS_PER_REQ);
569	return -EINVAL;	631	ret = -EINVAL;
		632	goto free_req;
570	}	633	}
571	/* Copy the header from the user buffer */	634	/* Copy the header from the user buffer */
572	ret = copy_from_user(&req->hdr, iovec[idx].iov_base + sizeof(info),	635	ret = copy_from_user(&req->hdr, iovec[idx].iov_base + sizeof(info),
@@ -634,7 +697,7 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 61 unmodified lines
634	idx++;	697	idx++;
635		698
636	/* Save all the IO vector structures */	699	/* Save all the IO vector structures */
637	while (i < req->data_iovs) {	700	for (i = 0; i < req->data_iovs; i++) {
638	INIT_LIST_HEAD(&req->iovs[i].list);	701	INIT_LIST_HEAD(&req->iovs[i].list);
639	memcpy(&req->iovs[i].iov, iovec + idx++, sizeof(struct iovec));	702	memcpy(&req->iovs[i].iov, iovec + idx++, sizeof(struct iovec));
640	ret = pin_vector_pages(req, &req->iovs[i]);	703	ret = pin_vector_pages(req, &req->iovs[i]);
@@ -642,7 +705,7 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 1 unmodified line
642	req->status = ret;	705	req->status = ret;
643	goto free_req;	706	goto free_req;
644	}	707	}
645	req->data_len += req->iovs[i++].iov.iov_len;	708	req->data_len += req->iovs[i].iov.iov_len;
646	}	709	}
647	SDMA_DBG(req, "total data length %u", req->data_len);	710	SDMA_DBG(req, "total data length %u", req->data_len);
648		711
@@ -686,9 +749,13 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 37 unmodified lines
686	idx++;	749	idx++;
687	}	750	}
688		751
		752	dlid = be16_to_cpu(req->hdr.lrh[1]);
		753	selector = dlid_to_selector(dlid);
		754
689	/* Have to select the engine */	755	/* Have to select the engine */
690	req->sde = sdma_select_engine_vl(dd,	756	req->sde = sdma_select_engine_vl(dd,
691	(u32)(uctxt->ctxt + fd->subctxt),	757	(u32)(uctxt->ctxt + fd->subctxt +
		758	selector),
692	vl);	759	vl);
693	if (!req->sde \|\| !sdma_running(req->sde)) {	760	if (!req->sde \|\| !sdma_running(req->sde)) {
694	ret = -ECOMM;	761	ret = -ECOMM;
@@ -766,14 +833,21 @@ static inline u32 compute_data_length(struct user_sdma_request *req, · 71 unmodified lines
766	* The size of the data of the first packet is in the header	833	* The size of the data of the first packet is in the header
767	* template. However, it includes the header and ICRC, which need	834	* template. However, it includes the header and ICRC, which need
768	* to be subtracted.	835	* to be subtracted.
		836	* The minimum representable packet data length in a header is 4 bytes,
		837	* therefore, when the data length request is less than 4 bytes, there's
		838	* only one packet, and the packet data length is equal to that of the
		839	* request data length.
769	* The size of the remaining packets is the minimum of the frag	840	* The size of the remaining packets is the minimum of the frag
770	* size (MTU) or remaining data in the request.	841	* size (MTU) or remaining data in the request.
771	*/	842	*/
772	u32 len;	843	u32 len;
773		844
774	if (!req->seqnum) {	845	if (!req->seqnum) {
775	len = ((be16_to_cpu(req->hdr.lrh[2]) << 2) -	846	if (req->data_len < sizeof(u32))
776	(sizeof(tx->hdr) - 4));	847	len = req->data_len;
		848	else
		849	len = ((be16_to_cpu(req->hdr.lrh[2]) << 2) -
		850	(sizeof(tx->hdr) - 4));
777	} else if (req_opcode(req->info.ctrl) == EXPECTED) {	851	} else if (req_opcode(req->info.ctrl) == EXPECTED) {
778	u32 tidlen = EXP_TID_GET(req->tids[req->tididx], LEN) *	852	u32 tidlen = EXP_TID_GET(req->tids[req->tididx], LEN) *
779	PAGE_SIZE;	853	PAGE_SIZE;
@@ -803,6 +877,13 @@ static inline u32 compute_data_length(struct user_sdma_request *req, · 23 unmodified lines
803	return len;	877	return len;
804	}	878	}
805		879
		880	static inline u32 pad_len(u32 len)
		881	{
		882	if (len & (sizeof(u32) - 1))
		883	len += sizeof(u32) - (len & (sizeof(u32) - 1));
		884	return len;
		885	}
		886
806	static inline u32 get_lrh_len(struct hfi1_pkt_header hdr, u32 len)	887	static inline u32 get_lrh_len(struct hfi1_pkt_header hdr, u32 len)
807	{	888	{
808	/* (Size of complete header - size of PBC) + 4B ICRC + data length */	889	/* (Size of complete header - size of PBC) + 4B ICRC + data length */
@@ -894,7 +975,8 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts) · 85 unmodified lines
894	if (test_bit(SDMA_REQ_HAVE_AHG, &req->flags)) {	975	if (test_bit(SDMA_REQ_HAVE_AHG, &req->flags)) {
895	if (!req->seqnum) {	976	if (!req->seqnum) {
896	u16 pbclen = le16_to_cpu(req->hdr.pbc[0]);	977	u16 pbclen = le16_to_cpu(req->hdr.pbc[0]);
897	u32 lrhlen = get_lrh_len(req->hdr, datalen);	978	u32 lrhlen = get_lrh_len(req->hdr,
		979	pad_len(datalen));
898	/*	980	/*
899	* Copy the request header into the tx header	981	* Copy the request header into the tx header
900	* because the HW needs a cacheline-aligned	982	* because the HW needs a cacheline-aligned
@@ -1048,39 +1130,24 @@ static inline int num_user_pages(const struct iovec *iov) · 147 unmodified lines
1048		1130
1049	static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)	1131	static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)
1050	{	1132	{
1051	u32 cleared = 0;	1133	struct evict_data evict_data;
1052	struct sdma_mmu_node node, ptr;
1053	struct list_head to_evict = LIST_HEAD_INIT(to_evict);
1054
1055	spin_lock(&pq->evict_lock);
1056	list_for_each_entry_safe_reverse(node, ptr, &pq->evict, list) {
1057	/* Make sure that no one is still using the node. */
1058	if (!atomic_read(&node->refcount)) {
1059	set_bit(SDMA_CACHE_NODE_EVICT, &node->flags);
1060	list_del_init(&node->list);
1061	list_add(&node->list, &to_evict);
1062	cleared += node->npages;
1063	if (cleared >= npages)
1064	break;
1065	}
1066	}
1067	spin_unlock(&pq->evict_lock);
1068
1069	list_for_each_entry_safe(node, ptr, &to_evict, list)
1070	hfi1_mmu_rb_remove(&pq->sdma_rb_root, &node->rb);
1071		1134
1072	return cleared;	1135	evict_data.cleared = 0;
		1136	evict_data.target = npages;
		1137	hfi1_mmu_rb_evict(pq->handler, &evict_data);
		1138	return evict_data.cleared;
1073	}	1139	}
1074		1140
1075	static int pin_vector_pages(struct user_sdma_request *req,	1141	static int pin_vector_pages(struct user_sdma_request *req,
1076	struct user_sdma_iovec *iovec) {	1142	struct user_sdma_iovec *iovec)
		1143	{
1077	int ret = 0, pinned, npages, cleared;	1144	int ret = 0, pinned, npages, cleared;
1078	struct page **pages;	1145	struct page **pages;
1079	struct hfi1_user_sdma_pkt_q *pq = req->pq;	1146	struct hfi1_user_sdma_pkt_q *pq = req->pq;
1080	struct sdma_mmu_node *node = NULL;	1147	struct sdma_mmu_node *node = NULL;
1081	struct mmu_rb_node *rb_node;	1148	struct mmu_rb_node *rb_node;
1082		1149
1083	rb_node = hfi1_mmu_rb_extract(&pq->sdma_rb_root,	1150	rb_node = hfi1_mmu_rb_extract(pq->handler,
1084	(unsigned long)iovec->iov.iov_base,	1151	(unsigned long)iovec->iov.iov_base,
1085	iovec->iov.iov_len);	1152	iovec->iov.iov_len);
1086	if (rb_node && !IS_ERR(rb_node))	1153	if (rb_node && !IS_ERR(rb_node))
@@ -1096,7 +1163,6 @@ static int pin_vector_pages(struct user_sdma_request *req, · 9 unmodified lines
1096	node->rb.addr = (unsigned long)iovec->iov.iov_base;	1163	node->rb.addr = (unsigned long)iovec->iov.iov_base;
1097	node->pq = pq;	1164	node->pq = pq;
1098	atomic_set(&node->refcount, 0);	1165	atomic_set(&node->refcount, 0);
1099	INIT_LIST_HEAD(&node->list);
1100	}	1166	}
1101		1167
1102	npages = num_user_pages(&iovec->iov);	1168	npages = num_user_pages(&iovec->iov);
@@ -1111,28 +1177,14 @@ static int pin_vector_pages(struct user_sdma_request *req, · 8 unmodified lines
1111		1177
1112	npages -= node->npages;	1178	npages -= node->npages;
1113		1179
1114	/*
1115	* If rb_node is NULL, it means that this is brand new node
1116	* and, therefore not on the eviction list.
1117	* If, however, the rb_node is non-NULL, it means that the
1118	* node is already in RB tree and, therefore on the eviction
1119	* list (nodes are unconditionally inserted in the eviction
1120	* list). In that case, we have to remove the node prior to
1121	* calling the eviction function in order to prevent it from
1122	* freeing this node.
1123	*/
1124	if (rb_node) {
1125	spin_lock(&pq->evict_lock);
1126	list_del_init(&node->list);
1127	spin_unlock(&pq->evict_lock);
1128	}
1129	retry:	1180	retry:
1130	if (!hfi1_can_pin_pages(pq->dd, pq->n_locked, npages)) {	1181	if (!hfi1_can_pin_pages(pq->dd, pq->mm,
		1182	atomic_read(&pq->n_locked), npages)) {
1131	cleared = sdma_cache_evict(pq, npages);	1183	cleared = sdma_cache_evict(pq, npages);
1132	if (cleared >= npages)	1184	if (cleared >= npages)
1133	goto retry;	1185	goto retry;
1134	}	1186	}
1135	pinned = hfi1_acquire_user_pages(	1187	pinned = hfi1_acquire_user_pages(pq->mm,
1136	((unsigned long)iovec->iov.iov_base +	1188	((unsigned long)iovec->iov.iov_base +
1137	(node->npages * PAGE_SIZE)), npages, 0,	1189	(node->npages * PAGE_SIZE)), npages, 0,
1138	pages + node->npages);	1190	pages + node->npages);
@@ -1142,7 +1194,7 @@ retry: · 3 unmodified lines
1142	goto bail;	1194	goto bail;
1143	}	1195	}
1144	if (pinned != npages) {	1196	if (pinned != npages) {
1145	unpin_vector_pages(current->mm, pages, node->npages,	1197	unpin_vector_pages(pq->mm, pages, node->npages,
1146	pinned);	1198	pinned);
1147	ret = -EFAULT;	1199	ret = -EFAULT;
1148	goto bail;	1200	goto bail;
@@ -1152,28 +1204,22 @@ retry: · 3 unmodified lines
1152	node->pages = pages;	1204	node->pages = pages;
1153	node->npages += pinned;	1205	node->npages += pinned;
1154	npages = node->npages;	1206	npages = node->npages;
1155	spin_lock(&pq->evict_lock);	1207	atomic_add(pinned, &pq->n_locked);
1156	list_add(&node->list, &pq->evict);
1157	pq->n_locked += pinned;
1158	spin_unlock(&pq->evict_lock);
1159	}	1208	}
1160	iovec->pages = node->pages;	1209	iovec->pages = node->pages;
1161	iovec->npages = npages;	1210	iovec->npages = npages;
1162	iovec->node = node;	1211	iovec->node = node;
1163		1212
1164	ret = hfi1_mmu_rb_insert(&req->pq->sdma_rb_root, &node->rb);	1213	ret = hfi1_mmu_rb_insert(req->pq->handler, &node->rb);
1165	if (ret) {	1214	if (ret) {
1166	spin_lock(&pq->evict_lock);	1215	atomic_sub(node->npages, &pq->n_locked);
1167	if (!list_empty(&node->list))	1216	iovec->node = NULL;
1168	list_del(&node->list);
1169	pq->n_locked -= node->npages;
1170	spin_unlock(&pq->evict_lock);
1171	goto bail;	1217	goto bail;
1172	}	1218	}
1173	return 0;	1219	return 0;
1174	bail:	1220	bail:
1175	if (rb_node)	1221	if (rb_node)
1176	unpin_vector_pages(current->mm, node->pages, 0, node->npages);	1222	unpin_vector_pages(pq->mm, node->pages, 0, node->npages);
1177	kfree(node);	1223	kfree(node);
1178	return ret;	1224	return ret;
1179	}	1225	}
@@ -1181,7 +1227,7 @@ bail: · 1 unmodified line
1181	static void unpin_vector_pages(struct mm_struct mm, struct page *pages,	1227	static void unpin_vector_pages(struct mm_struct mm, struct page *pages,
1182	unsigned start, unsigned npages)	1228	unsigned start, unsigned npages)
1183	{	1229	{
1184	hfi1_release_user_pages(mm, pages + start, npages, 0);	1230	hfi1_release_user_pages(mm, pages + start, npages, false);
1185	kfree(pages);	1231	kfree(pages);
1186	}	1232	}
1187		1233
@@ -1192,16 +1238,14 @@ static int check_header_template(struct user_sdma_request *req, · 4 unmodified lines
1192	/*	1238	/*
1193	* Perform safety checks for any type of packet:	1239	* Perform safety checks for any type of packet:
1194	* - transfer size is multiple of 64bytes	1240	* - transfer size is multiple of 64bytes
1195	* - packet length is multiple of 4bytes	1241	* - packet length is multiple of 4 bytes
1196	* - entire request length is multiple of 4bytes
1197	* - packet length is not larger than MTU size	1242	* - packet length is not larger than MTU size
1198	*	1243	*
1199	* These checks are only done for the first packet of the	1244	* These checks are only done for the first packet of the
1200	* transfer since the header is "given" to us by user space.	1245	* transfer since the header is "given" to us by user space.
1201	* For the remainder of the packets we compute the values.	1246	* For the remainder of the packets we compute the values.
1202	*/	1247	*/
1203	if (req->info.fragsize % PIO_BLOCK_SIZE \|\|	1248	if (req->info.fragsize % PIO_BLOCK_SIZE \|\| lrhlen & 0x3 \|\|
1204	lrhlen & 0x3 \|\| req->data_len & 0x3 \|\|
1205	lrhlen > get_lrh_len(*hdr, req->info.fragsize))	1249	lrhlen > get_lrh_len(*hdr, req->info.fragsize))
1206	return -EINVAL;	1250	return -EINVAL;
1207		1251
@@ -1263,7 +1307,7 @@ static int set_txreq_header(struct user_sdma_request *req, · 55 unmodified lines
1263	struct hfi1_pkt_header *hdr = &tx->hdr;	1307	struct hfi1_pkt_header *hdr = &tx->hdr;
1264	u16 pbclen;	1308	u16 pbclen;
1265	int ret;	1309	int ret;
1266	u32 tidval = 0, lrhlen = get_lrh_len(*hdr, datalen);	1310	u32 tidval = 0, lrhlen = get_lrh_len(*hdr, pad_len(datalen));
1267		1311
1268	/* Copy the header template to the request before modification */	1312	/* Copy the header template to the request before modification */
1269	memcpy(hdr, &req->hdr, sizeof(*hdr));	1313	memcpy(hdr, &req->hdr, sizeof(*hdr));
@@ -1374,7 +1418,7 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, · 104 unmodified lines
1374	struct hfi1_user_sdma_pkt_q *pq = req->pq;	1418	struct hfi1_user_sdma_pkt_q *pq = req->pq;
1375	struct hfi1_pkt_header *hdr = &req->hdr;	1419	struct hfi1_pkt_header *hdr = &req->hdr;
1376	u16 pbclen = le16_to_cpu(hdr->pbc[0]);	1420	u16 pbclen = le16_to_cpu(hdr->pbc[0]);
1377	u32 val32, tidval = 0, lrhlen = get_lrh_len(*hdr, len);	1421	u32 val32, tidval = 0, lrhlen = get_lrh_len(*hdr, pad_len(len));
1378		1422
1379	if (PBC2LRH(pbclen) != lrhlen) {	1423	if (PBC2LRH(pbclen) != lrhlen) {
1380	/* PBC.PbcLengthDWs */	1424	/* PBC.PbcLengthDWs */
@@ -1534,14 +1578,14 @@ static void user_sdma_free_request(struct user_sdma_request *req, bool unpin) · 153 unmodified lines
1534	continue;	1578	continue;
1535		1579
1536	if (unpin)	1580	if (unpin)
1537	hfi1_mmu_rb_remove(&req->pq->sdma_rb_root,	1581	hfi1_mmu_rb_remove(req->pq->handler,
1538	&node->rb);	1582	&node->rb);
1539	else	1583	else
1540	atomic_dec(&node->refcount);	1584	atomic_dec(&node->refcount);
1541	}	1585	}
1542	}	1586	}
1543	kfree(req->tids);	1587	kfree(req->tids);
1544	clear_bit(SDMA_REQ_IN_USE, &req->flags);	1588	clear_bit(req->info.comp_idx, req->pq->req_in_use);
1545	}	1589	}
1546		1590
1547	static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq,	1591	static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq,
@@ -1564,7 +1608,7 @@ static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr, · 16 unmodified lines
1564	return (bool)(node->addr == addr);	1608	return (bool)(node->addr == addr);
1565	}	1609	}
1566		1610
1567	static int sdma_rb_insert(struct rb_root root, struct mmu_rb_node mnode)	1611	static int sdma_rb_insert(void arg, struct mmu_rb_node mnode)
1568	{	1612	{
1569	struct sdma_mmu_node *node =	1613	struct sdma_mmu_node *node =
1570	container_of(mnode, struct sdma_mmu_node, rb);	1614	container_of(mnode, struct sdma_mmu_node, rb);
@@ -1573,48 +1617,45 @@ static int sdma_rb_insert(struct rb_root root, struct mmu_rb_node mnode) · 2 unmodified lines
1573	return 0;	1617	return 0;
1574	}	1618	}
1575		1619
1576	static void sdma_rb_remove(struct rb_root root, struct mmu_rb_node mnode,	1620	/*
1577	struct mm_struct *mm)	1621	* Return 1 to remove the node from the rb tree and call the remove op.
		1622	*
		1623	* Called with the rb tree lock held.
		1624	*/
		1625	static int sdma_rb_evict(void arg, struct mmu_rb_node mnode,
		1626	void evict_arg, bool stop)
		1627	{
		1628	struct sdma_mmu_node *node =
		1629	container_of(mnode, struct sdma_mmu_node, rb);
		1630	struct evict_data *evict_data = evict_arg;
		1631
		1632	/* is this node still being used? */
		1633	if (atomic_read(&node->refcount))
		1634	return 0; /* keep this node */
		1635
		1636	/* this node will be evicted, add its pages to our count */
		1637	evict_data->cleared += node->npages;
		1638
		1639	/* have enough pages been cleared? */
		1640	if (evict_data->cleared >= evict_data->target)
		1641	*stop = true;
		1642
		1643	return 1; /* remove this node */
		1644	}
		1645
		1646	static void sdma_rb_remove(void arg, struct mmu_rb_node mnode)
1578	{	1647	{
1579	struct sdma_mmu_node *node =	1648	struct sdma_mmu_node *node =
1580	container_of(mnode, struct sdma_mmu_node, rb);	1649	container_of(mnode, struct sdma_mmu_node, rb);
1581		1650
1582	spin_lock(&node->pq->evict_lock);	1651	atomic_sub(node->npages, &node->pq->n_locked);
1583	/*
1584	* We've been called by the MMU notifier but this node has been
1585	* scheduled for eviction. The eviction function will take care
1586	* of freeing this node.
1587	* We have to take the above lock first because we are racing
1588	* against the setting of the bit in the eviction function.
1589	*/
1590	if (mm && test_bit(SDMA_CACHE_NODE_EVICT, &node->flags)) {
1591	spin_unlock(&node->pq->evict_lock);
1592	return;
1593	}
1594		1652
1595	if (!list_empty(&node->list))	1653	unpin_vector_pages(node->pq->mm, node->pages, 0, node->npages);
1596	list_del(&node->list);
1597	node->pq->n_locked -= node->npages;
1598	spin_unlock(&node->pq->evict_lock);
1599		1654
1600	/*
1601	* If mm is set, we are being called by the MMU notifier and we
1602	* should not pass a mm_struct to unpin_vector_page(). This is to
1603	* prevent a deadlock when hfi1_release_user_pages() attempts to
1604	* take the mmap_sem, which the MMU notifier has already taken.
1605	*/
1606	unpin_vector_pages(mm ? NULL : current->mm, node->pages, 0,
1607	node->npages);
1608	/*
1609	* If called by the MMU notifier, we have to adjust the pinned
1610	* page count ourselves.
1611	*/
1612	if (mm)
1613	mm->pinned_vm -= node->npages;
1614	kfree(node);	1655	kfree(node);
1615	}	1656	}
1616		1657
1617	static int sdma_rb_invalidate(struct rb_root root, struct mmu_rb_node mnode)	1658	static int sdma_rb_invalidate(void arg, struct mmu_rb_node mnode)
1618	{	1659	{
1619	struct sdma_mmu_node *node =	1660	struct sdma_mmu_node *node =
1620	container_of(mnode, struct sdma_mmu_node, rb);	1661	container_of(mnode, struct sdma_mmu_node, rb);

drivers/infiniband/hw/hfi1/user_sdma.h+4−4

 	struct hfi1_devdata *dd;
 	struct kmem_cache *txreq_cache;
 	struct user_sdma_request *reqs;
+	unsigned long *req_in_use;
 	struct iowait busy;
 	unsigned state;
 	wait_queue_head_t wait;
 	unsigned long unpinned;
-	struct rb_root sdma_rb_root;
+	struct mmu_rb_handler *handler;
-	u32 n_locked;
+	atomic_t n_locked;
-	struct list_head evict;
+	struct mm_struct *mm;
-	spinlock_t evict_lock; /* protect evict and n_locked */
 };
 struct hfi1_user_sdma_comp_q {

drivers/infiniband/hw/hfi1/verbs.c+42−32

@@ -306,7 +306,10 @@ const enum ib_wc_opcode ib_hfi1_wc_opcode[] = { · 305 unmodified lines
306	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,	306	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
307	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,	307	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
308	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,	308	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
309	[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD	309	[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD,
		310	[IB_WR_SEND_WITH_INV] = IB_WC_SEND,
		311	[IB_WR_LOCAL_INV] = IB_WC_LOCAL_INV,
		312	[IB_WR_REG_MR] = IB_WC_REG_MR
310	};	313	};
311		314
312	/*	315	/*
@@ -378,6 +381,8 @@ static const opcode_handler opcode_handler_tbl[256] = { · 65 unmodified lines
378	[IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = &hfi1_rc_rcv,	381	[IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = &hfi1_rc_rcv,
379	[IB_OPCODE_RC_COMPARE_SWAP] = &hfi1_rc_rcv,	382	[IB_OPCODE_RC_COMPARE_SWAP] = &hfi1_rc_rcv,
380	[IB_OPCODE_RC_FETCH_ADD] = &hfi1_rc_rcv,	383	[IB_OPCODE_RC_FETCH_ADD] = &hfi1_rc_rcv,
		384	[IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = &hfi1_rc_rcv,
		385	[IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = &hfi1_rc_rcv,
381	/* UC */	386	/* UC */
382	[IB_OPCODE_UC_SEND_FIRST] = &hfi1_uc_rcv,	387	[IB_OPCODE_UC_SEND_FIRST] = &hfi1_uc_rcv,
383	[IB_OPCODE_UC_SEND_MIDDLE] = &hfi1_uc_rcv,	388	[IB_OPCODE_UC_SEND_MIDDLE] = &hfi1_uc_rcv,
@@ -540,19 +545,15 @@ void hfi1_skip_sge(struct rvt_sge_state *ss, u32 length, int release) · 156 unmodified lines
540	/*	545	/*
541	* Make sure the QP is ready and able to accept the given opcode.	546	* Make sure the QP is ready and able to accept the given opcode.
542	*/	547	*/
543	static inline int qp_ok(int opcode, struct hfi1_packet *packet)	548	static inline opcode_handler qp_ok(int opcode, struct hfi1_packet *packet)
544	{	549	{
545	struct hfi1_ibport *ibp;
546
547	if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))	550	if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))
548	goto dropit;	551	return NULL;
549	if (((opcode & RVT_OPCODE_QP_MASK) == packet->qp->allowed_ops) \|\|	552	if (((opcode & RVT_OPCODE_QP_MASK) == packet->qp->allowed_ops) \|\|
550	(opcode == IB_OPCODE_CNP))	553	(opcode == IB_OPCODE_CNP))
551	return 1;	554	return opcode_handler_tbl[opcode];
552	dropit:	555
553	ibp = &packet->rcd->ppd->ibport_data;	556	return NULL;
554	ibp->rvp.n_pkt_drops++;
555	return 0;
556	}	557	}
557		558
558	/**	559	/**
@@ -571,6 +572,7 @@ void hfi1_ib_rcv(struct hfi1_packet *packet) · 12 unmodified lines
571	struct hfi1_pportdata *ppd = rcd->ppd;	572	struct hfi1_pportdata *ppd = rcd->ppd;
572	struct hfi1_ibport *ibp = &ppd->ibport_data;	573	struct hfi1_ibport *ibp = &ppd->ibport_data;
573	struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi;	574	struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi;
		575	opcode_handler packet_handler;
574	unsigned long flags;	576	unsigned long flags;
575	u32 qp_num;	577	u32 qp_num;
576	int lnh;	578	int lnh;
@@ -616,8 +618,11 @@ void hfi1_ib_rcv(struct hfi1_packet *packet) · 39 unmodified lines
616	list_for_each_entry_rcu(p, &mcast->qp_list, list) {	618	list_for_each_entry_rcu(p, &mcast->qp_list, list) {
617	packet->qp = p->qp;	619	packet->qp = p->qp;
618	spin_lock_irqsave(&packet->qp->r_lock, flags);	620	spin_lock_irqsave(&packet->qp->r_lock, flags);
619	if (likely((qp_ok(opcode, packet))))	621	packet_handler = qp_ok(opcode, packet);
620	opcode_handler_tbl[opcode](packet);	622	if (likely(packet_handler))
		623	packet_handler(packet);
		624	else
		625	ibp->rvp.n_pkt_drops++;
621	spin_unlock_irqrestore(&packet->qp->r_lock, flags);	626	spin_unlock_irqrestore(&packet->qp->r_lock, flags);
622	}	627	}
623	/*	628	/*
@@ -634,8 +639,11 @@ void hfi1_ib_rcv(struct hfi1_packet *packet) · 10 unmodified lines
634	goto drop;	639	goto drop;
635	}	640	}
636	spin_lock_irqsave(&packet->qp->r_lock, flags);	641	spin_lock_irqsave(&packet->qp->r_lock, flags);
637	if (likely((qp_ok(opcode, packet))))	642	packet_handler = qp_ok(opcode, packet);
638	opcode_handler_tbl[opcode](packet);	643	if (likely(packet_handler))
		644	packet_handler(packet);
		645	else
		646	ibp->rvp.n_pkt_drops++;
639	spin_unlock_irqrestore(&packet->qp->r_lock, flags);	647	spin_unlock_irqrestore(&packet->qp->r_lock, flags);
640	rcu_read_unlock();	648	rcu_read_unlock();
641	}	649	}
@@ -808,19 +816,19 @@ static int build_verbs_tx_desc( · 166 unmodified lines
808	struct rvt_sge_state *ss,	816	struct rvt_sge_state *ss,
809	u32 length,	817	u32 length,
810	struct verbs_txreq *tx,	818	struct verbs_txreq *tx,
811	struct ahg_ib_header *ahdr,	819	struct hfi1_ahg_info *ahg_info,
812	u64 pbc)	820	u64 pbc)
813	{	821	{
814	int ret = 0;	822	int ret = 0;
815	struct hfi1_pio_header *phdr = &tx->phdr;	823	struct hfi1_sdma_header *phdr = &tx->phdr;
816	u16 hdrbytes = tx->hdr_dwords << 2;	824	u16 hdrbytes = tx->hdr_dwords << 2;
817		825
818	if (!ahdr->ahgcount) {	826	if (!ahg_info->ahgcount) {
819	ret = sdma_txinit_ahg(	827	ret = sdma_txinit_ahg(
820	&tx->txreq,	828	&tx->txreq,
821	ahdr->tx_flags,	829	ahg_info->tx_flags,
822	hdrbytes + length,	830	hdrbytes + length,
823	ahdr->ahgidx,	831	ahg_info->ahgidx,
824	0,	832	0,
825	NULL,	833	NULL,
826	0,	834	0,
@@ -838,11 +846,11 @@ static int build_verbs_tx_desc( · 11 unmodified lines
838	} else {	846	} else {
839	ret = sdma_txinit_ahg(	847	ret = sdma_txinit_ahg(
840	&tx->txreq,	848	&tx->txreq,
841	ahdr->tx_flags,	849	ahg_info->tx_flags,
842	length,	850	length,
843	ahdr->ahgidx,	851	ahg_info->ahgidx,
844	ahdr->ahgcount,	852	ahg_info->ahgcount,
845	ahdr->ahgdesc,	853	ahg_info->ahgdesc,
846	hdrbytes,	854	hdrbytes,
847	verbs_sdma_complete);	855	verbs_sdma_complete);
848	if (ret)	856	if (ret)
@@ -860,7 +868,7 @@ int hfi1_verbs_send_dma(struct rvt_qp qp, struct hfi1_pkt_state ps, · 11 unmodified lines
860	u64 pbc)	868	u64 pbc)
861	{	869	{
862	struct hfi1_qp_priv *priv = qp->priv;	870	struct hfi1_qp_priv *priv = qp->priv;
863	struct ahg_ib_header *ahdr = priv->s_hdr;	871	struct hfi1_ahg_info *ahg_info = priv->s_ahg;
864	u32 hdrwords = qp->s_hdrwords;	872	u32 hdrwords = qp->s_hdrwords;
865	struct rvt_sge_state *ss = qp->s_cur_sge;	873	struct rvt_sge_state *ss = qp->s_cur_sge;
866	u32 len = qp->s_cur_size;	874	u32 len = qp->s_cur_size;
@@ -888,7 +896,7 @@ int hfi1_verbs_send_dma(struct rvt_qp qp, struct hfi1_pkt_state ps, · 21 unmodified lines
888	plen);	896	plen);
889	}	897	}
890	tx->wqe = qp->s_wqe;	898	tx->wqe = qp->s_wqe;
891	ret = build_verbs_tx_desc(tx->sde, ss, len, tx, ahdr, pbc);	899	ret = build_verbs_tx_desc(tx->sde, ss, len, tx, ahg_info, pbc);
892	if (unlikely(ret))	900	if (unlikely(ret))
893	goto bail_build;	901	goto bail_build;
894	}	902	}
@@ -1300,13 +1308,15 @@ static void hfi1_fill_device_attr(struct hfi1_devdata *dd) · 405 unmodified lines
1300	rdi->dparms.props.device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR \|	1308	rdi->dparms.props.device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR \|
1301	IB_DEVICE_BAD_QKEY_CNTR \| IB_DEVICE_SHUTDOWN_PORT \|	1309	IB_DEVICE_BAD_QKEY_CNTR \| IB_DEVICE_SHUTDOWN_PORT \|
1302	IB_DEVICE_SYS_IMAGE_GUID \| IB_DEVICE_RC_RNR_NAK_GEN \|	1310	IB_DEVICE_SYS_IMAGE_GUID \| IB_DEVICE_RC_RNR_NAK_GEN \|
1303	IB_DEVICE_PORT_ACTIVE_EVENT \| IB_DEVICE_SRQ_RESIZE;	1311	IB_DEVICE_PORT_ACTIVE_EVENT \| IB_DEVICE_SRQ_RESIZE \|
		1312	IB_DEVICE_MEM_MGT_EXTENSIONS;
1304	rdi->dparms.props.page_size_cap = PAGE_SIZE;	1313	rdi->dparms.props.page_size_cap = PAGE_SIZE;
1305	rdi->dparms.props.vendor_id = dd->oui1 << 16 \| dd->oui2 << 8 \| dd->oui3;	1314	rdi->dparms.props.vendor_id = dd->oui1 << 16 \| dd->oui2 << 8 \| dd->oui3;
1306	rdi->dparms.props.vendor_part_id = dd->pcidev->device;	1315	rdi->dparms.props.vendor_part_id = dd->pcidev->device;
1307	rdi->dparms.props.hw_ver = dd->minrev;	1316	rdi->dparms.props.hw_ver = dd->minrev;
1308	rdi->dparms.props.sys_image_guid = ib_hfi1_sys_image_guid;	1317	rdi->dparms.props.sys_image_guid = ib_hfi1_sys_image_guid;
1309	rdi->dparms.props.max_mr_size = ~0ULL;	1318	rdi->dparms.props.max_mr_size = U64_MAX;
		1319	rdi->dparms.props.max_fast_reg_page_list_len = UINT_MAX;
1310	rdi->dparms.props.max_qp = hfi1_max_qps;	1320	rdi->dparms.props.max_qp = hfi1_max_qps;
1311	rdi->dparms.props.max_qp_wr = hfi1_max_qp_wrs;	1321	rdi->dparms.props.max_qp_wr = hfi1_max_qp_wrs;
1312	rdi->dparms.props.max_sge = hfi1_max_sges;	1322	rdi->dparms.props.max_sge = hfi1_max_sges;
@@ -1695,6 +1705,9 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) · 382 unmodified lines
1695	dd->verbs_dev.rdi.dparms.nports = dd->num_pports;	1705	dd->verbs_dev.rdi.dparms.nports = dd->num_pports;
1696	dd->verbs_dev.rdi.dparms.npkeys = hfi1_get_npkeys(dd);	1706	dd->verbs_dev.rdi.dparms.npkeys = hfi1_get_npkeys(dd);
1697		1707
		1708	/* post send table */
		1709	dd->verbs_dev.rdi.post_parms = hfi1_post_parms;
		1710
1698	ppd = dd->pport;	1711	ppd = dd->pport;
1699	for (i = 0; i < dd->num_pports; i++, ppd++)	1712	for (i = 0; i < dd->num_pports; i++, ppd++)
1700	rvt_init_port(&dd->verbs_dev.rdi,	1713	rvt_init_port(&dd->verbs_dev.rdi,
@@ -1745,8 +1758,7 @@ void hfi1_cnp_rcv(struct hfi1_packet *packet) · 44 unmodified lines
1745	struct rvt_qp *qp = packet->qp;	1758	struct rvt_qp *qp = packet->qp;
1746	u32 lqpn, rqpn = 0;	1759	u32 lqpn, rqpn = 0;
1747	u16 rlid = 0;	1760	u16 rlid = 0;
1748	u8 sl, sc5, sc4_bit, svc_type;	1761	u8 sl, sc5, svc_type;
1749	bool sc4_set = has_sc4_bit(packet);
1750		1762
1751	switch (packet->qp->ibqp.qp_type) {	1763	switch (packet->qp->ibqp.qp_type) {
1752	case IB_QPT_UC:	1764	case IB_QPT_UC:
@@ -1769,9 +1781,7 @@ void hfi1_cnp_rcv(struct hfi1_packet *packet) · 16 unmodified lines
1769	return;	1781	return;
1770	}	1782	}
1771		1783
1772	sc4_bit = sc4_set << 4;	1784	sc5 = hdr2sc((struct hfi1_message_header *)hdr, packet->rhf);
1773	sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf;
1774	sc5 \|= sc4_bit;
1775	sl = ibp->sc_to_sl[sc5];	1785	sl = ibp->sc_to_sl[sc5];
1776	lqpn = qp->ibqp.qp_num;	1786	lqpn = qp->ibqp.qp_num;
1777		1787

drivers/infiniband/hw/hfi1/verbs.h+3−5

 	} u;
 } __packed;
-struct ahg_ib_header {
+struct hfi1_ahg_info {
-	struct sdma_engine *sde;
 	u32 ahgdesc[2];
 	u16 tx_flags;
 	u8 ahgcount;
 	u8 ahgidx;
-	struct hfi1_ib_header ibh;
 };
-struct hfi1_pio_header {
+struct hfi1_sdma_header {
 	__le64 pbc;
 	struct hfi1_ib_header hdr;
 } __packed;
  * pair is made common
  */
 struct hfi1_qp_priv {
-	struct ahg_ib_header *s_hdr;              /* next header to send */
+	struct hfi1_ahg_info *s_ahg;              /* ahg info for next header */
 	struct sdma_engine *s_sde;                /* current sde */
 	struct send_context *s_sendcontext;       /* current sendcontext */
 	u8 s_sc;		                  /* SC[0..4] for next packet */

drivers/infiniband/hw/hfi1/verbs_txreq.h+1−1

@@ -56,7 +56,7 @@ · 55 unmodified lines
56	#include "iowait.h"	56	#include "iowait.h"
57		57
58	struct verbs_txreq {	58	struct verbs_txreq {
59	struct hfi1_pio_header phdr;	59	struct hfi1_sdma_header phdr;
60	struct sdma_txreq txreq;	60	struct sdma_txreq txreq;
61	struct rvt_qp *qp;	61	struct rvt_qp *qp;
62	struct rvt_swqe *wqe;	62	struct rvt_swqe *wqe;

drivers/infiniband/hw/qib/qib_qp.c+43−0

@@ -106,6 +106,49 @@ static u32 credit_table[31] = { · 105 unmodified lines
106	32768 /* 1E */	106	32768 /* 1E */
107	};	107	};
108		108
		109	const struct rvt_operation_params qib_post_parms[RVT_OPERATION_MAX] = {
		110	[IB_WR_RDMA_WRITE] = {
		111	.length = sizeof(struct ib_rdma_wr),
		112	.qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		113	},
		114
		115	[IB_WR_RDMA_READ] = {
		116	.length = sizeof(struct ib_rdma_wr),
		117	.qpt_support = BIT(IB_QPT_RC),
		118	.flags = RVT_OPERATION_ATOMIC,
		119	},
		120
		121	[IB_WR_ATOMIC_CMP_AND_SWP] = {
		122	.length = sizeof(struct ib_atomic_wr),
		123	.qpt_support = BIT(IB_QPT_RC),
		124	.flags = RVT_OPERATION_ATOMIC \| RVT_OPERATION_ATOMIC_SGE,
		125	},
		126
		127	[IB_WR_ATOMIC_FETCH_AND_ADD] = {
		128	.length = sizeof(struct ib_atomic_wr),
		129	.qpt_support = BIT(IB_QPT_RC),
		130	.flags = RVT_OPERATION_ATOMIC \| RVT_OPERATION_ATOMIC_SGE,
		131	},
		132
		133	[IB_WR_RDMA_WRITE_WITH_IMM] = {
		134	.length = sizeof(struct ib_rdma_wr),
		135	.qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		136	},
		137
		138	[IB_WR_SEND] = {
		139	.length = sizeof(struct ib_send_wr),
		140	.qpt_support = BIT(IB_QPT_UD) \| BIT(IB_QPT_SMI) \| BIT(IB_QPT_GSI) \|
		141	BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		142	},
		143
		144	[IB_WR_SEND_WITH_IMM] = {
		145	.length = sizeof(struct ib_send_wr),
		146	.qpt_support = BIT(IB_QPT_UD) \| BIT(IB_QPT_SMI) \| BIT(IB_QPT_GSI) \|
		147	BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
		148	},
		149
		150	};
		151
109	static void get_map_page(struct rvt_qpn_table qpt, struct rvt_qpn_map map,	152	static void get_map_page(struct rvt_qpn_table qpt, struct rvt_qpn_map map,
110	gfp_t gfp)	153	gfp_t gfp)
111	{	154	{

drivers/infiniband/hw/qib/qib_ud.c+6−2

 	}
 	if (ah_attr->ah_flags & IB_AH_GRH) {
-		qib_copy_sge(&qp->r_sge, &ah_attr->grh,
+		struct ib_grh grh;
-			     sizeof(struct ib_grh), 1);
+		struct ib_global_route grd = ah_attr->grh;
+		qib_make_grh(ibp, &grh, &grd, 0, 0);
+		qib_copy_sge(&qp->r_sge, &grh,
+			     sizeof(grh), 1);
 		wc.wc_flags |= IB_WC_GRH;
 	} else
 		qib_skip_sge(&qp->r_sge, sizeof(struct ib_grh), 1);

drivers/infiniband/hw/qib/qib_verbs.c+2−0

@@ -1582,6 +1582,8 @@ static void qib_fill_device_attr(struct qib_devdata *dd) · 1581 unmodified lines
1582	rdi->dparms.props.max_total_mcast_qp_attach =	1582	rdi->dparms.props.max_total_mcast_qp_attach =
1583	rdi->dparms.props.max_mcast_qp_attach *	1583	rdi->dparms.props.max_mcast_qp_attach *
1584	rdi->dparms.props.max_mcast_grp;	1584	rdi->dparms.props.max_mcast_grp;
		1585	/* post send table */
		1586	dd->verbs_dev.rdi.post_parms = qib_post_parms;
1585	}	1587	}
1586		1588
1587	/**	1589	/**

drivers/infiniband/hw/qib/qib_verbs.h+2−0

@@ -497,4 +497,6 @@ extern unsigned int ib_qib_max_srq_wrs; · 496 unmodified lines
497		497
498	extern const u32 ib_qib_rnr_table[];	498	extern const u32 ib_qib_rnr_table[];
499		499
		500	extern const struct rvt_operation_params qib_post_parms[];
		501
500	#endif /* QIB_VERBS_H */	502	#endif /* QIB_VERBS_H */

drivers/infiniband/sw/rdmavt/cq.c+1−0

@@ -510,6 +510,7 @@ int rvt_driver_cq_init(struct rvt_dev_info *rdi) · 509 unmodified lines
510		510
511	if (rdi->worker)	511	if (rdi->worker)
512	return 0;	512	return 0;
		513	spin_lock_init(&rdi->n_cqs_lock);
513	rdi->worker = kzalloc(sizeof(*rdi->worker), GFP_KERNEL);	514	rdi->worker = kzalloc(sizeof(*rdi->worker), GFP_KERNEL);
514	if (!rdi->worker)	515	if (!rdi->worker)
515	return -ENOMEM;	516	return -ENOMEM;

drivers/infiniband/sw/rdmavt/mr.c+122−2

@@ -140,6 +140,7 @@ static int rvt_init_mregion(struct rvt_mregion mr, struct ib_pd pd, · 139 unmodified lines
140	init_completion(&mr->comp);	140	init_completion(&mr->comp);
141	/* count returning the ptr to user */	141	/* count returning the ptr to user */
142	atomic_set(&mr->refcount, 1);	142	atomic_set(&mr->refcount, 1);
		143	atomic_set(&mr->lkey_invalid, 0);
143	mr->pd = pd;	144	mr->pd = pd;
144	mr->max_segs = count;	145	mr->max_segs = count;
145	return 0;	146	return 0;
@@ -479,6 +480,123 @@ struct ib_mr rvt_alloc_mr(struct ib_pd pd, · 333 unmodified lines
479	return &mr->ibmr;	480	return &mr->ibmr;
480	}	481	}
481		482
		483	/**
		484	* rvt_set_page - page assignment function called by ib_sg_to_pages
		485	* @ibmr: memory region
		486	* @addr: dma address of mapped page
		487	*
		488	* Return: 0 on success
		489	*/
		490	static int rvt_set_page(struct ib_mr *ibmr, u64 addr)
		491	{
		492	struct rvt_mr *mr = to_imr(ibmr);
		493	u32 ps = 1 << mr->mr.page_shift;
		494	u32 mapped_segs = mr->mr.length >> mr->mr.page_shift;
		495	int m, n;
		496
		497	if (unlikely(mapped_segs == mr->mr.max_segs))
		498	return -ENOMEM;
		499
		500	if (mr->mr.length == 0) {
		501	mr->mr.user_base = addr;
		502	mr->mr.iova = addr;
		503	}
		504
		505	m = mapped_segs / RVT_SEGSZ;
		506	n = mapped_segs % RVT_SEGSZ;
		507	mr->mr.map[m]->segs[n].vaddr = (void *)addr;
		508	mr->mr.map[m]->segs[n].length = ps;
		509	mr->mr.length += ps;
		510
		511	return 0;
		512	}
		513
		514	/**
		515	* rvt_map_mr_sg - map sg list and set it the memory region
		516	* @ibmr: memory region
		517	* @sg: dma mapped scatterlist
		518	* @sg_nents: number of entries in sg
		519	* @sg_offset: offset in bytes into sg
		520	*
		521	* Return: number of sg elements mapped to the memory region
		522	*/
		523	int rvt_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg,
		524	int sg_nents, unsigned int *sg_offset)
		525	{
		526	struct rvt_mr *mr = to_imr(ibmr);
		527
		528	mr->mr.length = 0;
		529	mr->mr.page_shift = PAGE_SHIFT;
		530	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
		531	rvt_set_page);
		532	}
		533
		534	/**
		535	* rvt_fast_reg_mr - fast register physical MR
		536	* @qp: the queue pair where the work request comes from
		537	* @ibmr: the memory region to be registered
		538	* @key: updated key for this memory region
		539	* @access: access flags for this memory region
		540	*
		541	* Returns 0 on success.
		542	*/
		543	int rvt_fast_reg_mr(struct rvt_qp qp, struct ib_mr ibmr, u32 key,
		544	int access)
		545	{
		546	struct rvt_mr *mr = to_imr(ibmr);
		547
		548	if (qp->ibqp.pd != mr->mr.pd)
		549	return -EACCES;
		550
		551	/* not applicable to dma MR or user MR */
		552	if (!mr->mr.lkey \|\| mr->umem)
		553	return -EINVAL;
		554
		555	if ((key & 0xFFFFFF00) != (mr->mr.lkey & 0xFFFFFF00))
		556	return -EINVAL;
		557
		558	ibmr->lkey = key;
		559	ibmr->rkey = key;
		560	mr->mr.lkey = key;
		561	mr->mr.access_flags = access;
		562	atomic_set(&mr->mr.lkey_invalid, 0);
		563
		564	return 0;
		565	}
		566	EXPORT_SYMBOL(rvt_fast_reg_mr);
		567
		568	/**
		569	* rvt_invalidate_rkey - invalidate an MR rkey
		570	* @qp: queue pair associated with the invalidate op
		571	* @rkey: rkey to invalidate
		572	*
		573	* Returns 0 on success.
		574	*/
		575	int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey)
		576	{
		577	struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device);
		578	struct rvt_lkey_table *rkt = &dev->lkey_table;
		579	struct rvt_mregion *mr;
		580
		581	if (rkey == 0)
		582	return -EINVAL;
		583
		584	rcu_read_lock();
		585	mr = rcu_dereference(
		586	rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]);
		587	if (unlikely(!mr \|\| mr->lkey != rkey \|\| qp->ibqp.pd != mr->pd))
		588	goto bail;
		589
		590	atomic_set(&mr->lkey_invalid, 1);
		591	rcu_read_unlock();
		592	return 0;
		593
		594	bail:
		595	rcu_read_unlock();
		596	return -EINVAL;
		597	}
		598	EXPORT_SYMBOL(rvt_invalidate_rkey);
		599
482	/**	600	/**
483	* rvt_alloc_fmr - allocate a fast memory region	601	* rvt_alloc_fmr - allocate a fast memory region
484	* @pd: the protection domain for this memory region	602	* @pd: the protection domain for this memory region
@@ -682,7 +800,8 @@ int rvt_lkey_ok(struct rvt_lkey_table rkt, struct rvt_pd pd, · 197 unmodified lines
682	}	800	}
683	mr = rcu_dereference(	801	mr = rcu_dereference(
684	rkt->table[(sge->lkey >> (32 - dev->dparms.lkey_table_size))]);	802	rkt->table[(sge->lkey >> (32 - dev->dparms.lkey_table_size))]);
685	if (unlikely(!mr \|\| mr->lkey != sge->lkey \|\| mr->pd != &pd->ibpd))	803	if (unlikely(!mr \|\| atomic_read(&mr->lkey_invalid) \|\|
		804	mr->lkey != sge->lkey \|\| mr->pd != &pd->ibpd))
686	goto bail;	805	goto bail;
687		806
688	off = sge->addr - mr->user_base;	807	off = sge->addr - mr->user_base;
@@ -782,7 +901,8 @@ int rvt_rkey_ok(struct rvt_qp qp, struct rvt_sge sge, · 93 unmodified lines
782		901
783	mr = rcu_dereference(	902	mr = rcu_dereference(
784	rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]);	903	rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]);
785	if (unlikely(!mr \|\| mr->lkey != rkey \|\| qp->ibqp.pd != mr->pd))	904	if (unlikely(!mr \|\| atomic_read(&mr->lkey_invalid) \|\|
		905	mr->lkey != rkey \|\| qp->ibqp.pd != mr->pd))
786	goto bail;	906	goto bail;
787		907
788	off = vaddr - mr->iova;	908	off = vaddr - mr->iova;

drivers/infiniband/sw/rdmavt/mr.h+2−0

 struct ib_mr *rvt_alloc_mr(struct ib_pd *pd,
 			   enum ib_mr_type mr_type,
 			   u32 max_num_sg);
+int rvt_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		  int sg_nents, unsigned int *sg_offset);
 struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
 			     struct ib_fmr_attr *fmr_attr);
 int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,

drivers/infiniband/sw/rdmavt/qp.c+181−65

@@ -435,8 +435,7 @@ static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends) · 434 unmodified lines
435	for (n = 0; n < rvt_max_atomic(rdi); n++) {	435	for (n = 0; n < rvt_max_atomic(rdi); n++) {
436	struct rvt_ack_entry *e = &qp->s_ack_queue[n];	436	struct rvt_ack_entry *e = &qp->s_ack_queue[n];
437		437
438	if (e->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST &&	438	if (e->rdma_sge.mr) {
439	e->rdma_sge.mr) {
440	rvt_put_mr(e->rdma_sge.mr);	439	rvt_put_mr(e->rdma_sge.mr);
441	e->rdma_sge.mr = NULL;	440	e->rdma_sge.mr = NULL;
442	}	441	}
@@ -584,6 +583,7 @@ static void rvt_reset_qp(struct rvt_dev_info rdi, struct rvt_qp qp, · 141 unmodified lines
584	qp->r_rq.wq->tail = 0;	583	qp->r_rq.wq->tail = 0;
585	}	584	}
586	qp->r_sge.num_sge = 0;	585	qp->r_sge.num_sge = 0;
		586	atomic_set(&qp->s_reserved_used, 0);
587	}	587	}
588		588
589	/**	589	/**
@@ -613,6 +613,7 @@ struct ib_qp rvt_create_qp(struct ib_pd ibpd, · 23 unmodified lines
613	struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device);	613	struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device);
614	void *priv = NULL;	614	void *priv = NULL;
615	gfp_t gfp;	615	gfp_t gfp;
		616	size_t sqsize;
616		617
617	if (!rdi)	618	if (!rdi)
618	return ERR_PTR(-EINVAL);	619	return ERR_PTR(-EINVAL);
@@ -643,7 +644,9 @@ struct ib_qp rvt_create_qp(struct ib_pd ibpd, · 24 unmodified lines
643	init_attr->cap.max_recv_wr == 0)	644	init_attr->cap.max_recv_wr == 0)
644	return ERR_PTR(-EINVAL);	645	return ERR_PTR(-EINVAL);
645	}	646	}
646		647	sqsize =
		648	init_attr->cap.max_send_wr + 1 +
		649	rdi->dparms.reserved_operations;
647	switch (init_attr->qp_type) {	650	switch (init_attr->qp_type) {
648	case IB_QPT_SMI:	651	case IB_QPT_SMI:
649	case IB_QPT_GSI:	652	case IB_QPT_GSI:
@@ -658,11 +661,11 @@ struct ib_qp rvt_create_qp(struct ib_pd ibpd, · 8 unmodified lines
658	sizeof(struct rvt_swqe);	661	sizeof(struct rvt_swqe);
659	if (gfp == GFP_NOIO)	662	if (gfp == GFP_NOIO)
660	swq = __vmalloc(	663	swq = __vmalloc(
661	(init_attr->cap.max_send_wr + 1) * sz,	664	sqsize * sz,
662	gfp \| __GFP_ZERO, PAGE_KERNEL);	665	gfp \| __GFP_ZERO, PAGE_KERNEL);
663	else	666	else
664	swq = vzalloc_node(	667	swq = vzalloc_node(
665	(init_attr->cap.max_send_wr + 1) * sz,	668	sqsize * sz,
666	rdi->dparms.node);	669	rdi->dparms.node);
667	if (!swq)	670	if (!swq)
668	return ERR_PTR(-ENOMEM);	671	return ERR_PTR(-ENOMEM);
@@ -741,13 +744,14 @@ struct ib_qp rvt_create_qp(struct ib_pd ibpd, · 72 unmodified lines
741	spin_lock_init(&qp->s_lock);	744	spin_lock_init(&qp->s_lock);
742	spin_lock_init(&qp->r_rq.lock);	745	spin_lock_init(&qp->r_rq.lock);
743	atomic_set(&qp->refcount, 0);	746	atomic_set(&qp->refcount, 0);
		747	atomic_set(&qp->local_ops_pending, 0);
744	init_waitqueue_head(&qp->wait);	748	init_waitqueue_head(&qp->wait);
745	init_timer(&qp->s_timer);	749	init_timer(&qp->s_timer);
746	qp->s_timer.data = (unsigned long)qp;	750	qp->s_timer.data = (unsigned long)qp;
747	INIT_LIST_HEAD(&qp->rspwait);	751	INIT_LIST_HEAD(&qp->rspwait);
748	qp->state = IB_QPS_RESET;	752	qp->state = IB_QPS_RESET;
749	qp->s_wq = swq;	753	qp->s_wq = swq;
750	qp->s_size = init_attr->cap.max_send_wr + 1;	754	qp->s_size = sqsize;
751	qp->s_avail = init_attr->cap.max_send_wr;	755	qp->s_avail = init_attr->cap.max_send_wr;
752	qp->s_max_sge = init_attr->cap.max_send_sge;	756	qp->s_max_sge = init_attr->cap.max_send_sge;
753	if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR)	757	if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR)
@@ -1332,7 +1336,8 @@ int rvt_query_qp(struct ib_qp ibqp, struct ib_qp_attr attr, · 578 unmodified lines
1332	attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask;	1336	attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask;
1333	attr->dest_qp_num = qp->remote_qpn;	1337	attr->dest_qp_num = qp->remote_qpn;
1334	attr->qp_access_flags = qp->qp_access_flags;	1338	attr->qp_access_flags = qp->qp_access_flags;
1335	attr->cap.max_send_wr = qp->s_size - 1;	1339	attr->cap.max_send_wr = qp->s_size - 1 -
		1340	rdi->dparms.reserved_operations;
1336	attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1;	1341	attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1;
1337	attr->cap.max_send_sge = qp->s_max_sge;	1342	attr->cap.max_send_sge = qp->s_max_sge;
1338	attr->cap.max_recv_sge = qp->r_rq.max_sge;	1343	attr->cap.max_recv_sge = qp->r_rq.max_sge;
@@ -1440,25 +1445,116 @@ int rvt_post_recv(struct ib_qp ibqp, struct ib_recv_wr wr, · 101 unmodified lines
1440	}	1445	}
1441		1446
1442	/**	1447	/**
1443	* qp_get_savail - return number of avail send entries	1448	* rvt_qp_valid_operation - validate post send wr request
		1449	* @qp - the qp
		1450	* @post-parms - the post send table for the driver
		1451	* @wr - the work request
		1452	*
		1453	* The routine validates the operation based on the
		1454	* validation table an returns the length of the operation
		1455	* which can extend beyond the ib_send_bw. Operation
		1456	* dependent flags key atomic operation validation.
1444	*	1457	*
		1458	* There is an exception for UD qps that validates the pd and
		1459	* overrides the length to include the additional UD specific
		1460	* length.
		1461	*
		1462	* Returns a negative error or the length of the work request
		1463	* for building the swqe.
		1464	*/
		1465	static inline int rvt_qp_valid_operation(
		1466	struct rvt_qp *qp,
		1467	const struct rvt_operation_params *post_parms,
		1468	struct ib_send_wr *wr)
		1469	{
		1470	int len;
		1471
		1472	if (wr->opcode >= RVT_OPERATION_MAX \|\| !post_parms[wr->opcode].length)
		1473	return -EINVAL;
		1474	if (!(post_parms[wr->opcode].qpt_support & BIT(qp->ibqp.qp_type)))
		1475	return -EINVAL;
		1476	if ((post_parms[wr->opcode].flags & RVT_OPERATION_PRIV) &&
		1477	ibpd_to_rvtpd(qp->ibqp.pd)->user)
		1478	return -EINVAL;
		1479	if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC_SGE &&
		1480	(wr->num_sge == 0 \|\|
		1481	wr->sg_list[0].length < sizeof(u64) \|\|
		1482	wr->sg_list[0].addr & (sizeof(u64) - 1)))
		1483	return -EINVAL;
		1484	if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC &&
		1485	!qp->s_max_rd_atomic)
		1486	return -EINVAL;
		1487	len = post_parms[wr->opcode].length;
		1488	/* UD specific */
		1489	if (qp->ibqp.qp_type != IB_QPT_UC &&
		1490	qp->ibqp.qp_type != IB_QPT_RC) {
		1491	if (qp->ibqp.pd != ud_wr(wr)->ah->pd)
		1492	return -EINVAL;
		1493	len = sizeof(struct ib_ud_wr);
		1494	}
		1495	return len;
		1496	}
		1497
		1498	/**
		1499	* rvt_qp_is_avail - determine queue capacity
1445	* @qp - the qp	1500	* @qp - the qp
		1501	* @rdi - the rdmavt device
		1502	* @reserved_op - is reserved operation
1446	*	1503	*
1447	* This assumes the s_hlock is held but the s_last	1504	* This assumes the s_hlock is held but the s_last
1448	* qp variable is uncontrolled.	1505	* qp variable is uncontrolled.
		1506	*
		1507	* For non reserved operations, the qp->s_avail
		1508	* may be changed.
		1509	*
		1510	* The return value is zero or a -ENOMEM.
1449	*/	1511	*/
1450	static inline u32 qp_get_savail(struct rvt_qp *qp)	1512	static inline int rvt_qp_is_avail(
		1513	struct rvt_qp *qp,
		1514	struct rvt_dev_info *rdi,
		1515	bool reserved_op)
1451	{	1516	{
1452	u32 slast;	1517	u32 slast;
1453	u32 ret;	1518	u32 avail;
1454		1519	u32 reserved_used;
		1520
		1521	/* see rvt_qp_wqe_unreserve() */
		1522	smp_mb__before_atomic();
		1523	reserved_used = atomic_read(&qp->s_reserved_used);
		1524	if (unlikely(reserved_op)) {
		1525	/* see rvt_qp_wqe_unreserve() */
		1526	smp_mb__before_atomic();
		1527	if (reserved_used >= rdi->dparms.reserved_operations)
		1528	return -ENOMEM;
		1529	return 0;
		1530	}
		1531	/* non-reserved operations */
		1532	if (likely(qp->s_avail))
		1533	return 0;
1455	smp_read_barrier_depends(); /* see rc.c */	1534	smp_read_barrier_depends(); /* see rc.c */
1456	slast = ACCESS_ONCE(qp->s_last);	1535	slast = ACCESS_ONCE(qp->s_last);
1457	if (qp->s_head >= slast)	1536	if (qp->s_head >= slast)
1458	ret = qp->s_size - (qp->s_head - slast);	1537	avail = qp->s_size - (qp->s_head - slast);
1459	else	1538	else
1460	ret = slast - qp->s_head;	1539	avail = slast - qp->s_head;
1461	return ret - 1;	1540
		1541	/* see rvt_qp_wqe_unreserve() */
		1542	smp_mb__before_atomic();
		1543	reserved_used = atomic_read(&qp->s_reserved_used);
		1544	avail = avail - 1 -
		1545	(rdi->dparms.reserved_operations - reserved_used);
		1546	/* insure we don't assign a negative s_avail */
		1547	if ((s32)avail <= 0)
		1548	return -ENOMEM;
		1549	qp->s_avail = avail;
		1550	if (WARN_ON(qp->s_avail >
		1551	(qp->s_size - 1 - rdi->dparms.reserved_operations)))
		1552	rvt_pr_err(rdi,
		1553	"More avail entries than QP RB size.\nQP: %u, size: %u, avail: %u\nhead: %u, tail: %u, cur: %u, acked: %u, last: %u",
		1554	qp->ibqp.qp_num, qp->s_size, qp->s_avail,
		1555	qp->s_head, qp->s_tail, qp->s_cur,
		1556	qp->s_acked, qp->s_last);
		1557	return 0;
1462	}	1558	}
1463		1559
1464	/**	1560	/**
@@ -1480,49 +1576,64 @@ static int rvt_post_one_wr(struct rvt_qp *qp, · 15 unmodified lines
1480	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);	1576	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
1481	u8 log_pmtu;	1577	u8 log_pmtu;
1482	int ret;	1578	int ret;
		1579	size_t cplen;
		1580	bool reserved_op;
		1581	int local_ops_delayed = 0;
		1582
		1583	BUILD_BUG_ON(IB_QPT_MAX >= (sizeof(u32) * BITS_PER_BYTE));
1483		1584
1484	/* IB spec says that num_sge == 0 is OK. */	1585	/* IB spec says that num_sge == 0 is OK. */
1485	if (unlikely(wr->num_sge > qp->s_max_sge))	1586	if (unlikely(wr->num_sge > qp->s_max_sge))
1486	return -EINVAL;	1587	return -EINVAL;
1487		1588
		1589	ret = rvt_qp_valid_operation(qp, rdi->post_parms, wr);
		1590	if (ret < 0)
		1591	return ret;
		1592	cplen = ret;
		1593
1488	/*	1594	/*
1489	* Don't allow RDMA reads or atomic operations on UC or	1595	* Local operations include fast register and local invalidate.
1490	* undefined operations.	1596	* Fast register needs to be processed immediately because the
1491	* Make sure buffer is large enough to hold the result for atomics.	1597	* registered lkey may be used by following work requests and the
		1598	* lkey needs to be valid at the time those requests are posted.
		1599	* Local invalidate can be processed immediately if fencing is
		1600	* not required and no previous local invalidate ops are pending.
		1601	* Signaled local operations that have been processed immediately
		1602	* need to have requests with "completion only" flags set posted
		1603	* to the send queue in order to generate completions.
1492	*/	1604	*/
1493	if (qp->ibqp.qp_type == IB_QPT_UC) {	1605	if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) {
1494	if ((unsigned)wr->opcode >= IB_WR_RDMA_READ)	1606	switch (wr->opcode) {
1495	return -EINVAL;	1607	case IB_WR_REG_MR:
1496	} else if (qp->ibqp.qp_type != IB_QPT_RC) {	1608	ret = rvt_fast_reg_mr(qp,
1497	/* Check IB_QPT_SMI, IB_QPT_GSI, IB_QPT_UD opcode */	1609	reg_wr(wr)->mr,
1498	if (wr->opcode != IB_WR_SEND &&	1610	reg_wr(wr)->key,
1499	wr->opcode != IB_WR_SEND_WITH_IMM)	1611	reg_wr(wr)->access);
1500	return -EINVAL;	1612	if (ret \|\| !(wr->send_flags & IB_SEND_SIGNALED))
1501	/* Check UD destination address PD */	1613	return ret;
1502	if (qp->ibqp.pd != ud_wr(wr)->ah->pd)	1614	break;
		1615	case IB_WR_LOCAL_INV:
		1616	if ((wr->send_flags & IB_SEND_FENCE) \|\|
		1617	atomic_read(&qp->local_ops_pending)) {
		1618	local_ops_delayed = 1;
		1619	} else {
		1620	ret = rvt_invalidate_rkey(
		1621	qp, wr->ex.invalidate_rkey);
		1622	if (ret \|\| !(wr->send_flags & IB_SEND_SIGNALED))
		1623	return ret;
		1624	}
		1625	break;
		1626	default:
1503	return -EINVAL;	1627	return -EINVAL;
1504	} else if ((unsigned)wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) {	1628	}
1505	return -EINVAL;
1506	} else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP &&
1507	(wr->num_sge == 0 \|\|
1508	wr->sg_list[0].length < sizeof(u64) \|\|
1509	wr->sg_list[0].addr & (sizeof(u64) - 1))) {
1510	return -EINVAL;
1511	} else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic) {
1512	return -EINVAL;
1513	}	1629	}
		1630
		1631	reserved_op = rdi->post_parms[wr->opcode].flags &
		1632	RVT_OPERATION_USE_RESERVE;
1514	/* check for avail */	1633	/* check for avail */
1515	if (unlikely(!qp->s_avail)) {	1634	ret = rvt_qp_is_avail(qp, rdi, reserved_op);
1516	qp->s_avail = qp_get_savail(qp);	1635	if (ret)
1517	if (WARN_ON(qp->s_avail > (qp->s_size - 1)))	1636	return ret;
1518	rvt_pr_err(rdi,
1519	"More avail entries than QP RB size.\nQP: %u, size: %u, avail: %u\nhead: %u, tail: %u, cur: %u, acked: %u, last: %u",
1520	qp->ibqp.qp_num, qp->s_size, qp->s_avail,
1521	qp->s_head, qp->s_tail, qp->s_cur,
1522	qp->s_acked, qp->s_last);
1523	if (!qp->s_avail)
1524	return -ENOMEM;
1525	}
1526	next = qp->s_head + 1;	1637	next = qp->s_head + 1;
1527	if (next >= qp->s_size)	1638	if (next >= qp->s_size)
1528	next = 0;	1639	next = 0;
@@ -1531,18 +1642,8 @@ static int rvt_post_one_wr(struct rvt_qp *qp, · 2 unmodified lines
1531	pd = ibpd_to_rvtpd(qp->ibqp.pd);	1642	pd = ibpd_to_rvtpd(qp->ibqp.pd);
1532	wqe = rvt_get_swqe_ptr(qp, qp->s_head);	1643	wqe = rvt_get_swqe_ptr(qp, qp->s_head);
1533		1644
1534	if (qp->ibqp.qp_type != IB_QPT_UC &&	1645	/* cplen has length from above */
1535	qp->ibqp.qp_type != IB_QPT_RC)	1646	memcpy(&wqe->wr, wr, cplen);
1536	memcpy(&wqe->ud_wr, ud_wr(wr), sizeof(wqe->ud_wr));
1537	else if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM \|\|
1538	wr->opcode == IB_WR_RDMA_WRITE \|\|
1539	wr->opcode == IB_WR_RDMA_READ)
1540	memcpy(&wqe->rdma_wr, rdma_wr(wr), sizeof(wqe->rdma_wr));
1541	else if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP \|\|
1542	wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
1543	memcpy(&wqe->atomic_wr, atomic_wr(wr), sizeof(wqe->atomic_wr));
1544	else
1545	memcpy(&wqe->wr, wr, sizeof(wqe->wr));
1546		1647
1547	wqe->length = 0;	1648	wqe->length = 0;
1548	j = 0;	1649	j = 0;
@@ -1585,14 +1686,29 @@ static int rvt_post_one_wr(struct rvt_qp *qp, · 36 unmodified lines
1585	atomic_inc(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount);	1686	atomic_inc(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount);
1586	}	1687	}
1587		1688
1588	wqe->ssn = qp->s_ssn++;	1689	if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) {
1589	wqe->psn = qp->s_next_psn;	1690	if (local_ops_delayed)
1590	wqe->lpsn = wqe->psn +	1691	atomic_inc(&qp->local_ops_pending);
1591	(wqe->length ? ((wqe->length - 1) >> log_pmtu) : 0);	1692	else
1592	qp->s_next_psn = wqe->lpsn + 1;	1693	wqe->wr.send_flags \|= RVT_SEND_COMPLETION_ONLY;
		1694	wqe->ssn = 0;
		1695	wqe->psn = 0;
		1696	wqe->lpsn = 0;
		1697	} else {
		1698	wqe->ssn = qp->s_ssn++;
		1699	wqe->psn = qp->s_next_psn;
		1700	wqe->lpsn = wqe->psn +
		1701	(wqe->length ?
		1702	((wqe->length - 1) >> log_pmtu) :
		1703	0);
		1704	qp->s_next_psn = wqe->lpsn + 1;
		1705	}
1593	trace_rvt_post_one_wr(qp, wqe);	1706	trace_rvt_post_one_wr(qp, wqe);
		1707	if (unlikely(reserved_op))
		1708	rvt_qp_wqe_reserve(qp, wqe);
		1709	else
		1710	qp->s_avail--;
1594	smp_wmb(); /* see request builders */	1711	smp_wmb(); /* see request builders */
1595	qp->s_avail--;
1596	qp->s_head = next;	1712	qp->s_head = next;
1597		1713
1598	return 0;	1714	return 0;

drivers/infiniband/sw/rdmavt/vt.c+9−1

 	REG_USER_MR,
 	DEREG_MR,
 	ALLOC_MR,
+	MAP_MR_SG,
 	ALLOC_FMR,
 	MAP_PHYS_FMR,
 	UNMAP_FMR,
 							 post_send),
 					   rvt_post_send))
 			if (!rdi->driver_f.schedule_send ||
-			    !rdi->driver_f.do_send)
+			    !rdi->driver_f.do_send ||
+			    !rdi->post_parms)
 				return -EINVAL;
 		break;
 				      rvt_alloc_mr);
 		break;
+	case MAP_MR_SG:
+		check_driver_override(rdi, offsetof(struct ib_device,
+						    map_mr_sg),
+				      rvt_map_mr_sg);
+		break;
 	case MAP_PHYS_FMR:
 		check_driver_override(rdi, offsetof(struct ib_device,
 						    map_phys_fmr),

drivers/infiniband/ulp/isert/ib_isert.c+0−2

 	attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
 	attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
 	attr.cap.max_send_sge = device->ib_device->attrs.max_sge;
-	isert_conn->max_sge = min(device->ib_device->attrs.max_sge,
-				  device->ib_device->attrs.max_sge_rd);
 	attr.cap.max_recv_sge = 1;
 	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 	attr.qp_type = IB_QPT_RC;

drivers/infiniband/ulp/isert/ib_isert.h+0−1

@@ -138,7 +138,6 @@ struct isert_conn { · 137 unmodified lines
138	u32 responder_resources;	138	u32 responder_resources;
139	u32 initiator_depth;	139	u32 initiator_depth;
140	bool pi_support;	140	bool pi_support;
141	u32 max_sge;
142	struct iser_rx_desc *login_req_buf;	141	struct iser_rx_desc *login_req_buf;
143	char *login_rsp_buf;	142	char *login_rsp_buf;
144	u64 login_req_dma;	143	u64 login_req_dma;

drivers/infiniband/ulp/srpt/ib_srpt.c+4−6

 	struct ib_qp_init_attr *qp_init;
 	struct srpt_port *sport = ch->sport;
 	struct srpt_device *sdev = sport->sdev;
+	const struct ib_device_attr *attrs = &sdev->device->attrs;
 	u32 srp_sq_size = sport->port_attrib.srp_sq_size;
 	int ret;
 	 */
 	qp_init->cap.max_send_wr = srp_sq_size / 2;
 	qp_init->cap.max_rdma_ctxs = srp_sq_size / 2;
-	qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
+	qp_init->cap.max_send_sge = min(attrs->max_sge, SRPT_MAX_SG_PER_WQE);
 	qp_init->port_num = ch->sport->port;
 	ch->qp = ib_create_qp(sdev->pd, qp_init);
 		container_of(cmd, struct srpt_send_ioctx, cmd);
 	struct srpt_rdma_ch *ch = ioctx->ch;
 	struct srpt_device *sdev = ch->sport->sdev;
-	struct ib_send_wr send_wr, *first_wr = NULL, *bad_wr;
+	struct ib_send_wr send_wr, *first_wr = &send_wr, *bad_wr;
 	struct ib_sge sge;
 	enum srpt_command_state state;
 	unsigned long flags;
 			struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
 			first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp,
-					ch->sport->port, NULL,
+					ch->sport->port, NULL, first_wr);
-					first_wr ? first_wr : &send_wr);
 		}
-	} else {
-		first_wr = &send_wr;
 	}
 	if (state != SRPT_STATE_MGMT)

drivers/infiniband/ulp/srpt/ib_srpt.h+5−1

 	SRP_LOGIN_RSP_MULTICHAN_MAINTAINED = 0x2,
 	SRPT_DEF_SG_TABLESIZE = 128,
-	SRPT_DEF_SG_PER_WQE = 16,
+	/*
+	 * An experimentally determined value that avoids that QP creation
+	 * fails due to "swiotlb buffer is full" on systems using the swiotlb.
+	 */
+	SRPT_MAX_SG_PER_WQE = 16,
 	MIN_SRPT_SQ_SIZE = 16,
 	DEF_SRPT_SQ_SIZE = 4096,

include/rdma/ib_verbs.h+6−0

 	struct ib_wq	**ind_tbl;
 };
+/*
+ * @max_write_sge: Maximum SGE elements per RDMA WRITE request.
+ * @max_read_sge:  Maximum SGE elements per RDMA READ request.
+ */
 struct ib_qp {
 	struct ib_device       *device;
 	struct ib_pd	       *pd;
 	void                  (*event_handler)(struct ib_event *, void *);
 	void		       *qp_context;
 	u32			qp_num;
+	u32			max_write_sge;
+	u32			max_read_sge;
 	enum ib_qp_type		qp_type;
 	struct ib_rwq_ind_table *rwq_ind_tbl;
 };

include/rdma/opa_port_info.h+0−16

 #if !defined(OPA_PORT_INFO_H)
 #define OPA_PORT_INFO_H
-/* Temporary until HFI driver is updated */
-#ifndef USE_PI_LED_ENABLE
-#define USE_PI_LED_ENABLE 0
-#endif
 #define OPA_PORT_LINK_MODE_NOP	0		/* No change */
 #define OPA_PORT_LINK_MODE_OPA	4		/* Port mode is OPA */
 	OPA_PI_MASK_MTU_CAP                       = 0x0F,
 };
-#if USE_PI_LED_ENABLE
 struct opa_port_states {
 	u8     reserved;
 	u8     ledenable_offlinereason;   /* 1 res, 1 bit, 6 bits */
 	u8     reserved2;
 	u8     portphysstate_portstate;   /* 4 bits, 4 bits */
 };
-#define PI_LED_ENABLE_SUP 1
-#else
-struct opa_port_states {
-	u8     reserved;
-	u8     offline_reason;            /* 2 res, 6 bits */
-	u8     reserved2;
-	u8     portphysstate_portstate;   /* 4 bits, 4 bits */
-};
-#define PI_LED_ENABLE_SUP 0
-#endif
 struct opa_port_state_info {
 	struct opa_port_states port_states;

include/rdma/rdma_vt.h+7−0

 	u32 max_mad_size;
 	u8 qos_shift;
 	u8 max_rdma_atomic;
+	u8 reserved_operations;
 };
 /* Protection domain */
 	/* Driver specific properties */
 	struct rvt_driver_params dparms;
+	/* post send table */
+	const struct rvt_operation_params *post_parms;
 	struct rvt_mregion __rcu *dma_mr;
 	struct rvt_lkey_table lkey_table;
 int rvt_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr);
 int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
 		  int port_index, u16 *pkey_table);
+int rvt_fast_reg_mr(struct rvt_qp *qp, struct ib_mr *ibmr, u32 key,
+		    int access);
+int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey);
 int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
 		u32 len, u64 vaddr, u32 rkey, int acc);
 int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,

include/rdma/rdmavt_mr.h+1−0

@@ -81,6 +81,7 @@ struct rvt_mregion { · 80 unmodified lines
81	u32 mapsz; /* size of the map array */	81	u32 mapsz; /* size of the map array */
82	u8 page_shift; /* 0 - non unform/non powerof2 sizes */	82	u8 page_shift; /* 0 - non unform/non powerof2 sizes */
83	u8 lkey_published; /* in global table */	83	u8 lkey_published; /* in global table */
		84	atomic_t lkey_invalid; /* true if current lkey is invalid */
84	struct completion comp; /* complete when refcount goes to zero */	85	struct completion comp; /* complete when refcount goes to zero */
85	atomic_t refcount;	86	atomic_t refcount;
86	struct rvt_segarray map[0]; / the segments */	87	struct rvt_segarray map[0]; / the segments */

include/rdma/rdmavt_qp.h+82−10

@@ -144,6 +144,12 @@ · 143 unmodified lines
144	#define RVT_PROCESS_OR_FLUSH_SEND \	144	#define RVT_PROCESS_OR_FLUSH_SEND \
145	(RVT_PROCESS_SEND_OK \| RVT_FLUSH_SEND)	145	(RVT_PROCESS_SEND_OK \| RVT_FLUSH_SEND)
146		146
		147	/*
		148	* Internal send flags
		149	*/
		150	#define RVT_SEND_RESERVE_USED IB_SEND_RESERVED_START
		151	#define RVT_SEND_COMPLETION_ONLY (IB_SEND_RESERVED_START << 1)
		152
147	/*	153	/*
148	* Send work request queue entry.	154	* Send work request queue entry.
149	* The size of the sg_list is determined when the QP is created and stored	155	* The size of the sg_list is determined when the QP is created and stored
@@ -216,23 +222,43 @@ struct rvt_mmap_info { · 66 unmodified lines
216	* to send a RDMA read response or atomic operation.	222	* to send a RDMA read response or atomic operation.
217	*/	223	*/
218	struct rvt_ack_entry {	224	struct rvt_ack_entry {
219	u8 opcode;	225	struct rvt_sge rdma_sge;
220	u8 sent;	226	u64 atomic_data;
221	u32 psn;	227	u32 psn;
222	u32 lpsn;	228	u32 lpsn;
223	union {	229	u8 opcode;
224	struct rvt_sge rdma_sge;	230	u8 sent;
225	u64 atomic_data;
226	};
227	};	231	};
228		232
229	#define RC_QP_SCALING_INTERVAL 5	233	#define RC_QP_SCALING_INTERVAL 5
230		234
231	/*	235	#define RVT_OPERATION_PRIV 0x00000001
232	* Variables prefixed with s_ are for the requester (sender).	236	#define RVT_OPERATION_ATOMIC 0x00000002
233	* Variables prefixed with r_ are for the responder (receiver).	237	#define RVT_OPERATION_ATOMIC_SGE 0x00000004
234	* Variables prefixed with ack_ are for responder replies.	238	#define RVT_OPERATION_LOCAL 0x00000008
		239	#define RVT_OPERATION_USE_RESERVE 0x00000010
		240
		241	#define RVT_OPERATION_MAX (IB_WR_RESERVED10 + 1)
		242
		243	/**
		244	* rvt_operation_params - op table entry
		245	* @length - the length to copy into the swqe entry
		246	* @qpt_support - a bit mask indicating QP type support
		247	* @flags - RVT_OPERATION flags (see above)
235	*	248	*
		249	* This supports table driven post send so that
		250	* the driver can have differing an potentially
		251	* different sets of operations.
		252	*
		253	**/
		254
		255	struct rvt_operation_params {
		256	size_t length;
		257	u32 qpt_support;
		258	u32 flags;
		259	};
		260
		261	/*
236	* Common variables are protected by both r_rq.lock and s_lock in that order	262	* Common variables are protected by both r_rq.lock and s_lock in that order
237	* which only happens in modify_qp() or changing the QP 'state'.	263	* which only happens in modify_qp() or changing the QP 'state'.
238	*/	264	*/
@@ -307,6 +333,7 @@ struct rvt_qp { · 68 unmodified lines
307	u32 s_next_psn; /* PSN for next request */	333	u32 s_next_psn; /* PSN for next request */
308	u32 s_avail; /* number of entries avail */	334	u32 s_avail; /* number of entries avail */
309	u32 s_ssn; /* SSN of tail entry */	335	u32 s_ssn; /* SSN of tail entry */
		336	atomic_t s_reserved_used; /* reserved entries in use */
310		337
311	spinlock_t s_lock ____cacheline_aligned_in_smp;	338	spinlock_t s_lock ____cacheline_aligned_in_smp;
312	u32 s_flags;	339	u32 s_flags;
@@ -343,6 +370,8 @@ struct rvt_qp { · 30 unmodified lines
343	struct rvt_sge_state s_ack_rdma_sge;	370	struct rvt_sge_state s_ack_rdma_sge;
344	struct timer_list s_timer;	371	struct timer_list s_timer;
345		372
		373	atomic_t local_ops_pending; /* number of fast_reg/local_inv reqs */
		374
346	/*	375	/*
347	* This sge list MUST be last. Do not add anything below here.	376	* This sge list MUST be last. Do not add anything below here.
348	*/	377	*/
@@ -436,6 +465,49 @@ static inline struct rvt_rwqe rvt_get_rwqe_ptr(struct rvt_rq rq, unsigned n) · 87 unmodified lines
436	rq->max_sge * sizeof(struct ib_sge)) * n);	465	rq->max_sge * sizeof(struct ib_sge)) * n);
437	}	466	}
438		467
		468	/**
		469	* rvt_qp_wqe_reserve - reserve operation
		470	* @qp - the rvt qp
		471	* @wqe - the send wqe
		472	*
		473	* This routine used in post send to record
		474	* a wqe relative reserved operation use.
		475	*/
		476	static inline void rvt_qp_wqe_reserve(
		477	struct rvt_qp *qp,
		478	struct rvt_swqe *wqe)
		479	{
		480	wqe->wr.send_flags \|= RVT_SEND_RESERVE_USED;
		481	atomic_inc(&qp->s_reserved_used);
		482	}
		483
		484	/**
		485	* rvt_qp_wqe_unreserve - clean reserved operation
		486	* @qp - the rvt qp
		487	* @wqe - the send wqe
		488	*
		489	* This decrements the reserve use count.
		490	*
		491	* This call MUST precede the change to
		492	* s_last to insure that post send sees a stable
		493	* s_avail.
		494	*
		495	* An smp_mp__after_atomic() is used to insure
		496	* the compiler does not juggle the order of the s_last
		497	* ring index and the decrementing of s_reserved_used.
		498	*/
		499	static inline void rvt_qp_wqe_unreserve(
		500	struct rvt_qp *qp,
		501	struct rvt_swqe *wqe)
		502	{
		503	if (unlikely(wqe->wr.send_flags & RVT_SEND_RESERVE_USED)) {
		504	wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED;
		505	atomic_dec(&qp->s_reserved_used);
		506	/* insure no compiler re-order up to s_last change */
		507	smp_mb__after_atomic();
		508	}
		509	}
		510
439	extern const int ib_rvt_state_ops[];	511	extern const int ib_rvt_state_ops[];
440		512
441	struct rvt_dev_info;	513	struct rvt_dev_info;

include/uapi/rdma/hfi/hfi1_user.h+1−1

@@ -75,7 +75,7 @@ · 74 unmodified lines
75	* may not be implemented; the user code must deal with this if it	75	* may not be implemented; the user code must deal with this if it
76	* cares, or it must abort after initialization reports the difference.	76	* cares, or it must abort after initialization reports the difference.
77	*/	77	*/
78	#define HFI1_USER_SWMINOR 1	78	#define HFI1_USER_SWMINOR 2
79		79
80	/*	80	/*
81	* We will encode the major/minor inside a single 32bit version number.	81	* We will encode the major/minor inside a single 32bit version number.

Merge tag 'for-linus-2' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code