Merge tag 'for-linus-2' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma

Pull second round of rdma updates from Doug Ledford: "This can be split out into just two categories: - fixes to the RDMA R/W API in regards to SG list length limits (about 5 patches) - fixes/features for the Intel hfi1 driver (everything else) The hfi1 driver is still being brought to full feature support by Intel, and they have a lot of people working on it, so that amounts to almost the entirety of this pull request" * tag 'for-linus-2' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma: (84 commits) IB/hfi1: Add cache evict LRU list IB/hfi1: Fix memory leak during unexpected shutdown IB/hfi1: Remove unneeded mm argument in remove function IB/hfi1: Consistently call ops->remove outside spinlock IB/hfi1: Use evict mmu rb operation IB/hfi1: Add evict operation to the mmu rb handler IB/hfi1: Fix TID caching actions IB/hfi1: Make the cache handler own its rb tree root IB/hfi1: Make use of mm consistent IB/hfi1: Fix user SDMA racy user request claim IB/hfi1: Fix error condition that needs to clean up IB/hfi1: Release node on insert failure IB/hfi1: Validate SDMA user iovector count IB/hfi1: Validate SDMA user request index IB/hfi1: Use the same capability state for all shared contexts IB/hfi1: Prevent null pointer dereference IB/hfi1: Rename TID mmu_rb_* functions IB/hfi1: Remove unneeded empty check in hfi1_mmu_rb_unregister() IB/hfi1: Restructure hfi1_file_open IB/hfi1: Make iovec loop index easy to understand ...

Linus Torvalds committed 10 years ago

Commit 84e39ee2 parents 0cda611 7c4176566 files · +4323 −3012Browse files

Unified · Split

66 files

drivers/infiniband/core/rw.c +11 −13
drivers/infiniband/core/verbs.c +9 −0
drivers/infiniband/hw/hfi1/Kconfig +2 −1
drivers/infiniband/hw/hfi1/Makefile +1 −1
drivers/infiniband/hw/hfi1/affinity.c +441 −126
drivers/infiniband/hw/hfi1/affinity.h +31 −7
drivers/infiniband/hw/hfi1/chip.c +193 −103
drivers/infiniband/hw/hfi1/chip.h +1 −4
drivers/infiniband/hw/hfi1/chip_registers.h +4 −0
drivers/infiniband/hw/hfi1/driver.c +28 −24
drivers/infiniband/hw/hfi1/file_ops.c +51 −40
drivers/infiniband/hw/hfi1/firmware.c +125 −0
drivers/infiniband/hw/hfi1/hfi.h +104 −17
drivers/infiniband/hw/hfi1/init.c +34 −17
drivers/infiniband/hw/hfi1/mad.c +21 −39
drivers/infiniband/hw/hfi1/mad.h +0 −7
drivers/infiniband/hw/hfi1/mmu_rb.c +149 −105
drivers/infiniband/hw/hfi1/mmu_rb.h +24 −13
drivers/infiniband/hw/hfi1/pcie.c +64 −4
drivers/infiniband/hw/hfi1/pio.c +17 −4
drivers/infiniband/hw/hfi1/platform.c +5 −15
drivers/infiniband/hw/hfi1/qp.c +65 −3
drivers/infiniband/hw/hfi1/qp.h +3 −1
drivers/infiniband/hw/hfi1/qsfp.c +302 −107
drivers/infiniband/hw/hfi1/qsfp.h +3 −0
drivers/infiniband/hw/hfi1/rc.c +65 −25
drivers/infiniband/hw/hfi1/ruc.c +40 −17
drivers/infiniband/hw/hfi1/sysfs.c +25 −0
drivers/infiniband/hw/hfi1/trace.h +7 −1326
drivers/infiniband/hw/hfi1/trace_ctxts.h +141 −0
drivers/infiniband/hw/hfi1/trace_dbg.h +155 −0
drivers/infiniband/hw/hfi1/trace_ibhdrs.h +209 −0
drivers/infiniband/hw/hfi1/trace_misc.h +81 −0
drivers/infiniband/hw/hfi1/trace_rc.h +123 −0
drivers/infiniband/hw/hfi1/trace_rx.h +322 −0
drivers/infiniband/hw/hfi1/trace_tx.h +642 −0
drivers/infiniband/hw/hfi1/twsi.c +0 −489
drivers/infiniband/hw/hfi1/twsi.h +0 −65
drivers/infiniband/hw/hfi1/uc.c +26 −35
drivers/infiniband/hw/hfi1/ud.c +29 −57
drivers/infiniband/hw/hfi1/user_exp_rcv.c +65 −59
drivers/infiniband/hw/hfi1/user_pages.c +10 −9
drivers/infiniband/hw/hfi1/user_sdma.c +171 −130
drivers/infiniband/hw/hfi1/user_sdma.h +4 −4
drivers/infiniband/hw/hfi1/verbs.c +42 −32
drivers/infiniband/hw/hfi1/verbs.h +3 −5
drivers/infiniband/hw/hfi1/verbs_txreq.h +1 −1
drivers/infiniband/hw/qib/qib_qp.c +43 −0
drivers/infiniband/hw/qib/qib_ud.c +6 −2
drivers/infiniband/hw/qib/qib_verbs.c +2 −0
drivers/infiniband/hw/qib/qib_verbs.h +2 −0
drivers/infiniband/sw/rdmavt/cq.c +1 −0
drivers/infiniband/sw/rdmavt/mr.c +122 −2
drivers/infiniband/sw/rdmavt/mr.h +2 −0
drivers/infiniband/sw/rdmavt/qp.c +181 −65
drivers/infiniband/sw/rdmavt/vt.c +9 −1
drivers/infiniband/ulp/isert/ib_isert.c +0 −2
drivers/infiniband/ulp/isert/ib_isert.h +0 −1
drivers/infiniband/ulp/srpt/ib_srpt.c +4 −6
drivers/infiniband/ulp/srpt/ib_srpt.h +5 −1
include/rdma/ib_verbs.h +6 −0
include/rdma/opa_port_info.h +0 −16
include/rdma/rdma_vt.h +7 −0
include/rdma/rdmavt_mr.h +1 −0
include/rdma/rdmavt_qp.h +82 −10
include/uapi/rdma/hfi/hfi1_user.h +1 −1

drivers/infiniband/core/rw.c+11−13

@@ -58,19 +58,13 @@ static inline bool rdma_rw_io_needs_mr(struct ib_device *dev, u8 port_num, · 57 unmodified lines
58	58	return false;
59	59	}
60	60
61		−static inline u32 rdma_rw_max_sge(struct ib_device *dev,
62		− enum dma_data_direction dir)
63		−{
64		− return dir == DMA_TO_DEVICE ?
65		− dev->attrs.max_sge : dev->attrs.max_sge_rd;
66		−}
67		−
68	61	static inline u32 rdma_rw_fr_page_list_len(struct ib_device *dev)
69	62	{
70	63	/* arbitrary limit to avoid allocating gigantic resources */
71	64	return min_t(u32, dev->attrs.max_fast_reg_page_list_len, 256);
72	65	}
73	66
	67	+/* Caller must have zero-initialized reg. /
74	68	static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,
75	69	struct rdma_rw_reg_ctx reg, struct scatterlist sg,
76	70	u32 sg_cnt, u32 offset)
@@ -114,6 +108,7 @@ static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx ctx, struct ib_qp qp, · 37 unmodified lines
114	108	u8 port_num, struct scatterlist *sg, u32 sg_cnt, u32 offset,
115	109	u64 remote_addr, u32 rkey, enum dma_data_direction dir)
116	110	{
	111	+ struct rdma_rw_reg_ctx *prev = NULL;
117	112	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device);
118	113	int i, j, ret = 0, count = 0;
119	114
@@ -125,7 +120,6 @@ static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx ctx, struct ib_qp qp, · 5 unmodified lines
125	120	}
126	121
127	122	for (i = 0; i < ctx->nr_ops; i++) {
128		− struct rdma_rw_reg_ctx *prev = i ? &ctx->reg[i - 1] : NULL;
129	123	struct rdma_rw_reg_ctx *reg = &ctx->reg[i];
130	124	u32 nents = min(sg_cnt, pages_per_mr);
131	125
@@ -162,9 +156,13 @@ static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx ctx, struct ib_qp qp, · 30 unmodified lines
162	156	sg_cnt -= nents;
163	157	for (j = 0; j < nents; j++)
164	158	sg = sg_next(sg);
	159	+ prev = reg;
165	160	offset = 0;
166	161	}
167	162
	163	+ if (prev)
	164	+ prev->wr.wr.next = NULL;
	165	+
168	166	ctx->type = RDMA_RW_MR;
169	167	return count;
170	168
@@ -181,7 +179,8 @@ static int rdma_rw_init_map_wrs(struct rdma_rw_ctx ctx, struct ib_qp qp, · 10 unmodified lines
181	179	u64 remote_addr, u32 rkey, enum dma_data_direction dir)
182	180	{
183	181	struct ib_device *dev = qp->pd->device;
184		− u32 max_sge = rdma_rw_max_sge(dev, dir);
	182	+ u32 max_sge = dir == DMA_TO_DEVICE ? qp->max_write_sge :
	183	+ qp->max_read_sge;
185	184	struct ib_sge *sge;
186	185	u32 total_len = 0, i, j;
187	186
@@ -205,11 +204,10 @@ static int rdma_rw_init_map_wrs(struct rdma_rw_ctx ctx, struct ib_qp qp, · 17 unmodified lines
205	204	rdma_wr->wr.opcode = IB_WR_RDMA_READ;
206	205	rdma_wr->remote_addr = remote_addr + total_len;
207	206	rdma_wr->rkey = rkey;
	207	+ rdma_wr->wr.num_sge = nr_sge;
208	208	rdma_wr->wr.sg_list = sge;
209	209
210	210	for (j = 0; j < nr_sge; j++, sg = sg_next(sg)) {
211		− rdma_wr->wr.num_sge++;
212		−
213	211	sge->addr = ib_sg_dma_address(dev, sg) + offset;
214	212	sge->length = ib_sg_dma_len(dev, sg) - offset;
215	213	sge->lkey = qp->pd->local_dma_lkey;
@@ -220,8 +218,8 @@ static int rdma_rw_init_map_wrs(struct rdma_rw_ctx ctx, struct ib_qp qp, · 4 unmodified lines
220	218	offset = 0;
221	219	}
222	220
223		− if (i + 1 < ctx->nr_ops)
224		− rdma_wr->wr.next = &ctx->map.wrs[i + 1].wr;
	221	+ rdma_wr->wr.next = i + 1 < ctx->nr_ops ?
	222	+ &ctx->map.wrs[i + 1].wr : NULL;
225	223	}
226	224
227	225	ctx->type = RDMA_RW_MULTI_WR;

drivers/infiniband/core/verbs.c+9−0

@@ -825,6 +825,15 @@ struct ib_qp ib_create_qp(struct ib_pd pd, · 824 unmodified lines
825	825	}
826	826	}
827	827
	828	+ /*
	829	+ * Note: all hw drivers guarantee that max_send_sge is lower than
	830	+ * the device RDMA WRITE SGE limit but not all hw drivers ensure that
	831	+ * max_send_sge <= max_sge_rd.
	832	+ */
	833	+ qp->max_write_sge = qp_init_attr->cap.max_send_sge;
	834	+ qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
	835	+ device->attrs.max_sge_rd);
	836	+
828	837	return qp;
829	838	}
830	839	EXPORT_SYMBOL(ib_create_qp);

drivers/infiniband/hw/hfi1/Kconfig+2−1

@@ -1,8 +1,9 @@
1	1	config INFINIBAND_HFI1
2	2	tristate "Intel OPA Gen1 support"
3		− depends on X86_64 && INFINIBAND_RDMAVT
	3	+ depends on X86_64 && INFINIBAND_RDMAVT && I2C
4	4	select MMU_NOTIFIER
5	5	select CRC32
	6	+ select I2C_ALGOBIT
6	7	---help---
7	8	This is a low-level driver for Intel OPA Gen1 adapter.
8	9	config HFI1_DEBUG_SDMA_ORDER

drivers/infiniband/hw/hfi1/Makefile+1−1

@@ -10,7 +10,7 @@ obj-$(CONFIG_INFINIBAND_HFI1) += hfi1.o · 9 unmodified lines
10	10	hfi1-y := affinity.o chip.o device.o driver.o efivar.o \
11	11	eprom.o file_ops.o firmware.o \
12	12	init.o intr.o mad.o mmu_rb.o pcie.o pio.o pio_copy.o platform.o \
13		− qp.o qsfp.o rc.o ruc.o sdma.o sysfs.o trace.o twsi.o \
	13	+ qp.o qsfp.o rc.o ruc.o sdma.o sysfs.o trace.o \
14	14	uc.o ud.o user_exp_rcv.o user_pages.o user_sdma.o verbs.o \
15	15	verbs_txreq.o
16	16	hfi1-$(CONFIG_DEBUG_FS) += debugfs.o

drivers/infiniband/hw/hfi1/affinity.c+441−126

@@ -47,12 +47,18 @@ · 46 unmodified lines
47	47	#include <linux/topology.h>
48	48	#include <linux/cpumask.h>
49	49	#include <linux/module.h>
	50	+#include <linux/cpumask.h>
50	51
51	52	#include "hfi.h"
52	53	#include "affinity.h"
53	54	#include "sdma.h"
54	55	#include "trace.h"
55	56
	57	+struct hfi1_affinity_node_list node_affinity = {
	58	+ .list = LIST_HEAD_INIT(node_affinity.list),
	59	+ .lock = __SPIN_LOCK_UNLOCKED(&node_affinity.lock),
	60	+};
	61	+
56	62	/* Name of IRQ types, indexed by enum irq_type */
57	63	static const char * const irq_type_names[] = {
58	64	"SDMA",
@@ -61,6 +67,9 @@ static const char * const irq_type_names[] = { · 2 unmodified lines
61	67	"OTHER",
62	68	};
63	69
	70	+/* Per NUMA node count of HFI devices */
	71	+static unsigned int *hfi1_per_node_cntr;
	72	+
64	73	static inline void init_cpu_mask_set(struct cpu_mask_set *set)
65	74	{
66	75	cpumask_clear(&set->mask);
@@ -69,47 +78,136 @@ static inline void init_cpu_mask_set(struct cpu_mask_set *set) · 2 unmodified lines
69	78	}
70	79
71	80	/* Initialize non-HT cpu cores mask */
72		−int init_real_cpu_mask(struct hfi1_devdata *dd)
	81	+void init_real_cpu_mask(void)
73	82	{
74		− struct hfi1_affinity *info;
75	83	int possible, curr_cpu, i, ht;
76	84
77		− info = kzalloc(sizeof(*info), GFP_KERNEL);
78		− if (!info)
79		− return -ENOMEM;
80		−
81		− cpumask_clear(&info->real_cpu_mask);
	85	+ cpumask_clear(&node_affinity.real_cpu_mask);
82	86
83	87	/* Start with cpu online mask as the real cpu mask */
84		− cpumask_copy(&info->real_cpu_mask, cpu_online_mask);
	88	+ cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
85	89
86	90	/*
87	91	* Remove HT cores from the real cpu mask. Do this in two steps below.
88	92	*/
89		− possible = cpumask_weight(&info->real_cpu_mask);
	93	+ possible = cpumask_weight(&node_affinity.real_cpu_mask);
90	94	ht = cpumask_weight(topology_sibling_cpumask(
91		− cpumask_first(&info->real_cpu_mask)));
	95	+ cpumask_first(&node_affinity.real_cpu_mask)));
92	96	/*
93	97	* Step 1. Skip over the first N HT siblings and use them as the
94	98	* "real" cores. Assumes that HT cores are not enumerated in
95	99	* succession (except in the single core case).
96	100	*/
97		− curr_cpu = cpumask_first(&info->real_cpu_mask);
	101	+ curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
98	102	for (i = 0; i < possible / ht; i++)
99		− curr_cpu = cpumask_next(curr_cpu, &info->real_cpu_mask);
	103	+ curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
100	104	/*
101	105	* Step 2. Remove the remaining HT siblings. Use cpumask_next() to
102	106	* skip any gaps.
103	107	*/
104	108	for (; i < possible; i++) {
105		− cpumask_clear_cpu(curr_cpu, &info->real_cpu_mask);
106		− curr_cpu = cpumask_next(curr_cpu, &info->real_cpu_mask);
	109	+ cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
	110	+ curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
	111	+ }
	112	+}
	113	+
	114	+int node_affinity_init(void)
	115	+{
	116	+ int node;
	117	+ struct pci_dev *dev = NULL;
	118	+ const struct pci_device_id *ids = hfi1_pci_tbl;
	119	+
	120	+ cpumask_clear(&node_affinity.proc.used);
	121	+ cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
	122	+
	123	+ node_affinity.proc.gen = 0;
	124	+ node_affinity.num_core_siblings =
	125	+ cpumask_weight(topology_sibling_cpumask(
	126	+ cpumask_first(&node_affinity.proc.mask)
	127	+ ));
	128	+ node_affinity.num_online_nodes = num_online_nodes();
	129	+ node_affinity.num_online_cpus = num_online_cpus();
	130	+
	131	+ /*
	132	+ * The real cpu mask is part of the affinity struct but it has to be
	133	+ * initialized early. It is needed to calculate the number of user
	134	+ * contexts in set_up_context_variables().
	135	+ */
	136	+ init_real_cpu_mask();
	137	+
	138	+ hfi1_per_node_cntr = kcalloc(num_possible_nodes(),
	139	+ sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
	140	+ if (!hfi1_per_node_cntr)
	141	+ return -ENOMEM;
	142	+
	143	+ while (ids->vendor) {
	144	+ dev = NULL;
	145	+ while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
	146	+ node = pcibus_to_node(dev->bus);
	147	+ if (node < 0)
	148	+ node = numa_node_id();
	149	+
	150	+ hfi1_per_node_cntr[node]++;
	151	+ }
	152	+ ids++;
107	153	}
108	154
109		− dd->affinity = info;
110	155	return 0;
111	156	}
112	157
	158	+void node_affinity_destroy(void)
	159	+{
	160	+ struct list_head pos, q;
	161	+ struct hfi1_affinity_node *entry;
	162	+
	163	+ spin_lock(&node_affinity.lock);
	164	+ list_for_each_safe(pos, q, &node_affinity.list) {
	165	+ entry = list_entry(pos, struct hfi1_affinity_node,
	166	+ list);
	167	+ list_del(pos);
	168	+ kfree(entry);
	169	+ }
	170	+ spin_unlock(&node_affinity.lock);
	171	+ kfree(hfi1_per_node_cntr);
	172	+}
	173	+
	174	+static struct hfi1_affinity_node *node_affinity_allocate(int node)
	175	+{
	176	+ struct hfi1_affinity_node *entry;
	177	+
	178	+ entry = kzalloc(sizeof(*entry), GFP_KERNEL);
	179	+ if (!entry)
	180	+ return NULL;
	181	+ entry->node = node;
	182	+ INIT_LIST_HEAD(&entry->list);
	183	+
	184	+ return entry;
	185	+}
	186	+
	187	+/*
	188	+ * It appends an entry to the list.
	189	+ * It must be called with node_affinity.lock held.
	190	+ */
	191	+static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
	192	+{
	193	+ list_add_tail(&entry->list, &node_affinity.list);
	194	+}
	195	+
	196	+/* It must be called with node_affinity.lock held */
	197	+static struct hfi1_affinity_node *node_affinity_lookup(int node)
	198	+{
	199	+ struct list_head *pos;
	200	+ struct hfi1_affinity_node *entry;
	201	+
	202	+ list_for_each(pos, &node_affinity.list) {
	203	+ entry = list_entry(pos, struct hfi1_affinity_node, list);
	204	+ if (entry->node == node)
	205	+ return entry;
	206	+ }
	207	+
	208	+ return NULL;
	209	+}
	210	+
113	211	/*
114	212	* Interrupt affinity.
115	213	*
@@ -121,10 +219,10 @@ int init_real_cpu_mask(struct hfi1_devdata *dd) · 5 unmodified lines
121	219	* to the node relative 1 as necessary.
122	220	*
123	221	*/
124		−void hfi1_dev_affinity_init(struct hfi1_devdata *dd)
	222	+int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
125	223	{
126	224	int node = pcibus_to_node(dd->pcidev->bus);
127		− struct hfi1_affinity *info = dd->affinity;
	225	+ struct hfi1_affinity_node *entry;
128	226	const struct cpumask *local_mask;
129	227	int curr_cpu, possible, i;
130	228
@@ -132,56 +230,93 @@ void hfi1_dev_affinity_init(struct hfi1_devdata *dd) · 1 unmodified line
132	230	node = numa_node_id();
133	231	dd->node = node;
134	232
135		− spin_lock_init(&info->lock);
136		−
137		− init_cpu_mask_set(&info->def_intr);
138		− init_cpu_mask_set(&info->rcv_intr);
139		− init_cpu_mask_set(&info->proc);
140		−
141	233	local_mask = cpumask_of_node(dd->node);
142	234	if (cpumask_first(local_mask) >= nr_cpu_ids)
143	235	local_mask = topology_core_cpumask(0);
144		− /* Use the "real" cpu mask of this node as the default */
145		− cpumask_and(&info->def_intr.mask, &info->real_cpu_mask, local_mask);
146		−
147		− /* fill in the receive list */
148		− possible = cpumask_weight(&info->def_intr.mask);
149		− curr_cpu = cpumask_first(&info->def_intr.mask);
150		− if (possible == 1) {
151		− /* only one CPU, everyone will use it */
152		− cpumask_set_cpu(curr_cpu, &info->rcv_intr.mask);
153		− } else {
154		− /*
155		− * Retain the first CPU in the default list for the control
156		− * context.
157		− */
158		− curr_cpu = cpumask_next(curr_cpu, &info->def_intr.mask);
159		− /*
160		− * Remove the remaining kernel receive queues from
161		− * the default list and add them to the receive list.
162		− */
163		− for (i = 0; i < dd->n_krcv_queues - 1; i++) {
164		− cpumask_clear_cpu(curr_cpu, &info->def_intr.mask);
165		− cpumask_set_cpu(curr_cpu, &info->rcv_intr.mask);
166		− curr_cpu = cpumask_next(curr_cpu, &info->def_intr.mask);
167		− if (curr_cpu >= nr_cpu_ids)
168		− break;
	236	+
	237	+ spin_lock(&node_affinity.lock);
	238	+ entry = node_affinity_lookup(dd->node);
	239	+ spin_unlock(&node_affinity.lock);
	240	+
	241	+ /*
	242	+ * If this is the first time this NUMA node's affinity is used,
	243	+ * create an entry in the global affinity structure and initialize it.
	244	+ */
	245	+ if (!entry) {
	246	+ entry = node_affinity_allocate(node);
	247	+ if (!entry) {
	248	+ dd_dev_err(dd,
	249	+ "Unable to allocate global affinity node\n");
	250	+ return -ENOMEM;
169	251	}
170		− }
	252	+ init_cpu_mask_set(&entry->def_intr);
	253	+ init_cpu_mask_set(&entry->rcv_intr);
	254	+ cpumask_clear(&entry->general_intr_mask);
	255	+ /* Use the "real" cpu mask of this node as the default */
	256	+ cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
	257	+ local_mask);
	258	+
	259	+ /* fill in the receive list */
	260	+ possible = cpumask_weight(&entry->def_intr.mask);
	261	+ curr_cpu = cpumask_first(&entry->def_intr.mask);
	262	+
	263	+ if (possible == 1) {
	264	+ /* only one CPU, everyone will use it */
	265	+ cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
	266	+ cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
	267	+ } else {
	268	+ /*
	269	+ * The general/control context will be the first CPU in
	270	+ * the default list, so it is removed from the default
	271	+ * list and added to the general interrupt list.
	272	+ */
	273	+ cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
	274	+ cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
	275	+ curr_cpu = cpumask_next(curr_cpu,
	276	+ &entry->def_intr.mask);
171	277
172		− cpumask_copy(&info->proc.mask, cpu_online_mask);
173		−}
	278	+ /*
	279	+ * Remove the remaining kernel receive queues from
	280	+ * the default list and add them to the receive list.
	281	+ */
	282	+ for (i = 0;
	283	+ i < (dd->n_krcv_queues - 1) *
	284	+ hfi1_per_node_cntr[dd->node];
	285	+ i++) {
	286	+ cpumask_clear_cpu(curr_cpu,
	287	+ &entry->def_intr.mask);
	288	+ cpumask_set_cpu(curr_cpu,
	289	+ &entry->rcv_intr.mask);
	290	+ curr_cpu = cpumask_next(curr_cpu,
	291	+ &entry->def_intr.mask);
	292	+ if (curr_cpu >= nr_cpu_ids)
	293	+ break;
	294	+ }
174	295
175		−void hfi1_dev_affinity_free(struct hfi1_devdata *dd)
176		−{
177		− kfree(dd->affinity);
	296	+ /*
	297	+ * If there ends up being 0 CPU cores leftover for SDMA
	298	+ * engines, use the same CPU cores as general/control
	299	+ * context.
	300	+ */
	301	+ if (cpumask_weight(&entry->def_intr.mask) == 0)
	302	+ cpumask_copy(&entry->def_intr.mask,
	303	+ &entry->general_intr_mask);
	304	+ }
	305	+
	306	+ spin_lock(&node_affinity.lock);
	307	+ node_affinity_add_tail(entry);
	308	+ spin_unlock(&node_affinity.lock);
	309	+ }
	310	+
	311	+ return 0;
178	312	}
179	313
180	314	int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix)
181	315	{
182	316	int ret;
183	317	cpumask_var_t diff;
184		− struct cpu_mask_set *set;
	318	+ struct hfi1_affinity_node *entry;
	319	+ struct cpu_mask_set *set = NULL;
185	320	struct sdma_engine *sde = NULL;
186	321	struct hfi1_ctxtdata *rcd = NULL;
187	322	char extra[64];
@@ -194,22 +329,25 @@ int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix) · 6 unmodified lines
194	329	if (!ret)
195	330	return -ENOMEM;
196	331
	332	+ spin_lock(&node_affinity.lock);
	333	+ entry = node_affinity_lookup(dd->node);
	334	+ spin_unlock(&node_affinity.lock);
	335	+
197	336	switch (msix->type) {
198	337	case IRQ_SDMA:
199	338	sde = (struct sdma_engine *)msix->arg;
200	339	scnprintf(extra, 64, "engine %u", sde->this_idx);
201		− /* fall through */
	340	+ set = &entry->def_intr;
	341	+ break;
202	342	case IRQ_GENERAL:
203		− set = &dd->affinity->def_intr;
	343	+ cpu = cpumask_first(&entry->general_intr_mask);
204	344	break;
205	345	case IRQ_RCVCTXT:
206	346	rcd = (struct hfi1_ctxtdata *)msix->arg;
207		− if (rcd->ctxt == HFI1_CTRL_CTXT) {
208		− set = &dd->affinity->def_intr;
209		− cpu = cpumask_first(&set->mask);
210		− } else {
211		− set = &dd->affinity->rcv_intr;
212		− }
	347	+ if (rcd->ctxt == HFI1_CTRL_CTXT)
	348	+ cpu = cpumask_first(&entry->general_intr_mask);
	349	+ else
	350	+ set = &entry->rcv_intr;
213	351	scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
214	352	break;
215	353	default:
@@ -218,12 +356,12 @@ int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix) · 2 unmodified lines
218	356	}
219	357
220	358	/*
221		− * The control receive context is placed on a particular CPU, which
222		− * is set above. Skip accounting for it. Everything else finds its
223		− * CPU here.
	359	+ * The general and control contexts are placed on a particular
	360	+ * CPU, which is set above. Skip accounting for it. Everything else
	361	+ * finds its CPU here.
224	362	*/
225		− if (cpu == -1) {
226		− spin_lock(&dd->affinity->lock);
	363	+ if (cpu == -1 && set) {
	364	+ spin_lock(&node_affinity.lock);
227	365	if (cpumask_equal(&set->mask, &set->used)) {
228	366	/*
229	367	* We've used up all the CPUs, bump up the generation
@@ -235,7 +373,7 @@ int hfi1_get_irq_affinity(struct hfi1_devdata dd, struct hfi1_msix_entry msix) · 5 unmodified lines
235	373	cpumask_andnot(diff, &set->mask, &set->used);
236	374	cpu = cpumask_first(diff);
237	375	cpumask_set_cpu(cpu, &set->used);
238		− spin_unlock(&dd->affinity->lock);
	376	+ spin_unlock(&node_affinity.lock);
239	377	}
240	378
241	379	switch (msix->type) {
@@ -263,43 +401,84 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd, · 21 unmodified lines
263	401	{
264	402	struct cpu_mask_set *set = NULL;
265	403	struct hfi1_ctxtdata *rcd;
	404	+ struct hfi1_affinity_node *entry;
	405	+
	406	+ spin_lock(&node_affinity.lock);
	407	+ entry = node_affinity_lookup(dd->node);
	408	+ spin_unlock(&node_affinity.lock);
266	409
267	410	switch (msix->type) {
268	411	case IRQ_SDMA:
	412	+ set = &entry->def_intr;
	413	+ break;
269	414	case IRQ_GENERAL:
270		− set = &dd->affinity->def_intr;
	415	+ /* Don't do accounting for general contexts */
271	416	break;
272	417	case IRQ_RCVCTXT:
273	418	rcd = (struct hfi1_ctxtdata *)msix->arg;
274		− /* only do accounting for non control contexts */
	419	+ /* Don't do accounting for control contexts */
275	420	if (rcd->ctxt != HFI1_CTRL_CTXT)
276		− set = &dd->affinity->rcv_intr;
	421	+ set = &entry->rcv_intr;
277	422	break;
278	423	default:
279	424	return;
280	425	}
281	426
282	427	if (set) {
283		− spin_lock(&dd->affinity->lock);
	428	+ spin_lock(&node_affinity.lock);
284	429	cpumask_andnot(&set->used, &set->used, &msix->mask);
285	430	if (cpumask_empty(&set->used) && set->gen) {
286	431	set->gen--;
287	432	cpumask_copy(&set->used, &set->mask);
288	433	}
289		− spin_unlock(&dd->affinity->lock);
	434	+ spin_unlock(&node_affinity.lock);
290	435	}
291	436
292	437	irq_set_affinity_hint(msix->msix.vector, NULL);
293	438	cpumask_clear(&msix->mask);
294	439	}
295	440
296		−int hfi1_get_proc_affinity(struct hfi1_devdata *dd, int node)
	441	+/* This should be called with node_affinity.lock held */
	442	+static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
	443	+ struct hfi1_affinity_node_list *affinity)
297	444	{
298		− int cpu = -1, ret;
299		− cpumask_var_t diff, mask, intrs;
	445	+ int possible, curr_cpu, i;
	446	+ uint num_cores_per_socket = node_affinity.num_online_cpus /
	447	+ affinity->num_core_siblings /
	448	+ node_affinity.num_online_nodes;
	449	+
	450	+ cpumask_copy(hw_thread_mask, &affinity->proc.mask);
	451	+ if (affinity->num_core_siblings > 0) {
	452	+ /* Removing other siblings not needed for now */
	453	+ possible = cpumask_weight(hw_thread_mask);
	454	+ curr_cpu = cpumask_first(hw_thread_mask);
	455	+ for (i = 0;
	456	+ i < num_cores_per_socket * node_affinity.num_online_nodes;
	457	+ i++)
	458	+ curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
	459	+
	460	+ for (; i < possible; i++) {
	461	+ cpumask_clear_cpu(curr_cpu, hw_thread_mask);
	462	+ curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
	463	+ }
	464	+
	465	+ /* Identifying correct HW threads within physical cores */
	466	+ cpumask_shift_left(hw_thread_mask, hw_thread_mask,
	467	+ num_cores_per_socket *
	468	+ node_affinity.num_online_nodes *
	469	+ hw_thread_no);
	470	+ }
	471	+}
	472	+
	473	+int hfi1_get_proc_affinity(int node)
	474	+{
	475	+ int cpu = -1, ret, i;
	476	+ struct hfi1_affinity_node *entry;
	477	+ cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
300	478	const struct cpumask *node_mask,
301	479	*proc_mask = tsk_cpus_allowed(current);
302		− struct cpu_mask_set *set = &dd->affinity->proc;
	480	+ struct hfi1_affinity_node_list *affinity = &node_affinity;
	481	+ struct cpu_mask_set *set = &affinity->proc;
303	482
304	483	/*
305	484	* check whether process/context affinity has already
@@ -325,22 +504,41 @@ int hfi1_get_proc_affinity(struct hfi1_devdata *dd, int node) · 19 unmodified lines
325	504
326	505	/*
327	506	* The process does not have a preset CPU affinity so find one to
328		− * recommend. We prefer CPUs on the same NUMA as the device.
	507	+ * recommend using the following algorithm:
	508	+ *
	509	+ * For each user process that is opening a context on HFI Y:
	510	+ * a) If all cores are filled, reinitialize the bitmask
	511	+ * b) Fill real cores first, then HT cores (First set of HT
	512	+ * cores on all physical cores, then second set of HT core,
	513	+ * and, so on) in the following order:
	514	+ *
	515	+ * 1. Same NUMA node as HFI Y and not running an IRQ
	516	+ * handler
	517	+ * 2. Same NUMA node as HFI Y and running an IRQ handler
	518	+ * 3. Different NUMA node to HFI Y and not running an IRQ
	519	+ * handler
	520	+ * 4. Different NUMA node to HFI Y and running an IRQ
	521	+ * handler
	522	+ * c) Mark core as filled in the bitmask. As user processes are
	523	+ * done, clear cores from the bitmask.
329	524	*/
330	525
331	526	ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
332	527	if (!ret)
333	528	goto done;
334		− ret = zalloc_cpumask_var(&mask, GFP_KERNEL);
	529	+ ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
335	530	if (!ret)
336	531	goto free_diff;
337		− ret = zalloc_cpumask_var(&intrs, GFP_KERNEL);
	532	+ ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
	533	+ if (!ret)
	534	+ goto free_hw_thread_mask;
	535	+ ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
338	536	if (!ret)
339		− goto free_mask;
	537	+ goto free_available_mask;
340	538
341		− spin_lock(&dd->affinity->lock);
	539	+ spin_lock(&affinity->lock);
342	540	/*
343		− * If we've used all available CPUs, clear the mask and start
	541	+ * If we've used all available HW threads, clear the mask and start
344	542	* overloading.
345	543	*/
346	544	if (cpumask_equal(&set->mask, &set->used)) {
@@ -348,81 +546,198 @@ int hfi1_get_proc_affinity(struct hfi1_devdata *dd, int node) · 1 unmodified line
348	546	cpumask_clear(&set->used);
349	547	}
350	548
351		− /* CPUs used by interrupt handlers */
352		− cpumask_copy(intrs, (dd->affinity->def_intr.gen ?
353		− &dd->affinity->def_intr.mask :
354		− &dd->affinity->def_intr.used));
355		− cpumask_or(intrs, intrs, (dd->affinity->rcv_intr.gen ?
356		− &dd->affinity->rcv_intr.mask :
357		− &dd->affinity->rcv_intr.used));
	549	+ /*
	550	+ * If NUMA node has CPUs used by interrupt handlers, include them in the
	551	+ * interrupt handler mask.
	552	+ */
	553	+ entry = node_affinity_lookup(node);
	554	+ if (entry) {
	555	+ cpumask_copy(intrs_mask, (entry->def_intr.gen ?
	556	+ &entry->def_intr.mask :
	557	+ &entry->def_intr.used));
	558	+ cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
	559	+ &entry->rcv_intr.mask :
	560	+ &entry->rcv_intr.used));
	561	+ cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
	562	+ }
358	563	hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
359		− cpumask_pr_args(intrs));
	564	+ cpumask_pr_args(intrs_mask));
	565	+
	566	+ cpumask_copy(hw_thread_mask, &set->mask);
360	567
361	568	/*
362		− * If we don't have a NUMA node requested, preference is towards
363		− * device NUMA node
	569	+ * If HT cores are enabled, identify which HW threads within the
	570	+ * physical cores should be used.
364	571	*/
365		− if (node == -1)
366		− node = dd->node;
	572	+ if (affinity->num_core_siblings > 0) {
	573	+ for (i = 0; i < affinity->num_core_siblings; i++) {
	574	+ find_hw_thread_mask(i, hw_thread_mask, affinity);
	575	+
	576	+ /*
	577	+ * If there's at least one available core for this HW
	578	+ * thread number, stop looking for a core.
	579	+ *
	580	+ * diff will always be not empty at least once in this
	581	+ * loop as the used mask gets reset when
	582	+ * (set->mask == set->used) before this loop.
	583	+ */
	584	+ cpumask_andnot(diff, hw_thread_mask, &set->used);
	585	+ if (!cpumask_empty(diff))
	586	+ break;
	587	+ }
	588	+ }
	589	+ hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
	590	+ cpumask_pr_args(hw_thread_mask));
	591	+
367	592	node_mask = cpumask_of_node(node);
368		− hfi1_cdbg(PROC, "device on NUMA %u, CPUs %*pbl", node,
	593	+ hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
369	594	cpumask_pr_args(node_mask));
370	595
371		− /* diff will hold all unused cpus */
372		− cpumask_andnot(diff, &set->mask, &set->used);
373		− hfi1_cdbg(PROC, "unused CPUs (all) %*pbl", cpumask_pr_args(diff));
374		−
375		− /* get cpumask of available CPUs on preferred NUMA */
376		− cpumask_and(mask, diff, node_mask);
377		− hfi1_cdbg(PROC, "available cpus on NUMA %*pbl", cpumask_pr_args(mask));
	596	+ /* Get cpumask of available CPUs on preferred NUMA */
	597	+ cpumask_and(available_mask, hw_thread_mask, node_mask);
	598	+ cpumask_andnot(available_mask, available_mask, &set->used);
	599	+ hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
	600	+ cpumask_pr_args(available_mask));
378	601
379	602	/*
380	603	* At first, we don't want to place processes on the same
381		− * CPUs as interrupt handlers.
	604	+ * CPUs as interrupt handlers. Then, CPUs running interrupt
	605	+ * handlers are used.
	606	+ *
	607	+ * 1) If diff is not empty, then there are CPUs not running
	608	+ * non-interrupt handlers available, so diff gets copied
	609	+ * over to available_mask.
	610	+ * 2) If diff is empty, then all CPUs not running interrupt
	611	+ * handlers are taken, so available_mask contains all
	612	+ * available CPUs running interrupt handlers.
	613	+ * 3) If available_mask is empty, then all CPUs on the
	614	+ * preferred NUMA node are taken, so other NUMA nodes are
	615	+ * used for process assignments using the same method as
	616	+ * the preferred NUMA node.
382	617	*/
383		− cpumask_andnot(diff, mask, intrs);
	618	+ cpumask_andnot(diff, available_mask, intrs_mask);
384	619	if (!cpumask_empty(diff))
385		− cpumask_copy(mask, diff);
	620	+ cpumask_copy(available_mask, diff);
386	621
387		− /*
388		− * if we don't have a cpu on the preferred NUMA, get
389		− * the list of the remaining available CPUs
390		− */
391		− if (cpumask_empty(mask)) {
392		− cpumask_andnot(diff, &set->mask, &set->used);
393		− cpumask_andnot(mask, diff, node_mask);
	622	+ /* If we don't have CPUs on the preferred node, use other NUMA nodes */
	623	+ if (cpumask_empty(available_mask)) {
	624	+ cpumask_andnot(available_mask, hw_thread_mask, &set->used);
	625	+ /* Excluding preferred NUMA cores */
	626	+ cpumask_andnot(available_mask, available_mask, node_mask);
	627	+ hfi1_cdbg(PROC,
	628	+ "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
	629	+ cpumask_pr_args(available_mask));
	630	+
	631	+ /*
	632	+ * At first, we don't want to place processes on the same
	633	+ * CPUs as interrupt handlers.
	634	+ */
	635	+ cpumask_andnot(diff, available_mask, intrs_mask);
	636	+ if (!cpumask_empty(diff))
	637	+ cpumask_copy(available_mask, diff);
394	638	}
395		− hfi1_cdbg(PROC, "possible CPUs for process %*pbl",
396		− cpumask_pr_args(mask));
	639	+ hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
	640	+ cpumask_pr_args(available_mask));
397	641
398		− cpu = cpumask_first(mask);
	642	+ cpu = cpumask_first(available_mask);
399	643	if (cpu >= nr_cpu_ids) /* empty */
400	644	cpu = -1;
401	645	else
402	646	cpumask_set_cpu(cpu, &set->used);
403		− spin_unlock(&dd->affinity->lock);
404		−
405		− free_cpumask_var(intrs);
406		−free_mask:
407		− free_cpumask_var(mask);
	647	+ spin_unlock(&affinity->lock);
	648	+ hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
	649	+
	650	+ free_cpumask_var(intrs_mask);
	651	+free_available_mask:
	652	+ free_cpumask_var(available_mask);
	653	+free_hw_thread_mask:
	654	+ free_cpumask_var(hw_thread_mask);
408	655	free_diff:
409	656	free_cpumask_var(diff);
410	657	done:
411	658	return cpu;
412	659	}
413	660
414		−void hfi1_put_proc_affinity(struct hfi1_devdata *dd, int cpu)
	661	+void hfi1_put_proc_affinity(int cpu)
415	662	{
416		− struct cpu_mask_set *set = &dd->affinity->proc;
	663	+ struct hfi1_affinity_node_list *affinity = &node_affinity;
	664	+ struct cpu_mask_set *set = &affinity->proc;
417	665
418	666	if (cpu < 0)
419	667	return;
420		− spin_lock(&dd->affinity->lock);
	668	+ spin_lock(&affinity->lock);
421	669	cpumask_clear_cpu(cpu, &set->used);
	670	+ hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
422	671	if (cpumask_empty(&set->used) && set->gen) {
423	672	set->gen--;
424	673	cpumask_copy(&set->used, &set->mask);
425	674	}
426		− spin_unlock(&dd->affinity->lock);
	675	+ spin_unlock(&affinity->lock);
427	676	}
428	677
	678	+/* Prevents concurrent reads and writes of the sdma_affinity attrib */
	679	+static DEFINE_MUTEX(sdma_affinity_mutex);
	680	+
	681	+int hfi1_set_sdma_affinity(struct hfi1_devdata dd, const char buf,
	682	+ size_t count)
	683	+{
	684	+ struct hfi1_affinity_node *entry;
	685	+ struct cpumask mask;
	686	+ int ret, i;
	687	+
	688	+ spin_lock(&node_affinity.lock);
	689	+ entry = node_affinity_lookup(dd->node);
	690	+ spin_unlock(&node_affinity.lock);
	691	+
	692	+ if (!entry)
	693	+ return -EINVAL;
	694	+
	695	+ ret = cpulist_parse(buf, &mask);
	696	+ if (ret)
	697	+ return ret;
	698	+
	699	+ if (!cpumask_subset(&mask, cpu_online_mask) \|\| cpumask_empty(&mask)) {
	700	+ dd_dev_warn(dd, "Invalid CPU mask\n");
	701	+ return -EINVAL;
	702	+ }
	703	+
	704	+ mutex_lock(&sdma_affinity_mutex);
	705	+ /* reset the SDMA interrupt affinity details */
	706	+ init_cpu_mask_set(&entry->def_intr);
	707	+ cpumask_copy(&entry->def_intr.mask, &mask);
	708	+ /*
	709	+ * Reassign the affinity for each SDMA interrupt.
	710	+ */
	711	+ for (i = 0; i < dd->num_msix_entries; i++) {
	712	+ struct hfi1_msix_entry *msix;
	713	+
	714	+ msix = &dd->msix_entries[i];
	715	+ if (msix->type != IRQ_SDMA)
	716	+ continue;
	717	+
	718	+ ret = hfi1_get_irq_affinity(dd, msix);
	719	+
	720	+ if (ret)
	721	+ break;
	722	+ }
	723	+
	724	+ mutex_unlock(&sdma_affinity_mutex);
	725	+ return ret ? ret : strnlen(buf, PAGE_SIZE);
	726	+}
	727	+
	728	+int hfi1_get_sdma_affinity(struct hfi1_devdata dd, char buf)
	729	+{
	730	+ struct hfi1_affinity_node *entry;
	731	+
	732	+ spin_lock(&node_affinity.lock);
	733	+ entry = node_affinity_lookup(dd->node);
	734	+ spin_unlock(&node_affinity.lock);
	735	+
	736	+ if (!entry)
	737	+ return -EINVAL;
	738	+
	739	+ mutex_lock(&sdma_affinity_mutex);
	740	+ cpumap_print_to_pagebuf(true, buf, &entry->def_intr.mask);
	741	+ mutex_unlock(&sdma_affinity_mutex);
	742	+ return strnlen(buf, PAGE_SIZE);
	743	+}

drivers/infiniband/hw/hfi1/affinity.h+31−7

@@ -73,7 +73,6 @@ struct cpu_mask_set { · 72 unmodified lines
73	73	struct hfi1_affinity {
74	74	struct cpu_mask_set def_intr;
75	75	struct cpu_mask_set rcv_intr;
76		− struct cpu_mask_set proc;
77	76	struct cpumask real_cpu_mask;
78	77	/* spin lock to protect affinity struct */
79	78	spinlock_t lock;
@@ -82,11 +81,9 @@ struct hfi1_affinity { · 2 unmodified lines
82	81	struct hfi1_msix_entry;
83	82
84	83	/* Initialize non-HT cpu cores mask */
85		−int init_real_cpu_mask(struct hfi1_devdata *);
	84	+void init_real_cpu_mask(void);
86	85	/* Initialize driver affinity data */
87		−void hfi1_dev_affinity_init(struct hfi1_devdata *);
88		−/* Free driver affinity data */
89		−void hfi1_dev_affinity_free(struct hfi1_devdata *);
	86	+int hfi1_dev_affinity_init(struct hfi1_devdata *);
90	87	/*
91	88	* Set IRQ affinity to a CPU. The function will determine the
92	89	* CPU and set the affinity to it.
@@ -101,8 +98,35 @@ void hfi1_put_irq_affinity(struct hfi1_devdata , struct hfi1_msix_entry ); · 8 unmodified lines
101	98	* Determine a CPU affinity for a user process, if the process does not
102	99	* have an affinity set yet.
103	100	*/
104		−int hfi1_get_proc_affinity(struct hfi1_devdata *, int);
	101	+int hfi1_get_proc_affinity(int);
105	102	/* Release a CPU used by a user process. */
106		−void hfi1_put_proc_affinity(struct hfi1_devdata *, int);
	103	+void hfi1_put_proc_affinity(int);
	104	+
	105	+int hfi1_get_sdma_affinity(struct hfi1_devdata dd, char buf);
	106	+int hfi1_set_sdma_affinity(struct hfi1_devdata dd, const char buf,
	107	+ size_t count);
	108	+
	109	+struct hfi1_affinity_node {
	110	+ int node;
	111	+ struct cpu_mask_set def_intr;
	112	+ struct cpu_mask_set rcv_intr;
	113	+ struct cpumask general_intr_mask;
	114	+ struct list_head list;
	115	+};
	116	+
	117	+struct hfi1_affinity_node_list {
	118	+ struct list_head list;
	119	+ struct cpumask real_cpu_mask;
	120	+ struct cpu_mask_set proc;
	121	+ int num_core_siblings;
	122	+ int num_online_nodes;
	123	+ int num_online_cpus;
	124	+ /* protect affinity node list */
	125	+ spinlock_t lock;
	126	+};
	127	+
	128	+int node_affinity_init(void);
	129	+void node_affinity_destroy(void);
	130	+extern struct hfi1_affinity_node_list node_affinity;
107	131
108	132	#endif /* _HFI1_AFFINITY_H */

drivers/infiniband/hw/hfi1/chip.c+193−103

@@ -63,6 +63,7 @@ · 62 unmodified lines
63	63	#include "efivar.h"
64	64	#include "platform.h"
65	65	#include "aspm.h"
	66	+#include "affinity.h"
66	67
67	68	#define NUM_IB_PORTS 1
68	69
@@ -121,6 +122,7 @@ struct flag_table { · 52 unmodified lines
121	122	#define SEC_SC_HALTED 0x4 /* per-context only */
122	123	#define SEC_SPC_FREEZE 0x8 /* per-HFI only */
123	124
	125	+#define DEFAULT_KRCVQS 2
124	126	#define MIN_KERNEL_KCTXTS 2
125	127	#define FIRST_KERNEL_KCTXT 1
126	128	/* sizes for both the QP and RSM map tables */
@@ -238,6 +240,9 @@ struct flag_table { · 111 unmodified lines
238	240	/* all CceStatus sub-block RXE pause bits */
239	241	#define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
240	242
	243	+#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
	244	+#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
	245	+
241	246	/*
242	247	* CCE Error flags.
243	248	*/
@@ -3947,6 +3952,28 @@ static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry, · 3703 unmodified lines
3947	3952	return dd->sw_send_dma_eng_err_status_cnt[0];
3948	3953	}
3949	3954
	3955	+static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry,
	3956	+ void *context, int vl, int mode,
	3957	+ u64 data)
	3958	+{
	3959	+ struct hfi1_devdata dd = (struct hfi1_devdata )context;
	3960	+
	3961	+ u64 val = 0;
	3962	+ u64 csr = entry->csr;
	3963	+
	3964	+ val = read_write_csr(dd, csr, mode, data);
	3965	+ if (mode == CNTR_MODE_R) {
	3966	+ val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ?
	3967	+ CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors;
	3968	+ } else if (mode == CNTR_MODE_W) {
	3969	+ dd->sw_rcv_bypass_packet_errors = 0;
	3970	+ } else {
	3971	+ dd_dev_err(dd, "Invalid cntr register access mode");
	3972	+ return 0;
	3973	+ }
	3974	+ return val;
	3975	+}
	3976	+
3950	3977	#define def_access_sw_cpu(cntr) \
3951	3978	static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \
3952	3979	void *context, int vl, int mode, u64 data) \
@@ -4020,7 +4047,8 @@ static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = { · 67 unmodified lines
4020	4047	CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4021	4048	[C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4022	4049	CNTR_SYNTH),
4023		−[C_DC_RCV_ERR] = DC_PERF_CNTR(DcRecvErr, DCC_ERR_PORTRCV_ERR_CNT, CNTR_SYNTH),
	4050	+[C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH,
	4051	+ access_dc_rcv_err_cnt),
4024	4052	[C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4025	4053	CNTR_SYNTH),
4026	4054	[C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
@@ -8798,30 +8826,6 @@ static int write_tx_settings(struct hfi1_devdata *dd, · 4771 unmodified lines
8798	8826	return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
8799	8827	}
8800	8828
8801		−static void check_fabric_firmware_versions(struct hfi1_devdata *dd)
8802		−{
8803		− u32 frame, version, prod_id;
8804		− int ret, lane;
8805		−
8806		− /* 4 lanes */
8807		− for (lane = 0; lane < 4; lane++) {
8808		− ret = read_8051_config(dd, SPICO_FW_VERSION, lane, &frame);
8809		− if (ret) {
8810		− dd_dev_err(dd,
8811		− "Unable to read lane %d firmware details\n",
8812		− lane);
8813		− continue;
8814		− }
8815		− version = (frame >> SPICO_ROM_VERSION_SHIFT)
8816		− & SPICO_ROM_VERSION_MASK;
8817		− prod_id = (frame >> SPICO_ROM_PROD_ID_SHIFT)
8818		− & SPICO_ROM_PROD_ID_MASK;
8819		− dd_dev_info(dd,
8820		− "Lane %d firmware: version 0x%04x, prod_id 0x%04x\n",
8821		− lane, version, prod_id);
8822		− }
8823		−}
8824		−
8825	8829	/*
8826	8830	* Read an idle LCB message.
8827	8831	*
@@ -9187,17 +9191,24 @@ static void wait_for_qsfp_init(struct hfi1_pportdata *ppd) · 359 unmodified lines
9187	9191	unsigned long timeout;
9188	9192
9189	9193	/*
9190		− * Check for QSFP interrupt for t_init (SFF 8679)
	9194	+ * Some QSFP cables have a quirk that asserts the IntN line as a side
	9195	+ * effect of power up on plug-in. We ignore this false positive
	9196	+ * interrupt until the module has finished powering up by waiting for
	9197	+ * a minimum timeout of the module inrush initialization time of
	9198	+ * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the
	9199	+ * module have stabilized.
	9200	+ */
	9201	+ msleep(500);
	9202	+
	9203	+ /*
	9204	+ * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1)
9191	9205	*/
9192	9206	timeout = jiffies + msecs_to_jiffies(2000);
9193	9207	while (1) {
9194	9208	mask = read_csr(dd, dd->hfi1_id ?
9195	9209	ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9196		− if (!(mask & QSFP_HFI0_INT_N)) {
9197		− write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR :
9198		− ASIC_QSFP1_CLEAR, QSFP_HFI0_INT_N);
	9210	+ if (!(mask & QSFP_HFI0_INT_N))
9199	9211	break;
9200		− }
9201	9212	if (time_after(jiffies, timeout)) {
9202	9213	dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9203	9214	__func__);
@@ -9213,10 +9224,17 @@ static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable) · 9 unmodified lines
9213	9224	u64 mask;
9214	9225
9215	9226	mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9216		− if (enable)
	9227	+ if (enable) {
	9228	+ /*
	9229	+ * Clear the status register to avoid an immediate interrupt
	9230	+ * when we re-enable the IntN pin
	9231	+ */
	9232	+ write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
	9233	+ QSFP_HFI0_INT_N);
9217	9234	mask \|= (u64)QSFP_HFI0_INT_N;
9218		− else
	9235	+ } else {
9219	9236	mask &= ~(u64)QSFP_HFI0_INT_N;
	9237	+ }
9220	9238	write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9221	9239	}
9222	9240
@@ -9630,14 +9648,6 @@ void hfi1_clear_tids(struct hfi1_ctxtdata *rcd) · 407 unmodified lines
9630	9648	hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9631	9649	}
9632	9650
9633		−int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
9634		− struct hfi1_ctxt_info *kinfo)
9635		−{
9636		− kinfo->runtime_flags = (HFI1_MISC_GET() << HFI1_CAP_USER_SHIFT) \|
9637		− HFI1_CAP_UGET(MASK) \| HFI1_CAP_KGET(K2U);
9638		− return 0;
9639		−}
9640		−
9641	9651	struct hfi1_message_header *hfi1_get_msgheader(
9642	9652	struct hfi1_devdata dd, __le32 rhf_addr)
9643	9653	{
@@ -9890,6 +9900,131 @@ static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs) · 246 unmodified lines
9890	9900	return 0;
9891	9901	}
9892	9902
	9903	+static const char *state_completed_string(u32 completed)
	9904	+{
	9905	+ static const char * const state_completed[] = {
	9906	+ "EstablishComm",
	9907	+ "OptimizeEQ",
	9908	+ "VerifyCap"
	9909	+ };
	9910	+
	9911	+ if (completed < ARRAY_SIZE(state_completed))
	9912	+ return state_completed[completed];
	9913	+
	9914	+ return "unknown";
	9915	+}
	9916	+
	9917	+static const char all_lanes_dead_timeout_expired[] =
	9918	+ "All lanes were inactive – was the interconnect media removed?";
	9919	+static const char tx_out_of_policy[] =
	9920	+ "Passing lanes on local port do not meet the local link width policy";
	9921	+static const char no_state_complete[] =
	9922	+ "State timeout occurred before link partner completed the state";
	9923	+static const char * const state_complete_reasons[] = {
	9924	+ [0x00] = "Reason unknown",
	9925	+ [0x01] = "Link was halted by driver, refer to LinkDownReason",
	9926	+ [0x02] = "Link partner reported failure",
	9927	+ [0x10] = "Unable to achieve frame sync on any lane",
	9928	+ [0x11] =
	9929	+ "Unable to find a common bit rate with the link partner",
	9930	+ [0x12] =
	9931	+ "Unable to achieve frame sync on sufficient lanes to meet the local link width policy",
	9932	+ [0x13] =
	9933	+ "Unable to identify preset equalization on sufficient lanes to meet the local link width policy",
	9934	+ [0x14] = no_state_complete,
	9935	+ [0x15] =
	9936	+ "State timeout occurred before link partner identified equalization presets",
	9937	+ [0x16] =
	9938	+ "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy",
	9939	+ [0x17] = tx_out_of_policy,
	9940	+ [0x20] = all_lanes_dead_timeout_expired,
	9941	+ [0x21] =
	9942	+ "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy",
	9943	+ [0x22] = no_state_complete,
	9944	+ [0x23] =
	9945	+ "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy",
	9946	+ [0x24] = tx_out_of_policy,
	9947	+ [0x30] = all_lanes_dead_timeout_expired,
	9948	+ [0x31] =
	9949	+ "State timeout occurred waiting for host to process received frames",
	9950	+ [0x32] = no_state_complete,
	9951	+ [0x33] =
	9952	+ "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy",
	9953	+ [0x34] = tx_out_of_policy,
	9954	+};
	9955	+
	9956	+static const char state_complete_reason_code_string(struct hfi1_pportdata ppd,
	9957	+ u32 code)
	9958	+{
	9959	+ const char *str = NULL;
	9960	+
	9961	+ if (code < ARRAY_SIZE(state_complete_reasons))
	9962	+ str = state_complete_reasons[code];
	9963	+
	9964	+ if (str)
	9965	+ return str;
	9966	+ return "Reserved";
	9967	+}
	9968	+
	9969	+/* describe the given last state complete frame */
	9970	+static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame,
	9971	+ const char *prefix)
	9972	+{
	9973	+ struct hfi1_devdata *dd = ppd->dd;
	9974	+ u32 success;
	9975	+ u32 state;
	9976	+ u32 reason;
	9977	+ u32 lanes;
	9978	+
	9979	+ /*
	9980	+ * Decode frame:
	9981	+ * [ 0: 0] - success
	9982	+ * [ 3: 1] - state
	9983	+ * [ 7: 4] - next state timeout
	9984	+ * [15: 8] - reason code
	9985	+ * [31:16] - lanes
	9986	+ */
	9987	+ success = frame & 0x1;
	9988	+ state = (frame >> 1) & 0x7;
	9989	+ reason = (frame >> 8) & 0xff;
	9990	+ lanes = (frame >> 16) & 0xffff;
	9991	+
	9992	+ dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n",
	9993	+ prefix, frame);
	9994	+ dd_dev_err(dd, " last reported state state: %s (0x%x)\n",
	9995	+ state_completed_string(state), state);
	9996	+ dd_dev_err(dd, " state successfully completed: %s\n",
	9997	+ success ? "yes" : "no");
	9998	+ dd_dev_err(dd, " fail reason 0x%x: %s\n",
	9999	+ reason, state_complete_reason_code_string(ppd, reason));
	10000	+ dd_dev_err(dd, " passing lane mask: 0x%x", lanes);
	10001	+}
	10002	+
	10003	+/*
	10004	+ * Read the last state complete frames and explain them. This routine
	10005	+ * expects to be called if the link went down during link negotiation
	10006	+ * and initialization (LNI). That is, anywhere between polling and link up.
	10007	+ */
	10008	+static void check_lni_states(struct hfi1_pportdata *ppd)
	10009	+{
	10010	+ u32 last_local_state;
	10011	+ u32 last_remote_state;
	10012	+
	10013	+ read_last_local_state(ppd->dd, &last_local_state);
	10014	+ read_last_remote_state(ppd->dd, &last_remote_state);
	10015	+
	10016	+ /*
	10017	+ * Don't report anything if there is nothing to report. A value of
	10018	+ * 0 means the link was taken down while polling and there was no
	10019	+ * training in-process.
	10020	+ */
	10021	+ if (last_local_state == 0 && last_remote_state == 0)
	10022	+ return;
	10023	+
	10024	+ decode_state_complete(ppd, last_local_state, "transmitted");
	10025	+ decode_state_complete(ppd, last_remote_state, "received");
	10026	+}
	10027	+
9893	10028	/*
9894	10029	* Helper for set_link_state(). Do not call except from that routine.
9895	10030	* Expects ppd->hls_mutex to be held.
@@ -9902,8 +10037,6 @@ static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason) · 6 unmodified lines
9902	10037	{
9903	10038	struct hfi1_devdata *dd = ppd->dd;
9904	10039	u32 pstate, previous_state;
9905		− u32 last_local_state;
9906		− u32 last_remote_state;
9907	10040	int ret;
9908	10041	int do_transition;
9909	10042	int do_wait;
@@ -10003,12 +10136,7 @@ static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason) · 93 unmodified lines
10003	10136	} else if (previous_state
10004	10137	& (HLS_DN_POLL \| HLS_VERIFY_CAP \| HLS_GOING_UP)) {
10005	10138	/* went down while attempting link up */
10006		− /* byte 1 of last__state is the failure reason /
10007		− read_last_local_state(dd, &last_local_state);
10008		− read_last_remote_state(dd, &last_remote_state);
10009		− dd_dev_err(dd,
10010		− "LNI failure last states: local 0x%08x, remote 0x%08x\n",
10011		− last_local_state, last_remote_state);
	10139	+ check_lni_states(ppd);
10012	10140	}
10013	10141
10014	10142	/* the active link width (downgrade) is 0 on link down */
@@ -11668,9 +11796,6 @@ static void free_cntrs(struct hfi1_devdata *dd) · 1653 unmodified lines
11668	11796	dd->cntrnames = NULL;
11669	11797	}
11670	11798
11671		−#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
11672		−#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
11673		−
11674	11799	static u64 read_dev_port_cntr(struct hfi1_devdata dd, struct cntr_entry entry,
11675	11800	u64 psval, void context, int vl)
11676	11801	{
@@ -12325,37 +12450,6 @@ u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd) · 648 unmodified lines
12325	12450	return ib_pstate;
12326	12451	}
12327	12452
12328		−/*
12329		− * Read/modify/write ASIC_QSFP register bits as selected by mask
12330		− * data: 0 or 1 in the positions depending on what needs to be written
12331		− * dir: 0 for read, 1 for write
12332		− * mask: select by setting
12333		− * I2CCLK (bit 0)
12334		− * I2CDATA (bit 1)
12335		− */
12336		−u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
12337		− u32 mask)
12338		−{
12339		− u64 qsfp_oe, target_oe;
12340		−
12341		− target_oe = target ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
12342		− if (mask) {
12343		− /* We are writing register bits, so lock access */
12344		− dir &= mask;
12345		− data &= mask;
12346		−
12347		− qsfp_oe = read_csr(dd, target_oe);
12348		− qsfp_oe = (qsfp_oe & ~(u64)mask) \| (u64)dir;
12349		− write_csr(dd, target_oe, qsfp_oe);
12350		− }
12351		− /* We are exclusively reading bits here, but it is unlikely
12352		− * we'll get valid data when we set the direction of the pin
12353		− * in the same call, so read should call this function again
12354		− * to get valid data
12355		− */
12356		− return read_csr(dd, target ? ASIC_QSFP2_IN : ASIC_QSFP1_IN);
12357		−}
12358		−
12359	12453	#define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
12360	12454	(r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12361	12455
@@ -12780,7 +12874,6 @@ static int set_up_context_variables(struct hfi1_devdata *dd) · 418 unmodified lines
12780	12874
12781	12875	/*
12782	12876	* Kernel receive contexts:
12783		− * - min of 2 or 1 context/numa (excluding control context)
12784	12877	* - Context 0 - control context (VL15/multicast/error)
12785	12878	* - Context 1 - first kernel context
12786	12879	* - Context 2 - second kernel context
@@ -12794,9 +12887,7 @@ static int set_up_context_variables(struct hfi1_devdata *dd) · 7 unmodified lines
12794	12887	*/
12795	12888	num_kernel_contexts = n_krcvqs + 1;
12796	12889	else
12797		− num_kernel_contexts = num_online_nodes() + 1;
12798		− num_kernel_contexts =
12799		− max_t(int, MIN_KERNEL_KCTXTS, num_kernel_contexts);
	12890	+ num_kernel_contexts = DEFAULT_KRCVQS + 1;
12800	12891	/*
12801	12892	* Every kernel receive context needs an ACK send context.
12802	12893	* one send context is allocated for each VL{0-7} and VL15
@@ -12815,7 +12906,7 @@ static int set_up_context_variables(struct hfi1_devdata *dd) · 12 unmodified lines
12815	12906	*/
12816	12907	if (num_user_contexts < 0)
12817	12908	num_user_contexts =
12818		− cpumask_weight(&dd->affinity->real_cpu_mask);
	12909	+ cpumask_weight(&node_affinity.real_cpu_mask);
12819	12910
12820	12911	total_contexts = num_kernel_contexts + num_user_contexts;
12821	12912
@@ -14141,6 +14232,11 @@ static int init_asic_data(struct hfi1_devdata *dd) · 1319 unmodified lines
14141	14232	}
14142	14233	dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
14143	14234	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
	14235	+
	14236	+ /* first one through - set up i2c devices */
	14237	+ if (!peer)
	14238	+ ret = set_up_i2c(dd, dd->asic_data);
	14239	+
14144	14240	return ret;
14145	14241	}
14146	14242
@@ -14445,19 +14541,6 @@ struct hfi1_devdata hfi1_init_dd(struct pci_dev pdev, · 298 unmodified lines
14445	14541	(dd->revision >> CCE_REVISION_SW_SHIFT)
14446	14542	& CCE_REVISION_SW_MASK);
14447	14543
14448		− /*
14449		− * The real cpu mask is part of the affinity struct but has to be
14450		− * initialized earlier than the rest of the affinity struct because it
14451		− * is needed to calculate the number of user contexts in
14452		− * set_up_context_variables(). However, hfi1_dev_affinity_init(),
14453		− * which initializes the rest of the affinity struct members,
14454		− * depends on set_up_context_variables() for the number of kernel
14455		− * contexts, so it cannot be called before set_up_context_variables().
14456		− */
14457		− ret = init_real_cpu_mask(dd);
14458		− if (ret)
14459		− goto bail_cleanup;
14460		−
14461	14544	ret = set_up_context_variables(dd);
14462	14545	if (ret)
14463	14546	goto bail_cleanup;
@@ -14471,7 +14554,9 @@ struct hfi1_devdata hfi1_init_dd(struct pci_dev pdev, · 7 unmodified lines
14471	14554	/* set up KDETH QP prefix in both RX and TX CSRs */
14472	14555	init_kdeth_qp(dd);
14473	14556
14474		− hfi1_dev_affinity_init(dd);
	14557	+ ret = hfi1_dev_affinity_init(dd);
	14558	+ if (ret)
	14559	+ goto bail_cleanup;
14475	14560
14476	14561	/* send contexts must be set up before receive contexts */
14477	14562	ret = init_send_contexts(dd);
@@ -14508,8 +14593,14 @@ struct hfi1_devdata hfi1_init_dd(struct pci_dev pdev, · 30 unmodified lines
14508	14593	/* set up LCB access - must be after set_up_interrupts() */
14509	14594	init_lcb_access(dd);
14510	14595
	14596	+ /*
	14597	+ * Serial number is created from the base guid:
	14598	+ * [27:24] = base guid [38:35]
	14599	+ * [23: 0] = base guid [23: 0]
	14600	+ */
14511	14601	snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
14512		− dd->base_guid & 0xFFFFFF);
	14602	+ (dd->base_guid & 0xFFFFFF) \|
	14603	+ ((dd->base_guid >> 11) & 0xF000000));
14513	14604
14514	14605	dd->oui1 = dd->base_guid >> 56 & 0xFF;
14515	14606	dd->oui2 = dd->base_guid >> 48 & 0xFF;
@@ -14518,7 +14609,6 @@ struct hfi1_devdata hfi1_init_dd(struct pci_dev pdev, · 2 unmodified lines
14518	14609	ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
14519	14610	if (ret)
14520	14611	goto bail_clear_intr;
14521		− check_fabric_firmware_versions(dd);
14522	14612
14523	14613	thermal_init(dd);
14524	14614

drivers/infiniband/hw/hfi1/chip.h+1−4

@@ -640,6 +640,7 @@ extern uint platform_config_load; · 639 unmodified lines
640	640	/* SBus commands */
641	641	#define RESET_SBUS_RECEIVER 0x20
642	642	#define WRITE_SBUS_RECEIVER 0x21
	643	+#define READ_SBUS_RECEIVER 0x22
643	644	void sbus_request(struct hfi1_devdata *dd,
644	645	u8 receiver_addr, u8 data_addr, u8 command, u32 data_in);
645	646	int sbus_request_slow(struct hfi1_devdata *dd,
@@ -1336,10 +1337,6 @@ void hfi1_start_cleanup(struct hfi1_devdata *dd); · 690 unmodified lines
1336	1337	void hfi1_clear_tids(struct hfi1_ctxtdata *rcd);
1337	1338	struct hfi1_message_header *hfi1_get_msgheader(
1338	1339	struct hfi1_devdata dd, __le32 rhf_addr);
1339		−int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
1340		− struct hfi1_ctxt_info *kinfo);
1341		−u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
1342		− u32 mask);
1343	1340	int hfi1_init_ctxt(struct send_context *sc);
1344	1341	void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
1345	1342	u32 type, unsigned long pa, u16 order);

drivers/infiniband/hw/hfi1/chip_registers.h+4−0

@@ -471,6 +471,10 @@ · 470 unmodified lines
471	471	#define ASIC_STS_SBUS_RESULT (ASIC + 0x000000000010)
472	472	#define ASIC_STS_SBUS_RESULT_DONE_SMASK 0x1ull
473	473	#define ASIC_STS_SBUS_RESULT_RCV_DATA_VALID_SMASK 0x2ull
	474	+#define ASIC_STS_SBUS_RESULT_RESULT_CODE_SHIFT 2
	475	+#define ASIC_STS_SBUS_RESULT_RESULT_CODE_MASK 0x7ull
	476	+#define ASIC_STS_SBUS_RESULT_DATA_OUT_SHIFT 32
	477	+#define ASIC_STS_SBUS_RESULT_DATA_OUT_MASK 0xFFFFFFFFull
474	478	#define ASIC_STS_THERM (ASIC + 0x000000000058)
475	479	#define ASIC_STS_THERM_CRIT_TEMP_MASK 0x7FFull
476	480	#define ASIC_STS_THERM_CRIT_TEMP_SHIFT 18

drivers/infiniband/hw/hfi1/driver.c+28−24

@@ -392,9 +392,7 @@ static void rcv_hdrerr(struct hfi1_ctxtdata rcd, struct hfi1_pportdata ppd, · 391 unmodified lines
392	392	u16 rlid;
393	393	u8 svc_type, sl, sc5;
394	394
395		− sc5 = (be16_to_cpu(rhdr->lrh[0]) >> 12) & 0xf;
396		− if (rhf_dc_info(packet->rhf))
397		− sc5 \|= 0x10;
	395	+ sc5 = hdr2sc(rhdr, packet->rhf);
398	396	sl = ibp->sc_to_sl[sc5];
399	397
400	398	lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK;
@@ -450,14 +448,20 @@ static inline void init_packet(struct hfi1_ctxtdata *rcd, · 49 unmodified lines
450	448	packet->rcv_flags = 0;
451	449	}
452	450
453		−static void process_ecn(struct rvt_qp qp, struct hfi1_ib_header hdr,
454		− struct hfi1_other_headers *ohdr,
455		− u64 rhf, u32 bth1, struct ib_grh *grh)
	451	+void hfi1_process_ecn_slowpath(struct rvt_qp qp, struct hfi1_packet pkt,
	452	+ bool do_cnp)
456	453	{
457	454	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
458		− u32 rqpn = 0;
459		− u16 rlid;
460		− u8 sc5, svc_type;
	455	+ struct hfi1_ib_header *hdr = pkt->hdr;
	456	+ struct hfi1_other_headers *ohdr = pkt->ohdr;
	457	+ struct ib_grh *grh = NULL;
	458	+ u32 rqpn = 0, bth1;
	459	+ u16 rlid, dlid = be16_to_cpu(hdr->lrh[1]);
	460	+ u8 sc, svc_type;
	461	+ bool is_mcast = false;
	462	+
	463	+ if (pkt->rcv_flags & HFI1_HAS_GRH)
	464	+ grh = &hdr->u.l.grh;
461	465
462	466	switch (qp->ibqp.qp_type) {
463	467	case IB_QPT_SMI:
@@ -466,6 +470,8 @@ static void process_ecn(struct rvt_qp qp, struct hfi1_ib_header hdr, · 2 unmodified lines
466	470	rlid = be16_to_cpu(hdr->lrh[3]);
467	471	rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
468	472	svc_type = IB_CC_SVCTYPE_UD;
	473	+ is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
	474	+ (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
469	475	break;
470	476	case IB_QPT_UC:
471	477	rlid = qp->remote_ah_attr.dlid;
@@ -481,24 +487,23 @@ static void process_ecn(struct rvt_qp qp, struct hfi1_ib_header hdr, · 9 unmodified lines
481	487	return;
482	488	}
483	489
484		− sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf;
485		− if (rhf_dc_info(rhf))
486		− sc5 \|= 0x10;
	490	+ sc = hdr2sc((struct hfi1_message_header *)hdr, pkt->rhf);
487	491
488		− if (bth1 & HFI1_FECN_SMASK) {
	492	+ bth1 = be32_to_cpu(ohdr->bth[1]);
	493	+ if (do_cnp && (bth1 & HFI1_FECN_SMASK)) {
489	494	u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);
490		− u16 dlid = be16_to_cpu(hdr->lrh[1]);
491	495
492		− return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc5, grh);
	496	+ return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh);
493	497	}
494	498
495		− if (bth1 & HFI1_BECN_SMASK) {
	499	+ if (!is_mcast && (bth1 & HFI1_BECN_SMASK)) {
496	500	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
497	501	u32 lqpn = bth1 & RVT_QPN_MASK;
498		− u8 sl = ibp->sc_to_sl[sc5];
	502	+ u8 sl = ibp->sc_to_sl[sc];
499	503
500	504	process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
501	505	}
	506	+
502	507	}
503	508
504	509	struct ps_mdata {
@@ -596,7 +601,6 @@ static void __prescan_rxq(struct hfi1_packet *packet) · 91 unmodified lines
596	601	struct rvt_qp *qp;
597	602	struct hfi1_ib_header *hdr;
598	603	struct hfi1_other_headers *ohdr;
599		− struct ib_grh *grh = NULL;
600	604	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
601	605	u64 rhf = rhf_to_cpu(rhf_addr);
602	606	u32 etype = rhf_rcv_type(rhf), qpn, bth1;
@@ -616,14 +620,13 @@ static void __prescan_rxq(struct hfi1_packet *packet) · 13 unmodified lines
616	620	hfi1_get_msgheader(dd, rhf_addr);
617	621	lnh = be16_to_cpu(hdr->lrh[0]) & 3;
618	622
619		− if (lnh == HFI1_LRH_BTH) {
	623	+ if (lnh == HFI1_LRH_BTH)
620	624	ohdr = &hdr->u.oth;
621		− } else if (lnh == HFI1_LRH_GRH) {
	625	+ else if (lnh == HFI1_LRH_GRH)
622	626	ohdr = &hdr->u.l.oth;
623		− grh = &hdr->u.l.grh;
624		− } else {
	627	+ else
625	628	goto next; /* just in case */
626		− }
	629	+
627	630	bth1 = be32_to_cpu(ohdr->bth[1]);
628	631	is_ecn = !!(bth1 & (HFI1_FECN_SMASK \| HFI1_BECN_SMASK));
629	632
@@ -639,7 +642,7 @@ static void __prescan_rxq(struct hfi1_packet *packet) · 9 unmodified lines
639	642	goto next;
640	643	}
641	644
642		− process_ecn(qp, hdr, ohdr, rhf, bth1, grh);
	645	+ process_ecn(qp, packet, true);
643	646	rcu_read_unlock();
644	647
645	648	/* turn off BECN, FECN */
@@ -1362,6 +1365,7 @@ int process_receive_bypass(struct hfi1_packet *packet) · 716 unmodified lines
1362	1365
1363	1366	dd_dev_err(packet->rcd->dd,
1364	1367	"Bypass packets are not supported in normal operation. Dropping\n");
	1368	+ incr_cntr64(&packet->rcd->dd->sw_rcv_bypass_packet_errors);
1365	1369	return RHF_RCV_CONTINUE;
1366	1370	}
1367	1371

drivers/infiniband/hw/hfi1/file_ops.c+51−40

@@ -168,6 +168,7 @@ static inline int is_valid_mmap(u64 token) · 167 unmodified lines
168	168
169	169	static int hfi1_file_open(struct inode inode, struct file fp)
170	170	{
	171	+ struct hfi1_filedata *fd;
171	172	struct hfi1_devdata *dd = container_of(inode->i_cdev,
172	173	struct hfi1_devdata,
173	174	user_cdev);
@@ -176,10 +177,17 @@ static int hfi1_file_open(struct inode inode, struct file fp) · 2 unmodified lines
176	177	kobject_get(&dd->kobj);
177	178
178	179	/* The real work is performed later in assign_ctxt() */
179		− fp->private_data = kzalloc(sizeof(struct hfi1_filedata), GFP_KERNEL);
180		− if (fp->private_data) /* no cpu affinity by default */
181		− ((struct hfi1_filedata *)fp->private_data)->rec_cpu_num = -1;
182		− return fp->private_data ? 0 : -ENOMEM;
	180	+
	181	+ fd = kzalloc(sizeof(*fd), GFP_KERNEL);
	182	+
	183	+ if (fd) {
	184	+ fd->rec_cpu_num = -1; /* no cpu affinity by default */
	185	+ fd->mm = current->mm;
	186	+ }
	187	+
	188	+ fp->private_data = fd;
	189	+
	190	+ return fd ? 0 : -ENOMEM;
183	191	}
184	192
185	193	static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
@@ -392,41 +400,38 @@ static ssize_t hfi1_write_iter(struct kiocb kiocb, struct iov_iter from) · 206 unmodified lines
392	400	struct hfi1_filedata *fd = kiocb->ki_filp->private_data;
393	401	struct hfi1_user_sdma_pkt_q *pq = fd->pq;
394	402	struct hfi1_user_sdma_comp_q *cq = fd->cq;
395		− int ret = 0, done = 0, reqs = 0;
	403	+ int done = 0, reqs = 0;
396	404	unsigned long dim = from->nr_segs;
397	405
398		− if (!cq \|\| !pq) {
399		− ret = -EIO;
400		− goto done;
401		− }
	406	+ if (!cq \|\| !pq)
	407	+ return -EIO;
402	408
403		− if (!iter_is_iovec(from) \|\| !dim) {
404		− ret = -EINVAL;
405		− goto done;
406		− }
	409	+ if (!iter_is_iovec(from) \|\| !dim)
	410	+ return -EINVAL;
407	411
408	412	hfi1_cdbg(SDMA, "SDMA request from %u:%u (%lu)",
409	413	fd->uctxt->ctxt, fd->subctxt, dim);
410	414
411		− if (atomic_read(&pq->n_reqs) == pq->n_max_reqs) {
412		− ret = -ENOSPC;
413		− goto done;
414		− }
	415	+ if (atomic_read(&pq->n_reqs) == pq->n_max_reqs)
	416	+ return -ENOSPC;
415	417
416	418	while (dim) {
	419	+ int ret;
417	420	unsigned long count = 0;
418	421
419	422	ret = hfi1_user_sdma_process_request(
420	423	kiocb->ki_filp, (struct iovec *)(from->iov + done),
421	424	dim, &count);
422		− if (ret)
423		− goto done;
	425	+ if (ret) {
	426	+ reqs = ret;
	427	+ break;
	428	+ }
424	429	dim -= count;
425	430	done += count;
426	431	reqs++;
427	432	}
428		−done:
429		− return ret ? ret : reqs;
	433	+
	434	+ return reqs;
430	435	}
431	436
432	437	static int hfi1_file_mmap(struct file fp, struct vm_area_struct vma)
@@ -718,7 +723,7 @@ static int hfi1_file_close(struct inode inode, struct file fp) · 285 unmodified lines
718	723	hfi1_user_sdma_free_queues(fdata);
719	724
720	725	/* release the cpu */
721		− hfi1_put_proc_affinity(dd, fdata->rec_cpu_num);
	726	+ hfi1_put_proc_affinity(fdata->rec_cpu_num);
722	727
723	728	/*
724	729	* Clear any left over, unhandled events so the next process that
@@ -730,7 +735,6 @@ static int hfi1_file_close(struct inode inode, struct file fp) · 5 unmodified lines
730	735
731	736	if (--uctxt->cnt) {
732	737	uctxt->active_slaves &= ~(1 << fdata->subctxt);
733		− uctxt->subpid[fdata->subctxt] = 0;
734	738	mutex_unlock(&hfi1_mutex);
735	739	goto done;
736	740	}
@@ -756,7 +760,6 @@ static int hfi1_file_close(struct inode inode, struct file fp) · 19 unmodified lines
756	760	write_kctxt_csr(dd, uctxt->sc->hw_context, SEND_CTXT_CHECK_ENABLE,
757	761	hfi1_pkt_default_send_ctxt_mask(dd, uctxt->sc->type));
758	762	sc_disable(uctxt->sc);
759		− uctxt->pid = 0;
760	763	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
761	764
762	765	dd->rcd[uctxt->ctxt] = NULL;
@@ -818,9 +821,10 @@ static int assign_ctxt(struct file fp, struct hfi1_user_info uinfo) · 55 unmodified lines
818	821	ret = find_shared_ctxt(fp, uinfo);
819	822	if (ret < 0)
820	823	goto done_unlock;
821		− if (ret)
822		− fd->rec_cpu_num = hfi1_get_proc_affinity(
823		− fd->uctxt->dd, fd->uctxt->numa_id);
	824	+ if (ret) {
	825	+ fd->rec_cpu_num =
	826	+ hfi1_get_proc_affinity(fd->uctxt->numa_id);
	827	+ }
824	828	}
825	829
826	830	/*
@@ -895,7 +899,6 @@ static int find_shared_ctxt(struct file *fp, · 68 unmodified lines
895	899	}
896	900	fd->uctxt = uctxt;
897	901	fd->subctxt = uctxt->cnt++;
898		− uctxt->subpid[fd->subctxt] = current->pid;
899	902	uctxt->active_slaves \|= 1 << fd->subctxt;
900	903	ret = 1;
901	904	goto done;
@@ -932,7 +935,11 @@ static int allocate_ctxt(struct file fp, struct hfi1_devdata dd, · 30 unmodified lines
932	935	if (ctxt == dd->num_rcv_contexts)
933	936	return -EBUSY;
934	937
935		− fd->rec_cpu_num = hfi1_get_proc_affinity(dd, -1);
	938	+ /*
	939	+ * If we don't have a NUMA node requested, preference is towards
	940	+ * device NUMA node.
	941	+ */
	942	+ fd->rec_cpu_num = hfi1_get_proc_affinity(dd->node);
936	943	if (fd->rec_cpu_num != -1)
937	944	numa = cpu_to_node(fd->rec_cpu_num);
938	945	else
@@ -976,8 +983,7 @@ static int allocate_ctxt(struct file fp, struct hfi1_devdata dd, · 37 unmodified lines
976	983	return ret;
977	984	}
978	985	uctxt->userversion = uinfo->userversion;
979		− uctxt->pid = current->pid;
980		− uctxt->flags = HFI1_CAP_UGET(MASK);
	986	+ uctxt->flags = hfi1_cap_mask; /* save current flag state */
981	987	init_waitqueue_head(&uctxt->wait);
982	988	strlcpy(uctxt->comm, current->comm, sizeof(uctxt->comm));
983	989	memcpy(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid));
@@ -1080,18 +1086,18 @@ static int user_init(struct file *fp) · 96 unmodified lines
1080	1086	hfi1_set_ctxt_jkey(uctxt->dd, uctxt->ctxt, uctxt->jkey);
1081	1087
1082	1088	rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;
1083		− if (HFI1_CAP_KGET_MASK(uctxt->flags, HDRSUPP))
	1089	+ if (HFI1_CAP_UGET_MASK(uctxt->flags, HDRSUPP))
1084	1090	rcvctrl_ops \|= HFI1_RCVCTRL_TIDFLOW_ENB;
1085	1091	/*
1086	1092	* Ignore the bit in the flags for now until proper
1087	1093	* support for multiple packet per rcv array entry is
1088	1094	* added.
1089	1095	*/
1090		− if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))
	1096	+ if (!HFI1_CAP_UGET_MASK(uctxt->flags, MULTI_PKT_EGR))
1091	1097	rcvctrl_ops \|= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
1092		− if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))
	1098	+ if (HFI1_CAP_UGET_MASK(uctxt->flags, NODROP_EGR_FULL))
1093	1099	rcvctrl_ops \|= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
1094		− if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
	1100	+ if (HFI1_CAP_UGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
1095	1101	rcvctrl_ops \|= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
1096	1102	/*
1097	1103	* The RcvCtxtCtrl.TailUpd bit has to be explicitly written.
@@ -1099,7 +1105,7 @@ static int user_init(struct file *fp) · 1 unmodified line
1099	1105	* uses of the chip or ctxt. Therefore, add the rcvctrl op
1100	1106	* for both cases.
1101	1107	*/
1102		− if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))
	1108	+ if (HFI1_CAP_UGET_MASK(uctxt->flags, DMA_RTAIL))
1103	1109	rcvctrl_ops \|= HFI1_RCVCTRL_TAILUPD_ENB;
1104	1110	else
1105	1111	rcvctrl_ops \|= HFI1_RCVCTRL_TAILUPD_DIS;
@@ -1122,9 +1128,14 @@ static int get_ctxt_info(struct file fp, void __user ubase, __u32 len) · 16 unmodified lines
1122	1128	int ret = 0;
1123	1129
1124	1130	memset(&cinfo, 0, sizeof(cinfo));
1125		− ret = hfi1_get_base_kinfo(uctxt, &cinfo);
1126		− if (ret < 0)
1127		− goto done;
	1131	+ cinfo.runtime_flags = (((uctxt->flags >> HFI1_CAP_MISC_SHIFT) &
	1132	+ HFI1_CAP_MISC_MASK) << HFI1_CAP_USER_SHIFT) \|
	1133	+ HFI1_CAP_UGET_MASK(uctxt->flags, MASK) \|
	1134	+ HFI1_CAP_KGET_MASK(uctxt->flags, K2U);
	1135	+ /* adjust flag if this fd is not able to cache */
	1136	+ if (!fd->handler)
	1137	+ cinfo.runtime_flags \|= HFI1_CAP_TID_UNMAP; /* no caching */
	1138	+
1128	1139	cinfo.num_active = hfi1_count_active_units();
1129	1140	cinfo.unit = uctxt->dd->unit;
1130	1141	cinfo.ctxt = uctxt->ctxt;
@@ -1146,7 +1157,7 @@ static int get_ctxt_info(struct file fp, void __user ubase, __u32 len) · 15 unmodified lines
1146	1157	trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, cinfo);
1147	1158	if (copy_to_user(ubase, &cinfo, sizeof(cinfo)))
1148	1159	ret = -EFAULT;
1149		−done:
	1160	+
1150	1161	return ret;
1151	1162	}
1152	1163

drivers/infiniband/hw/hfi1/firmware.c+125−0

@@ -206,6 +206,9 @@ static const struct firmware *platform_config; · 205 unmodified lines
206	206	/* the number of fabric SerDes on the SBus */
207	207	#define NUM_FABRIC_SERDES 4
208	208
	209	+/* ASIC_STS_SBUS_RESULT.RESULT_CODE value */
	210	+#define SBUS_READ_COMPLETE 0x4
	211	+
209	212	/* SBus fabric SerDes addresses, one set per HFI */
210	213	static const u8 fabric_serdes_addrs[2][NUM_FABRIC_SERDES] = {
211	214	{ 0x01, 0x02, 0x03, 0x04 },
@@ -240,6 +243,7 @@ static const u8 all_pcie_serdes_broadcast = 0xe0; · 28 unmodified lines
240	243	static void dispose_one_firmware(struct firmware_details *fdet);
241	244	static int load_fabric_serdes_firmware(struct hfi1_devdata *dd,
242	245	struct firmware_details *fdet);
	246	+static void dump_fw_version(struct hfi1_devdata *dd);
243	247
244	248	/*
245	249	* Read a single 64-bit value from 8051 data memory.
@@ -1078,6 +1082,44 @@ void sbus_request(struct hfi1_devdata *dd, · 832 unmodified lines
1078	1082	ASIC_CFG_SBUS_REQUEST_RECEIVER_ADDR_SHIFT));
1079	1083	}
1080	1084
	1085	+/*
	1086	+ * Read a value from the SBus.
	1087	+ *
	1088	+ * Requires the caller to be in fast mode
	1089	+ */
	1090	+static u32 sbus_read(struct hfi1_devdata *dd, u8 receiver_addr, u8 data_addr,
	1091	+ u32 data_in)
	1092	+{
	1093	+ u64 reg;
	1094	+ int retries;
	1095	+ int success = 0;
	1096	+ u32 result = 0;
	1097	+ u32 result_code = 0;
	1098	+
	1099	+ sbus_request(dd, receiver_addr, data_addr, READ_SBUS_RECEIVER, data_in);
	1100	+
	1101	+ for (retries = 0; retries < 100; retries++) {
	1102	+ usleep_range(1000, 1200); /* arbitrary */
	1103	+ reg = read_csr(dd, ASIC_STS_SBUS_RESULT);
	1104	+ result_code = (reg >> ASIC_STS_SBUS_RESULT_RESULT_CODE_SHIFT)
	1105	+ & ASIC_STS_SBUS_RESULT_RESULT_CODE_MASK;
	1106	+ if (result_code != SBUS_READ_COMPLETE)
	1107	+ continue;
	1108	+
	1109	+ success = 1;
	1110	+ result = (reg >> ASIC_STS_SBUS_RESULT_DATA_OUT_SHIFT)
	1111	+ & ASIC_STS_SBUS_RESULT_DATA_OUT_MASK;
	1112	+ break;
	1113	+ }
	1114	+
	1115	+ if (!success) {
	1116	+ dd_dev_err(dd, "%s: read failed, result code 0x%x\n", __func__,
	1117	+ result_code);
	1118	+ }
	1119	+
	1120	+ return result;
	1121	+}
	1122	+
1081	1123	/*
1082	1124	* Turn off the SBus and fabric serdes spicos.
1083	1125	*
@@ -1636,6 +1678,7 @@ int load_firmware(struct hfi1_devdata *dd) · 552 unmodified lines
1636	1678	return ret;
1637	1679	}
1638	1680
	1681	+ dump_fw_version(dd);
1639	1682	return 0;
1640	1683	}
1641	1684
@@ -2054,3 +2097,85 @@ void read_guid(struct hfi1_devdata *dd) · 412 unmodified lines
2054	2097	dd_dev_info(dd, "GUID %llx",
2055	2098	(unsigned long long)dd->base_guid);
2056	2099	}
	2100	+
	2101	+/* read and display firmware version info */
	2102	+static void dump_fw_version(struct hfi1_devdata *dd)
	2103	+{
	2104	+ u32 pcie_vers[NUM_PCIE_SERDES];
	2105	+ u32 fabric_vers[NUM_FABRIC_SERDES];
	2106	+ u32 sbus_vers;
	2107	+ int i;
	2108	+ int all_same;
	2109	+ int ret;
	2110	+ u8 rcv_addr;
	2111	+
	2112	+ ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
	2113	+ if (ret) {
	2114	+ dd_dev_err(dd, "Unable to acquire SBus to read firmware versions\n");
	2115	+ return;
	2116	+ }
	2117	+
	2118	+ /* set fast mode */
	2119	+ set_sbus_fast_mode(dd);
	2120	+
	2121	+ /* read version for SBus Master */
	2122	+ sbus_request(dd, SBUS_MASTER_BROADCAST, 0x02, WRITE_SBUS_RECEIVER, 0);
	2123	+ sbus_request(dd, SBUS_MASTER_BROADCAST, 0x07, WRITE_SBUS_RECEIVER, 0x1);
	2124	+ /* wait for interrupt to be processed */
	2125	+ usleep_range(10000, 11000);
	2126	+ sbus_vers = sbus_read(dd, SBUS_MASTER_BROADCAST, 0x08, 0x1);
	2127	+ dd_dev_info(dd, "SBus Master firmware version 0x%08x\n", sbus_vers);
	2128	+
	2129	+ /* read version for PCIe SerDes */
	2130	+ all_same = 1;
	2131	+ pcie_vers[0] = 0;
	2132	+ for (i = 0; i < NUM_PCIE_SERDES; i++) {
	2133	+ rcv_addr = pcie_serdes_addrs[dd->hfi1_id][i];
	2134	+ sbus_request(dd, rcv_addr, 0x03, WRITE_SBUS_RECEIVER, 0);
	2135	+ /* wait for interrupt to be processed */
	2136	+ usleep_range(10000, 11000);
	2137	+ pcie_vers[i] = sbus_read(dd, rcv_addr, 0x04, 0x0);
	2138	+ if (i > 0 && pcie_vers[0] != pcie_vers[i])
	2139	+ all_same = 0;
	2140	+ }
	2141	+
	2142	+ if (all_same) {
	2143	+ dd_dev_info(dd, "PCIe SerDes firmware version 0x%x\n",
	2144	+ pcie_vers[0]);
	2145	+ } else {
	2146	+ dd_dev_warn(dd, "PCIe SerDes do not have the same firmware version\n");
	2147	+ for (i = 0; i < NUM_PCIE_SERDES; i++) {
	2148	+ dd_dev_info(dd,
	2149	+ "PCIe SerDes lane %d firmware version 0x%x\n",
	2150	+ i, pcie_vers[i]);
	2151	+ }
	2152	+ }
	2153	+
	2154	+ /* read version for fabric SerDes */
	2155	+ all_same = 1;
	2156	+ fabric_vers[0] = 0;
	2157	+ for (i = 0; i < NUM_FABRIC_SERDES; i++) {
	2158	+ rcv_addr = fabric_serdes_addrs[dd->hfi1_id][i];
	2159	+ sbus_request(dd, rcv_addr, 0x03, WRITE_SBUS_RECEIVER, 0);
	2160	+ /* wait for interrupt to be processed */
	2161	+ usleep_range(10000, 11000);
	2162	+ fabric_vers[i] = sbus_read(dd, rcv_addr, 0x04, 0x0);
	2163	+ if (i > 0 && fabric_vers[0] != fabric_vers[i])
	2164	+ all_same = 0;
	2165	+ }
	2166	+
	2167	+ if (all_same) {
	2168	+ dd_dev_info(dd, "Fabric SerDes firmware version 0x%x\n",
	2169	+ fabric_vers[0]);
	2170	+ } else {
	2171	+ dd_dev_warn(dd, "Fabric SerDes do not have the same firmware version\n");
	2172	+ for (i = 0; i < NUM_FABRIC_SERDES; i++) {
	2173	+ dd_dev_info(dd,
	2174	+ "Fabric SerDes lane %d firmware version 0x%x\n",
	2175	+ i, fabric_vers[i]);
	2176	+ }
	2177	+ }
	2178	+
	2179	+ clear_sbus_fast_mode(dd);
	2180	+ release_chip_resource(dd, CR_SBUS);
	2181	+}

drivers/infiniband/hw/hfi1/hfi.h+104−17

@@ -62,6 +62,8 @@ · 61 unmodified lines
62	62	#include <linux/cdev.h>
63	63	#include <linux/delay.h>
64	64	#include <linux/kthread.h>
	65	+#include <linux/i2c.h>
	66	+#include <linux/i2c-algo-bit.h>
65	67	#include <rdma/rdma_vt.h>
66	68
67	69	#include "chip_registers.h"
@@ -253,7 +255,7 @@ struct hfi1_ctxtdata { · 185 unmodified lines
253	255	/* chip offset of PIO buffers for this ctxt */
254	256	u32 piobufs;
255	257	/* per-context configuration flags */
256		− u32 flags;
	258	+ unsigned long flags;
257	259	/* per-context event flags for fileops/intr communication */
258	260	unsigned long event_flags;
259	261	/* WAIT_RCV that timed out, no interrupt */
@@ -268,9 +270,6 @@ struct hfi1_ctxtdata { · 8 unmodified lines
268	270	u32 urgent;
269	271	/* saved total number of polled urgent packets for poll edge trigger */
270	272	u32 urgent_poll;
271		− /* pid of process using this ctxt */
272		− pid_t pid;
273		− pid_t subpid[HFI1_MAX_SHARED_CTXTS];
274	273	/* same size as task_struct .comm[], command that opened context */
275	274	char comm[TASK_COMM_LEN];
276	275	/* so file ops can get at unit */
@@ -366,11 +365,6 @@ struct hfi1_packet { · 89 unmodified lines
366	365	u8 etype;
367	366	};
368	367
369		−static inline bool has_sc4_bit(struct hfi1_packet *p)
370		−{
371		− return !!rhf_dc_info(p->rhf);
372		−}
373		−
374	368	/*
375	369	* Private data for snoop/capture support.
376	370	*/
@@ -805,10 +799,19 @@ struct hfi1_temp { · 428 unmodified lines
805	799	u8 triggers; /* temperature triggers */
806	800	};
807	801
	802	+struct hfi1_i2c_bus {
	803	+ struct hfi1_devdata controlling_dd; / current controlling device */
	804	+ struct i2c_adapter adapter; /* bus details */
	805	+ struct i2c_algo_bit_data algo; /* bus algorithm details */
	806	+ int num; /* bus number, 0 or 1 */
	807	+};
	808	+
808	809	/* common data between shared ASIC HFIs */
809	810	struct hfi1_asic_data {
810	811	struct hfi1_devdata dds[2]; / back pointers */
811	812	struct mutex asic_resource_mutex;
	813	+ struct hfi1_i2c_bus *i2c_bus0;
	814	+ struct hfi1_i2c_bus *i2c_bus1;
812	815	};
813	816
814	817	/* device data struct now contains only "general per-device" info.
@@ -1128,7 +1131,8 @@ struct hfi1_devdata { · 313 unmodified lines
1128	1131	NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS];
1129	1132	/* Software counter that aggregates all cce_err_status errors */
1130	1133	u64 sw_cce_err_status_aggregate;
1131		−
	1134	+ /* Software counter that aggregates all bypass packet rcv errors */
	1135	+ u64 sw_rcv_bypass_packet_errors;
1132	1136	/* receive interrupt functions */
1133	1137	rhf_rcv_function_ptr *rhf_rcv_function_map;
1134	1138	rhf_rcv_function_ptr normal_rhf_rcv_functions[8];
@@ -1184,6 +1188,7 @@ struct hfi1_devdata { · 49 unmodified lines
1184	1188
1185	1189	struct tid_rb_node;
1186	1190	struct mmu_rb_node;
	1191	+struct mmu_rb_handler;
1187	1192
1188	1193	/* Private data for file operations */
1189	1194	struct hfi1_filedata {
@@ -1194,7 +1199,7 @@ struct hfi1_filedata { · 4 unmodified lines
1194	1199	/* for cpu affinity; -1 if none */
1195	1200	int rec_cpu_num;
1196	1201	u32 tid_n_pinned;
1197		− struct rb_root tid_rb_root;
	1202	+ struct mmu_rb_handler *handler;
1198	1203	struct tid_rb_node **entry_to_rb;
1199	1204	spinlock_t tid_lock; /* protect tid_[limit,used] counters */
1200	1205	u32 tid_limit;
@@ -1203,6 +1208,7 @@ struct hfi1_filedata { · 2 unmodified lines
1203	1208	u32 invalid_tid_idx;
1204	1209	/* protect invalid_tids array and invalid_tid_idx */
1205	1210	spinlock_t invalid_lock;
	1211	+ struct mm_struct *mm;
1206	1212	};
1207	1213
1208	1214	extern struct list_head hfi1_dev_list;
@@ -1236,6 +1242,8 @@ int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *, int); · 27 unmodified lines
1236	1242	int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *, int);
1237	1243	void set_all_slowpath(struct hfi1_devdata *dd);
1238	1244
	1245	+extern const struct pci_device_id hfi1_pci_tbl[];
	1246	+
1239	1247	/* receive packet handler dispositions */
1240	1248	#define RCV_PKT_OK 0x0 /* keep going */
1241	1249	#define RCV_PKT_LIMIT 0x1 /* stop, hit limit, start thread */
@@ -1261,7 +1269,7 @@ void receive_interrupt_work(struct work_struct *work); · 19 unmodified lines
1261	1269	static inline int hdr2sc(struct hfi1_message_header *hdr, u64 rhf)
1262	1270	{
1263	1271	return ((be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf) \|
1264		− ((!!(rhf & RHF_DC_INFO_SMASK)) << 4);
	1272	+ ((!!(rhf_dc_info(rhf))) << 4);
1265	1273	}
1266	1274
1267	1275	static inline u16 generate_jkey(kuid_t uid)
@@ -1571,6 +1579,22 @@ static inline struct hfi1_ibport to_iport(struct ib_device ibdev, u8 port) · 303 unmodified lines
1571	1579	return &dd->pport[pidx].ibport_data;
1572	1580	}
1573	1581
	1582	+void hfi1_process_ecn_slowpath(struct rvt_qp qp, struct hfi1_packet pkt,
	1583	+ bool do_cnp);
	1584	+static inline bool process_ecn(struct rvt_qp qp, struct hfi1_packet pkt,
	1585	+ bool do_cnp)
	1586	+{
	1587	+ struct hfi1_other_headers *ohdr = pkt->ohdr;
	1588	+ u32 bth1;
	1589	+
	1590	+ bth1 = be32_to_cpu(ohdr->bth[1]);
	1591	+ if (unlikely(bth1 & (HFI1_BECN_SMASK \| HFI1_FECN_SMASK))) {
	1592	+ hfi1_process_ecn_slowpath(qp, pkt, do_cnp);
	1593	+ return bth1 & HFI1_FECN_SMASK;
	1594	+ }
	1595	+ return false;
	1596	+}
	1597	+
1574	1598	/*
1575	1599	* Return the indexed PKEY from the port PKEY table.
1576	1600	*/
@@ -1588,14 +1612,23 @@ static inline u16 hfi1_get_pkey(struct hfi1_ibport *ibp, unsigned index) · 11 unmodified lines
1588	1612	}
1589	1613
1590	1614	/*
1591		− * Readers of cc_state must call get_cc_state() under rcu_read_lock().
1592		− * Writers of cc_state must call get_cc_state() under cc_state_lock.
	1615	+ * Called by readers of cc_state only, must call under rcu_read_lock().
1593	1616	*/
1594	1617	static inline struct cc_state get_cc_state(struct hfi1_pportdata ppd)
1595	1618	{
1596	1619	return rcu_dereference(ppd->cc_state);
1597	1620	}
1598	1621
	1622	+/*
	1623	+ * Called by writers of cc_state only, must call under cc_state_lock.
	1624	+ */
	1625	+static inline
	1626	+struct cc_state get_cc_state_protected(struct hfi1_pportdata ppd)
	1627	+{
	1628	+ return rcu_dereference_protected(ppd->cc_state,
	1629	+ lockdep_is_held(&ppd->cc_state_lock));
	1630	+}
	1631	+
1599	1632	/*
1600	1633	* values for dd->flags (_device_ related flags)
1601	1634	*/
@@ -1671,9 +1704,12 @@ void shutdown_led_override(struct hfi1_pportdata *ppd); · 69 unmodified lines
1671	1704	*/
1672	1705	#define DEFAULT_RCVHDR_ENTSIZE 32
1673	1706
1674		−bool hfi1_can_pin_pages(struct hfi1_devdata *, u32, u32);
1675		−int hfi1_acquire_user_pages(unsigned long, size_t, bool, struct page **);
1676		−void hfi1_release_user_pages(struct mm_struct , struct page *, size_t, bool);
	1707	+bool hfi1_can_pin_pages(struct hfi1_devdata dd, struct mm_struct mm,
	1708	+ u32 nlocked, u32 npages);
	1709	+int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr,
	1710	+ size_t npages, bool writable, struct page **pages);
	1711	+void hfi1_release_user_pages(struct mm_struct mm, struct page *p,
	1712	+ size_t npages, bool dirty);
1677	1713
1678	1714	static inline void clear_rcvhdrtail(const struct hfi1_ctxtdata *rcd)
1679	1715	{
@@ -1949,4 +1985,55 @@ static inline u32 qsfp_resource(struct hfi1_devdata *dd) · 269 unmodified lines
1949	1985
1950	1986	int hfi1_tempsense_rd(struct hfi1_devdata dd, struct hfi1_temp temp);
1951	1987
	1988	+#define DD_DEV_ENTRY(dd) __string(dev, dev_name(&(dd)->pcidev->dev))
	1989	+#define DD_DEV_ASSIGN(dd) __assign_str(dev, dev_name(&(dd)->pcidev->dev))
	1990	+
	1991	+#define packettype_name(etype) { RHF_RCV_TYPE_##etype, #etype }
	1992	+#define show_packettype(etype) \
	1993	+__print_symbolic(etype, \
	1994	+ packettype_name(EXPECTED), \
	1995	+ packettype_name(EAGER), \
	1996	+ packettype_name(IB), \
	1997	+ packettype_name(ERROR), \
	1998	+ packettype_name(BYPASS))
	1999	+
	2000	+#define ib_opcode_name(opcode) { IB_OPCODE_##opcode, #opcode }
	2001	+#define show_ib_opcode(opcode) \
	2002	+__print_symbolic(opcode, \
	2003	+ ib_opcode_name(RC_SEND_FIRST), \
	2004	+ ib_opcode_name(RC_SEND_MIDDLE), \
	2005	+ ib_opcode_name(RC_SEND_LAST), \
	2006	+ ib_opcode_name(RC_SEND_LAST_WITH_IMMEDIATE), \
	2007	+ ib_opcode_name(RC_SEND_ONLY), \
	2008	+ ib_opcode_name(RC_SEND_ONLY_WITH_IMMEDIATE), \
	2009	+ ib_opcode_name(RC_RDMA_WRITE_FIRST), \
	2010	+ ib_opcode_name(RC_RDMA_WRITE_MIDDLE), \
	2011	+ ib_opcode_name(RC_RDMA_WRITE_LAST), \
	2012	+ ib_opcode_name(RC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
	2013	+ ib_opcode_name(RC_RDMA_WRITE_ONLY), \
	2014	+ ib_opcode_name(RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
	2015	+ ib_opcode_name(RC_RDMA_READ_REQUEST), \
	2016	+ ib_opcode_name(RC_RDMA_READ_RESPONSE_FIRST), \
	2017	+ ib_opcode_name(RC_RDMA_READ_RESPONSE_MIDDLE), \
	2018	+ ib_opcode_name(RC_RDMA_READ_RESPONSE_LAST), \
	2019	+ ib_opcode_name(RC_RDMA_READ_RESPONSE_ONLY), \
	2020	+ ib_opcode_name(RC_ACKNOWLEDGE), \
	2021	+ ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE), \
	2022	+ ib_opcode_name(RC_COMPARE_SWAP), \
	2023	+ ib_opcode_name(RC_FETCH_ADD), \
	2024	+ ib_opcode_name(UC_SEND_FIRST), \
	2025	+ ib_opcode_name(UC_SEND_MIDDLE), \
	2026	+ ib_opcode_name(UC_SEND_LAST), \
	2027	+ ib_opcode_name(UC_SEND_LAST_WITH_IMMEDIATE), \
	2028	+ ib_opcode_name(UC_SEND_ONLY), \
	2029	+ ib_opcode_name(UC_SEND_ONLY_WITH_IMMEDIATE), \
	2030	+ ib_opcode_name(UC_RDMA_WRITE_FIRST), \
	2031	+ ib_opcode_name(UC_RDMA_WRITE_MIDDLE), \
	2032	+ ib_opcode_name(UC_RDMA_WRITE_LAST), \
	2033	+ ib_opcode_name(UC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
	2034	+ ib_opcode_name(UC_RDMA_WRITE_ONLY), \
	2035	+ ib_opcode_name(UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
	2036	+ ib_opcode_name(UD_SEND_ONLY), \
	2037	+ ib_opcode_name(UD_SEND_ONLY_WITH_IMMEDIATE), \
	2038	+ ib_opcode_name(CNP))
1952	2039	#endif /* _HFI1_KERNEL_H */

drivers/infiniband/hw/hfi1/init.c+34−17

@@ -64,6 +64,7 @@ · 63 unmodified lines
64	64	#include "debugfs.h"
65	65	#include "verbs.h"
66	66	#include "aspm.h"
	67	+#include "affinity.h"
67	68
68	69	#undef pr_fmt
69	70	#define pr_fmt(fmt) DRIVER_NAME ": " fmt
@@ -474,8 +475,9 @@ static enum hrtimer_restart cca_timer_fn(struct hrtimer *t) · 404 unmodified lines
474	475	void hfi1_init_pportdata(struct pci_dev pdev, struct hfi1_pportdata ppd,
475	476	struct hfi1_devdata *dd, u8 hw_pidx, u8 port)
476	477	{
477		− int i, size;
	478	+ int i;
478	479	uint default_pkey_idx;
	480	+ struct cc_state *cc_state;
479	481
480	482	ppd->dd = dd;
481	483	ppd->hw_pidx = hw_pidx;
@@ -526,9 +528,9 @@ void hfi1_init_pportdata(struct pci_dev pdev, struct hfi1_pportdata ppd, · 44 unmodified lines
526	528
527	529	spin_lock_init(&ppd->cc_state_lock);
528	530	spin_lock_init(&ppd->cc_log_lock);
529		− size = sizeof(struct cc_state);
530		− RCU_INIT_POINTER(ppd->cc_state, kzalloc(size, GFP_KERNEL));
531		− if (!rcu_dereference(ppd->cc_state))
	531	+ cc_state = kzalloc(sizeof(*cc_state), GFP_KERNEL);
	532	+ RCU_INIT_POINTER(ppd->cc_state, cc_state);
	533	+ if (!cc_state)
532	534	goto bail;
533	535	return;
534	536
@@ -972,39 +974,49 @@ void hfi1_free_ctxtdata(struct hfi1_devdata dd, struct hfi1_ctxtdata rcd) · 437 unmodified lines
972	974
973	975	/*
974	976	* Release our hold on the shared asic data. If we are the last one,
975		− * free the structure. Must be holding hfi1_devs_lock.
	977	+ * return the structure to be finalized outside the lock. Must be
	978	+ * holding hfi1_devs_lock.
976	979	*/
977		−static void release_asic_data(struct hfi1_devdata *dd)
	980	+static struct hfi1_asic_data release_asic_data(struct hfi1_devdata dd)
978	981	{
	982	+ struct hfi1_asic_data *ad;
979	983	int other;
980	984
981	985	if (!dd->asic_data)
982		− return;
	986	+ return NULL;
983	987	dd->asic_data->dds[dd->hfi1_id] = NULL;
984	988	other = dd->hfi1_id ? 0 : 1;
985		− if (!dd->asic_data->dds[other]) {
986		− /* we are the last holder, free it */
987		− kfree(dd->asic_data);
988		− }
	989	+ ad = dd->asic_data;
989	990	dd->asic_data = NULL;
	991	+ /* return NULL if the other dd still has a link */
	992	+ return ad->dds[other] ? NULL : ad;
	993	+}
	994	+
	995	+static void finalize_asic_data(struct hfi1_devdata *dd,
	996	+ struct hfi1_asic_data *ad)
	997	+{
	998	+ clean_up_i2c(dd, ad);
	999	+ kfree(ad);
990	1000	}
991	1001
992	1002	static void __hfi1_free_devdata(struct kobject *kobj)
993	1003	{
994	1004	struct hfi1_devdata *dd =
995	1005	container_of(kobj, struct hfi1_devdata, kobj);
	1006	+ struct hfi1_asic_data *ad;
996	1007	unsigned long flags;
997	1008
998	1009	spin_lock_irqsave(&hfi1_devs_lock, flags);
999	1010	idr_remove(&hfi1_unit_table, dd->unit);
1000	1011	list_del(&dd->list);
1001		− release_asic_data(dd);
	1012	+ ad = release_asic_data(dd);
1002	1013	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
	1014	+ if (ad)
	1015	+ finalize_asic_data(dd, ad);
1003	1016	free_platform_config(dd);
1004	1017	rcu_barrier(); /* wait for rcu callbacks to complete */
1005	1018	free_percpu(dd->int_counter);
1006	1019	free_percpu(dd->rcv_limit);
1007		− hfi1_dev_affinity_free(dd);
1008	1020	free_percpu(dd->send_schedule);
1009	1021	rvt_dealloc_device(&dd->verbs_dev.rdi);
1010	1022	}
@@ -1162,7 +1174,7 @@ static int init_one(struct pci_dev , const struct pci_device_id ); · 151 unmodified lines
1162	1174	#define DRIVER_LOAD_MSG "Intel " DRIVER_NAME " loaded: "
1163	1175	#define PFX DRIVER_NAME ": "
1164	1176
1165		−static const struct pci_device_id hfi1_pci_tbl[] = {
	1177	+const struct pci_device_id hfi1_pci_tbl[] = {
1166	1178	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL0) },
1167	1179	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL1) },
1168	1180	{ 0, }
@@ -1198,6 +1210,10 @@ static int __init hfi1_mod_init(void) · 29 unmodified lines
1198	1210	if (ret)
1199	1211	goto bail;
1200	1212
	1213	+ ret = node_affinity_init();
	1214	+ if (ret)
	1215	+ goto bail;
	1216	+
1201	1217	/* validate max MTU before any devices start */
1202	1218	if (!valid_opa_max_mtu(hfi1_max_mtu)) {
1203	1219	pr_err("Invalid max_mtu 0x%x, using 0x%x instead\n",
@@ -1278,6 +1294,7 @@ module_init(hfi1_mod_init); · 74 unmodified lines
1278	1294	static void __exit hfi1_mod_cleanup(void)
1279	1295	{
1280	1296	pci_unregister_driver(&hfi1_pci_driver);
	1297	+ node_affinity_destroy();
1281	1298	hfi1_wss_exit();
1282	1299	hfi1_dbg_exit();
1283	1300	hfi1_cpulist_count = 0;
@@ -1311,7 +1328,7 @@ static void cleanup_device_data(struct hfi1_devdata *dd) · 27 unmodified lines
1311	1328	hrtimer_cancel(&ppd->cca_timer[i].hrtimer);
1312	1329
1313	1330	spin_lock(&ppd->cc_state_lock);
1314		− cc_state = get_cc_state(ppd);
	1331	+ cc_state = get_cc_state_protected(ppd);
1315	1332	RCU_INIT_POINTER(ppd->cc_state, NULL);
1316	1333	spin_unlock(&ppd->cc_state_lock);
1317	1334
@@ -1760,8 +1777,8 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd) · 442 unmodified lines
1760	1777
1761	1778	hfi1_cdbg(PROC,
1762	1779	"ctxt%u: Alloced %u rcv tid entries @ %uKB, total %zuKB\n",
1763		− rcd->ctxt, rcd->egrbufs.alloced, rcd->egrbufs.rcvtid_size,
1764		− rcd->egrbufs.size);
	1780	+ rcd->ctxt, rcd->egrbufs.alloced,
	1781	+ rcd->egrbufs.rcvtid_size / 1024, rcd->egrbufs.size / 1024);
1765	1782
1766	1783	/*
1767	1784	* Set the contexts rcv array head update threshold to the closest

drivers/infiniband/hw/hfi1/mad.c+21−39

@@ -588,7 +588,6 @@ static int __subn_get_opa_portinfo(struct opa_smp smp, u32 am, u8 data, · 587 unmodified lines
588	588
589	589	pi->port_phys_conf = (ppd->port_type & 0xf);
590	590
591		−#if PI_LED_ENABLE_SUP
592	591	pi->port_states.ledenable_offlinereason = ppd->neighbor_normal << 4;
593	592	pi->port_states.ledenable_offlinereason \|=
594	593	ppd->is_sm_config_started << 5;
@@ -602,11 +601,6 @@ static int __subn_get_opa_portinfo(struct opa_smp smp, u32 am, u8 data, · 7 unmodified lines
602	601	pi->port_states.ledenable_offlinereason \|= is_beaconing_active << 6;
603	602	pi->port_states.ledenable_offlinereason \|=
604	603	ppd->offline_disabled_reason;
605		−#else
606		− pi->port_states.offline_reason = ppd->neighbor_normal << 4;
607		− pi->port_states.offline_reason \|= ppd->is_sm_config_started << 5;
608		− pi->port_states.offline_reason \|= ppd->offline_disabled_reason;
609		−#endif /* PI_LED_ENABLE_SUP */
610	604
611	605	pi->port_states.portphysstate_portstate =
612	606	(hfi1_ibphys_portstate(ppd) << 4) \| state;
@@ -1752,17 +1746,11 @@ static int __subn_get_opa_psi(struct opa_smp smp, u32 am, u8 data, · 1139 unmodified lines
1752	1746	if (start_of_sm_config && (lstate == IB_PORT_INIT))
1753	1747	ppd->is_sm_config_started = 1;
1754	1748
1755		−#if PI_LED_ENABLE_SUP
1756	1749	psi->port_states.ledenable_offlinereason = ppd->neighbor_normal << 4;
1757	1750	psi->port_states.ledenable_offlinereason \|=
1758	1751	ppd->is_sm_config_started << 5;
1759	1752	psi->port_states.ledenable_offlinereason \|=
1760	1753	ppd->offline_disabled_reason;
1761		−#else
1762		− psi->port_states.offline_reason = ppd->neighbor_normal << 4;
1763		− psi->port_states.offline_reason \|= ppd->is_sm_config_started << 5;
1764		− psi->port_states.offline_reason \|= ppd->offline_disabled_reason;
1765		−#endif /* PI_LED_ENABLE_SUP */
1766	1754
1767	1755	psi->port_states.portphysstate_portstate =
1768	1756	(hfi1_ibphys_portstate(ppd) << 4) \| (lstate & 0xf);
@@ -2430,14 +2418,9 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, · 661 unmodified lines
2430	2418	rsp->port_rcv_remote_physical_errors =
2431	2419	cpu_to_be64(read_dev_cntr(dd, C_DC_RMT_PHY_ERR,
2432	2420	CNTR_INVALID_VL));
2433		− tmp = read_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL);
2434		− tmp2 = tmp + read_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL);
2435		− if (tmp2 < tmp) {
2436		− /* overflow/wrapped */
2437		− rsp->local_link_integrity_errors = cpu_to_be64(~0);
2438		− } else {
2439		− rsp->local_link_integrity_errors = cpu_to_be64(tmp2);
2440		− }
	2421	+ rsp->local_link_integrity_errors =
	2422	+ cpu_to_be64(read_dev_cntr(dd, C_DC_RX_REPLAY,
	2423	+ CNTR_INVALID_VL));
2441	2424	tmp = read_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL);
2442	2425	tmp2 = tmp + read_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT,
2443	2426	CNTR_INVALID_VL);
@@ -2499,6 +2482,9 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, · 55 unmodified lines
2499	2482	cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_BCN_VL,
2500	2483	idx_from_vl(vl)));
2501	2484
	2485	+ rsp->vls[vfi].port_vl_xmit_discards =
	2486	+ cpu_to_be64(read_port_cntr(ppd, C_SW_XMIT_DSCD_VL,
	2487	+ idx_from_vl(vl)));
2502	2488	vlinfo++;
2503	2489	vfi++;
2504	2490	}
@@ -2529,9 +2515,8 @@ static u64 get_error_counter_summary(struct ib_device *ibdev, u8 port, · 24 unmodified lines
2529	2515	error_counter_summary += read_dev_cntr(dd, C_DC_RMT_PHY_ERR,
2530	2516	CNTR_INVALID_VL);
2531	2517	/* local link integrity must be right-shifted by the lli resolution */
2532		− tmp = read_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL);
2533		− tmp += read_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL);
2534		− error_counter_summary += (tmp >> res_lli);
	2518	+ error_counter_summary += (read_dev_cntr(dd, C_DC_RX_REPLAY,
	2519	+ CNTR_INVALID_VL) >> res_lli);
2535	2520	/* link error recovery must b right-shifted by the ler resolution */
2536	2521	tmp = read_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL);
2537	2522	tmp += read_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT, CNTR_INVALID_VL);
@@ -2800,14 +2785,9 @@ static void pma_get_opa_port_ectrs(struct ib_device *ibdev, · 262 unmodified lines
2800	2785	rsp->port_rcv_constraint_errors =
2801	2786	cpu_to_be64(read_port_cntr(ppd, C_SW_RCV_CSTR_ERR,
2802	2787	CNTR_INVALID_VL));
2803		− tmp = read_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL);
2804		− tmp2 = tmp + read_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL);
2805		− if (tmp2 < tmp) {
2806		− /* overflow/wrapped */
2807		− rsp->local_link_integrity_errors = cpu_to_be64(~0);
2808		− } else {
2809		− rsp->local_link_integrity_errors = cpu_to_be64(tmp2);
2810		− }
	2788	+ rsp->local_link_integrity_errors =
	2789	+ cpu_to_be64(read_dev_cntr(dd, C_DC_RX_REPLAY,
	2790	+ CNTR_INVALID_VL));
2811	2791	rsp->excessive_buffer_overruns =
2812	2792	cpu_to_be64(read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL));
2813	2793	}
@@ -2883,14 +2863,17 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, · 69 unmodified lines
2883	2863	tmp = read_dev_cntr(dd, C_DC_UNC_ERR, CNTR_INVALID_VL);
2884	2864
2885	2865	rsp->uncorrectable_errors = tmp < 0x100 ? (tmp & 0xff) : 0xff;
2886		−
	2866	+ rsp->port_rcv_errors =
	2867	+ cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_ERR, CNTR_INVALID_VL));
2887	2868	vlinfo = &rsp->vls[0];
2888	2869	vfi = 0;
2889	2870	vl_select_mask = be32_to_cpu(req->vl_select_mask);
2890	2871	for_each_set_bit(vl, (unsigned long *)&(vl_select_mask),
2891	2872	8 * sizeof(req->vl_select_mask)) {
2892	2873	memset(vlinfo, 0, sizeof(*vlinfo));
2893		− /* vlinfo->vls[vfi].port_vl_xmit_discards ??? */
	2874	+ rsp->vls[vfi].port_vl_xmit_discards =
	2875	+ cpu_to_be64(read_port_cntr(ppd, C_SW_XMIT_DSCD_VL,
	2876	+ idx_from_vl(vl)));
2894	2877	vlinfo += 1;
2895	2878	vfi++;
2896	2879	}
@@ -3162,10 +3145,8 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, · 265 unmodified lines
3162	3145	if (counter_select & CS_PORT_RCV_REMOTE_PHYSICAL_ERRORS)
3163	3146	write_dev_cntr(dd, C_DC_RMT_PHY_ERR, CNTR_INVALID_VL, 0);
3164	3147
3165		− if (counter_select & CS_LOCAL_LINK_INTEGRITY_ERRORS) {
3166		− write_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL, 0);
	3148	+ if (counter_select & CS_LOCAL_LINK_INTEGRITY_ERRORS)
3167	3149	write_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL, 0);
3168		− }
3169	3150
3170	3151	if (counter_select & CS_LINK_ERROR_RECOVERY) {
3171	3152	write_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL, 0);
@@ -3223,7 +3204,9 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, · 51 unmodified lines
3223	3204	/* if (counter_select & CS_PORT_MARK_FECN)
3224	3205	* write_csr(dd, DCC_PRF_PORT_VL_MARK_FECN_CNT + offset, 0);
3225	3206	*/
3226		− /* port_vl_xmit_discards ??? */
	3207	+ if (counter_select & C_SW_XMIT_DSCD_VL)
	3208	+ write_port_cntr(ppd, C_SW_XMIT_DSCD_VL,
	3209	+ idx_from_vl(vl), 0);
3227	3210	}
3228	3211
3229	3212	if (resp_len)
@@ -3392,7 +3375,7 @@ static void apply_cc_state(struct hfi1_pportdata *ppd) · 162 unmodified lines
3392	3375	*/
3393	3376	spin_lock(&ppd->cc_state_lock);
3394	3377
3395		− old_cc_state = get_cc_state(ppd);
	3378	+ old_cc_state = get_cc_state_protected(ppd);
3396	3379	if (!old_cc_state) {
3397	3380	/* never active, or shutting down */
3398	3381	spin_unlock(&ppd->cc_state_lock);
@@ -3960,7 +3943,6 @@ void clear_linkup_counters(struct hfi1_devdata *dd) · 561 unmodified lines
3960	3943	write_dev_cntr(dd, C_DC_SEQ_CRC_CNT, CNTR_INVALID_VL, 0);
3961	3944	write_dev_cntr(dd, C_DC_REINIT_FROM_PEER_CNT, CNTR_INVALID_VL, 0);
3962	3945	/* LocalLinkIntegrityErrors */
3963		− write_dev_cntr(dd, C_DC_TX_REPLAY, CNTR_INVALID_VL, 0);
3964	3946	write_dev_cntr(dd, C_DC_RX_REPLAY, CNTR_INVALID_VL, 0);
3965	3947	/* ExcessiveBufferOverruns */
3966	3948	write_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL, 0);

drivers/infiniband/hw/hfi1/mad.h+0−7

@@ -48,15 +48,8 @@ · 47 unmodified lines
48	48	#define _HFI1_MAD_H
49	49
50	50	#include <rdma/ib_pma.h>
51		−#define USE_PI_LED_ENABLE 1 /*
52		− * use led enabled bit in struct
53		− * opa_port_states, if available
54		− */
55	51	#include <rdma/opa_smi.h>
56	52	#include <rdma/opa_port_info.h>
57		−#ifndef PI_LED_ENABLE_SUP
58		−#define PI_LED_ENABLE_SUP 0
59		−#endif
60	53	#include "opa_compat.h"
61	54
62	55	/*

drivers/infiniband/hw/hfi1/mmu_rb.c+149−105

@@ -53,19 +53,20 @@ · 52 unmodified lines
53	53	#include "trace.h"
54	54
55	55	struct mmu_rb_handler {
56		− struct list_head list;
57	56	struct mmu_notifier mn;
58		− struct rb_root *root;
	57	+ struct rb_root root;
	58	+ void *ops_arg;
59	59	spinlock_t lock; /* protect the RB tree */
60	60	struct mmu_rb_ops *ops;
	61	+ struct mm_struct *mm;
	62	+ struct list_head lru_list;
	63	+ struct work_struct del_work;
	64	+ struct list_head del_list;
	65	+ struct workqueue_struct *wq;
61	66	};
62	67
63		−static LIST_HEAD(mmu_rb_handlers);
64		−static DEFINE_SPINLOCK(mmu_rb_lock); /* protect mmu_rb_handlers list */
65		−
66	68	static unsigned long mmu_node_start(struct mmu_rb_node *);
67	69	static unsigned long mmu_node_last(struct mmu_rb_node *);
68		−static struct mmu_rb_handler find_mmu_handler(struct rb_root );
69	70	static inline void mmu_notifier_page(struct mmu_notifier , struct mm_struct ,
70	71	unsigned long);
71	72	static inline void mmu_notifier_range_start(struct mmu_notifier *,
@@ -76,6 +77,9 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *, · 4 unmodified lines
76	77	unsigned long, unsigned long);
77	78	static struct mmu_rb_node __mmu_rb_search(struct mmu_rb_handler ,
78	79	unsigned long, unsigned long);
	80	+static void do_remove(struct mmu_rb_handler *handler,
	81	+ struct list_head *del_list);
	82	+static void handle_remove(struct work_struct *work);
79	83
80	84	static struct mmu_notifier_ops mn_opts = {
81	85	.invalidate_page = mmu_notifier_page,
@@ -95,73 +99,79 @@ static unsigned long mmu_node_last(struct mmu_rb_node *node) · 13 unmodified lines
95	99	return PAGE_ALIGN(node->addr + node->len) - 1;
96	100	}
97	101
98		−int hfi1_mmu_rb_register(struct rb_root root, struct mmu_rb_ops ops)
	102	+int hfi1_mmu_rb_register(void ops_arg, struct mm_struct mm,
	103	+ struct mmu_rb_ops *ops,
	104	+ struct workqueue_struct *wq,
	105	+ struct mmu_rb_handler **handler)
99	106	{
100	107	struct mmu_rb_handler *handlr;
101		−
102		− if (!ops->invalidate)
103		− return -EINVAL;
	108	+ int ret;
104	109
105	110	handlr = kmalloc(sizeof(*handlr), GFP_KERNEL);
106	111	if (!handlr)
107	112	return -ENOMEM;
108	113
109		− handlr->root = root;
	114	+ handlr->root = RB_ROOT;
110	115	handlr->ops = ops;
	116	+ handlr->ops_arg = ops_arg;
111	117	INIT_HLIST_NODE(&handlr->mn.hlist);
112	118	spin_lock_init(&handlr->lock);
113	119	handlr->mn.ops = &mn_opts;
114		− spin_lock(&mmu_rb_lock);
115		− list_add_tail_rcu(&handlr->list, &mmu_rb_handlers);
116		− spin_unlock(&mmu_rb_lock);
	120	+ handlr->mm = mm;
	121	+ INIT_WORK(&handlr->del_work, handle_remove);
	122	+ INIT_LIST_HEAD(&handlr->del_list);
	123	+ INIT_LIST_HEAD(&handlr->lru_list);
	124	+ handlr->wq = wq;
	125	+
	126	+ ret = mmu_notifier_register(&handlr->mn, handlr->mm);
	127	+ if (ret) {
	128	+ kfree(handlr);
	129	+ return ret;
	130	+ }
117	131
118		− return mmu_notifier_register(&handlr->mn, current->mm);
	132	+ *handler = handlr;
	133	+ return 0;
119	134	}
120	135
121		−void hfi1_mmu_rb_unregister(struct rb_root *root)
	136	+void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)
122	137	{
123		− struct mmu_rb_handler *handler = find_mmu_handler(root);
	138	+ struct mmu_rb_node *rbnode;
	139	+ struct rb_node *node;
124	140	unsigned long flags;
125		−
126		− if (!handler)
127		− return;
	141	+ struct list_head del_list;
128	142
129	143	/* Unregister first so we don't get any more notifications. */
130		− if (current->mm)
131		− mmu_notifier_unregister(&handler->mn, current->mm);
	144	+ mmu_notifier_unregister(&handler->mn, handler->mm);
132	145
133		− spin_lock(&mmu_rb_lock);
134		− list_del_rcu(&handler->list);
135		− spin_unlock(&mmu_rb_lock);
136		− synchronize_rcu();
	146	+ /*
	147	+ * Make sure the wq delete handler is finished running. It will not
	148	+ * be triggered once the mmu notifiers are unregistered above.
	149	+ */
	150	+ flush_work(&handler->del_work);
	151	+
	152	+ INIT_LIST_HEAD(&del_list);
137	153
138	154	spin_lock_irqsave(&handler->lock, flags);
139		− if (!RB_EMPTY_ROOT(root)) {
140		− struct rb_node *node;
141		− struct mmu_rb_node *rbnode;
142		−
143		− while ((node = rb_first(root))) {
144		− rbnode = rb_entry(node, struct mmu_rb_node, node);
145		− rb_erase(node, root);
146		− if (handler->ops->remove)
147		− handler->ops->remove(root, rbnode, NULL);
148		− }
	155	+ while ((node = rb_first(&handler->root))) {
	156	+ rbnode = rb_entry(node, struct mmu_rb_node, node);
	157	+ rb_erase(node, &handler->root);
	158	+ /* move from LRU list to delete list */
	159	+ list_move(&rbnode->list, &del_list);
149	160	}
150	161	spin_unlock_irqrestore(&handler->lock, flags);
151	162
	163	+ do_remove(handler, &del_list);
	164	+
152	165	kfree(handler);
153	166	}
154	167
155		−int hfi1_mmu_rb_insert(struct rb_root root, struct mmu_rb_node mnode)
	168	+int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
	169	+ struct mmu_rb_node *mnode)
156	170	{
157		− struct mmu_rb_handler *handler = find_mmu_handler(root);
158	171	struct mmu_rb_node *node;
159	172	unsigned long flags;
160	173	int ret = 0;
161	174
162		− if (!handler)
163		− return -EINVAL;
164		−
165	175	spin_lock_irqsave(&handler->lock, flags);
166	176	hfi1_cdbg(MMU, "Inserting node addr 0x%llx, len %u", mnode->addr,
167	177	mnode->len);
@@ -170,12 +180,13 @@ int hfi1_mmu_rb_insert(struct rb_root root, struct mmu_rb_node mnode) · 2 unmodified lines
170	180	ret = -EINVAL;
171	181	goto unlock;
172	182	}
173		− __mmu_int_rb_insert(mnode, root);
	183	+ __mmu_int_rb_insert(mnode, &handler->root);
	184	+ list_add(&mnode->list, &handler->lru_list);
174	185
175		− if (handler->ops->insert) {
176		− ret = handler->ops->insert(root, mnode);
177		− if (ret)
178		− __mmu_int_rb_remove(mnode, root);
	186	+ ret = handler->ops->insert(handler->ops_arg, mnode);
	187	+ if (ret) {
	188	+ __mmu_int_rb_remove(mnode, &handler->root);
	189	+ list_del(&mnode->list); /* remove from LRU list */
179	190	}
180	191	unlock:
181	192	spin_unlock_irqrestore(&handler->lock, flags);
@@ -191,10 +202,10 @@ static struct mmu_rb_node __mmu_rb_search(struct mmu_rb_handler handler, · 9 unmodified lines
191	202
192	203	hfi1_cdbg(MMU, "Searching for addr 0x%llx, len %u", addr, len);
193	204	if (!handler->ops->filter) {
194		− node = __mmu_int_rb_iter_first(handler->root, addr,
	205	+ node = __mmu_int_rb_iter_first(&handler->root, addr,
195	206	(addr + len) - 1);
196	207	} else {
197		− for (node = __mmu_int_rb_iter_first(handler->root, addr,
	208	+ for (node = __mmu_int_rb_iter_first(&handler->root, addr,
198	209	(addr + len) - 1);
199	210	node;
200	211	node = __mmu_int_rb_iter_next(node, addr,
@@ -206,82 +217,72 @@ static struct mmu_rb_node __mmu_rb_search(struct mmu_rb_handler handler, · 5 unmodified lines
206	217	return node;
207	218	}
208	219
209		−/* Caller must not hold handler lock. */
210		−static void __mmu_rb_remove(struct mmu_rb_handler *handler,
211		− struct mmu_rb_node node, struct mm_struct mm)
212		−{
213		− unsigned long flags;
214		−
215		− /* Validity of handler and node pointers has been checked by caller. */
216		− hfi1_cdbg(MMU, "Removing node addr 0x%llx, len %u", node->addr,
217		− node->len);
218		− spin_lock_irqsave(&handler->lock, flags);
219		− __mmu_int_rb_remove(node, handler->root);
220		− spin_unlock_irqrestore(&handler->lock, flags);
221		−
222		− if (handler->ops->remove)
223		− handler->ops->remove(handler->root, node, mm);
224		−}
225		−
226		−struct mmu_rb_node hfi1_mmu_rb_search(struct rb_root root, unsigned long addr,
227		− unsigned long len)
	220	+struct mmu_rb_node hfi1_mmu_rb_extract(struct mmu_rb_handler handler,
	221	+ unsigned long addr, unsigned long len)
228	222	{
229		− struct mmu_rb_handler *handler = find_mmu_handler(root);
230	223	struct mmu_rb_node *node;
231	224	unsigned long flags;
232	225
233		− if (!handler)
234		− return ERR_PTR(-EINVAL);
235		−
236	226	spin_lock_irqsave(&handler->lock, flags);
237	227	node = __mmu_rb_search(handler, addr, len);
	228	+ if (node) {
	229	+ __mmu_int_rb_remove(node, &handler->root);
	230	+ list_del(&node->list); /* remove from LRU list */
	231	+ }
238	232	spin_unlock_irqrestore(&handler->lock, flags);
239	233
240	234	return node;
241	235	}
242	236
243		−struct mmu_rb_node hfi1_mmu_rb_extract(struct rb_root root,
244		− unsigned long addr, unsigned long len)
	237	+void hfi1_mmu_rb_evict(struct mmu_rb_handler handler, void evict_arg)
245	238	{
246		− struct mmu_rb_handler *handler = find_mmu_handler(root);
247		− struct mmu_rb_node *node;
	239	+ struct mmu_rb_node rbnode, ptr;
	240	+ struct list_head del_list;
248	241	unsigned long flags;
	242	+ bool stop = false;
249	243
250		− if (!handler)
251		− return ERR_PTR(-EINVAL);
	244	+ INIT_LIST_HEAD(&del_list);
252	245
253	246	spin_lock_irqsave(&handler->lock, flags);
254		− node = __mmu_rb_search(handler, addr, len);
255		− if (node)
256		− __mmu_int_rb_remove(node, handler->root);
	247	+ list_for_each_entry_safe_reverse(rbnode, ptr, &handler->lru_list,
	248	+ list) {
	249	+ if (handler->ops->evict(handler->ops_arg, rbnode, evict_arg,
	250	+ &stop)) {
	251	+ __mmu_int_rb_remove(rbnode, &handler->root);
	252	+ /* move from LRU list to delete list */
	253	+ list_move(&rbnode->list, &del_list);
	254	+ }
	255	+ if (stop)
	256	+ break;
	257	+ }
257	258	spin_unlock_irqrestore(&handler->lock, flags);
258	259
259		− return node;
	260	+ while (!list_empty(&del_list)) {
	261	+ rbnode = list_first_entry(&del_list, struct mmu_rb_node, list);
	262	+ list_del(&rbnode->list);
	263	+ handler->ops->remove(handler->ops_arg, rbnode);
	264	+ }
260	265	}
261	266
262		−void hfi1_mmu_rb_remove(struct rb_root root, struct mmu_rb_node node)
	267	+/*
	268	+ * It is up to the caller to ensure that this function does not race with the
	269	+ * mmu invalidate notifier which may be calling the users remove callback on
	270	+ * 'node'.
	271	+ */
	272	+void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
	273	+ struct mmu_rb_node *node)
263	274	{
264		− struct mmu_rb_handler *handler = find_mmu_handler(root);
265		−
266		− if (!handler \|\| !node)
267		− return;
268		−
269		− __mmu_rb_remove(handler, node, NULL);
270		−}
	275	+ unsigned long flags;
271	276
272		−static struct mmu_rb_handler find_mmu_handler(struct rb_root root)
273		−{
274		− struct mmu_rb_handler *handler;
	277	+ /* Validity of handler and node pointers has been checked by caller. */
	278	+ hfi1_cdbg(MMU, "Removing node addr 0x%llx, len %u", node->addr,
	279	+ node->len);
	280	+ spin_lock_irqsave(&handler->lock, flags);
	281	+ __mmu_int_rb_remove(node, &handler->root);
	282	+ list_del(&node->list); /* remove from LRU list */
	283	+ spin_unlock_irqrestore(&handler->lock, flags);
275	284
276		− rcu_read_lock();
277		− list_for_each_entry_rcu(handler, &mmu_rb_handlers, list) {
278		− if (handler->root == root)
279		− goto unlock;
280		− }
281		− handler = NULL;
282		−unlock:
283		− rcu_read_unlock();
284		− return handler;
	285	+ handler->ops->remove(handler->ops_arg, node);
285	286	}
286	287
287	288	static inline void mmu_notifier_page(struct mmu_notifier *mn,
@@ -304,9 +305,10 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn, · 16 unmodified lines
304	305	{
305	306	struct mmu_rb_handler *handler =
306	307	container_of(mn, struct mmu_rb_handler, mn);
307		− struct rb_root *root = handler->root;
	308	+ struct rb_root *root = &handler->root;
308	309	struct mmu_rb_node node, ptr = NULL;
309	310	unsigned long flags;
	311	+ bool added = false;
310	312
311	313	spin_lock_irqsave(&handler->lock, flags);
312	314	for (node = __mmu_int_rb_iter_first(root, start, end - 1);
@@ -315,11 +317,53 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn, · 2 unmodified lines
315	317	ptr = __mmu_int_rb_iter_next(node, start, end - 1);
316	318	hfi1_cdbg(MMU, "Invalidating node addr 0x%llx, len %u",
317	319	node->addr, node->len);
318		− if (handler->ops->invalidate(root, node)) {
	320	+ if (handler->ops->invalidate(handler->ops_arg, node)) {
319	321	__mmu_int_rb_remove(node, root);
320		− if (handler->ops->remove)
321		− handler->ops->remove(root, node, mm);
	322	+ /* move from LRU list to delete list */
	323	+ list_move(&node->list, &handler->del_list);
	324	+ added = true;
322	325	}
323	326	}
324	327	spin_unlock_irqrestore(&handler->lock, flags);
	328	+
	329	+ if (added)
	330	+ queue_work(handler->wq, &handler->del_work);
	331	+}
	332	+
	333	+/*
	334	+ * Call the remove function for the given handler and the list. This
	335	+ * is expected to be called with a delete list extracted from handler.
	336	+ * The caller should not be holding the handler lock.
	337	+ */
	338	+static void do_remove(struct mmu_rb_handler *handler,
	339	+ struct list_head *del_list)
	340	+{
	341	+ struct mmu_rb_node *node;
	342	+
	343	+ while (!list_empty(del_list)) {
	344	+ node = list_first_entry(del_list, struct mmu_rb_node, list);
	345	+ list_del(&node->list);
	346	+ handler->ops->remove(handler->ops_arg, node);
	347	+ }
	348	+}
	349	+
	350	+/*
	351	+ * Work queue function to remove all nodes that have been queued up to
	352	+ * be removed. The key feature is that mm->mmap_sem is not being held
	353	+ * and the remove callback can sleep while taking it, if needed.
	354	+ */
	355	+static void handle_remove(struct work_struct *work)
	356	+{
	357	+ struct mmu_rb_handler *handler = container_of(work,
	358	+ struct mmu_rb_handler,
	359	+ del_work);
	360	+ struct list_head del_list;
	361	+ unsigned long flags;
	362	+
	363	+ /* remove anything that is queued to get removed */
	364	+ spin_lock_irqsave(&handler->lock, flags);
	365	+ list_replace_init(&handler->del_list, &del_list);
	366	+ spin_unlock_irqrestore(&handler->lock, flags);
	367	+
	368	+ do_remove(handler, &del_list);
325	369	}

drivers/infiniband/hw/hfi1/mmu_rb.h+24−13

@@ -54,23 +54,34 @@ struct mmu_rb_node { · 53 unmodified lines
54	54	unsigned long len;
55	55	unsigned long __last;
56	56	struct rb_node node;
	57	+ struct list_head list;
57	58	};
58	59
	60	+/*
	61	+ * NOTE: filter, insert, invalidate, and evict must not sleep. Only remove is
	62	+ * allowed to sleep.
	63	+ */
59	64	struct mmu_rb_ops {
60		− bool (filter)(struct mmu_rb_node , unsigned long, unsigned long);
61		− int (insert)(struct rb_root , struct mmu_rb_node *);
62		− void (remove)(struct rb_root , struct mmu_rb_node *,
63		− struct mm_struct *);
64		− int (invalidate)(struct rb_root , struct mmu_rb_node *);
	65	+ bool (filter)(struct mmu_rb_node node, unsigned long addr,
	66	+ unsigned long len);
	67	+ int (insert)(void ops_arg, struct mmu_rb_node *mnode);
	68	+ void (remove)(void ops_arg, struct mmu_rb_node *mnode);
	69	+ int (invalidate)(void ops_arg, struct mmu_rb_node *node);
	70	+ int (evict)(void ops_arg, struct mmu_rb_node *mnode,
	71	+ void evict_arg, bool stop);
65	72	};
66	73
67		−int hfi1_mmu_rb_register(struct rb_root root, struct mmu_rb_ops ops);
68		−void hfi1_mmu_rb_unregister(struct rb_root *);
69		−int hfi1_mmu_rb_insert(struct rb_root , struct mmu_rb_node );
70		−void hfi1_mmu_rb_remove(struct rb_root , struct mmu_rb_node );
71		−struct mmu_rb_node hfi1_mmu_rb_search(struct rb_root , unsigned long,
72		− unsigned long);
73		−struct mmu_rb_node hfi1_mmu_rb_extract(struct rb_root , unsigned long,
74		− unsigned long);
	74	+int hfi1_mmu_rb_register(void ops_arg, struct mm_struct mm,
	75	+ struct mmu_rb_ops *ops,
	76	+ struct workqueue_struct *wq,
	77	+ struct mmu_rb_handler **handler);
	78	+void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler);
	79	+int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
	80	+ struct mmu_rb_node *mnode);
	81	+void hfi1_mmu_rb_evict(struct mmu_rb_handler handler, void evict_arg);
	82	+void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
	83	+ struct mmu_rb_node *mnode);
	84	+struct mmu_rb_node hfi1_mmu_rb_extract(struct mmu_rb_handler handler,
	85	+ unsigned long addr, unsigned long len);
75	86
76	87	#endif /* _HFI1_MMU_RB_H */

drivers/infiniband/hw/hfi1/pcie.c+64−4

@@ -679,6 +679,10 @@ static uint pcie_pset = UNSET_PSET; · 678 unmodified lines
679	679	module_param(pcie_pset, uint, S_IRUGO);
680	680	MODULE_PARM_DESC(pcie_pset, "PCIe Eq Pset value to use, range is 0-10");
681	681
	682	+static uint pcie_ctle = 1; /* discrete on, integrated off */
	683	+module_param(pcie_ctle, uint, S_IRUGO);
	684	+MODULE_PARM_DESC(pcie_ctle, "PCIe static CTLE mode, bit 0 - discrete on/off, bit 1 - integrated on/off");
	685	+
682	686	/* equalization columns */
683	687	#define PREC 0
684	688	#define ATTN 1
@@ -716,6 +720,36 @@ static const u8 integrated_preliminary_eq[11][3] = { · 31 unmodified lines
716	720	{ 0x00, 0x1e, 0x0a }, /* p10 */
717	721	};
718	722
	723	+static const u8 discrete_ctle_tunings[11][4] = {
	724	+ /* DC LF HF BW */
	725	+ { 0x48, 0x0b, 0x04, 0x04 }, /* p0 */
	726	+ { 0x60, 0x05, 0x0f, 0x0a }, /* p1 */
	727	+ { 0x50, 0x09, 0x06, 0x06 }, /* p2 */
	728	+ { 0x68, 0x05, 0x0f, 0x0a }, /* p3 */
	729	+ { 0x80, 0x05, 0x0f, 0x0a }, /* p4 */
	730	+ { 0x70, 0x05, 0x0f, 0x0a }, /* p5 */
	731	+ { 0x68, 0x05, 0x0f, 0x0a }, /* p6 */
	732	+ { 0x38, 0x0f, 0x00, 0x00 }, /* p7 */
	733	+ { 0x48, 0x09, 0x06, 0x06 }, /* p8 */
	734	+ { 0x60, 0x05, 0x0f, 0x0a }, /* p9 */
	735	+ { 0x38, 0x0f, 0x00, 0x00 }, /* p10 */
	736	+};
	737	+
	738	+static const u8 integrated_ctle_tunings[11][4] = {
	739	+ /* DC LF HF BW */
	740	+ { 0x38, 0x0f, 0x00, 0x00 }, /* p0 */
	741	+ { 0x38, 0x0f, 0x00, 0x00 }, /* p1 */
	742	+ { 0x38, 0x0f, 0x00, 0x00 }, /* p2 */
	743	+ { 0x38, 0x0f, 0x00, 0x00 }, /* p3 */
	744	+ { 0x58, 0x0a, 0x05, 0x05 }, /* p4 */
	745	+ { 0x48, 0x0a, 0x05, 0x05 }, /* p5 */
	746	+ { 0x40, 0x0a, 0x05, 0x05 }, /* p6 */
	747	+ { 0x38, 0x0f, 0x00, 0x00 }, /* p7 */
	748	+ { 0x38, 0x0f, 0x00, 0x00 }, /* p8 */
	749	+ { 0x38, 0x09, 0x06, 0x06 }, /* p9 */
	750	+ { 0x38, 0x0e, 0x01, 0x01 }, /* p10 */
	751	+};
	752	+
719	753	/* helper to format the value to write to hardware */
720	754	#define eq_value(pre, curr, post) \
721	755	((((u32)(pre)) << \
@@ -951,11 +985,14 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd) · 229 unmodified lines
951	985	u32 status, err;
952	986	int ret;
953	987	int do_retry, retry_count = 0;
	988	+ int intnum = 0;
954	989	uint default_pset;
955	990	u16 target_vector, target_speed;
956	991	u16 lnkctl2, vendor;
957	992	u8 div;
958	993	const u8 (*eq)[3];
	994	+ const u8 (*ctle_tunings)[4];
	995	+ uint static_ctle_mode;
959	996	int return_error = 0;
960	997
961	998	/* PCIe Gen3 is for the ASIC only */
@@ -1089,6 +1126,9 @@ retry: · 127 unmodified lines
1089	1126	div = 3;
1090	1127	eq = discrete_preliminary_eq;
1091	1128	default_pset = DEFAULT_DISCRETE_PSET;
	1129	+ ctle_tunings = discrete_ctle_tunings;
	1130	+ /* bit 0 - discrete on/off */
	1131	+ static_ctle_mode = pcie_ctle & 0x1;
1092	1132	} else {
1093	1133	/* 400mV, FS=29, LF = 9 */
1094	1134	fs = 29;
@@ -1096,6 +1136,9 @@ retry: · 1 unmodified line
1096	1136	div = 1;
1097	1137	eq = integrated_preliminary_eq;
1098	1138	default_pset = DEFAULT_MCP_PSET;
	1139	+ ctle_tunings = integrated_ctle_tunings;
	1140	+ /* bit 1 - integrated on/off */
	1141	+ static_ctle_mode = (pcie_ctle >> 1) & 0x1;
1099	1142	}
1100	1143	pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL101,
1101	1144	(fs <<
@@ -1135,16 +1178,33 @@ retry: · 33 unmodified lines
1135	1178	* step 5c: Program gasket interrupts
1136	1179	*/
1137	1180	/* set the Rx Bit Rate to REFCLK ratio */
1138		− write_gasket_interrupt(dd, 0, 0x0006, 0x0050);
	1181	+ write_gasket_interrupt(dd, intnum++, 0x0006, 0x0050);
1139	1182	/* disable pCal for PCIe Gen3 RX equalization */
1140		− write_gasket_interrupt(dd, 1, 0x0026, 0x5b01);
	1183	+ /* select adaptive or static CTLE */
	1184	+ write_gasket_interrupt(dd, intnum++, 0x0026,
	1185	+ 0x5b01 \| (static_ctle_mode << 3));
1141	1186	/*
1142	1187	* Enable iCal for PCIe Gen3 RX equalization, and set which
1143	1188	* evaluation of RX_EQ_EVAL will launch the iCal procedure.
1144	1189	*/
1145		− write_gasket_interrupt(dd, 2, 0x0026, 0x5202);
	1190	+ write_gasket_interrupt(dd, intnum++, 0x0026, 0x5202);
	1191	+
	1192	+ if (static_ctle_mode) {
	1193	+ /* apply static CTLE tunings */
	1194	+ u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw;
	1195	+
	1196	+ pcie_dc = ctle_tunings[pcie_pset][0];
	1197	+ pcie_lf = ctle_tunings[pcie_pset][1];
	1198	+ pcie_hf = ctle_tunings[pcie_pset][2];
	1199	+ pcie_bw = ctle_tunings[pcie_pset][3];
	1200	+ write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 \| pcie_dc);
	1201	+ write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 \| pcie_lf);
	1202	+ write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 \| pcie_hf);
	1203	+ write_gasket_interrupt(dd, intnum++, 0x0026, 0x5500 \| pcie_bw);
	1204	+ }
	1205	+
1146	1206	/* terminate list */
1147		− write_gasket_interrupt(dd, 3, 0x0000, 0x0000);
	1207	+ write_gasket_interrupt(dd, intnum++, 0x0000, 0x0000);
1148	1208
1149	1209	/*
1150	1210	* step 5d: program XMT margin

drivers/infiniband/hw/hfi1/pio.c+17−4

@@ -1952,13 +1952,17 @@ int init_pervl_scs(struct hfi1_devdata *dd) · 1951 unmodified lines
1952	1952	dd->vld[15].sc = sc_alloc(dd, SC_VL15,
1953	1953	dd->rcd[0]->rcvhdrqentsize, dd->node);
1954	1954	if (!dd->vld[15].sc)
1955		− goto nomem;
	1955	+ return -ENOMEM;
	1956	+
1956	1957	hfi1_init_ctxt(dd->vld[15].sc);
1957	1958	dd->vld[15].mtu = enum_to_mtu(OPA_MTU_2048);
1958	1959
1959		− dd->kernel_send_context = kmalloc_node(dd->num_send_contexts *
	1960	+ dd->kernel_send_context = kzalloc_node(dd->num_send_contexts *
1960	1961	sizeof(struct send_context *),
1961	1962	GFP_KERNEL, dd->node);
	1963	+ if (!dd->kernel_send_context)
	1964	+ goto freesc15;
	1965	+
1962	1966	dd->kernel_send_context[0] = dd->vld[15].sc;
1963	1967
1964	1968	for (i = 0; i < num_vls; i++) {
@@ -2010,12 +2014,21 @@ int init_pervl_scs(struct hfi1_devdata *dd) · 45 unmodified lines
2010	2014	if (pio_map_init(dd, ppd->port - 1, num_vls, NULL))
2011	2015	goto nomem;
2012	2016	return 0;
	2017	+
2013	2018	nomem:
2014		− sc_free(dd->vld[15].sc);
2015		− for (i = 0; i < num_vls; i++)
	2019	+ for (i = 0; i < num_vls; i++) {
2016	2020	sc_free(dd->vld[i].sc);
	2021	+ dd->vld[i].sc = NULL;
	2022	+ }
	2023	+
2017	2024	for (i = num_vls; i < INIT_SC_PER_VL * num_vls; i++)
2018	2025	sc_free(dd->kernel_send_context[i + 1]);
	2026	+
	2027	+ kfree(dd->kernel_send_context);
	2028	+ dd->kernel_send_context = NULL;
	2029	+
	2030	+freesc15:
	2031	+ sc_free(dd->vld[15].sc);
2019	2032	return -ENOMEM;
2020	2033	}
2021	2034

drivers/infiniband/hw/hfi1/platform.c+5−15

@@ -537,20 +537,6 @@ static void apply_tunings( · 536 unmodified lines
537	537	u8 precur = 0, attn = 0, postcur = 0, external_device_config = 0;
538	538	u8 *cache = ppd->qsfp_info.cache;
539	539
540		− /* Enable external device config if channel is limiting active */
541		− read_8051_config(ppd->dd, LINK_OPTIMIZATION_SETTINGS,
542		− GENERAL_CONFIG, &config_data);
543		− config_data &= ~(0xff << ENABLE_EXT_DEV_CONFIG_SHIFT);
544		− config_data \|= ((u32)limiting_active << ENABLE_EXT_DEV_CONFIG_SHIFT);
545		− ret = load_8051_config(ppd->dd, LINK_OPTIMIZATION_SETTINGS,
546		− GENERAL_CONFIG, config_data);
547		− if (ret != HCMD_SUCCESS)
548		− dd_dev_err(
549		− ppd->dd,
550		− "%s: Failed to set enable external device config\n",
551		− __func__);
552		−
553		− config_data = 0; /* re-init */
554	540	/* Pass tuning method to 8051 */
555	541	read_8051_config(ppd->dd, LINK_TUNING_PARAMETERS, GENERAL_CONFIG,
556	542	&config_data);
@@ -638,9 +624,13 @@ static int tune_active_qsfp(struct hfi1_pportdata ppd, u32 ptr_tx_preset, · 81 unmodified lines
638	624	if (ret)
639	625	return ret;
640	626
	627	+ /*
	628	+ * We'll change the QSFP memory contents from here on out, thus we set a
	629	+ * flag here to remind ourselves to reset the QSFP module. This prevents
	630	+ * reuse of stale settings established in our previous pass through.
	631	+ */
641	632	if (ppd->qsfp_info.reset_needed) {
642	633	reset_qsfp(ppd);
643		− ppd->qsfp_info.reset_needed = 0;
644	634	refresh_qsfp_cache(ppd, &ppd->qsfp_info);
645	635	} else {
646	636	ppd->qsfp_info.reset_needed = 1;

drivers/infiniband/hw/hfi1/qp.c+65−3

@@ -52,6 +52,7 @@ · 51 unmodified lines
52	52	#include <linux/seq_file.h>
53	53	#include <rdma/rdma_vt.h>
54	54	#include <rdma/rdmavt_qp.h>
	55	+#include <rdma/ib_verbs.h>
55	56
56	57	#include "hfi.h"
57	58	#include "qp.h"
@@ -115,6 +116,66 @@ static const u16 credit_table[31] = { · 57 unmodified lines
115	116	32768 /* 1E */
116	117	};
117	118
	119	+const struct rvt_operation_params hfi1_post_parms[RVT_OPERATION_MAX] = {
	120	+[IB_WR_RDMA_WRITE] = {
	121	+ .length = sizeof(struct ib_rdma_wr),
	122	+ .qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	123	+},
	124	+
	125	+[IB_WR_RDMA_READ] = {
	126	+ .length = sizeof(struct ib_rdma_wr),
	127	+ .qpt_support = BIT(IB_QPT_RC),
	128	+ .flags = RVT_OPERATION_ATOMIC,
	129	+},
	130	+
	131	+[IB_WR_ATOMIC_CMP_AND_SWP] = {
	132	+ .length = sizeof(struct ib_atomic_wr),
	133	+ .qpt_support = BIT(IB_QPT_RC),
	134	+ .flags = RVT_OPERATION_ATOMIC \| RVT_OPERATION_ATOMIC_SGE,
	135	+},
	136	+
	137	+[IB_WR_ATOMIC_FETCH_AND_ADD] = {
	138	+ .length = sizeof(struct ib_atomic_wr),
	139	+ .qpt_support = BIT(IB_QPT_RC),
	140	+ .flags = RVT_OPERATION_ATOMIC \| RVT_OPERATION_ATOMIC_SGE,
	141	+},
	142	+
	143	+[IB_WR_RDMA_WRITE_WITH_IMM] = {
	144	+ .length = sizeof(struct ib_rdma_wr),
	145	+ .qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	146	+},
	147	+
	148	+[IB_WR_SEND] = {
	149	+ .length = sizeof(struct ib_send_wr),
	150	+ .qpt_support = BIT(IB_QPT_UD) \| BIT(IB_QPT_SMI) \| BIT(IB_QPT_GSI) \|
	151	+ BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	152	+},
	153	+
	154	+[IB_WR_SEND_WITH_IMM] = {
	155	+ .length = sizeof(struct ib_send_wr),
	156	+ .qpt_support = BIT(IB_QPT_UD) \| BIT(IB_QPT_SMI) \| BIT(IB_QPT_GSI) \|
	157	+ BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	158	+},
	159	+
	160	+[IB_WR_REG_MR] = {
	161	+ .length = sizeof(struct ib_reg_wr),
	162	+ .qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	163	+ .flags = RVT_OPERATION_LOCAL,
	164	+},
	165	+
	166	+[IB_WR_LOCAL_INV] = {
	167	+ .length = sizeof(struct ib_send_wr),
	168	+ .qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	169	+ .flags = RVT_OPERATION_LOCAL,
	170	+},
	171	+
	172	+[IB_WR_SEND_WITH_INV] = {
	173	+ .length = sizeof(struct ib_send_wr),
	174	+ .qpt_support = BIT(IB_QPT_RC),
	175	+},
	176	+
	177	+};
	178	+
118	179	static void flush_tx_list(struct rvt_qp *qp)
119	180	{
120	181	struct hfi1_qp_priv *priv = qp->priv;
@@ -745,8 +806,9 @@ void qp_priv_alloc(struct rvt_dev_info rdi, struct rvt_qp *qp, · 624 unmodified lines
745	806
746	807	priv->owner = qp;
747	808
748		− priv->s_hdr = kzalloc_node(sizeof(*priv->s_hdr), gfp, rdi->dparms.node);
749		− if (!priv->s_hdr) {
	809	+ priv->s_ahg = kzalloc_node(sizeof(*priv->s_ahg), gfp,
	810	+ rdi->dparms.node);
	811	+ if (!priv->s_ahg) {
750	812	kfree(priv);
751	813	return ERR_PTR(-ENOMEM);
752	814	}
@@ -759,7 +821,7 @@ void qp_priv_free(struct rvt_dev_info rdi, struct rvt_qp qp) · 6 unmodified lines
759	821	{
760	822	struct hfi1_qp_priv *priv = qp->priv;
761	823
762		− kfree(priv->s_hdr);
	824	+ kfree(priv->s_ahg);
763	825	kfree(priv);
764	826	}
765	827

drivers/infiniband/hw/hfi1/qp.h+3−1

@@ -54,6 +54,8 @@ · 53 unmodified lines
54	54
55	55	extern unsigned int hfi1_qp_table_size;
56	56
	57	+extern const struct rvt_operation_params hfi1_post_parms[];
	58	+
57	59	/*
58	60	* free_ahg - clear ahg from QP
59	61	*/
@@ -61,7 +63,7 @@ static inline void clear_ahg(struct rvt_qp *qp) · 1 unmodified line
61	63	{
62	64	struct hfi1_qp_priv *priv = qp->priv;
63	65
64		− priv->s_hdr->ahgcount = 0;
	66	+ priv->s_ahg->ahgcount = 0;
65	67	qp->s_flags &= ~(RVT_S_AHG_VALID \| RVT_S_AHG_CLEAR);
66	68	if (priv->s_sde && qp->s_ahgidx >= 0)
67	69	sdma_ahg_free(priv->s_sde, qp->s_ahgidx);

drivers/infiniband/hw/hfi1/qsfp.c+302−107

@@ -50,46 +50,285 @@ · 49 unmodified lines
50	50	#include <linux/vmalloc.h>
51	51
52	52	#include "hfi.h"
53		−#include "twsi.h"
	53	+
	54	+/* for the given bus number, return the CSR for reading an i2c line */
	55	+static inline u32 i2c_in_csr(u32 bus_num)
	56	+{
	57	+ return bus_num ? ASIC_QSFP2_IN : ASIC_QSFP1_IN;
	58	+}
	59	+
	60	+/* for the given bus number, return the CSR for writing an i2c line */
	61	+static inline u32 i2c_oe_csr(u32 bus_num)
	62	+{
	63	+ return bus_num ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
	64	+}
	65	+
	66	+static void hfi1_setsda(void *data, int state)
	67	+{
	68	+ struct hfi1_i2c_bus bus = (struct hfi1_i2c_bus )data;
	69	+ struct hfi1_devdata *dd = bus->controlling_dd;
	70	+ u64 reg;
	71	+ u32 target_oe;
	72	+
	73	+ target_oe = i2c_oe_csr(bus->num);
	74	+ reg = read_csr(dd, target_oe);
	75	+ /*
	76	+ * The OE bit value is inverted and connected to the pin. When
	77	+ * OE is 0 the pin is left to be pulled up, when the OE is 1
	78	+ * the pin is driven low. This matches the "open drain" or "open
	79	+ * collector" convention.
	80	+ */
	81	+ if (state)
	82	+ reg &= ~QSFP_HFI0_I2CDAT;
	83	+ else
	84	+ reg \|= QSFP_HFI0_I2CDAT;
	85	+ write_csr(dd, target_oe, reg);
	86	+ /* do a read to force the write into the chip */
	87	+ (void)read_csr(dd, target_oe);
	88	+}
	89	+
	90	+static void hfi1_setscl(void *data, int state)
	91	+{
	92	+ struct hfi1_i2c_bus bus = (struct hfi1_i2c_bus )data;
	93	+ struct hfi1_devdata *dd = bus->controlling_dd;
	94	+ u64 reg;
	95	+ u32 target_oe;
	96	+
	97	+ target_oe = i2c_oe_csr(bus->num);
	98	+ reg = read_csr(dd, target_oe);
	99	+ /*
	100	+ * The OE bit value is inverted and connected to the pin. When
	101	+ * OE is 0 the pin is left to be pulled up, when the OE is 1
	102	+ * the pin is driven low. This matches the "open drain" or "open
	103	+ * collector" convention.
	104	+ */
	105	+ if (state)
	106	+ reg &= ~QSFP_HFI0_I2CCLK;
	107	+ else
	108	+ reg \|= QSFP_HFI0_I2CCLK;
	109	+ write_csr(dd, target_oe, reg);
	110	+ /* do a read to force the write into the chip */
	111	+ (void)read_csr(dd, target_oe);
	112	+}
	113	+
	114	+static int hfi1_getsda(void *data)
	115	+{
	116	+ struct hfi1_i2c_bus bus = (struct hfi1_i2c_bus )data;
	117	+ u64 reg;
	118	+ u32 target_in;
	119	+
	120	+ hfi1_setsda(data, 1); /* clear OE so we do not pull line down */
	121	+ udelay(2); /* 1us pull up + 250ns hold */
	122	+
	123	+ target_in = i2c_in_csr(bus->num);
	124	+ reg = read_csr(bus->controlling_dd, target_in);
	125	+ return !!(reg & QSFP_HFI0_I2CDAT);
	126	+}
	127	+
	128	+static int hfi1_getscl(void *data)
	129	+{
	130	+ struct hfi1_i2c_bus bus = (struct hfi1_i2c_bus )data;
	131	+ u64 reg;
	132	+ u32 target_in;
	133	+
	134	+ hfi1_setscl(data, 1); /* clear OE so we do not pull line down */
	135	+ udelay(2); /* 1us pull up + 250ns hold */
	136	+
	137	+ target_in = i2c_in_csr(bus->num);
	138	+ reg = read_csr(bus->controlling_dd, target_in);
	139	+ return !!(reg & QSFP_HFI0_I2CCLK);
	140	+}
54	141
55	142	/*
56		− * QSFP support for hfi driver, using "Two Wire Serial Interface" driver
57		− * in twsi.c
	143	+ * Allocate and initialize the given i2c bus number.
	144	+ * Returns NULL on failure.
58	145	*/
59		−#define I2C_MAX_RETRY 4
	146	+static struct hfi1_i2c_bus init_i2c_bus(struct hfi1_devdata dd,
	147	+ struct hfi1_asic_data *ad, int num)
	148	+{
	149	+ struct hfi1_i2c_bus *bus;
	150	+ int ret;
	151	+
	152	+ bus = kzalloc(sizeof(*bus), GFP_KERNEL);
	153	+ if (!bus)
	154	+ return NULL;
	155	+
	156	+ bus->controlling_dd = dd;
	157	+ bus->num = num; /* our bus number */
	158	+
	159	+ bus->algo.setsda = hfi1_setsda;
	160	+ bus->algo.setscl = hfi1_setscl;
	161	+ bus->algo.getsda = hfi1_getsda;
	162	+ bus->algo.getscl = hfi1_getscl;
	163	+ bus->algo.udelay = 5;
	164	+ bus->algo.timeout = usecs_to_jiffies(50);
	165	+ bus->algo.data = bus;
	166	+
	167	+ bus->adapter.owner = THIS_MODULE;
	168	+ bus->adapter.algo_data = &bus->algo;
	169	+ bus->adapter.dev.parent = &dd->pcidev->dev;
	170	+ snprintf(bus->adapter.name, sizeof(bus->adapter.name),
	171	+ "hfi1_i2c%d", num);
	172	+
	173	+ ret = i2c_bit_add_bus(&bus->adapter);
	174	+ if (ret) {
	175	+ dd_dev_info(dd, "%s: unable to add i2c bus %d, err %d\n",
	176	+ __func__, num, ret);
	177	+ kfree(bus);
	178	+ return NULL;
	179	+ }
	180	+
	181	+ return bus;
	182	+}
60	183
61	184	/*
62		− * Raw i2c write. No set-up or lock checking.
	185	+ * Initialize i2c buses.
	186	+ * Return 0 on success, -errno on error.
63	187	*/
64		−static int __i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr,
65		− int offset, void *bp, int len)
	188	+int set_up_i2c(struct hfi1_devdata dd, struct hfi1_asic_data ad)
66	189	{
67		− struct hfi1_devdata *dd = ppd->dd;
68		− int ret, cnt;
69		− u8 *buff = bp;
	190	+ ad->i2c_bus0 = init_i2c_bus(dd, ad, 0);
	191	+ ad->i2c_bus1 = init_i2c_bus(dd, ad, 1);
	192	+ if (!ad->i2c_bus0 \|\| !ad->i2c_bus1)
	193	+ return -ENOMEM;
	194	+ return 0;
	195	+};
70	196
71		− cnt = 0;
72		− while (cnt < len) {
73		− int wlen = len - cnt;
	197	+static void clean_i2c_bus(struct hfi1_i2c_bus *bus)
	198	+{
	199	+ if (bus) {
	200	+ i2c_del_adapter(&bus->adapter);
	201	+ kfree(bus);
	202	+ }
	203	+}
74	204
75		− ret = hfi1_twsi_blk_wr(dd, target, i2c_addr, offset,
76		− buff + cnt, wlen);
77		− if (ret) {
78		− /* hfi1_twsi_blk_wr() 1 for error, else 0 */
79		− return -EIO;
80		− }
81		− offset += wlen;
82		− cnt += wlen;
	205	+void clean_up_i2c(struct hfi1_devdata dd, struct hfi1_asic_data ad)
	206	+{
	207	+ clean_i2c_bus(ad->i2c_bus0);
	208	+ ad->i2c_bus0 = NULL;
	209	+ clean_i2c_bus(ad->i2c_bus1);
	210	+ ad->i2c_bus1 = NULL;
	211	+}
	212	+
	213	+static int i2c_bus_write(struct hfi1_devdata dd, struct hfi1_i2c_bus i2c,
	214	+ u8 slave_addr, int offset, int offset_size,
	215	+ u8 *data, u16 len)
	216	+{
	217	+ int ret;
	218	+ int num_msgs;
	219	+ u8 offset_bytes[2];
	220	+ struct i2c_msg msgs[2];
	221	+
	222	+ switch (offset_size) {
	223	+ case 0:
	224	+ num_msgs = 1;
	225	+ msgs[0].addr = slave_addr;
	226	+ msgs[0].flags = 0;
	227	+ msgs[0].len = len;
	228	+ msgs[0].buf = data;
	229	+ break;
	230	+ case 2:
	231	+ offset_bytes[1] = (offset >> 8) & 0xff;
	232	+ /* fall through */
	233	+ case 1:
	234	+ num_msgs = 2;
	235	+ offset_bytes[0] = offset & 0xff;
	236	+
	237	+ msgs[0].addr = slave_addr;
	238	+ msgs[0].flags = 0;
	239	+ msgs[0].len = offset_size;
	240	+ msgs[0].buf = offset_bytes;
	241	+
	242	+ msgs[1].addr = slave_addr;
	243	+ msgs[1].flags = I2C_M_NOSTART,
	244	+ msgs[1].len = len;
	245	+ msgs[1].buf = data;
	246	+ break;
	247	+ default:
	248	+ return -EINVAL;
83	249	}
84	250
85		− /* Must wait min 20us between qsfp i2c transactions */
86		− udelay(20);
	251	+ i2c->controlling_dd = dd;
	252	+ ret = i2c_transfer(&i2c->adapter, msgs, num_msgs);
	253	+ if (ret != num_msgs) {
	254	+ dd_dev_err(dd, "%s: bus %d, i2c slave 0x%x, offset 0x%x, len 0x%x; write failed, ret %d\n",
	255	+ __func__, i2c->num, slave_addr, offset, len, ret);
	256	+ return ret < 0 ? ret : -EIO;
	257	+ }
	258	+ return 0;
	259	+}
	260	+
	261	+static int i2c_bus_read(struct hfi1_devdata dd, struct hfi1_i2c_bus bus,
	262	+ u8 slave_addr, int offset, int offset_size,
	263	+ u8 *data, u16 len)
	264	+{
	265	+ int ret;
	266	+ int num_msgs;
	267	+ u8 offset_bytes[2];
	268	+ struct i2c_msg msgs[2];
	269	+
	270	+ switch (offset_size) {
	271	+ case 0:
	272	+ num_msgs = 1;
	273	+ msgs[0].addr = slave_addr;
	274	+ msgs[0].flags = I2C_M_RD;
	275	+ msgs[0].len = len;
	276	+ msgs[0].buf = data;
	277	+ break;
	278	+ case 2:
	279	+ offset_bytes[1] = (offset >> 8) & 0xff;
	280	+ /* fall through */
	281	+ case 1:
	282	+ num_msgs = 2;
	283	+ offset_bytes[0] = offset & 0xff;
	284	+
	285	+ msgs[0].addr = slave_addr;
	286	+ msgs[0].flags = 0;
	287	+ msgs[0].len = offset_size;
	288	+ msgs[0].buf = offset_bytes;
	289	+
	290	+ msgs[1].addr = slave_addr;
	291	+ msgs[1].flags = I2C_M_RD,
	292	+ msgs[1].len = len;
	293	+ msgs[1].buf = data;
	294	+ break;
	295	+ default:
	296	+ return -EINVAL;
	297	+ }
87	298
88		− return cnt;
	299	+ bus->controlling_dd = dd;
	300	+ ret = i2c_transfer(&bus->adapter, msgs, num_msgs);
	301	+ if (ret != num_msgs) {
	302	+ dd_dev_err(dd, "%s: bus %d, i2c slave 0x%x, offset 0x%x, len 0x%x; read failed, ret %d\n",
	303	+ __func__, bus->num, slave_addr, offset, len, ret);
	304	+ return ret < 0 ? ret : -EIO;
	305	+ }
	306	+ return 0;
	307	+}
	308	+
	309	+/*
	310	+ * Raw i2c write. No set-up or lock checking.
	311	+ *
	312	+ * Return 0 on success, -errno on error.
	313	+ */
	314	+static int __i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr,
	315	+ int offset, void *bp, int len)
	316	+{
	317	+ struct hfi1_devdata *dd = ppd->dd;
	318	+ struct hfi1_i2c_bus *bus;
	319	+ u8 slave_addr;
	320	+ int offset_size;
	321	+
	322	+ bus = target ? dd->asic_data->i2c_bus1 : dd->asic_data->i2c_bus0;
	323	+ slave_addr = (i2c_addr & 0xff) >> 1; /* convert to 7-bit addr */
	324	+ offset_size = (i2c_addr >> 8) & 0x3;
	325	+ return i2c_bus_write(dd, bus, slave_addr, offset, offset_size, bp, len);
89	326	}
90	327
91	328	/*
92	329	* Caller must hold the i2c chain resource.
	330	+ *
	331	+ * Return number of bytes written, or -errno.
93	332	*/
94	333	int i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset,
95	334	void *bp, int len)
@@ -99,63 +338,36 @@ int i2c_write(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset, · 3 unmodified lines
99	338	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))
100	339	return -EACCES;
101	340
102		− /* make sure the TWSI bus is in a sane state */
103		− ret = hfi1_twsi_reset(ppd->dd, target);
104		− if (ret) {
105		− hfi1_dev_porterr(ppd->dd, ppd->port,
106		− "I2C chain %d write interface reset failed\n",
107		− target);
	341	+ ret = __i2c_write(ppd, target, i2c_addr, offset, bp, len);
	342	+ if (ret)
108	343	return ret;
109		− }
110	344
111		− return __i2c_write(ppd, target, i2c_addr, offset, bp, len);
	345	+ return len;
112	346	}
113	347
114	348	/*
115	349	* Raw i2c read. No set-up or lock checking.
	350	+ *
	351	+ * Return 0 on success, -errno on error.
116	352	*/
117	353	static int __i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr,
118	354	int offset, void *bp, int len)
119	355	{
120	356	struct hfi1_devdata *dd = ppd->dd;
121		− int ret, cnt, pass = 0;
122		− int orig_offset = offset;
123		−
124		− cnt = 0;
125		− while (cnt < len) {
126		− int rlen = len - cnt;
127		−
128		− ret = hfi1_twsi_blk_rd(dd, target, i2c_addr, offset,
129		− bp + cnt, rlen);
130		− /* Some QSFP's fail first try. Retry as experiment */
131		− if (ret && cnt == 0 && ++pass < I2C_MAX_RETRY)
132		− continue;
133		− if (ret) {
134		− /* hfi1_twsi_blk_rd() 1 for error, else 0 */
135		− ret = -EIO;
136		− goto exit;
137		− }
138		− offset += rlen;
139		− cnt += rlen;
140		− }
141		−
142		− ret = cnt;
143		−
144		−exit:
145		− if (ret < 0) {
146		− hfi1_dev_porterr(dd, ppd->port,
147		− "I2C chain %d read failed, addr 0x%x, offset 0x%x, len %d\n",
148		− target, i2c_addr, orig_offset, len);
149		− }
150		−
151		− /* Must wait min 20us between qsfp i2c transactions */
152		− udelay(20);
153		−
154		− return ret;
	357	+ struct hfi1_i2c_bus *bus;
	358	+ u8 slave_addr;
	359	+ int offset_size;
	360	+
	361	+ bus = target ? dd->asic_data->i2c_bus1 : dd->asic_data->i2c_bus0;
	362	+ slave_addr = (i2c_addr & 0xff) >> 1; /* convert to 7-bit addr */
	363	+ offset_size = (i2c_addr >> 8) & 0x3;
	364	+ return i2c_bus_read(dd, bus, slave_addr, offset, offset_size, bp, len);
155	365	}
156	366
157	367	/*
158	368	* Caller must hold the i2c chain resource.
	369	+ *
	370	+ * Return number of bytes read, or -errno.
159	371	*/
160	372	int i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset,
161	373	void *bp, int len)
@@ -165,16 +377,11 @@ int i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset, · 3 unmodified lines
165	377	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))
166	378	return -EACCES;
167	379
168		− /* make sure the TWSI bus is in a sane state */
169		− ret = hfi1_twsi_reset(ppd->dd, target);
170		− if (ret) {
171		− hfi1_dev_porterr(ppd->dd, ppd->port,
172		− "I2C chain %d read interface reset failed\n",
173		− target);
	380	+ ret = __i2c_read(ppd, target, i2c_addr, offset, bp, len);
	381	+ if (ret)
174	382	return ret;
175		− }
176	383
177		− return __i2c_read(ppd, target, i2c_addr, offset, bp, len);
	384	+ return len;
178	385	}
179	386
180	387	/*
@@ -182,6 +389,8 @@ int i2c_read(struct hfi1_pportdata *ppd, u32 target, int i2c_addr, int offset, · 1 unmodified line
182	389	* by writing @addr = ((256 * n) + m)
183	390	*
184	391	* Caller must hold the i2c chain resource.
	392	+ *
	393	+ * Return number of bytes written or -errno.
185	394	*/
186	395	int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp,
187	396	int len)
@@ -189,21 +398,12 @@ int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 1 unmodified line
189	398	int count = 0;
190	399	int offset;
191	400	int nwrite;
192		− int ret;
	401	+ int ret = 0;
193	402	u8 page;
194	403
195	404	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))
196	405	return -EACCES;
197	406
198		− /* make sure the TWSI bus is in a sane state */
199		− ret = hfi1_twsi_reset(ppd->dd, target);
200		− if (ret) {
201		− hfi1_dev_porterr(ppd->dd, ppd->port,
202		− "QSFP chain %d write interface reset failed\n",
203		− target);
204		− return ret;
205		− }
206		−
207	407	while (count < len) {
208	408	/*
209	409	* Set the qsfp page based on a zero-based address
@@ -213,11 +413,12 @@ int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 3 unmodified lines
213	413
214	414	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,
215	415	QSFP_PAGE_SELECT_BYTE_OFFS, &page, 1);
216		− if (ret != 1) {
	416	+ /* QSFPs require a 5-10msec delay after write operations */
	417	+ mdelay(5);
	418	+ if (ret) {
217	419	hfi1_dev_porterr(ppd->dd, ppd->port,
218	420	"QSFP chain %d can't write QSFP_PAGE_SELECT_BYTE: %d\n",
219	421	target, ret);
220		− ret = -EIO;
221	422	break;
222	423	}
223	424
@@ -229,11 +430,13 @@ int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 5 unmodified lines
229	430
230	431	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,
231	432	offset, bp + count, nwrite);
232		− if (ret <= 0) /* stop on error or nothing written */
	433	+ /* QSFPs require a 5-10msec delay after write operations */
	434	+ mdelay(5);
	435	+ if (ret) /* stop on error */
233	436	break;
234	437
235		− count += ret;
236		− addr += ret;
	438	+ count += nwrite;
	439	+ addr += nwrite;
237	440	}
238	441
239	442	if (ret < 0)
@@ -243,7 +446,7 @@ int qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 3 unmodified lines
243	446
244	447	/*
245	448	* Perform a stand-alone single QSFP write. Acquire the resource, do the
246		− * read, then release the resource.
	449	+ * write, then release the resource.
247	450	*/
248	451	int one_qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp,
249	452	int len)
@@ -266,6 +469,8 @@ int one_qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 16 unmodified lines
266	469	* by reading @addr = ((256 * n) + m)
267	470	*
268	471	* Caller must hold the i2c chain resource.
	472	+ *
	473	+ * Return the number of bytes read or -errno.
269	474	*/
270	475	int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp,
271	476	int len)
@@ -273,21 +478,12 @@ int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 1 unmodified line
273	478	int count = 0;
274	479	int offset;
275	480	int nread;
276		− int ret;
	481	+ int ret = 0;
277	482	u8 page;
278	483
279	484	if (!check_chip_resource(ppd->dd, i2c_target(target), __func__))
280	485	return -EACCES;
281	486
282		− /* make sure the TWSI bus is in a sane state */
283		− ret = hfi1_twsi_reset(ppd->dd, target);
284		− if (ret) {
285		− hfi1_dev_porterr(ppd->dd, ppd->port,
286		− "QSFP chain %d read interface reset failed\n",
287		− target);
288		− return ret;
289		− }
290		−
291	487	while (count < len) {
292	488	/*
293	489	* Set the qsfp page based on a zero-based address
@@ -296,11 +492,12 @@ int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 2 unmodified lines
296	492	page = (u8)(addr / QSFP_PAGESIZE);
297	493	ret = __i2c_write(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,
298	494	QSFP_PAGE_SELECT_BYTE_OFFS, &page, 1);
299		− if (ret != 1) {
	495	+ /* QSFPs require a 5-10msec delay after write operations */
	496	+ mdelay(5);
	497	+ if (ret) {
300	498	hfi1_dev_porterr(ppd->dd, ppd->port,
301	499	"QSFP chain %d can't write QSFP_PAGE_SELECT_BYTE: %d\n",
302	500	target, ret);
303		− ret = -EIO;
304	501	break;
305	502	}
306	503
@@ -310,15 +507,13 @@ int qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 3 unmodified lines
310	507	if (((addr % QSFP_RW_BOUNDARY) + nread) > QSFP_RW_BOUNDARY)
311	508	nread = QSFP_RW_BOUNDARY - (addr % QSFP_RW_BOUNDARY);
312	509
313		− /* QSFPs require a 5-10msec delay after write operations */
314		− mdelay(5);
315	510	ret = __i2c_read(ppd, target, QSFP_DEV \| QSFP_OFFSET_SIZE,
316	511	offset, bp + count, nread);
317		− if (ret <= 0) /* stop on error or nothing read */
	512	+ if (ret) /* stop on error */
318	513	break;
319	514
320		− count += ret;
321		− addr += ret;
	515	+ count += nread;
	516	+ addr += nread;
322	517	}
323	518
324	519	if (ret < 0)

drivers/infiniband/hw/hfi1/qsfp.h+3−0

@@ -238,3 +238,6 @@ int one_qsfp_write(struct hfi1_pportdata ppd, u32 target, int addr, void bp, · 237 unmodified lines
238	238	int len);
239	239	int one_qsfp_read(struct hfi1_pportdata ppd, u32 target, int addr, void bp,
240	240	int len);
	241	+struct hfi1_asic_data;
	242	+int set_up_i2c(struct hfi1_devdata dd, struct hfi1_asic_data ad);
	243	+void clean_up_i2c(struct hfi1_devdata dd, struct hfi1_asic_data ad);

drivers/infiniband/hw/hfi1/rc.c+65−25

@@ -477,6 +477,37 @@ int hfi1_make_rc_req(struct rvt_qp qp, struct hfi1_pkt_state ps) · 476 unmodified lines
477	477	qp->s_flags \|= RVT_S_WAIT_FENCE;
478	478	goto bail;
479	479	}
	480	+ /*
	481	+ * Local operations are processed immediately
	482	+ * after all prior requests have completed
	483	+ */
	484	+ if (wqe->wr.opcode == IB_WR_REG_MR \|\|
	485	+ wqe->wr.opcode == IB_WR_LOCAL_INV) {
	486	+ int local_ops = 0;
	487	+ int err = 0;
	488	+
	489	+ if (qp->s_last != qp->s_cur)
	490	+ goto bail;
	491	+ if (++qp->s_cur == qp->s_size)
	492	+ qp->s_cur = 0;
	493	+ if (++qp->s_tail == qp->s_size)
	494	+ qp->s_tail = 0;
	495	+ if (!(wqe->wr.send_flags &
	496	+ RVT_SEND_COMPLETION_ONLY)) {
	497	+ err = rvt_invalidate_rkey(
	498	+ qp,
	499	+ wqe->wr.ex.invalidate_rkey);
	500	+ local_ops = 1;
	501	+ }
	502	+ hfi1_send_complete(qp, wqe,
	503	+ err ? IB_WC_LOC_PROT_ERR
	504	+ : IB_WC_SUCCESS);
	505	+ if (local_ops)
	506	+ atomic_dec(&qp->local_ops_pending);
	507	+ qp->s_hdrwords = 0;
	508	+ goto done_free_tx;
	509	+ }
	510	+
480	511	newreq = 1;
481	512	qp->s_psn = wqe->psn;
482	513	}
@@ -491,6 +522,7 @@ int hfi1_make_rc_req(struct rvt_qp qp, struct hfi1_pkt_state ps) · 8 unmodified lines
491	522	switch (wqe->wr.opcode) {
492	523	case IB_WR_SEND:
493	524	case IB_WR_SEND_WITH_IMM:
	525	+ case IB_WR_SEND_WITH_INV:
494	526	/* If no credit, return. */
495	527	if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
496	528	cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
@@ -504,11 +536,17 @@ int hfi1_make_rc_req(struct rvt_qp qp, struct hfi1_pkt_state ps) · 7 unmodified lines
504	536	}
505	537	if (wqe->wr.opcode == IB_WR_SEND) {
506	538	qp->s_state = OP(SEND_ONLY);
507		− } else {
	539	+ } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
508	540	qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
509	541	/* Immediate data comes after the BTH */
510	542	ohdr->u.imm_data = wqe->wr.ex.imm_data;
511	543	hwords += 1;
	544	+ } else {
	545	+ qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE);
	546	+ /* Invalidate rkey comes after the BTH */
	547	+ ohdr->u.ieth = cpu_to_be32(
	548	+ wqe->wr.ex.invalidate_rkey);
	549	+ hwords += 1;
512	550	}
513	551	if (wqe->wr.send_flags & IB_SEND_SOLICITED)
514	552	bth0 \|= IB_BTH_SOLICITED;
@@ -671,11 +709,16 @@ int hfi1_make_rc_req(struct rvt_qp qp, struct hfi1_pkt_state ps) · 156 unmodified lines
671	709	}
672	710	if (wqe->wr.opcode == IB_WR_SEND) {
673	711	qp->s_state = OP(SEND_LAST);
674		− } else {
	712	+ } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
675	713	qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
676	714	/* Immediate data comes after the BTH */
677	715	ohdr->u.imm_data = wqe->wr.ex.imm_data;
678	716	hwords += 1;
	717	+ } else {
	718	+ qp->s_state = OP(SEND_LAST_WITH_INVALIDATE);
	719	+ /* invalidate data comes after the BTH */
	720	+ ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey);
	721	+ hwords += 1;
679	722	}
680	723	if (wqe->wr.send_flags & IB_SEND_SOLICITED)
681	724	bth0 \|= IB_BTH_SOLICITED;
@@ -1047,7 +1090,7 @@ void hfi1_rc_timeout(unsigned long arg) · 365 unmodified lines
1047	1090	ibp->rvp.n_rc_timeouts++;
1048	1091	qp->s_flags &= ~RVT_S_TIMER;
1049	1092	del_timer(&qp->s_timer);
1050		− trace_hfi1_rc_timeout(qp, qp->s_last_psn + 1);
	1093	+ trace_hfi1_timeout(qp, qp->s_last_psn + 1);
1051	1094	restart_rc(qp, qp->s_last_psn + 1, 1);
1052	1095	hfi1_schedule_send(qp);
1053	1096	}
@@ -1171,7 +1214,7 @@ void hfi1_rc_send_complete(struct rvt_qp qp, struct hfi1_ib_header hdr) · 117 unmodified lines
1171	1214	* If we were waiting for sends to complete before re-sending,
1172	1215	* and they are now complete, restart sending.
1173	1216	*/
1174		− trace_hfi1_rc_sendcomplete(qp, psn);
	1217	+ trace_hfi1_sendcomplete(qp, psn);
1175	1218	if (qp->s_flags & RVT_S_WAIT_PSN &&
1176	1219	cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1177	1220	qp->s_flags &= ~RVT_S_WAIT_PSN;
@@ -1567,7 +1610,7 @@ static void rc_rcv_resp(struct hfi1_ibport *ibp, · 389 unmodified lines
1567	1610
1568	1611	spin_lock_irqsave(&qp->s_lock, flags);
1569	1612
1570		− trace_hfi1_rc_ack(qp, psn);
	1613	+ trace_hfi1_ack(qp, psn);
1571	1614
1572	1615	/* Ignore invalid responses. */
1573	1616	smp_read_barrier_depends(); /* see post_one_send */
@@ -1782,7 +1825,7 @@ static noinline int rc_rcv_error(struct hfi1_other_headers ohdr, void data, · 208 unmodified lines
1782	1825	u8 i, prev;
1783	1826	int old_req;
1784	1827
1785		− trace_hfi1_rc_rcv_error(qp, psn);
	1828	+ trace_hfi1_rcv_error(qp, psn);
1786	1829	if (diff > 0) {
1787	1830	/*
1788	1831	* Packet sequence error.
@@ -2086,7 +2129,6 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) · 297 unmodified lines
2086	2129	u32 tlen = packet->tlen;
2087	2130	struct rvt_qp *qp = packet->qp;
2088	2131	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
2089		− struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
2090	2132	struct hfi1_other_headers *ohdr = packet->ohdr;
2091	2133	u32 bth0, opcode;
2092	2134	u32 hdrsize = packet->hlen;
@@ -2097,30 +2139,15 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) · 4 unmodified lines
2097	2139	int diff;
2098	2140	struct ib_reth *reth;
2099	2141	unsigned long flags;
2100		− u32 bth1;
2101	2142	int ret, is_fecn = 0;
2102	2143	int copy_last = 0;
	2144	+ u32 rkey;
2103	2145
2104	2146	bth0 = be32_to_cpu(ohdr->bth[0]);
2105	2147	if (hfi1_ruc_check_hdr(ibp, hdr, rcv_flags & HFI1_HAS_GRH, qp, bth0))
2106	2148	return;
2107	2149
2108		− bth1 = be32_to_cpu(ohdr->bth[1]);
2109		− if (unlikely(bth1 & (HFI1_BECN_SMASK \| HFI1_FECN_SMASK))) {
2110		− if (bth1 & HFI1_BECN_SMASK) {
2111		− u16 rlid = qp->remote_ah_attr.dlid;
2112		− u32 lqpn, rqpn;
2113		−
2114		− lqpn = qp->ibqp.qp_num;
2115		− rqpn = qp->remote_qpn;
2116		− process_becn(
2117		− ppd,
2118		− qp->remote_ah_attr.sl,
2119		− rlid, lqpn, rqpn,
2120		− IB_CC_SVCTYPE_RC);
2121		− }
2122		− is_fecn = bth1 & HFI1_FECN_SMASK;
2123		− }
	2150	+ is_fecn = process_ecn(qp, packet, false);
2124	2151
2125	2152	psn = be32_to_cpu(ohdr->bth[2]);
2126	2153	opcode = (bth0 >> 24) & 0xff;
@@ -2154,7 +2181,8 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) · 27 unmodified lines
2154	2181	case OP(SEND_MIDDLE):
2155	2182	if (opcode == OP(SEND_MIDDLE) \|\|
2156	2183	opcode == OP(SEND_LAST) \|\|
2157		− opcode == OP(SEND_LAST_WITH_IMMEDIATE))
	2184	+ opcode == OP(SEND_LAST_WITH_IMMEDIATE) \|\|
	2185	+ opcode == OP(SEND_LAST_WITH_INVALIDATE))
2158	2186	break;
2159	2187	goto nack_inv;
2160	2188
@@ -2170,6 +2198,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) · 9 unmodified lines
2170	2198	if (opcode == OP(SEND_MIDDLE) \|\|
2171	2199	opcode == OP(SEND_LAST) \|\|
2172	2200	opcode == OP(SEND_LAST_WITH_IMMEDIATE) \|\|
	2201	+ opcode == OP(SEND_LAST_WITH_INVALIDATE) \|\|
2173	2202	opcode == OP(RDMA_WRITE_MIDDLE) \|\|
2174	2203	opcode == OP(RDMA_WRITE_LAST) \|\|
2175	2204	opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
@@ -2218,6 +2247,7 @@ send_middle: · 42 unmodified lines
2218	2247
2219	2248	case OP(SEND_ONLY):
2220	2249	case OP(SEND_ONLY_WITH_IMMEDIATE):
	2250	+ case OP(SEND_ONLY_WITH_INVALIDATE):
2221	2251	ret = hfi1_rvt_get_rwqe(qp, 0);
2222	2252	if (ret < 0)
2223	2253	goto nack_op_err;
@@ -2226,12 +2256,22 @@ send_middle: · 2 unmodified lines
2226	2256	qp->r_rcv_len = 0;
2227	2257	if (opcode == OP(SEND_ONLY))
2228	2258	goto no_immediate_data;
	2259	+ if (opcode == OP(SEND_ONLY_WITH_INVALIDATE))
	2260	+ goto send_last_inv;
2229	2261	/* FALLTHROUGH for SEND_ONLY_WITH_IMMEDIATE */
2230	2262	case OP(SEND_LAST_WITH_IMMEDIATE):
2231	2263	send_last_imm:
2232	2264	wc.ex.imm_data = ohdr->u.imm_data;
2233	2265	wc.wc_flags = IB_WC_WITH_IMM;
2234	2266	goto send_last;
	2267	+ case OP(SEND_LAST_WITH_INVALIDATE):
	2268	+send_last_inv:
	2269	+ rkey = be32_to_cpu(ohdr->u.ieth);
	2270	+ if (rvt_invalidate_rkey(qp, rkey))
	2271	+ goto no_immediate_data;
	2272	+ wc.ex.invalidate_rkey = rkey;
	2273	+ wc.wc_flags = IB_WC_WITH_INVALIDATE;
	2274	+ goto send_last;
2235	2275	case OP(RDMA_WRITE_LAST):
2236	2276	copy_last = ibpd_to_rvtpd(qp->ibqp.pd)->user;
2237	2277	/* fall through */

drivers/infiniband/hw/hfi1/ruc.c+40−17

@@ -372,6 +372,7 @@ static void ruc_loopback(struct rvt_qp *sqp) · 371 unmodified lines
372	372	int ret;
373	373	int copy_last = 0;
374	374	u32 to;
	375	+ int local_ops = 0;
375	376
376	377	rcu_read_lock();
377	378
@@ -440,11 +441,31 @@ again: · 62 unmodified lines
440	441	sqp->s_sge.num_sge = wqe->wr.num_sge;
441	442	sqp->s_len = wqe->length;
442	443	switch (wqe->wr.opcode) {
	444	+ case IB_WR_REG_MR:
	445	+ goto send_comp;
	446	+
	447	+ case IB_WR_LOCAL_INV:
	448	+ if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
	449	+ if (rvt_invalidate_rkey(sqp,
	450	+ wqe->wr.ex.invalidate_rkey))
	451	+ send_status = IB_WC_LOC_PROT_ERR;
	452	+ local_ops = 1;
	453	+ }
	454	+ goto send_comp;
	455	+
	456	+ case IB_WR_SEND_WITH_INV:
	457	+ if (!rvt_invalidate_rkey(qp, wqe->wr.ex.invalidate_rkey)) {
	458	+ wc.wc_flags = IB_WC_WITH_INVALIDATE;
	459	+ wc.ex.invalidate_rkey = wqe->wr.ex.invalidate_rkey;
	460	+ }
	461	+ goto send;
	462	+
443	463	case IB_WR_SEND_WITH_IMM:
444	464	wc.wc_flags = IB_WC_WITH_IMM;
445	465	wc.ex.imm_data = wqe->wr.ex.imm_data;
446	466	/* FALLTHROUGH */
447	467	case IB_WR_SEND:
	468	+send:
448	469	ret = hfi1_rvt_get_rwqe(qp, 0);
449	470	if (ret < 0)
450	471	goto op_err;
@@ -583,6 +604,10 @@ send_comp: · 132 unmodified lines
583	604	flush_send:
584	605	sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
585	606	hfi1_send_complete(sqp, wqe, send_status);
	607	+ if (local_ops) {
	608	+ atomic_dec(&sqp->local_ops_pending);
	609	+ local_ops = 0;
	610	+ }
586	611	goto again;
587	612
588	613	rnr_nak:
@@ -683,10 +708,10 @@ u32 hfi1_make_grh(struct hfi1_ibport ibp, struct ib_grh hdr, · 94 unmodified lines
683	708	return sizeof(struct ib_grh) / sizeof(u32);
684	709	}
685	710
686		−#define BTH2_OFFSET (offsetof(struct hfi1_pio_header, hdr.u.oth.bth[2]) / 4)
	711	+#define BTH2_OFFSET (offsetof(struct hfi1_sdma_header, hdr.u.oth.bth[2]) / 4)
687	712
688	713	/**
689		− * build_ahg - create ahg in s_hdr
	714	+ * build_ahg - create ahg in s_ahg
690	715	* @qp: a pointer to QP
691	716	* @npsn: the next PSN for the request/response
692	717	*
@@ -708,19 +733,18 @@ static inline void build_ahg(struct rvt_qp *qp, u32 npsn) · 15 unmodified lines
708	733	qp->s_ahgidx = sdma_ahg_alloc(priv->s_sde);
709	734	if (qp->s_ahgidx >= 0) {
710	735	qp->s_ahgpsn = npsn;
711		− priv->s_hdr->tx_flags \|= SDMA_TXREQ_F_AHG_COPY;
	736	+ priv->s_ahg->tx_flags \|= SDMA_TXREQ_F_AHG_COPY;
712	737	/* save to protect a change in another thread */
713		− priv->s_hdr->sde = priv->s_sde;
714		− priv->s_hdr->ahgidx = qp->s_ahgidx;
	738	+ priv->s_ahg->ahgidx = qp->s_ahgidx;
715	739	qp->s_flags \|= RVT_S_AHG_VALID;
716	740	}
717	741	} else {
718	742	/* subsequent middle after valid */
719	743	if (qp->s_ahgidx >= 0) {
720		− priv->s_hdr->tx_flags \|= SDMA_TXREQ_F_USE_AHG;
721		− priv->s_hdr->ahgidx = qp->s_ahgidx;
722		− priv->s_hdr->ahgcount++;
723		− priv->s_hdr->ahgdesc[0] =
	744	+ priv->s_ahg->tx_flags \|= SDMA_TXREQ_F_USE_AHG;
	745	+ priv->s_ahg->ahgidx = qp->s_ahgidx;
	746	+ priv->s_ahg->ahgcount++;
	747	+ priv->s_ahg->ahgdesc[0] =
724	748	sdma_build_ahg_descriptor(
725	749	(__force u16)cpu_to_be16((u16)npsn),
726	750	BTH2_OFFSET,
@@ -728,8 +752,8 @@ static inline void build_ahg(struct rvt_qp *qp, u32 npsn) · 1 unmodified line
728	752	16);
729	753	if ((npsn & 0xffff0000) !=
730	754	(qp->s_ahgpsn & 0xffff0000)) {
731		− priv->s_hdr->ahgcount++;
732		− priv->s_hdr->ahgdesc[1] =
	755	+ priv->s_ahg->ahgcount++;
	756	+ priv->s_ahg->ahgdesc[1] =
733	757	sdma_build_ahg_descriptor(
734	758	(__force u16)cpu_to_be16(
735	759	(u16)(npsn >> 16)),
@@ -766,7 +790,7 @@ void hfi1_make_ruc_header(struct rvt_qp qp, struct hfi1_other_headers ohdr, · 30 unmodified lines
766	790	}
767	791	lrh0 \|= (priv->s_sc & 0xf) << 12 \| (qp->remote_ah_attr.sl & 0xf) << 4;
768	792	/*
769		− * reset s_hdr/AHG fields
	793	+ * reset s_ahg/AHG fields
770	794	*
771	795	* This insures that the ahgentry/ahgcount
772	796	* are at a non-AHG default to protect
@@ -776,10 +800,9 @@ void hfi1_make_ruc_header(struct rvt_qp qp, struct hfi1_other_headers ohdr, · 3 unmodified lines
776	800	* build_ahg() will modify as appropriate
777	801	* to use the AHG feature.
778	802	*/
779		− priv->s_hdr->tx_flags = 0;
780		− priv->s_hdr->ahgcount = 0;
781		− priv->s_hdr->ahgidx = 0;
782		− priv->s_hdr->sde = NULL;
	803	+ priv->s_ahg->tx_flags = 0;
	804	+ priv->s_ahg->ahgcount = 0;
	805	+ priv->s_ahg->ahgidx = 0;
783	806	if (qp->s_mig_state == IB_MIG_MIGRATED)
784	807	bth0 \|= IB_BTH_MIG_REQ;
785	808	else
@@ -890,7 +913,7 @@ void hfi1_do_send(struct rvt_qp *qp) · 104 unmodified lines
890	913	*/
891	914	if (hfi1_verbs_send(qp, &ps))
892	915	return;
893		− /* Record that s_hdr is empty. */
	916	+ /* Record that s_ahg is empty. */
894	917	qp->s_hdrwords = 0;
895	918	/* allow other tasks to run */
896	919	if (unlikely(time_after(jiffies, timeout))) {

drivers/infiniband/hw/hfi1/sysfs.c+25−0

@@ -49,6 +49,7 @@ · 48 unmodified lines
49	49	#include "hfi.h"
50	50	#include "mad.h"
51	51	#include "trace.h"
	52	+#include "affinity.h"
52	53
53	54	/*
54	55	* Start of per-port congestion control structures and support code
@@ -622,6 +623,27 @@ static ssize_t show_tempsense(struct device *device, · 567 unmodified lines
622	623	return ret;
623	624	}
624	625
	626	+static ssize_t show_sdma_affinity(struct device *device,
	627	+ struct device_attribute attr, char buf)
	628	+{
	629	+ struct hfi1_ibdev *dev =
	630	+ container_of(device, struct hfi1_ibdev, rdi.ibdev.dev);
	631	+ struct hfi1_devdata *dd = dd_from_dev(dev);
	632	+
	633	+ return hfi1_get_sdma_affinity(dd, buf);
	634	+}
	635	+
	636	+static ssize_t store_sdma_affinity(struct device *device,
	637	+ struct device_attribute *attr,
	638	+ const char *buf, size_t count)
	639	+{
	640	+ struct hfi1_ibdev *dev =
	641	+ container_of(device, struct hfi1_ibdev, rdi.ibdev.dev);
	642	+ struct hfi1_devdata *dd = dd_from_dev(dev);
	643	+
	644	+ return hfi1_set_sdma_affinity(dd, buf, count);
	645	+}
	646	+
625	647	/*
626	648	* end of per-unit (or driver, in some cases, but replicated
627	649	* per unit) functions
@@ -636,6 +658,8 @@ static DEVICE_ATTR(serial, S_IRUGO, show_serial, NULL); · 8 unmodified lines
636	658	static DEVICE_ATTR(boardversion, S_IRUGO, show_boardversion, NULL);
637	659	static DEVICE_ATTR(tempsense, S_IRUGO, show_tempsense, NULL);
638	660	static DEVICE_ATTR(chip_reset, S_IWUSR, NULL, store_chip_reset);
	661	+static DEVICE_ATTR(sdma_affinity, S_IWUSR \| S_IRUGO, show_sdma_affinity,
	662	+ store_sdma_affinity);
639	663
640	664	static struct device_attribute *hfi1_attributes[] = {
641	665	&dev_attr_hw_rev,
@@ -646,6 +670,7 @@ static struct device_attribute *hfi1_attributes[] = { · 4 unmodified lines
646	670	&dev_attr_boardversion,
647	671	&dev_attr_tempsense,
648	672	&dev_attr_chip_reset,
	673	+ &dev_attr_sdma_affinity,
649	674	};
650	675
651	676	int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,

drivers/infiniband/hw/hfi1/trace.h+7−1326

@@ -44,1329 +44,10 @@ · 43 unmodified lines
44	44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45	45	*
46	46	*/
47		−#undef TRACE_SYSTEM_VAR
48		−#define TRACE_SYSTEM_VAR hfi1
49		−
50		−#if !defined(__HFI1_TRACE_H) \|\| defined(TRACE_HEADER_MULTI_READ)
51		−#define __HFI1_TRACE_H
52		−
53		−#include <linux/tracepoint.h>
54		−#include <linux/trace_seq.h>
55		−
56		−#include "hfi.h"
57		−#include "mad.h"
58		−#include "sdma.h"
59		−
60		−#define DD_DEV_ENTRY(dd) __string(dev, dev_name(&(dd)->pcidev->dev))
61		−#define DD_DEV_ASSIGN(dd) __assign_str(dev, dev_name(&(dd)->pcidev->dev))
62		−
63		−#define packettype_name(etype) { RHF_RCV_TYPE_##etype, #etype }
64		−#define show_packettype(etype) \
65		−__print_symbolic(etype, \
66		− packettype_name(EXPECTED), \
67		− packettype_name(EAGER), \
68		− packettype_name(IB), \
69		− packettype_name(ERROR), \
70		− packettype_name(BYPASS))
71		−
72		−#undef TRACE_SYSTEM
73		−#define TRACE_SYSTEM hfi1_rx
74		−
75		−TRACE_EVENT(hfi1_rcvhdr,
76		− TP_PROTO(struct hfi1_devdata *dd,
77		− u32 ctxt,
78		− u64 eflags,
79		− u32 etype,
80		− u32 hlen,
81		− u32 tlen,
82		− u32 updegr,
83		− u32 etail
84		− ),
85		− TP_ARGS(dd, ctxt, eflags, etype, hlen, tlen, updegr, etail),
86		− TP_STRUCT__entry(DD_DEV_ENTRY(dd)
87		− __field(u64, eflags)
88		− __field(u32, ctxt)
89		− __field(u32, etype)
90		− __field(u32, hlen)
91		− __field(u32, tlen)
92		− __field(u32, updegr)
93		− __field(u32, etail)
94		− ),
95		− TP_fast_assign(DD_DEV_ASSIGN(dd);
96		− __entry->eflags = eflags;
97		− __entry->ctxt = ctxt;
98		− __entry->etype = etype;
99		− __entry->hlen = hlen;
100		− __entry->tlen = tlen;
101		− __entry->updegr = updegr;
102		− __entry->etail = etail;
103		− ),
104		− TP_printk(
105		− "[%s] ctxt %d eflags 0x%llx etype %d,%s hlen %d tlen %d updegr %d etail %d",
106		− __get_str(dev),
107		− __entry->ctxt,
108		− __entry->eflags,
109		− __entry->etype, show_packettype(__entry->etype),
110		− __entry->hlen,
111		− __entry->tlen,
112		− __entry->updegr,
113		− __entry->etail
114		− )
115		−);
116		−
117		−TRACE_EVENT(hfi1_receive_interrupt,
118		− TP_PROTO(struct hfi1_devdata *dd, u32 ctxt),
119		− TP_ARGS(dd, ctxt),
120		− TP_STRUCT__entry(DD_DEV_ENTRY(dd)
121		− __field(u32, ctxt)
122		− __field(u8, slow_path)
123		− __field(u8, dma_rtail)
124		− ),
125		− TP_fast_assign(DD_DEV_ASSIGN(dd);
126		− __entry->ctxt = ctxt;
127		− if (dd->rcd[ctxt]->do_interrupt ==
128		− &handle_receive_interrupt) {
129		− __entry->slow_path = 1;
130		− __entry->dma_rtail = 0xFF;
131		− } else if (dd->rcd[ctxt]->do_interrupt ==
132		− &handle_receive_interrupt_dma_rtail){
133		− __entry->dma_rtail = 1;
134		− __entry->slow_path = 0;
135		− } else if (dd->rcd[ctxt]->do_interrupt ==
136		− &handle_receive_interrupt_nodma_rtail) {
137		− __entry->dma_rtail = 0;
138		− __entry->slow_path = 0;
139		− }
140		− ),
141		− TP_printk("[%s] ctxt %d SlowPath: %d DmaRtail: %d",
142		− __get_str(dev),
143		− __entry->ctxt,
144		− __entry->slow_path,
145		− __entry->dma_rtail
146		− )
147		−);
148		−
149		−TRACE_EVENT(hfi1_exp_tid_reg,
150		− TP_PROTO(unsigned ctxt, u16 subctxt, u32 rarr,
151		− u32 npages, unsigned long va, unsigned long pa,
152		− dma_addr_t dma),
153		− TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
154		− TP_STRUCT__entry(
155		− __field(unsigned, ctxt)
156		− __field(u16, subctxt)
157		− __field(u32, rarr)
158		− __field(u32, npages)
159		− __field(unsigned long, va)
160		− __field(unsigned long, pa)
161		− __field(dma_addr_t, dma)
162		− ),
163		− TP_fast_assign(
164		− __entry->ctxt = ctxt;
165		− __entry->subctxt = subctxt;
166		− __entry->rarr = rarr;
167		− __entry->npages = npages;
168		− __entry->va = va;
169		− __entry->pa = pa;
170		− __entry->dma = dma;
171		− ),
172		− TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
173		− __entry->ctxt,
174		− __entry->subctxt,
175		− __entry->rarr,
176		− __entry->npages,
177		− __entry->pa,
178		− __entry->va,
179		− __entry->dma
180		− )
181		− );
182		−
183		−TRACE_EVENT(hfi1_exp_tid_unreg,
184		− TP_PROTO(unsigned ctxt, u16 subctxt, u32 rarr, u32 npages,
185		− unsigned long va, unsigned long pa, dma_addr_t dma),
186		− TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
187		− TP_STRUCT__entry(
188		− __field(unsigned, ctxt)
189		− __field(u16, subctxt)
190		− __field(u32, rarr)
191		− __field(u32, npages)
192		− __field(unsigned long, va)
193		− __field(unsigned long, pa)
194		− __field(dma_addr_t, dma)
195		− ),
196		− TP_fast_assign(
197		− __entry->ctxt = ctxt;
198		− __entry->subctxt = subctxt;
199		− __entry->rarr = rarr;
200		− __entry->npages = npages;
201		− __entry->va = va;
202		− __entry->pa = pa;
203		− __entry->dma = dma;
204		− ),
205		− TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
206		− __entry->ctxt,
207		− __entry->subctxt,
208		− __entry->rarr,
209		− __entry->npages,
210		− __entry->pa,
211		− __entry->va,
212		− __entry->dma
213		− )
214		− );
215		−
216		−TRACE_EVENT(hfi1_exp_tid_inval,
217		− TP_PROTO(unsigned ctxt, u16 subctxt, unsigned long va, u32 rarr,
218		− u32 npages, dma_addr_t dma),
219		− TP_ARGS(ctxt, subctxt, va, rarr, npages, dma),
220		− TP_STRUCT__entry(
221		− __field(unsigned, ctxt)
222		− __field(u16, subctxt)
223		− __field(unsigned long, va)
224		− __field(u32, rarr)
225		− __field(u32, npages)
226		− __field(dma_addr_t, dma)
227		− ),
228		− TP_fast_assign(
229		− __entry->ctxt = ctxt;
230		− __entry->subctxt = subctxt;
231		− __entry->va = va;
232		− __entry->rarr = rarr;
233		− __entry->npages = npages;
234		− __entry->dma = dma;
235		− ),
236		− TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx dma: 0x%llx",
237		− __entry->ctxt,
238		− __entry->subctxt,
239		− __entry->rarr,
240		− __entry->npages,
241		− __entry->va,
242		− __entry->dma
243		− )
244		− );
245		−
246		−TRACE_EVENT(hfi1_mmu_invalidate,
247		− TP_PROTO(unsigned ctxt, u16 subctxt, const char *type,
248		− unsigned long start, unsigned long end),
249		− TP_ARGS(ctxt, subctxt, type, start, end),
250		− TP_STRUCT__entry(
251		− __field(unsigned, ctxt)
252		− __field(u16, subctxt)
253		− __string(type, type)
254		− __field(unsigned long, start)
255		− __field(unsigned long, end)
256		− ),
257		− TP_fast_assign(
258		− __entry->ctxt = ctxt;
259		− __entry->subctxt = subctxt;
260		− __assign_str(type, type);
261		− __entry->start = start;
262		− __entry->end = end;
263		− ),
264		− TP_printk("[%3u:%02u] MMU Invalidate (%s) 0x%lx - 0x%lx",
265		− __entry->ctxt,
266		− __entry->subctxt,
267		− __get_str(type),
268		− __entry->start,
269		− __entry->end
270		− )
271		− );
272		−
273		−#undef TRACE_SYSTEM
274		−#define TRACE_SYSTEM hfi1_tx
275		−
276		−TRACE_EVENT(hfi1_piofree,
277		− TP_PROTO(struct send_context *sc, int extra),
278		− TP_ARGS(sc, extra),
279		− TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
280		− __field(u32, sw_index)
281		− __field(u32, hw_context)
282		− __field(int, extra)
283		− ),
284		− TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
285		− __entry->sw_index = sc->sw_index;
286		− __entry->hw_context = sc->hw_context;
287		− __entry->extra = extra;
288		− ),
289		− TP_printk("[%s] ctxt %u(%u) extra %d",
290		− __get_str(dev),
291		− __entry->sw_index,
292		− __entry->hw_context,
293		− __entry->extra
294		− )
295		−);
296		−
297		−TRACE_EVENT(hfi1_wantpiointr,
298		− TP_PROTO(struct send_context *sc, u32 needint, u64 credit_ctrl),
299		− TP_ARGS(sc, needint, credit_ctrl),
300		− TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
301		− __field(u32, sw_index)
302		− __field(u32, hw_context)
303		− __field(u32, needint)
304		− __field(u64, credit_ctrl)
305		− ),
306		− TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
307		− __entry->sw_index = sc->sw_index;
308		− __entry->hw_context = sc->hw_context;
309		− __entry->needint = needint;
310		− __entry->credit_ctrl = credit_ctrl;
311		− ),
312		− TP_printk("[%s] ctxt %u(%u) on %d credit_ctrl 0x%llx",
313		− __get_str(dev),
314		− __entry->sw_index,
315		− __entry->hw_context,
316		− __entry->needint,
317		− (unsigned long long)__entry->credit_ctrl
318		− )
319		−);
320		−
321		−DECLARE_EVENT_CLASS(hfi1_qpsleepwakeup_template,
322		− TP_PROTO(struct rvt_qp *qp, u32 flags),
323		− TP_ARGS(qp, flags),
324		− TP_STRUCT__entry(
325		− DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
326		− __field(u32, qpn)
327		− __field(u32, flags)
328		− __field(u32, s_flags)
329		− ),
330		− TP_fast_assign(
331		− DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device))
332		− __entry->flags = flags;
333		− __entry->qpn = qp->ibqp.qp_num;
334		− __entry->s_flags = qp->s_flags;
335		− ),
336		− TP_printk(
337		− "[%s] qpn 0x%x flags 0x%x s_flags 0x%x",
338		− __get_str(dev),
339		− __entry->qpn,
340		− __entry->flags,
341		− __entry->s_flags
342		− )
343		−);
344		−
345		−DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpwakeup,
346		− TP_PROTO(struct rvt_qp *qp, u32 flags),
347		− TP_ARGS(qp, flags));
348		−
349		−DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpsleep,
350		− TP_PROTO(struct rvt_qp *qp, u32 flags),
351		− TP_ARGS(qp, flags));
352		−
353		−#undef TRACE_SYSTEM
354		−#define TRACE_SYSTEM hfi1_ibhdrs
355		−
356		−u8 ibhdr_exhdr_len(struct hfi1_ib_header *hdr);
357		−const char parse_everbs_hdrs(struct trace_seq p, u8 opcode, void *ehdrs);
358		−
359		−#define __parse_ib_ehdrs(op, ehdrs) parse_everbs_hdrs(p, op, ehdrs)
360		−
361		−const char parse_sdma_flags(struct trace_seq p, u64 desc0, u64 desc1);
362		−
363		−#define __parse_sdma_flags(desc0, desc1) parse_sdma_flags(p, desc0, desc1)
364		−
365		−#define lrh_name(lrh) { HFI1_##lrh, #lrh }
366		−#define show_lnh(lrh) \
367		−__print_symbolic(lrh, \
368		− lrh_name(LRH_BTH), \
369		− lrh_name(LRH_GRH))
370		−
371		−#define ib_opcode_name(opcode) { IB_OPCODE_##opcode, #opcode }
372		−#define show_ib_opcode(opcode) \
373		−__print_symbolic(opcode, \
374		− ib_opcode_name(RC_SEND_FIRST), \
375		− ib_opcode_name(RC_SEND_MIDDLE), \
376		− ib_opcode_name(RC_SEND_LAST), \
377		− ib_opcode_name(RC_SEND_LAST_WITH_IMMEDIATE), \
378		− ib_opcode_name(RC_SEND_ONLY), \
379		− ib_opcode_name(RC_SEND_ONLY_WITH_IMMEDIATE), \
380		− ib_opcode_name(RC_RDMA_WRITE_FIRST), \
381		− ib_opcode_name(RC_RDMA_WRITE_MIDDLE), \
382		− ib_opcode_name(RC_RDMA_WRITE_LAST), \
383		− ib_opcode_name(RC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
384		− ib_opcode_name(RC_RDMA_WRITE_ONLY), \
385		− ib_opcode_name(RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
386		− ib_opcode_name(RC_RDMA_READ_REQUEST), \
387		− ib_opcode_name(RC_RDMA_READ_RESPONSE_FIRST), \
388		− ib_opcode_name(RC_RDMA_READ_RESPONSE_MIDDLE), \
389		− ib_opcode_name(RC_RDMA_READ_RESPONSE_LAST), \
390		− ib_opcode_name(RC_RDMA_READ_RESPONSE_ONLY), \
391		− ib_opcode_name(RC_ACKNOWLEDGE), \
392		− ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE), \
393		− ib_opcode_name(RC_COMPARE_SWAP), \
394		− ib_opcode_name(RC_FETCH_ADD), \
395		− ib_opcode_name(RC_SEND_LAST_WITH_INVALIDATE), \
396		− ib_opcode_name(RC_SEND_ONLY_WITH_INVALIDATE), \
397		− ib_opcode_name(UC_SEND_FIRST), \
398		− ib_opcode_name(UC_SEND_MIDDLE), \
399		− ib_opcode_name(UC_SEND_LAST), \
400		− ib_opcode_name(UC_SEND_LAST_WITH_IMMEDIATE), \
401		− ib_opcode_name(UC_SEND_ONLY), \
402		− ib_opcode_name(UC_SEND_ONLY_WITH_IMMEDIATE), \
403		− ib_opcode_name(UC_RDMA_WRITE_FIRST), \
404		− ib_opcode_name(UC_RDMA_WRITE_MIDDLE), \
405		− ib_opcode_name(UC_RDMA_WRITE_LAST), \
406		− ib_opcode_name(UC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
407		− ib_opcode_name(UC_RDMA_WRITE_ONLY), \
408		− ib_opcode_name(UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
409		− ib_opcode_name(UD_SEND_ONLY), \
410		− ib_opcode_name(UD_SEND_ONLY_WITH_IMMEDIATE), \
411		− ib_opcode_name(CNP))
412		−
413		−#define LRH_PRN "vl %d lver %d sl %d lnh %d,%s dlid %.4x len %d slid %.4x"
414		−#define BTH_PRN \
415		− "op 0x%.2x,%s se %d m %d pad %d tver %d pkey 0x%.4x " \
416		− "f %d b %d qpn 0x%.6x a %d psn 0x%.8x"
417		−#define EHDR_PRN "%s"
418		−
419		−DECLARE_EVENT_CLASS(hfi1_ibhdr_template,
420		− TP_PROTO(struct hfi1_devdata *dd,
421		− struct hfi1_ib_header *hdr),
422		− TP_ARGS(dd, hdr),
423		− TP_STRUCT__entry(
424		− DD_DEV_ENTRY(dd)
425		− /* LRH */
426		− __field(u8, vl)
427		− __field(u8, lver)
428		− __field(u8, sl)
429		− __field(u8, lnh)
430		− __field(u16, dlid)
431		− __field(u16, len)
432		− __field(u16, slid)
433		− /* BTH */
434		− __field(u8, opcode)
435		− __field(u8, se)
436		− __field(u8, m)
437		− __field(u8, pad)
438		− __field(u8, tver)
439		− __field(u16, pkey)
440		− __field(u8, f)
441		− __field(u8, b)
442		− __field(u32, qpn)
443		− __field(u8, a)
444		− __field(u32, psn)
445		− /* extended headers */
446		− __dynamic_array(u8, ehdrs, ibhdr_exhdr_len(hdr))
447		− ),
448		− TP_fast_assign(
449		− struct hfi1_other_headers *ohdr;
450		−
451		− DD_DEV_ASSIGN(dd);
452		− /* LRH */
453		− __entry->vl =
454		− (u8)(be16_to_cpu(hdr->lrh[0]) >> 12);
455		− __entry->lver =
456		− (u8)(be16_to_cpu(hdr->lrh[0]) >> 8) & 0xf;
457		− __entry->sl =
458		− (u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
459		− __entry->lnh =
460		− (u8)(be16_to_cpu(hdr->lrh[0]) & 3);
461		− __entry->dlid =
462		− be16_to_cpu(hdr->lrh[1]);
463		− /* allow for larger len */
464		− __entry->len =
465		− be16_to_cpu(hdr->lrh[2]);
466		− __entry->slid =
467		− be16_to_cpu(hdr->lrh[3]);
468		− /* BTH */
469		− if (__entry->lnh == HFI1_LRH_BTH)
470		− ohdr = &hdr->u.oth;
471		− else
472		− ohdr = &hdr->u.l.oth;
473		− __entry->opcode =
474		− (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
475		− __entry->se =
476		− (be32_to_cpu(ohdr->bth[0]) >> 23) & 1;
477		− __entry->m =
478		− (be32_to_cpu(ohdr->bth[0]) >> 22) & 1;
479		− __entry->pad =
480		− (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
481		− __entry->tver =
482		− (be32_to_cpu(ohdr->bth[0]) >> 16) & 0xf;
483		− __entry->pkey =
484		− be32_to_cpu(ohdr->bth[0]) & 0xffff;
485		− __entry->f =
486		− (be32_to_cpu(ohdr->bth[1]) >> HFI1_FECN_SHIFT) &
487		− HFI1_FECN_MASK;
488		− __entry->b =
489		− (be32_to_cpu(ohdr->bth[1]) >> HFI1_BECN_SHIFT) &
490		− HFI1_BECN_MASK;
491		− __entry->qpn =
492		− be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
493		− __entry->a =
494		− (be32_to_cpu(ohdr->bth[2]) >> 31) & 1;
495		− /* allow for larger PSN */
496		− __entry->psn =
497		− be32_to_cpu(ohdr->bth[2]) & 0x7fffffff;
498		− /* extended headers */
499		− memcpy(__get_dynamic_array(ehdrs), &ohdr->u,
500		− ibhdr_exhdr_len(hdr));
501		− ),
502		− TP_printk("[%s] " LRH_PRN " " BTH_PRN " " EHDR_PRN,
503		− __get_str(dev),
504		− /* LRH */
505		− __entry->vl,
506		− __entry->lver,
507		− __entry->sl,
508		− __entry->lnh, show_lnh(__entry->lnh),
509		− __entry->dlid,
510		− __entry->len,
511		− __entry->slid,
512		− /* BTH */
513		− __entry->opcode, show_ib_opcode(__entry->opcode),
514		− __entry->se,
515		− __entry->m,
516		− __entry->pad,
517		− __entry->tver,
518		− __entry->pkey,
519		− __entry->f,
520		− __entry->b,
521		− __entry->qpn,
522		− __entry->a,
523		− __entry->psn,
524		− /* extended headers */
525		− __parse_ib_ehdrs(
526		− __entry->opcode,
527		− (void *)__get_dynamic_array(ehdrs))
528		− )
529		−);
530		−
531		−DEFINE_EVENT(hfi1_ibhdr_template, input_ibhdr,
532		− TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
533		− TP_ARGS(dd, hdr));
534		−
535		−DEFINE_EVENT(hfi1_ibhdr_template, pio_output_ibhdr,
536		− TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
537		− TP_ARGS(dd, hdr));
538		−
539		−DEFINE_EVENT(hfi1_ibhdr_template, ack_output_ibhdr,
540		− TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
541		− TP_ARGS(dd, hdr));
542		−
543		−DEFINE_EVENT(hfi1_ibhdr_template, sdma_output_ibhdr,
544		− TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
545		− TP_ARGS(dd, hdr));
546		−
547		−#define SNOOP_PRN \
548		− "slid %.4x dlid %.4x qpn 0x%.6x opcode 0x%.2x,%s " \
549		− "svc lvl %d pkey 0x%.4x [header = %d bytes] [data = %d bytes]"
550		−
551		−#undef TRACE_SYSTEM
552		−#define TRACE_SYSTEM hfi1_snoop
553		−
554		−TRACE_EVENT(snoop_capture,
555		− TP_PROTO(struct hfi1_devdata *dd,
556		− int hdr_len,
557		− struct hfi1_ib_header *hdr,
558		− int data_len,
559		− void *data),
560		− TP_ARGS(dd, hdr_len, hdr, data_len, data),
561		− TP_STRUCT__entry(
562		− DD_DEV_ENTRY(dd)
563		− __field(u16, slid)
564		− __field(u16, dlid)
565		− __field(u32, qpn)
566		− __field(u8, opcode)
567		− __field(u8, sl)
568		− __field(u16, pkey)
569		− __field(u32, hdr_len)
570		− __field(u32, data_len)
571		− __field(u8, lnh)
572		− __dynamic_array(u8, raw_hdr, hdr_len)
573		− __dynamic_array(u8, raw_pkt, data_len)
574		− ),
575		− TP_fast_assign(
576		− struct hfi1_other_headers *ohdr;
577		−
578		− __entry->lnh = (u8)(be16_to_cpu(hdr->lrh[0]) & 3);
579		− if (__entry->lnh == HFI1_LRH_BTH)
580		− ohdr = &hdr->u.oth;
581		− else
582		− ohdr = &hdr->u.l.oth;
583		− DD_DEV_ASSIGN(dd);
584		− __entry->slid = be16_to_cpu(hdr->lrh[3]);
585		− __entry->dlid = be16_to_cpu(hdr->lrh[1]);
586		− __entry->qpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
587		− __entry->opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
588		− __entry->sl = (u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
589		− __entry->pkey = be32_to_cpu(ohdr->bth[0]) & 0xffff;
590		− __entry->hdr_len = hdr_len;
591		− __entry->data_len = data_len;
592		− memcpy(__get_dynamic_array(raw_hdr), hdr, hdr_len);
593		− memcpy(__get_dynamic_array(raw_pkt), data, data_len);
594		− ),
595		− TP_printk(
596		− "[%s] " SNOOP_PRN,
597		− __get_str(dev),
598		− __entry->slid,
599		− __entry->dlid,
600		− __entry->qpn,
601		− __entry->opcode,
602		− show_ib_opcode(__entry->opcode),
603		− __entry->sl,
604		− __entry->pkey,
605		− __entry->hdr_len,
606		− __entry->data_len
607		− )
608		−);
609		−
610		−#undef TRACE_SYSTEM
611		−#define TRACE_SYSTEM hfi1_ctxts
612		−
613		−#define UCTXT_FMT \
614		− "cred:%u, credaddr:0x%llx, piobase:0x%llx, rcvhdr_cnt:%u, " \
615		− "rcvbase:0x%llx, rcvegrc:%u, rcvegrb:0x%llx"
616		−TRACE_EVENT(hfi1_uctxtdata,
617		− TP_PROTO(struct hfi1_devdata dd, struct hfi1_ctxtdata uctxt),
618		− TP_ARGS(dd, uctxt),
619		− TP_STRUCT__entry(DD_DEV_ENTRY(dd)
620		− __field(unsigned, ctxt)
621		− __field(u32, credits)
622		− __field(u64, hw_free)
623		− __field(u64, piobase)
624		− __field(u16, rcvhdrq_cnt)
625		− __field(u64, rcvhdrq_phys)
626		− __field(u32, eager_cnt)
627		− __field(u64, rcvegr_phys)
628		− ),
629		− TP_fast_assign(DD_DEV_ASSIGN(dd);
630		− __entry->ctxt = uctxt->ctxt;
631		− __entry->credits = uctxt->sc->credits;
632		− __entry->hw_free = (u64)uctxt->sc->hw_free;
633		− __entry->piobase = (u64)uctxt->sc->base_addr;
634		− __entry->rcvhdrq_cnt = uctxt->rcvhdrq_cnt;
635		− __entry->rcvhdrq_phys = uctxt->rcvhdrq_phys;
636		− __entry->eager_cnt = uctxt->egrbufs.alloced;
637		− __entry->rcvegr_phys =
638		− uctxt->egrbufs.rcvtids[0].phys;
639		− ),
640		− TP_printk("[%s] ctxt %u " UCTXT_FMT,
641		− __get_str(dev),
642		− __entry->ctxt,
643		− __entry->credits,
644		− __entry->hw_free,
645		− __entry->piobase,
646		− __entry->rcvhdrq_cnt,
647		− __entry->rcvhdrq_phys,
648		− __entry->eager_cnt,
649		− __entry->rcvegr_phys
650		− )
651		−);
652		−
653		−#define CINFO_FMT \
654		− "egrtids:%u, egr_size:%u, hdrq_cnt:%u, hdrq_size:%u, sdma_ring_size:%u"
655		−TRACE_EVENT(hfi1_ctxt_info,
656		− TP_PROTO(struct hfi1_devdata *dd, unsigned ctxt, unsigned subctxt,
657		− struct hfi1_ctxt_info cinfo),
658		− TP_ARGS(dd, ctxt, subctxt, cinfo),
659		− TP_STRUCT__entry(DD_DEV_ENTRY(dd)
660		− __field(unsigned, ctxt)
661		− __field(unsigned, subctxt)
662		− __field(u16, egrtids)
663		− __field(u16, rcvhdrq_cnt)
664		− __field(u16, rcvhdrq_size)
665		− __field(u16, sdma_ring_size)
666		− __field(u32, rcvegr_size)
667		− ),
668		− TP_fast_assign(DD_DEV_ASSIGN(dd);
669		− __entry->ctxt = ctxt;
670		− __entry->subctxt = subctxt;
671		− __entry->egrtids = cinfo.egrtids;
672		− __entry->rcvhdrq_cnt = cinfo.rcvhdrq_cnt;
673		− __entry->rcvhdrq_size = cinfo.rcvhdrq_entsize;
674		− __entry->sdma_ring_size = cinfo.sdma_ring_size;
675		− __entry->rcvegr_size = cinfo.rcvegr_size;
676		− ),
677		− TP_printk("[%s] ctxt %u:%u " CINFO_FMT,
678		− __get_str(dev),
679		− __entry->ctxt,
680		− __entry->subctxt,
681		− __entry->egrtids,
682		− __entry->rcvegr_size,
683		− __entry->rcvhdrq_cnt,
684		− __entry->rcvhdrq_size,
685		− __entry->sdma_ring_size
686		− )
687		−);
688		−
689		−#undef TRACE_SYSTEM
690		−#define TRACE_SYSTEM hfi1_sma
691		−
692		−#define BCT_FORMAT \
693		− "shared_limit %x vls 0-7 [%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x] 15 [%x,%x]"
694		−
695		−#define BCT(field) \
696		− be16_to_cpu( \
697		− ((struct buffer_control *)__get_dynamic_array(bct))->field \
698		− )
699		−
700		−DECLARE_EVENT_CLASS(hfi1_bct_template,
701		− TP_PROTO(struct hfi1_devdata *dd,
702		− struct buffer_control *bc),
703		− TP_ARGS(dd, bc),
704		− TP_STRUCT__entry(DD_DEV_ENTRY(dd)
705		− __dynamic_array(u8, bct, sizeof(*bc))
706		− ),
707		− TP_fast_assign(DD_DEV_ASSIGN(dd);
708		− memcpy(__get_dynamic_array(bct), bc,
709		− sizeof(*bc));
710		− ),
711		− TP_printk(BCT_FORMAT,
712		− BCT(overall_shared_limit),
713		−
714		− BCT(vl[0].dedicated),
715		− BCT(vl[0].shared),
716		−
717		− BCT(vl[1].dedicated),
718		− BCT(vl[1].shared),
719		−
720		− BCT(vl[2].dedicated),
721		− BCT(vl[2].shared),
722		−
723		− BCT(vl[3].dedicated),
724		− BCT(vl[3].shared),
725		−
726		− BCT(vl[4].dedicated),
727		− BCT(vl[4].shared),
728		−
729		− BCT(vl[5].dedicated),
730		− BCT(vl[5].shared),
731		−
732		− BCT(vl[6].dedicated),
733		− BCT(vl[6].shared),
734		−
735		− BCT(vl[7].dedicated),
736		− BCT(vl[7].shared),
737		−
738		− BCT(vl[15].dedicated),
739		− BCT(vl[15].shared)
740		− )
741		−);
742		−
743		−DEFINE_EVENT(hfi1_bct_template, bct_set,
744		− TP_PROTO(struct hfi1_devdata dd, struct buffer_control bc),
745		− TP_ARGS(dd, bc));
746		−
747		−DEFINE_EVENT(hfi1_bct_template, bct_get,
748		− TP_PROTO(struct hfi1_devdata dd, struct buffer_control bc),
749		− TP_ARGS(dd, bc));
750		−
751		−#undef TRACE_SYSTEM
752		−#define TRACE_SYSTEM hfi1_sdma
753		−
754		−TRACE_EVENT(hfi1_sdma_descriptor,
755		− TP_PROTO(struct sdma_engine *sde,
756		− u64 desc0,
757		− u64 desc1,
758		− u16 e,
759		− void *descp),
760		− TP_ARGS(sde, desc0, desc1, e, descp),
761		− TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
762		− __field(void *, descp)
763		− __field(u64, desc0)
764		− __field(u64, desc1)
765		− __field(u16, e)
766		− __field(u8, idx)
767		− ),
768		− TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
769		− __entry->desc0 = desc0;
770		− __entry->desc1 = desc1;
771		− __entry->idx = sde->this_idx;
772		− __entry->descp = descp;
773		− __entry->e = e;
774		− ),
775		− TP_printk(
776		− "[%s] SDE(%u) flags:%s addr:0x%016llx gen:%u len:%u d0:%016llx d1:%016llx to %p,%u",
777		− __get_str(dev),
778		− __entry->idx,
779		− __parse_sdma_flags(__entry->desc0, __entry->desc1),
780		− (__entry->desc0 >> SDMA_DESC0_PHY_ADDR_SHIFT) &
781		− SDMA_DESC0_PHY_ADDR_MASK,
782		− (u8)((__entry->desc1 >> SDMA_DESC1_GENERATION_SHIFT) &
783		− SDMA_DESC1_GENERATION_MASK),
784		− (u16)((__entry->desc0 >> SDMA_DESC0_BYTE_COUNT_SHIFT) &
785		− SDMA_DESC0_BYTE_COUNT_MASK),
786		− __entry->desc0,
787		− __entry->desc1,
788		− __entry->descp,
789		− __entry->e
790		− )
791		−);
792		−
793		−TRACE_EVENT(hfi1_sdma_engine_select,
794		− TP_PROTO(struct hfi1_devdata *dd, u32 sel, u8 vl, u8 idx),
795		− TP_ARGS(dd, sel, vl, idx),
796		− TP_STRUCT__entry(DD_DEV_ENTRY(dd)
797		− __field(u32, sel)
798		− __field(u8, vl)
799		− __field(u8, idx)
800		− ),
801		− TP_fast_assign(DD_DEV_ASSIGN(dd);
802		− __entry->sel = sel;
803		− __entry->vl = vl;
804		− __entry->idx = idx;
805		− ),
806		− TP_printk("[%s] selecting SDE %u sel 0x%x vl %u",
807		− __get_str(dev),
808		− __entry->idx,
809		− __entry->sel,
810		− __entry->vl
811		− )
812		−);
813		−
814		−DECLARE_EVENT_CLASS(hfi1_sdma_engine_class,
815		− TP_PROTO(struct sdma_engine *sde, u64 status),
816		− TP_ARGS(sde, status),
817		− TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
818		− __field(u64, status)
819		− __field(u8, idx)
820		− ),
821		− TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
822		− __entry->status = status;
823		− __entry->idx = sde->this_idx;
824		− ),
825		− TP_printk("[%s] SDE(%u) status %llx",
826		− __get_str(dev),
827		− __entry->idx,
828		− (unsigned long long)__entry->status
829		− )
830		−);
831		−
832		−DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_interrupt,
833		− TP_PROTO(struct sdma_engine *sde, u64 status),
834		− TP_ARGS(sde, status)
835		−);
836		−
837		−DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_progress,
838		− TP_PROTO(struct sdma_engine *sde, u64 status),
839		− TP_ARGS(sde, status)
840		−);
841		−
842		−DECLARE_EVENT_CLASS(hfi1_sdma_ahg_ad,
843		− TP_PROTO(struct sdma_engine *sde, int aidx),
844		− TP_ARGS(sde, aidx),
845		− TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
846		− __field(int, aidx)
847		− __field(u8, idx)
848		− ),
849		− TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
850		− __entry->idx = sde->this_idx;
851		− __entry->aidx = aidx;
852		− ),
853		− TP_printk("[%s] SDE(%u) aidx %d",
854		− __get_str(dev),
855		− __entry->idx,
856		− __entry->aidx
857		− )
858		−);
859		−
860		−DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_allocate,
861		− TP_PROTO(struct sdma_engine *sde, int aidx),
862		− TP_ARGS(sde, aidx));
863		−
864		−DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_deallocate,
865		− TP_PROTO(struct sdma_engine *sde, int aidx),
866		− TP_ARGS(sde, aidx));
867		−
868		−#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
869		−TRACE_EVENT(hfi1_sdma_progress,
870		− TP_PROTO(struct sdma_engine *sde,
871		− u16 hwhead,
872		− u16 swhead,
873		− struct sdma_txreq *txp
874		− ),
875		− TP_ARGS(sde, hwhead, swhead, txp),
876		− TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
877		− __field(u64, sn)
878		− __field(u16, hwhead)
879		− __field(u16, swhead)
880		− __field(u16, txnext)
881		− __field(u16, tx_tail)
882		− __field(u16, tx_head)
883		− __field(u8, idx)
884		− ),
885		− TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
886		− __entry->hwhead = hwhead;
887		− __entry->swhead = swhead;
888		− __entry->tx_tail = sde->tx_tail;
889		− __entry->tx_head = sde->tx_head;
890		− __entry->txnext = txp ? txp->next_descq_idx : ~0;
891		− __entry->idx = sde->this_idx;
892		− __entry->sn = txp ? txp->sn : ~0;
893		− ),
894		− TP_printk(
895		− "[%s] SDE(%u) sn %llu hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
896		− __get_str(dev),
897		− __entry->idx,
898		− __entry->sn,
899		− __entry->hwhead,
900		− __entry->swhead,
901		− __entry->txnext,
902		− __entry->tx_head,
903		− __entry->tx_tail
904		− )
905		−);
906		−#else
907		−TRACE_EVENT(hfi1_sdma_progress,
908		− TP_PROTO(struct sdma_engine *sde,
909		− u16 hwhead, u16 swhead,
910		− struct sdma_txreq *txp
911		− ),
912		− TP_ARGS(sde, hwhead, swhead, txp),
913		− TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
914		− __field(u16, hwhead)
915		− __field(u16, swhead)
916		− __field(u16, txnext)
917		− __field(u16, tx_tail)
918		− __field(u16, tx_head)
919		− __field(u8, idx)
920		− ),
921		− TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
922		− __entry->hwhead = hwhead;
923		− __entry->swhead = swhead;
924		− __entry->tx_tail = sde->tx_tail;
925		− __entry->tx_head = sde->tx_head;
926		− __entry->txnext = txp ? txp->next_descq_idx : ~0;
927		− __entry->idx = sde->this_idx;
928		− ),
929		− TP_printk(
930		− "[%s] SDE(%u) hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
931		− __get_str(dev),
932		− __entry->idx,
933		− __entry->hwhead,
934		− __entry->swhead,
935		− __entry->txnext,
936		− __entry->tx_head,
937		− __entry->tx_tail
938		− )
939		−);
940		−#endif
941		−
942		−DECLARE_EVENT_CLASS(hfi1_sdma_sn,
943		− TP_PROTO(struct sdma_engine *sde, u64 sn),
944		− TP_ARGS(sde, sn),
945		− TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
946		− __field(u64, sn)
947		− __field(u8, idx)
948		− ),
949		− TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
950		− __entry->sn = sn;
951		− __entry->idx = sde->this_idx;
952		− ),
953		− TP_printk("[%s] SDE(%u) sn %llu",
954		− __get_str(dev),
955		− __entry->idx,
956		− __entry->sn
957		− )
958		−);
959		−
960		−DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_out_sn,
961		− TP_PROTO(
962		− struct sdma_engine *sde,
963		− u64 sn
964		− ),
965		− TP_ARGS(sde, sn)
966		−);
967		−
968		−DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_in_sn,
969		− TP_PROTO(struct sdma_engine *sde, u64 sn),
970		− TP_ARGS(sde, sn)
971		−);
972		−
973		−#define USDMA_HDR_FORMAT \
974		− "[%s:%u:%u:%u] PBC=(0x%x 0x%x) LRH=(0x%x 0x%x) BTH=(0x%x 0x%x 0x%x) KDETH=(0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x) TIDVal=0x%x"
975		−
976		−TRACE_EVENT(hfi1_sdma_user_header,
977		− TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
978		− struct hfi1_pkt_header *hdr, u32 tidval),
979		− TP_ARGS(dd, ctxt, subctxt, req, hdr, tidval),
980		− TP_STRUCT__entry(
981		− DD_DEV_ENTRY(dd)
982		− __field(u16, ctxt)
983		− __field(u8, subctxt)
984		− __field(u16, req)
985		− __field(__le32, pbc0)
986		− __field(__le32, pbc1)
987		− __field(__be32, lrh0)
988		− __field(__be32, lrh1)
989		− __field(__be32, bth0)
990		− __field(__be32, bth1)
991		− __field(__be32, bth2)
992		− __field(__le32, kdeth0)
993		− __field(__le32, kdeth1)
994		− __field(__le32, kdeth2)
995		− __field(__le32, kdeth3)
996		− __field(__le32, kdeth4)
997		− __field(__le32, kdeth5)
998		− __field(__le32, kdeth6)
999		− __field(__le32, kdeth7)
1000		− __field(__le32, kdeth8)
1001		− __field(u32, tidval)
1002		− ),
1003		− TP_fast_assign(
1004		− __le32 pbc = (__le32 )hdr->pbc;
1005		− __be32 lrh = (__be32 )hdr->lrh;
1006		− __be32 bth = (__be32 )hdr->bth;
1007		− __le32 kdeth = (__le32 )&hdr->kdeth;
1008		−
1009		− DD_DEV_ASSIGN(dd);
1010		− __entry->ctxt = ctxt;
1011		− __entry->subctxt = subctxt;
1012		− __entry->req = req;
1013		− __entry->pbc0 = pbc[0];
1014		− __entry->pbc1 = pbc[1];
1015		− __entry->lrh0 = be32_to_cpu(lrh[0]);
1016		− __entry->lrh1 = be32_to_cpu(lrh[1]);
1017		− __entry->bth0 = be32_to_cpu(bth[0]);
1018		− __entry->bth1 = be32_to_cpu(bth[1]);
1019		− __entry->bth2 = be32_to_cpu(bth[2]);
1020		− __entry->kdeth0 = kdeth[0];
1021		− __entry->kdeth1 = kdeth[1];
1022		− __entry->kdeth2 = kdeth[2];
1023		− __entry->kdeth3 = kdeth[3];
1024		− __entry->kdeth4 = kdeth[4];
1025		− __entry->kdeth5 = kdeth[5];
1026		− __entry->kdeth6 = kdeth[6];
1027		− __entry->kdeth7 = kdeth[7];
1028		− __entry->kdeth8 = kdeth[8];
1029		− __entry->tidval = tidval;
1030		− ),
1031		− TP_printk(USDMA_HDR_FORMAT,
1032		− __get_str(dev),
1033		− __entry->ctxt,
1034		− __entry->subctxt,
1035		− __entry->req,
1036		− __entry->pbc1,
1037		− __entry->pbc0,
1038		− __entry->lrh0,
1039		− __entry->lrh1,
1040		− __entry->bth0,
1041		− __entry->bth1,
1042		− __entry->bth2,
1043		− __entry->kdeth0,
1044		− __entry->kdeth1,
1045		− __entry->kdeth2,
1046		− __entry->kdeth3,
1047		− __entry->kdeth4,
1048		− __entry->kdeth5,
1049		− __entry->kdeth6,
1050		− __entry->kdeth7,
1051		− __entry->kdeth8,
1052		− __entry->tidval
1053		− )
1054		− );
1055		−
1056		−#define SDMA_UREQ_FMT \
1057		− "[%s:%u:%u] ver/op=0x%x, iovcnt=%u, npkts=%u, frag=%u, idx=%u"
1058		−TRACE_EVENT(hfi1_sdma_user_reqinfo,
1059		− TP_PROTO(struct hfi1_devdata dd, u16 ctxt, u8 subctxt, u16 i),
1060		− TP_ARGS(dd, ctxt, subctxt, i),
1061		− TP_STRUCT__entry(
1062		− DD_DEV_ENTRY(dd);
1063		− __field(u16, ctxt)
1064		− __field(u8, subctxt)
1065		− __field(u8, ver_opcode)
1066		− __field(u8, iovcnt)
1067		− __field(u16, npkts)
1068		− __field(u16, fragsize)
1069		− __field(u16, comp_idx)
1070		− ),
1071		− TP_fast_assign(
1072		− DD_DEV_ASSIGN(dd);
1073		− __entry->ctxt = ctxt;
1074		− __entry->subctxt = subctxt;
1075		− __entry->ver_opcode = i[0] & 0xff;
1076		− __entry->iovcnt = (i[0] >> 8) & 0xff;
1077		− __entry->npkts = i[1];
1078		− __entry->fragsize = i[2];
1079		− __entry->comp_idx = i[3];
1080		− ),
1081		− TP_printk(SDMA_UREQ_FMT,
1082		− __get_str(dev),
1083		− __entry->ctxt,
1084		− __entry->subctxt,
1085		− __entry->ver_opcode,
1086		− __entry->iovcnt,
1087		− __entry->npkts,
1088		− __entry->fragsize,
1089		− __entry->comp_idx
1090		− )
1091		− );
1092		−
1093		−#define usdma_complete_name(st) { st, #st }
1094		−#define show_usdma_complete_state(st) \
1095		− __print_symbolic(st, \
1096		− usdma_complete_name(FREE), \
1097		− usdma_complete_name(QUEUED), \
1098		− usdma_complete_name(COMPLETE), \
1099		− usdma_complete_name(ERROR))
1100		−
1101		−TRACE_EVENT(hfi1_sdma_user_completion,
1102		− TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 idx,
1103		− u8 state, int code),
1104		− TP_ARGS(dd, ctxt, subctxt, idx, state, code),
1105		− TP_STRUCT__entry(
1106		− DD_DEV_ENTRY(dd)
1107		− __field(u16, ctxt)
1108		− __field(u8, subctxt)
1109		− __field(u16, idx)
1110		− __field(u8, state)
1111		− __field(int, code)
1112		− ),
1113		− TP_fast_assign(
1114		− DD_DEV_ASSIGN(dd);
1115		− __entry->ctxt = ctxt;
1116		− __entry->subctxt = subctxt;
1117		− __entry->idx = idx;
1118		− __entry->state = state;
1119		− __entry->code = code;
1120		− ),
1121		− TP_printk("[%s:%u:%u:%u] SDMA completion state %s (%d)",
1122		− __get_str(dev), __entry->ctxt, __entry->subctxt,
1123		− __entry->idx, show_usdma_complete_state(__entry->state),
1124		− __entry->code)
1125		− );
1126		−
1127		−const char print_u32_array(struct trace_seq , u32 *, int);
1128		−#define __print_u32_hex(arr, len) print_u32_array(p, arr, len)
1129		−
1130		−TRACE_EVENT(hfi1_sdma_user_header_ahg,
1131		− TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
1132		− u8 sde, u8 ahgidx, u32 *ahg, int len, u32 tidval),
1133		− TP_ARGS(dd, ctxt, subctxt, req, sde, ahgidx, ahg, len, tidval),
1134		− TP_STRUCT__entry(
1135		− DD_DEV_ENTRY(dd)
1136		− __field(u16, ctxt)
1137		− __field(u8, subctxt)
1138		− __field(u16, req)
1139		− __field(u8, sde)
1140		− __field(u8, idx)
1141		− __field(int, len)
1142		− __field(u32, tidval)
1143		− __array(u32, ahg, 10)
1144		− ),
1145		− TP_fast_assign(
1146		− DD_DEV_ASSIGN(dd);
1147		− __entry->ctxt = ctxt;
1148		− __entry->subctxt = subctxt;
1149		− __entry->req = req;
1150		− __entry->sde = sde;
1151		− __entry->idx = ahgidx;
1152		− __entry->len = len;
1153		− __entry->tidval = tidval;
1154		− memcpy(__entry->ahg, ahg, len * sizeof(u32));
1155		− ),
1156		− TP_printk("[%s:%u:%u:%u] (SDE%u/AHG%u) ahg[0-%d]=(%s) TIDVal=0x%x",
1157		− __get_str(dev),
1158		− __entry->ctxt,
1159		− __entry->subctxt,
1160		− __entry->req,
1161		− __entry->sde,
1162		− __entry->idx,
1163		− __entry->len - 1,
1164		− __print_u32_hex(__entry->ahg, __entry->len),
1165		− __entry->tidval
1166		− )
1167		− );
1168		−
1169		−TRACE_EVENT(hfi1_sdma_state,
1170		− TP_PROTO(struct sdma_engine *sde,
1171		− const char *cstate,
1172		− const char *nstate
1173		− ),
1174		− TP_ARGS(sde, cstate, nstate),
1175		− TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
1176		− __string(curstate, cstate)
1177		− __string(newstate, nstate)
1178		− ),
1179		− TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
1180		− __assign_str(curstate, cstate);
1181		− __assign_str(newstate, nstate);
1182		− ),
1183		− TP_printk("[%s] current state %s new state %s",
1184		− __get_str(dev),
1185		− __get_str(curstate),
1186		− __get_str(newstate)
1187		− )
1188		−);
1189		−
1190		−#undef TRACE_SYSTEM
1191		−#define TRACE_SYSTEM hfi1_rc
1192		−
1193		−DECLARE_EVENT_CLASS(hfi1_rc_template,
1194		− TP_PROTO(struct rvt_qp *qp, u32 psn),
1195		− TP_ARGS(qp, psn),
1196		− TP_STRUCT__entry(
1197		− DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
1198		− __field(u32, qpn)
1199		− __field(u32, s_flags)
1200		− __field(u32, psn)
1201		− __field(u32, s_psn)
1202		− __field(u32, s_next_psn)
1203		− __field(u32, s_sending_psn)
1204		− __field(u32, s_sending_hpsn)
1205		− __field(u32, r_psn)
1206		− ),
1207		− TP_fast_assign(
1208		− DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device))
1209		− __entry->qpn = qp->ibqp.qp_num;
1210		− __entry->s_flags = qp->s_flags;
1211		− __entry->psn = psn;
1212		− __entry->s_psn = qp->s_psn;
1213		− __entry->s_next_psn = qp->s_next_psn;
1214		− __entry->s_sending_psn = qp->s_sending_psn;
1215		− __entry->s_sending_hpsn = qp->s_sending_hpsn;
1216		− __entry->r_psn = qp->r_psn;
1217		− ),
1218		− TP_printk(
1219		− "[%s] qpn 0x%x s_flags 0x%x psn 0x%x s_psn 0x%x s_next_psn 0x%x s_sending_psn 0x%x sending_hpsn 0x%x r_psn 0x%x",
1220		− __get_str(dev),
1221		− __entry->qpn,
1222		− __entry->s_flags,
1223		− __entry->psn,
1224		− __entry->s_psn,
1225		− __entry->s_next_psn,
1226		− __entry->s_sending_psn,
1227		− __entry->s_sending_hpsn,
1228		− __entry->r_psn
1229		− )
1230		−);
1231		−
1232		−DEFINE_EVENT(hfi1_rc_template, hfi1_rc_sendcomplete,
1233		− TP_PROTO(struct rvt_qp *qp, u32 psn),
1234		− TP_ARGS(qp, psn)
1235		−);
1236		−
1237		−DEFINE_EVENT(hfi1_rc_template, hfi1_rc_ack,
1238		− TP_PROTO(struct rvt_qp *qp, u32 psn),
1239		− TP_ARGS(qp, psn)
1240		−);
1241		−
1242		−DEFINE_EVENT(hfi1_rc_template, hfi1_rc_timeout,
1243		− TP_PROTO(struct rvt_qp *qp, u32 psn),
1244		− TP_ARGS(qp, psn)
1245		−);
1246		−
1247		−DEFINE_EVENT(hfi1_rc_template, hfi1_rc_rcv_error,
1248		− TP_PROTO(struct rvt_qp *qp, u32 psn),
1249		− TP_ARGS(qp, psn)
1250		−);
1251		−
1252		−#undef TRACE_SYSTEM
1253		−#define TRACE_SYSTEM hfi1_misc
1254		−
1255		−TRACE_EVENT(hfi1_interrupt,
1256		− TP_PROTO(struct hfi1_devdata dd, const struct is_table is_entry,
1257		− int src),
1258		− TP_ARGS(dd, is_entry, src),
1259		− TP_STRUCT__entry(DD_DEV_ENTRY(dd)
1260		− __array(char, buf, 64)
1261		− __field(int, src)
1262		− ),
1263		− TP_fast_assign(DD_DEV_ASSIGN(dd)
1264		− is_entry->is_name(__entry->buf, 64,
1265		− src - is_entry->start);
1266		− __entry->src = src;
1267		− ),
1268		− TP_printk("[%s] source: %s [%d]", __get_str(dev), __entry->buf,
1269		− __entry->src)
1270		−);
1271		−
1272		−/*
1273		− * Note:
1274		− * This produces a REALLY ugly trace in the console output when the string is
1275		− * too long.
1276		− */
1277		−
1278		−#undef TRACE_SYSTEM
1279		−#define TRACE_SYSTEM hfi1_trace
1280		−
1281		−#define MAX_MSG_LEN 512
1282		−
1283		−DECLARE_EVENT_CLASS(hfi1_trace_template,
1284		− TP_PROTO(const char function, struct va_format vaf),
1285		− TP_ARGS(function, vaf),
1286		− TP_STRUCT__entry(__string(function, function)
1287		− __dynamic_array(char, msg, MAX_MSG_LEN)
1288		− ),
1289		− TP_fast_assign(__assign_str(function, function);
1290		− WARN_ON_ONCE(vsnprintf
1291		− (__get_dynamic_array(msg),
1292		− MAX_MSG_LEN, vaf->fmt,
1293		− *vaf->va) >=
1294		− MAX_MSG_LEN);
1295		− ),
1296		− TP_printk("(%s) %s",
1297		− __get_str(function),
1298		− __get_str(msg))
1299		−);
1300		−
1301		−/*
1302		− * It may be nice to macroize the __hfi1_trace but the va_* stuff requires an
1303		− * actual function to work and can not be in a macro.
1304		− */
1305		−#define __hfi1_trace_def(lvl) \
1306		−void __hfi1_trace_##lvl(const char funct, char fmt, ...); \
1307		− \
1308		−DEFINE_EVENT(hfi1_trace_template, hfi1_ ##lvl, \
1309		− TP_PROTO(const char function, struct va_format vaf), \
1310		− TP_ARGS(function, vaf))
1311		−
1312		−#define __hfi1_trace_fn(lvl) \
1313		−void __hfi1_trace_##lvl(const char func, char fmt, ...) \
1314		−{ \
1315		− struct va_format vaf = { \
1316		− .fmt = fmt, \
1317		− }; \
1318		− va_list args; \
1319		− \
1320		− va_start(args, fmt); \
1321		− vaf.va = &args; \
1322		− trace_hfi1_ ##lvl(func, &vaf); \
1323		− va_end(args); \
1324		− return; \
1325		−}
1326		−
1327		−/*
1328		− * To create a new trace level simply define it below and as a __hfi1_trace_fn
1329		− * in trace.c. This will create all the hooks for calling
1330		− * hfi1_cdbg(LVL, fmt, ...); as well as take care of all
1331		− * the debugfs stuff.
1332		− */
1333		−__hfi1_trace_def(PKT);
1334		−__hfi1_trace_def(PROC);
1335		−__hfi1_trace_def(SDMA);
1336		−__hfi1_trace_def(LINKVERB);
1337		−__hfi1_trace_def(DEBUG);
1338		−__hfi1_trace_def(SNOOP);
1339		−__hfi1_trace_def(CNTR);
1340		−__hfi1_trace_def(PIO);
1341		−__hfi1_trace_def(DC8051);
1342		−__hfi1_trace_def(FIRMWARE);
1343		−__hfi1_trace_def(RCVCTRL);
1344		−__hfi1_trace_def(TID);
1345		−__hfi1_trace_def(MMU);
1346		−__hfi1_trace_def(IOCTL);
1347		−
1348		−#define hfi1_cdbg(which, fmt, ...) \
1349		− __hfi1_trace_##which(__func__, fmt, ##__VA_ARGS__)
1350		−
1351		−#define hfi1_dbg(fmt, ...) \
1352		− hfi1_cdbg(DEBUG, fmt, ##__VA_ARGS__)
1353		−
1354		−/*
1355		− * Define HFI1_EARLY_DBG at compile time or here to enable early trace
1356		− * messages. Do not check in an enablement for this.
1357		− */
1358		−
1359		−#ifdef HFI1_EARLY_DBG
1360		−#define hfi1_dbg_early(fmt, ...) \
1361		− trace_printk(fmt, ##__VA_ARGS__)
1362		−#else
1363		−#define hfi1_dbg_early(fmt, ...)
1364		−#endif
1365		−
1366		−#endif /* __HFI1_TRACE_H */
1367		−
1368		−#undef TRACE_INCLUDE_PATH
1369		−#undef TRACE_INCLUDE_FILE
1370		−#define TRACE_INCLUDE_PATH .
1371		−#define TRACE_INCLUDE_FILE trace
1372		−#include <trace/define_trace.h>
	47	+#include "trace_dbg.h"
	48	+#include "trace_misc.h"
	49	+#include "trace_ctxts.h"
	50	+#include "trace_ibhdrs.h"
	51	+#include "trace_rc.h"
	52	+#include "trace_rx.h"
	53	+#include "trace_tx.h"

drivers/infiniband/hw/hfi1/trace_ctxts.h+141−0

@@ -0,0 +1,141 @@
	1	+/*
	2	+* Copyright(c) 2015, 2016 Intel Corporation.
	3	+*
	4	+* This file is provided under a dual BSD/GPLv2 license. When using or
	5	+* redistributing this file, you may do so under either license.
	6	+*
	7	+* GPL LICENSE SUMMARY
	8	+*
	9	+* This program is free software; you can redistribute it and/or modify
	10	+* it under the terms of version 2 of the GNU General Public License as
	11	+* published by the Free Software Foundation.
	12	+*
	13	+* This program is distributed in the hope that it will be useful, but
	14	+* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+* General Public License for more details.
	17	+*
	18	+* BSD LICENSE
	19	+*
	20	+* Redistribution and use in source and binary forms, with or without
	21	+* modification, are permitted provided that the following conditions
	22	+* are met:
	23	+*
	24	+* - Redistributions of source code must retain the above copyright
	25	+* notice, this list of conditions and the following disclaimer.
	26	+* - Redistributions in binary form must reproduce the above copyright
	27	+* notice, this list of conditions and the following disclaimer in
	28	+* the documentation and/or other materials provided with the
	29	+* distribution.
	30	+* - Neither the name of Intel Corporation nor the names of its
	31	+* contributors may be used to endorse or promote products derived
	32	+* from this software without specific prior written permission.
	33	+*
	34	+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	+*
	46	+*/
	47	+#if !defined(__HFI1_TRACE_CTXTS_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	+#define __HFI1_TRACE_CTXTS_H
	49	+
	50	+#include <linux/tracepoint.h>
	51	+#include <linux/trace_seq.h>
	52	+
	53	+#include "hfi.h"
	54	+
	55	+#undef TRACE_SYSTEM
	56	+#define TRACE_SYSTEM hfi1_ctxts
	57	+
	58	+#define UCTXT_FMT \
	59	+ "cred:%u, credaddr:0x%llx, piobase:0x%p, rcvhdr_cnt:%u, " \
	60	+ "rcvbase:0x%llx, rcvegrc:%u, rcvegrb:0x%llx"
	61	+TRACE_EVENT(hfi1_uctxtdata,
	62	+ TP_PROTO(struct hfi1_devdata dd, struct hfi1_ctxtdata uctxt),
	63	+ TP_ARGS(dd, uctxt),
	64	+ TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	65	+ __field(unsigned int, ctxt)
	66	+ __field(u32, credits)
	67	+ __field(u64, hw_free)
	68	+ __field(void __iomem *, piobase)
	69	+ __field(u16, rcvhdrq_cnt)
	70	+ __field(u64, rcvhdrq_phys)
	71	+ __field(u32, eager_cnt)
	72	+ __field(u64, rcvegr_phys)
	73	+ ),
	74	+ TP_fast_assign(DD_DEV_ASSIGN(dd);
	75	+ __entry->ctxt = uctxt->ctxt;
	76	+ __entry->credits = uctxt->sc->credits;
	77	+ __entry->hw_free = le64_to_cpu(*uctxt->sc->hw_free);
	78	+ __entry->piobase = uctxt->sc->base_addr;
	79	+ __entry->rcvhdrq_cnt = uctxt->rcvhdrq_cnt;
	80	+ __entry->rcvhdrq_phys = uctxt->rcvhdrq_phys;
	81	+ __entry->eager_cnt = uctxt->egrbufs.alloced;
	82	+ __entry->rcvegr_phys =
	83	+ uctxt->egrbufs.rcvtids[0].phys;
	84	+ ),
	85	+ TP_printk("[%s] ctxt %u " UCTXT_FMT,
	86	+ __get_str(dev),
	87	+ __entry->ctxt,
	88	+ __entry->credits,
	89	+ __entry->hw_free,
	90	+ __entry->piobase,
	91	+ __entry->rcvhdrq_cnt,
	92	+ __entry->rcvhdrq_phys,
	93	+ __entry->eager_cnt,
	94	+ __entry->rcvegr_phys
	95	+ )
	96	+);
	97	+
	98	+#define CINFO_FMT \
	99	+ "egrtids:%u, egr_size:%u, hdrq_cnt:%u, hdrq_size:%u, sdma_ring_size:%u"
	100	+TRACE_EVENT(hfi1_ctxt_info,
	101	+ TP_PROTO(struct hfi1_devdata *dd, unsigned int ctxt,
	102	+ unsigned int subctxt,
	103	+ struct hfi1_ctxt_info cinfo),
	104	+ TP_ARGS(dd, ctxt, subctxt, cinfo),
	105	+ TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	106	+ __field(unsigned int, ctxt)
	107	+ __field(unsigned int, subctxt)
	108	+ __field(u16, egrtids)
	109	+ __field(u16, rcvhdrq_cnt)
	110	+ __field(u16, rcvhdrq_size)
	111	+ __field(u16, sdma_ring_size)
	112	+ __field(u32, rcvegr_size)
	113	+ ),
	114	+ TP_fast_assign(DD_DEV_ASSIGN(dd);
	115	+ __entry->ctxt = ctxt;
	116	+ __entry->subctxt = subctxt;
	117	+ __entry->egrtids = cinfo.egrtids;
	118	+ __entry->rcvhdrq_cnt = cinfo.rcvhdrq_cnt;
	119	+ __entry->rcvhdrq_size = cinfo.rcvhdrq_entsize;
	120	+ __entry->sdma_ring_size = cinfo.sdma_ring_size;
	121	+ __entry->rcvegr_size = cinfo.rcvegr_size;
	122	+ ),
	123	+ TP_printk("[%s] ctxt %u:%u " CINFO_FMT,
	124	+ __get_str(dev),
	125	+ __entry->ctxt,
	126	+ __entry->subctxt,
	127	+ __entry->egrtids,
	128	+ __entry->rcvegr_size,
	129	+ __entry->rcvhdrq_cnt,
	130	+ __entry->rcvhdrq_size,
	131	+ __entry->sdma_ring_size
	132	+ )
	133	+);
	134	+
	135	+#endif /* __HFI1_TRACE_CTXTS_H */
	136	+
	137	+#undef TRACE_INCLUDE_PATH
	138	+#undef TRACE_INCLUDE_FILE
	139	+#define TRACE_INCLUDE_PATH .
	140	+#define TRACE_INCLUDE_FILE trace_ctxts
	141	+#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_dbg.h+155−0

@@ -0,0 +1,155 @@
	1	+/*
	2	+* Copyright(c) 2015, 2016 Intel Corporation.
	3	+*
	4	+* This file is provided under a dual BSD/GPLv2 license. When using or
	5	+* redistributing this file, you may do so under either license.
	6	+*
	7	+* GPL LICENSE SUMMARY
	8	+*
	9	+* This program is free software; you can redistribute it and/or modify
	10	+* it under the terms of version 2 of the GNU General Public License as
	11	+* published by the Free Software Foundation.
	12	+*
	13	+* This program is distributed in the hope that it will be useful, but
	14	+* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+* General Public License for more details.
	17	+*
	18	+* BSD LICENSE
	19	+*
	20	+* Redistribution and use in source and binary forms, with or without
	21	+* modification, are permitted provided that the following conditions
	22	+* are met:
	23	+*
	24	+* - Redistributions of source code must retain the above copyright
	25	+* notice, this list of conditions and the following disclaimer.
	26	+* - Redistributions in binary form must reproduce the above copyright
	27	+* notice, this list of conditions and the following disclaimer in
	28	+* the documentation and/or other materials provided with the
	29	+* distribution.
	30	+* - Neither the name of Intel Corporation nor the names of its
	31	+* contributors may be used to endorse or promote products derived
	32	+* from this software without specific prior written permission.
	33	+*
	34	+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	+*
	46	+*/
	47	+#if !defined(__HFI1_TRACE_EXTRA_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	+#define __HFI1_TRACE_EXTRA_H
	49	+
	50	+#include <linux/tracepoint.h>
	51	+#include <linux/trace_seq.h>
	52	+
	53	+#include "hfi.h"
	54	+
	55	+/*
	56	+ * Note:
	57	+ * This produces a REALLY ugly trace in the console output when the string is
	58	+ * too long.
	59	+ */
	60	+
	61	+#undef TRACE_SYSTEM
	62	+#define TRACE_SYSTEM hfi1_dbg
	63	+
	64	+#define MAX_MSG_LEN 512
	65	+
	66	+DECLARE_EVENT_CLASS(hfi1_trace_template,
	67	+ TP_PROTO(const char function, struct va_format vaf),
	68	+ TP_ARGS(function, vaf),
	69	+ TP_STRUCT__entry(__string(function, function)
	70	+ __dynamic_array(char, msg, MAX_MSG_LEN)
	71	+ ),
	72	+ TP_fast_assign(__assign_str(function, function);
	73	+ WARN_ON_ONCE(vsnprintf
	74	+ (__get_dynamic_array(msg),
	75	+ MAX_MSG_LEN, vaf->fmt,
	76	+ *vaf->va) >=
	77	+ MAX_MSG_LEN);
	78	+ ),
	79	+ TP_printk("(%s) %s",
	80	+ __get_str(function),
	81	+ __get_str(msg))
	82	+);
	83	+
	84	+/*
	85	+ * It may be nice to macroize the __hfi1_trace but the va_* stuff requires an
	86	+ * actual function to work and can not be in a macro.
	87	+ */
	88	+#define __hfi1_trace_def(lvl) \
	89	+void __hfi1_trace_##lvl(const char funct, char fmt, ...); \
	90	+ \
	91	+DEFINE_EVENT(hfi1_trace_template, hfi1_ ##lvl, \
	92	+ TP_PROTO(const char function, struct va_format vaf), \
	93	+ TP_ARGS(function, vaf))
	94	+
	95	+#define __hfi1_trace_fn(lvl) \
	96	+void __hfi1_trace_##lvl(const char func, char fmt, ...) \
	97	+{ \
	98	+ struct va_format vaf = { \
	99	+ .fmt = fmt, \
	100	+ }; \
	101	+ va_list args; \
	102	+ \
	103	+ va_start(args, fmt); \
	104	+ vaf.va = &args; \
	105	+ trace_hfi1_ ##lvl(func, &vaf); \
	106	+ va_end(args); \
	107	+ return; \
	108	+}
	109	+
	110	+/*
	111	+ * To create a new trace level simply define it below and as a __hfi1_trace_fn
	112	+ * in trace.c. This will create all the hooks for calling
	113	+ * hfi1_cdbg(LVL, fmt, ...); as well as take care of all
	114	+ * the debugfs stuff.
	115	+ */
	116	+__hfi1_trace_def(PKT);
	117	+__hfi1_trace_def(PROC);
	118	+__hfi1_trace_def(SDMA);
	119	+__hfi1_trace_def(LINKVERB);
	120	+__hfi1_trace_def(DEBUG);
	121	+__hfi1_trace_def(SNOOP);
	122	+__hfi1_trace_def(CNTR);
	123	+__hfi1_trace_def(PIO);
	124	+__hfi1_trace_def(DC8051);
	125	+__hfi1_trace_def(FIRMWARE);
	126	+__hfi1_trace_def(RCVCTRL);
	127	+__hfi1_trace_def(TID);
	128	+__hfi1_trace_def(MMU);
	129	+__hfi1_trace_def(IOCTL);
	130	+
	131	+#define hfi1_cdbg(which, fmt, ...) \
	132	+ __hfi1_trace_##which(__func__, fmt, ##__VA_ARGS__)
	133	+
	134	+#define hfi1_dbg(fmt, ...) \
	135	+ hfi1_cdbg(DEBUG, fmt, ##__VA_ARGS__)
	136	+
	137	+/*
	138	+ * Define HFI1_EARLY_DBG at compile time or here to enable early trace
	139	+ * messages. Do not check in an enablement for this.
	140	+ */
	141	+
	142	+#ifdef HFI1_EARLY_DBG
	143	+#define hfi1_dbg_early(fmt, ...) \
	144	+ trace_printk(fmt, ##__VA_ARGS__)
	145	+#else
	146	+#define hfi1_dbg_early(fmt, ...)
	147	+#endif
	148	+
	149	+#endif /* __HFI1_TRACE_EXTRA_H */
	150	+
	151	+#undef TRACE_INCLUDE_PATH
	152	+#undef TRACE_INCLUDE_FILE
	153	+#define TRACE_INCLUDE_PATH .
	154	+#define TRACE_INCLUDE_FILE trace_dbg
	155	+#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_ibhdrs.h+209−0

@@ -0,0 +1,209 @@
	1	+/*
	2	+ * Copyright(c) 2015, 2016 Intel Corporation.
	3	+ *
	4	+ * This file is provided under a dual BSD/GPLv2 license. When using or
	5	+ * redistributing this file, you may do so under either license.
	6	+ *
	7	+ * GPL LICENSE SUMMARY
	8	+ *
	9	+ * This program is free software; you can redistribute it and/or modify
	10	+ * it under the terms of version 2 of the GNU General Public License as
	11	+ * published by the Free Software Foundation.
	12	+ *
	13	+ * This program is distributed in the hope that it will be useful, but
	14	+ * WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+ * General Public License for more details.
	17	+ *
	18	+ * BSD LICENSE
	19	+ *
	20	+ * Redistribution and use in source and binary forms, with or without
	21	+ * modification, are permitted provided that the following conditions
	22	+ * are met:
	23	+ *
	24	+ * - Redistributions of source code must retain the above copyright
	25	+ * notice, this list of conditions and the following disclaimer.
	26	+ * - Redistributions in binary form must reproduce the above copyright
	27	+ * notice, this list of conditions and the following disclaimer in
	28	+ * the documentation and/or other materials provided with the
	29	+ * distribution.
	30	+ * - Neither the name of Intel Corporation nor the names of its
	31	+ * contributors may be used to endorse or promote products derived
	32	+ * from this software without specific prior written permission.
	33	+ *
	34	+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	+ *
	46	+ */
	47	+#if !defined(__HFI1_TRACE_IBHDRS_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	+#define __HFI1_TRACE_IBHDRS_H
	49	+
	50	+#include <linux/tracepoint.h>
	51	+#include <linux/trace_seq.h>
	52	+
	53	+#include "hfi.h"
	54	+
	55	+#undef TRACE_SYSTEM
	56	+#define TRACE_SYSTEM hfi1_ibhdrs
	57	+
	58	+u8 ibhdr_exhdr_len(struct hfi1_ib_header *hdr);
	59	+const char parse_everbs_hdrs(struct trace_seq p, u8 opcode, void *ehdrs);
	60	+
	61	+#define __parse_ib_ehdrs(op, ehdrs) parse_everbs_hdrs(p, op, ehdrs)
	62	+
	63	+#define lrh_name(lrh) { HFI1_##lrh, #lrh }
	64	+#define show_lnh(lrh) \
	65	+__print_symbolic(lrh, \
	66	+ lrh_name(LRH_BTH), \
	67	+ lrh_name(LRH_GRH))
	68	+
	69	+#define LRH_PRN "vl %d lver %d sl %d lnh %d,%s dlid %.4x len %d slid %.4x"
	70	+#define BTH_PRN \
	71	+ "op 0x%.2x,%s se %d m %d pad %d tver %d pkey 0x%.4x " \
	72	+ "f %d b %d qpn 0x%.6x a %d psn 0x%.8x"
	73	+#define EHDR_PRN "%s"
	74	+
	75	+DECLARE_EVENT_CLASS(hfi1_ibhdr_template,
	76	+ TP_PROTO(struct hfi1_devdata *dd,
	77	+ struct hfi1_ib_header *hdr),
	78	+ TP_ARGS(dd, hdr),
	79	+ TP_STRUCT__entry(
	80	+ DD_DEV_ENTRY(dd)
	81	+ /* LRH */
	82	+ __field(u8, vl)
	83	+ __field(u8, lver)
	84	+ __field(u8, sl)
	85	+ __field(u8, lnh)
	86	+ __field(u16, dlid)
	87	+ __field(u16, len)
	88	+ __field(u16, slid)
	89	+ /* BTH */
	90	+ __field(u8, opcode)
	91	+ __field(u8, se)
	92	+ __field(u8, m)
	93	+ __field(u8, pad)
	94	+ __field(u8, tver)
	95	+ __field(u16, pkey)
	96	+ __field(u8, f)
	97	+ __field(u8, b)
	98	+ __field(u32, qpn)
	99	+ __field(u8, a)
	100	+ __field(u32, psn)
	101	+ /* extended headers */
	102	+ __dynamic_array(u8, ehdrs, ibhdr_exhdr_len(hdr))
	103	+ ),
	104	+ TP_fast_assign(
	105	+ struct hfi1_other_headers *ohdr;
	106	+
	107	+ DD_DEV_ASSIGN(dd);
	108	+ /* LRH */
	109	+ __entry->vl =
	110	+ (u8)(be16_to_cpu(hdr->lrh[0]) >> 12);
	111	+ __entry->lver =
	112	+ (u8)(be16_to_cpu(hdr->lrh[0]) >> 8) & 0xf;
	113	+ __entry->sl =
	114	+ (u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
	115	+ __entry->lnh =
	116	+ (u8)(be16_to_cpu(hdr->lrh[0]) & 3);
	117	+ __entry->dlid =
	118	+ be16_to_cpu(hdr->lrh[1]);
	119	+ /* allow for larger len */
	120	+ __entry->len =
	121	+ be16_to_cpu(hdr->lrh[2]);
	122	+ __entry->slid =
	123	+ be16_to_cpu(hdr->lrh[3]);
	124	+ /* BTH */
	125	+ if (__entry->lnh == HFI1_LRH_BTH)
	126	+ ohdr = &hdr->u.oth;
	127	+ else
	128	+ ohdr = &hdr->u.l.oth;
	129	+ __entry->opcode =
	130	+ (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
	131	+ __entry->se =
	132	+ (be32_to_cpu(ohdr->bth[0]) >> 23) & 1;
	133	+ __entry->m =
	134	+ (be32_to_cpu(ohdr->bth[0]) >> 22) & 1;
	135	+ __entry->pad =
	136	+ (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
	137	+ __entry->tver =
	138	+ (be32_to_cpu(ohdr->bth[0]) >> 16) & 0xf;
	139	+ __entry->pkey =
	140	+ be32_to_cpu(ohdr->bth[0]) & 0xffff;
	141	+ __entry->f =
	142	+ (be32_to_cpu(ohdr->bth[1]) >> HFI1_FECN_SHIFT) &
	143	+ HFI1_FECN_MASK;
	144	+ __entry->b =
	145	+ (be32_to_cpu(ohdr->bth[1]) >> HFI1_BECN_SHIFT) &
	146	+ HFI1_BECN_MASK;
	147	+ __entry->qpn =
	148	+ be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
	149	+ __entry->a =
	150	+ (be32_to_cpu(ohdr->bth[2]) >> 31) & 1;
	151	+ /* allow for larger PSN */
	152	+ __entry->psn =
	153	+ be32_to_cpu(ohdr->bth[2]) & 0x7fffffff;
	154	+ /* extended headers */
	155	+ memcpy(__get_dynamic_array(ehdrs), &ohdr->u,
	156	+ ibhdr_exhdr_len(hdr));
	157	+ ),
	158	+ TP_printk("[%s] " LRH_PRN " " BTH_PRN " " EHDR_PRN,
	159	+ __get_str(dev),
	160	+ /* LRH */
	161	+ __entry->vl,
	162	+ __entry->lver,
	163	+ __entry->sl,
	164	+ __entry->lnh, show_lnh(__entry->lnh),
	165	+ __entry->dlid,
	166	+ __entry->len,
	167	+ __entry->slid,
	168	+ /* BTH */
	169	+ __entry->opcode, show_ib_opcode(__entry->opcode),
	170	+ __entry->se,
	171	+ __entry->m,
	172	+ __entry->pad,
	173	+ __entry->tver,
	174	+ __entry->pkey,
	175	+ __entry->f,
	176	+ __entry->b,
	177	+ __entry->qpn,
	178	+ __entry->a,
	179	+ __entry->psn,
	180	+ /* extended headers */
	181	+ __parse_ib_ehdrs(
	182	+ __entry->opcode,
	183	+ (void *)__get_dynamic_array(ehdrs))
	184	+ )
	185	+);
	186	+
	187	+DEFINE_EVENT(hfi1_ibhdr_template, input_ibhdr,
	188	+ TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
	189	+ TP_ARGS(dd, hdr));
	190	+
	191	+DEFINE_EVENT(hfi1_ibhdr_template, pio_output_ibhdr,
	192	+ TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
	193	+ TP_ARGS(dd, hdr));
	194	+
	195	+DEFINE_EVENT(hfi1_ibhdr_template, ack_output_ibhdr,
	196	+ TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
	197	+ TP_ARGS(dd, hdr));
	198	+
	199	+DEFINE_EVENT(hfi1_ibhdr_template, sdma_output_ibhdr,
	200	+ TP_PROTO(struct hfi1_devdata dd, struct hfi1_ib_header hdr),
	201	+ TP_ARGS(dd, hdr));
	202	+
	203	+#endif /* __HFI1_TRACE_IBHDRS_H */
	204	+
	205	+#undef TRACE_INCLUDE_PATH
	206	+#undef TRACE_INCLUDE_FILE
	207	+#define TRACE_INCLUDE_PATH .
	208	+#define TRACE_INCLUDE_FILE trace_ibhdrs
	209	+#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_misc.h+81−0

@@ -0,0 +1,81 @@
	1	+/*
	2	+* Copyright(c) 2015, 2016 Intel Corporation.
	3	+*
	4	+* This file is provided under a dual BSD/GPLv2 license. When using or
	5	+* redistributing this file, you may do so under either license.
	6	+*
	7	+* GPL LICENSE SUMMARY
	8	+*
	9	+* This program is free software; you can redistribute it and/or modify
	10	+* it under the terms of version 2 of the GNU General Public License as
	11	+* published by the Free Software Foundation.
	12	+*
	13	+* This program is distributed in the hope that it will be useful, but
	14	+* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+* General Public License for more details.
	17	+*
	18	+* BSD LICENSE
	19	+*
	20	+* Redistribution and use in source and binary forms, with or without
	21	+* modification, are permitted provided that the following conditions
	22	+* are met:
	23	+*
	24	+* - Redistributions of source code must retain the above copyright
	25	+* notice, this list of conditions and the following disclaimer.
	26	+* - Redistributions in binary form must reproduce the above copyright
	27	+* notice, this list of conditions and the following disclaimer in
	28	+* the documentation and/or other materials provided with the
	29	+* distribution.
	30	+* - Neither the name of Intel Corporation nor the names of its
	31	+* contributors may be used to endorse or promote products derived
	32	+* from this software without specific prior written permission.
	33	+*
	34	+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	+*
	46	+*/
	47	+#if !defined(__HFI1_TRACE_MISC_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	+#define __HFI1_TRACE_MISC_H
	49	+
	50	+#include <linux/tracepoint.h>
	51	+#include <linux/trace_seq.h>
	52	+
	53	+#include "hfi.h"
	54	+
	55	+#undef TRACE_SYSTEM
	56	+#define TRACE_SYSTEM hfi1_misc
	57	+
	58	+TRACE_EVENT(hfi1_interrupt,
	59	+ TP_PROTO(struct hfi1_devdata dd, const struct is_table is_entry,
	60	+ int src),
	61	+ TP_ARGS(dd, is_entry, src),
	62	+ TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	63	+ __array(char, buf, 64)
	64	+ __field(int, src)
	65	+ ),
	66	+ TP_fast_assign(DD_DEV_ASSIGN(dd)
	67	+ is_entry->is_name(__entry->buf, 64,
	68	+ src - is_entry->start);
	69	+ __entry->src = src;
	70	+ ),
	71	+ TP_printk("[%s] source: %s [%d]", __get_str(dev), __entry->buf,
	72	+ __entry->src)
	73	+);
	74	+
	75	+#endif /* __HFI1_TRACE_MISC_H */
	76	+
	77	+#undef TRACE_INCLUDE_PATH
	78	+#undef TRACE_INCLUDE_FILE
	79	+#define TRACE_INCLUDE_PATH .
	80	+#define TRACE_INCLUDE_FILE trace_misc
	81	+#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_rc.h+123−0

@@ -0,0 +1,123 @@
	1	+/*
	2	+* Copyright(c) 2015, 2016 Intel Corporation.
	3	+*
	4	+* This file is provided under a dual BSD/GPLv2 license. When using or
	5	+* redistributing this file, you may do so under either license.
	6	+*
	7	+* GPL LICENSE SUMMARY
	8	+*
	9	+* This program is free software; you can redistribute it and/or modify
	10	+* it under the terms of version 2 of the GNU General Public License as
	11	+* published by the Free Software Foundation.
	12	+*
	13	+* This program is distributed in the hope that it will be useful, but
	14	+* WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+* General Public License for more details.
	17	+*
	18	+* BSD LICENSE
	19	+*
	20	+* Redistribution and use in source and binary forms, with or without
	21	+* modification, are permitted provided that the following conditions
	22	+* are met:
	23	+*
	24	+* - Redistributions of source code must retain the above copyright
	25	+* notice, this list of conditions and the following disclaimer.
	26	+* - Redistributions in binary form must reproduce the above copyright
	27	+* notice, this list of conditions and the following disclaimer in
	28	+* the documentation and/or other materials provided with the
	29	+* distribution.
	30	+* - Neither the name of Intel Corporation nor the names of its
	31	+* contributors may be used to endorse or promote products derived
	32	+* from this software without specific prior written permission.
	33	+*
	34	+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	+*
	46	+*/
	47	+#if !defined(__HFI1_TRACE_RC_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	+#define __HFI1_TRACE_RC_H
	49	+
	50	+#include <linux/tracepoint.h>
	51	+#include <linux/trace_seq.h>
	52	+
	53	+#include "hfi.h"
	54	+
	55	+#undef TRACE_SYSTEM
	56	+#define TRACE_SYSTEM hfi1_rc
	57	+
	58	+DECLARE_EVENT_CLASS(hfi1_rc_template,
	59	+ TP_PROTO(struct rvt_qp *qp, u32 psn),
	60	+ TP_ARGS(qp, psn),
	61	+ TP_STRUCT__entry(
	62	+ DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
	63	+ __field(u32, qpn)
	64	+ __field(u32, s_flags)
	65	+ __field(u32, psn)
	66	+ __field(u32, s_psn)
	67	+ __field(u32, s_next_psn)
	68	+ __field(u32, s_sending_psn)
	69	+ __field(u32, s_sending_hpsn)
	70	+ __field(u32, r_psn)
	71	+ ),
	72	+ TP_fast_assign(
	73	+ DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device))
	74	+ __entry->qpn = qp->ibqp.qp_num;
	75	+ __entry->s_flags = qp->s_flags;
	76	+ __entry->psn = psn;
	77	+ __entry->s_psn = qp->s_psn;
	78	+ __entry->s_next_psn = qp->s_next_psn;
	79	+ __entry->s_sending_psn = qp->s_sending_psn;
	80	+ __entry->s_sending_hpsn = qp->s_sending_hpsn;
	81	+ __entry->r_psn = qp->r_psn;
	82	+ ),
	83	+ TP_printk(
	84	+ "[%s] qpn 0x%x s_flags 0x%x psn 0x%x s_psn 0x%x s_next_psn 0x%x s_sending_psn 0x%x sending_hpsn 0x%x r_psn 0x%x",
	85	+ __get_str(dev),
	86	+ __entry->qpn,
	87	+ __entry->s_flags,
	88	+ __entry->psn,
	89	+ __entry->s_psn,
	90	+ __entry->s_next_psn,
	91	+ __entry->s_sending_psn,
	92	+ __entry->s_sending_hpsn,
	93	+ __entry->r_psn
	94	+ )
	95	+);
	96	+
	97	+DEFINE_EVENT(hfi1_rc_template, hfi1_sendcomplete,
	98	+ TP_PROTO(struct rvt_qp *qp, u32 psn),
	99	+ TP_ARGS(qp, psn)
	100	+);
	101	+
	102	+DEFINE_EVENT(hfi1_rc_template, hfi1_ack,
	103	+ TP_PROTO(struct rvt_qp *qp, u32 psn),
	104	+ TP_ARGS(qp, psn)
	105	+);
	106	+
	107	+DEFINE_EVENT(hfi1_rc_template, hfi1_timeout,
	108	+ TP_PROTO(struct rvt_qp *qp, u32 psn),
	109	+ TP_ARGS(qp, psn)
	110	+);
	111	+
	112	+DEFINE_EVENT(hfi1_rc_template, hfi1_rcv_error,
	113	+ TP_PROTO(struct rvt_qp *qp, u32 psn),
	114	+ TP_ARGS(qp, psn)
	115	+);
	116	+
	117	+#endif /* __HFI1_TRACE_RC_H */
	118	+
	119	+#undef TRACE_INCLUDE_PATH
	120	+#undef TRACE_INCLUDE_FILE
	121	+#define TRACE_INCLUDE_PATH .
	122	+#define TRACE_INCLUDE_FILE trace_rc
	123	+#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_rx.h+322−0

@@ -0,0 +1,322 @@
	1	+/*
	2	+ * Copyright(c) 2015, 2016 Intel Corporation.
	3	+ *
	4	+ * This file is provided under a dual BSD/GPLv2 license. When using or
	5	+ * redistributing this file, you may do so under either license.
	6	+ *
	7	+ * GPL LICENSE SUMMARY
	8	+ *
	9	+ * This program is free software; you can redistribute it and/or modify
	10	+ * it under the terms of version 2 of the GNU General Public License as
	11	+ * published by the Free Software Foundation.
	12	+ *
	13	+ * This program is distributed in the hope that it will be useful, but
	14	+ * WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+ * General Public License for more details.
	17	+ *
	18	+ * BSD LICENSE
	19	+ *
	20	+ * Redistribution and use in source and binary forms, with or without
	21	+ * modification, are permitted provided that the following conditions
	22	+ * are met:
	23	+ *
	24	+ * - Redistributions of source code must retain the above copyright
	25	+ * notice, this list of conditions and the following disclaimer.
	26	+ * - Redistributions in binary form must reproduce the above copyright
	27	+ * notice, this list of conditions and the following disclaimer in
	28	+ * the documentation and/or other materials provided with the
	29	+ * distribution.
	30	+ * - Neither the name of Intel Corporation nor the names of its
	31	+ * contributors may be used to endorse or promote products derived
	32	+ * from this software without specific prior written permission.
	33	+ *
	34	+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	+ *
	46	+ */
	47	+#if !defined(__HFI1_TRACE_RX_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	+#define __HFI1_TRACE_RX_H
	49	+
	50	+#include <linux/tracepoint.h>
	51	+#include <linux/trace_seq.h>
	52	+
	53	+#include "hfi.h"
	54	+
	55	+#undef TRACE_SYSTEM
	56	+#define TRACE_SYSTEM hfi1_rx
	57	+
	58	+TRACE_EVENT(hfi1_rcvhdr,
	59	+ TP_PROTO(struct hfi1_devdata *dd,
	60	+ u32 ctxt,
	61	+ u64 eflags,
	62	+ u32 etype,
	63	+ u32 hlen,
	64	+ u32 tlen,
	65	+ u32 updegr,
	66	+ u32 etail
	67	+ ),
	68	+ TP_ARGS(dd, ctxt, eflags, etype, hlen, tlen, updegr, etail),
	69	+ TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	70	+ __field(u64, eflags)
	71	+ __field(u32, ctxt)
	72	+ __field(u32, etype)
	73	+ __field(u32, hlen)
	74	+ __field(u32, tlen)
	75	+ __field(u32, updegr)
	76	+ __field(u32, etail)
	77	+ ),
	78	+ TP_fast_assign(DD_DEV_ASSIGN(dd);
	79	+ __entry->eflags = eflags;
	80	+ __entry->ctxt = ctxt;
	81	+ __entry->etype = etype;
	82	+ __entry->hlen = hlen;
	83	+ __entry->tlen = tlen;
	84	+ __entry->updegr = updegr;
	85	+ __entry->etail = etail;
	86	+ ),
	87	+ TP_printk(
	88	+ "[%s] ctxt %d eflags 0x%llx etype %d,%s hlen %d tlen %d updegr %d etail %d",
	89	+ __get_str(dev),
	90	+ __entry->ctxt,
	91	+ __entry->eflags,
	92	+ __entry->etype, show_packettype(__entry->etype),
	93	+ __entry->hlen,
	94	+ __entry->tlen,
	95	+ __entry->updegr,
	96	+ __entry->etail
	97	+ )
	98	+);
	99	+
	100	+TRACE_EVENT(hfi1_receive_interrupt,
	101	+ TP_PROTO(struct hfi1_devdata *dd, u32 ctxt),
	102	+ TP_ARGS(dd, ctxt),
	103	+ TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	104	+ __field(u32, ctxt)
	105	+ __field(u8, slow_path)
	106	+ __field(u8, dma_rtail)
	107	+ ),
	108	+ TP_fast_assign(DD_DEV_ASSIGN(dd);
	109	+ __entry->ctxt = ctxt;
	110	+ if (dd->rcd[ctxt]->do_interrupt ==
	111	+ &handle_receive_interrupt) {
	112	+ __entry->slow_path = 1;
	113	+ __entry->dma_rtail = 0xFF;
	114	+ } else if (dd->rcd[ctxt]->do_interrupt ==
	115	+ &handle_receive_interrupt_dma_rtail){
	116	+ __entry->dma_rtail = 1;
	117	+ __entry->slow_path = 0;
	118	+ } else if (dd->rcd[ctxt]->do_interrupt ==
	119	+ &handle_receive_interrupt_nodma_rtail) {
	120	+ __entry->dma_rtail = 0;
	121	+ __entry->slow_path = 0;
	122	+ }
	123	+ ),
	124	+ TP_printk("[%s] ctxt %d SlowPath: %d DmaRtail: %d",
	125	+ __get_str(dev),
	126	+ __entry->ctxt,
	127	+ __entry->slow_path,
	128	+ __entry->dma_rtail
	129	+ )
	130	+);
	131	+
	132	+TRACE_EVENT(hfi1_exp_tid_reg,
	133	+ TP_PROTO(unsigned int ctxt, u16 subctxt, u32 rarr,
	134	+ u32 npages, unsigned long va, unsigned long pa,
	135	+ dma_addr_t dma),
	136	+ TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
	137	+ TP_STRUCT__entry(
	138	+ __field(unsigned int, ctxt)
	139	+ __field(u16, subctxt)
	140	+ __field(u32, rarr)
	141	+ __field(u32, npages)
	142	+ __field(unsigned long, va)
	143	+ __field(unsigned long, pa)
	144	+ __field(dma_addr_t, dma)
	145	+ ),
	146	+ TP_fast_assign(
	147	+ __entry->ctxt = ctxt;
	148	+ __entry->subctxt = subctxt;
	149	+ __entry->rarr = rarr;
	150	+ __entry->npages = npages;
	151	+ __entry->va = va;
	152	+ __entry->pa = pa;
	153	+ __entry->dma = dma;
	154	+ ),
	155	+ TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
	156	+ __entry->ctxt,
	157	+ __entry->subctxt,
	158	+ __entry->rarr,
	159	+ __entry->npages,
	160	+ __entry->pa,
	161	+ __entry->va,
	162	+ __entry->dma
	163	+ )
	164	+ );
	165	+
	166	+TRACE_EVENT(hfi1_exp_tid_unreg,
	167	+ TP_PROTO(unsigned int ctxt, u16 subctxt, u32 rarr, u32 npages,
	168	+ unsigned long va, unsigned long pa, dma_addr_t dma),
	169	+ TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
	170	+ TP_STRUCT__entry(
	171	+ __field(unsigned int, ctxt)
	172	+ __field(u16, subctxt)
	173	+ __field(u32, rarr)
	174	+ __field(u32, npages)
	175	+ __field(unsigned long, va)
	176	+ __field(unsigned long, pa)
	177	+ __field(dma_addr_t, dma)
	178	+ ),
	179	+ TP_fast_assign(
	180	+ __entry->ctxt = ctxt;
	181	+ __entry->subctxt = subctxt;
	182	+ __entry->rarr = rarr;
	183	+ __entry->npages = npages;
	184	+ __entry->va = va;
	185	+ __entry->pa = pa;
	186	+ __entry->dma = dma;
	187	+ ),
	188	+ TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
	189	+ __entry->ctxt,
	190	+ __entry->subctxt,
	191	+ __entry->rarr,
	192	+ __entry->npages,
	193	+ __entry->pa,
	194	+ __entry->va,
	195	+ __entry->dma
	196	+ )
	197	+ );
	198	+
	199	+TRACE_EVENT(hfi1_exp_tid_inval,
	200	+ TP_PROTO(unsigned int ctxt, u16 subctxt, unsigned long va, u32 rarr,
	201	+ u32 npages, dma_addr_t dma),
	202	+ TP_ARGS(ctxt, subctxt, va, rarr, npages, dma),
	203	+ TP_STRUCT__entry(
	204	+ __field(unsigned int, ctxt)
	205	+ __field(u16, subctxt)
	206	+ __field(unsigned long, va)
	207	+ __field(u32, rarr)
	208	+ __field(u32, npages)
	209	+ __field(dma_addr_t, dma)
	210	+ ),
	211	+ TP_fast_assign(
	212	+ __entry->ctxt = ctxt;
	213	+ __entry->subctxt = subctxt;
	214	+ __entry->va = va;
	215	+ __entry->rarr = rarr;
	216	+ __entry->npages = npages;
	217	+ __entry->dma = dma;
	218	+ ),
	219	+ TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx dma: 0x%llx",
	220	+ __entry->ctxt,
	221	+ __entry->subctxt,
	222	+ __entry->rarr,
	223	+ __entry->npages,
	224	+ __entry->va,
	225	+ __entry->dma
	226	+ )
	227	+ );
	228	+
	229	+TRACE_EVENT(hfi1_mmu_invalidate,
	230	+ TP_PROTO(unsigned int ctxt, u16 subctxt, const char *type,
	231	+ unsigned long start, unsigned long end),
	232	+ TP_ARGS(ctxt, subctxt, type, start, end),
	233	+ TP_STRUCT__entry(
	234	+ __field(unsigned int, ctxt)
	235	+ __field(u16, subctxt)
	236	+ __string(type, type)
	237	+ __field(unsigned long, start)
	238	+ __field(unsigned long, end)
	239	+ ),
	240	+ TP_fast_assign(
	241	+ __entry->ctxt = ctxt;
	242	+ __entry->subctxt = subctxt;
	243	+ __assign_str(type, type);
	244	+ __entry->start = start;
	245	+ __entry->end = end;
	246	+ ),
	247	+ TP_printk("[%3u:%02u] MMU Invalidate (%s) 0x%lx - 0x%lx",
	248	+ __entry->ctxt,
	249	+ __entry->subctxt,
	250	+ __get_str(type),
	251	+ __entry->start,
	252	+ __entry->end
	253	+ )
	254	+ );
	255	+
	256	+#define SNOOP_PRN \
	257	+ "slid %.4x dlid %.4x qpn 0x%.6x opcode 0x%.2x,%s " \
	258	+ "svc lvl %d pkey 0x%.4x [header = %d bytes] [data = %d bytes]"
	259	+
	260	+TRACE_EVENT(snoop_capture,
	261	+ TP_PROTO(struct hfi1_devdata *dd,
	262	+ int hdr_len,
	263	+ struct hfi1_ib_header *hdr,
	264	+ int data_len,
	265	+ void *data),
	266	+ TP_ARGS(dd, hdr_len, hdr, data_len, data),
	267	+ TP_STRUCT__entry(
	268	+ DD_DEV_ENTRY(dd)
	269	+ __field(u16, slid)
	270	+ __field(u16, dlid)
	271	+ __field(u32, qpn)
	272	+ __field(u8, opcode)
	273	+ __field(u8, sl)
	274	+ __field(u16, pkey)
	275	+ __field(u32, hdr_len)
	276	+ __field(u32, data_len)
	277	+ __field(u8, lnh)
	278	+ __dynamic_array(u8, raw_hdr, hdr_len)
	279	+ __dynamic_array(u8, raw_pkt, data_len)
	280	+ ),
	281	+ TP_fast_assign(
	282	+ struct hfi1_other_headers *ohdr;
	283	+
	284	+ __entry->lnh = (u8)(be16_to_cpu(hdr->lrh[0]) & 3);
	285	+ if (__entry->lnh == HFI1_LRH_BTH)
	286	+ ohdr = &hdr->u.oth;
	287	+ else
	288	+ ohdr = &hdr->u.l.oth;
	289	+ DD_DEV_ASSIGN(dd);
	290	+ __entry->slid = be16_to_cpu(hdr->lrh[3]);
	291	+ __entry->dlid = be16_to_cpu(hdr->lrh[1]);
	292	+ __entry->qpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
	293	+ __entry->opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
	294	+ __entry->sl = (u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
	295	+ __entry->pkey = be32_to_cpu(ohdr->bth[0]) & 0xffff;
	296	+ __entry->hdr_len = hdr_len;
	297	+ __entry->data_len = data_len;
	298	+ memcpy(__get_dynamic_array(raw_hdr), hdr, hdr_len);
	299	+ memcpy(__get_dynamic_array(raw_pkt), data, data_len);
	300	+ ),
	301	+ TP_printk(
	302	+ "[%s] " SNOOP_PRN,
	303	+ __get_str(dev),
	304	+ __entry->slid,
	305	+ __entry->dlid,
	306	+ __entry->qpn,
	307	+ __entry->opcode,
	308	+ show_ib_opcode(__entry->opcode),
	309	+ __entry->sl,
	310	+ __entry->pkey,
	311	+ __entry->hdr_len,
	312	+ __entry->data_len
	313	+ )
	314	+);
	315	+
	316	+#endif /* __HFI1_TRACE_RX_H */
	317	+
	318	+#undef TRACE_INCLUDE_PATH
	319	+#undef TRACE_INCLUDE_FILE
	320	+#define TRACE_INCLUDE_PATH .
	321	+#define TRACE_INCLUDE_FILE trace_rx
	322	+#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/trace_tx.h+642−0

@@ -0,0 +1,642 @@
	1	+/*
	2	+ * Copyright(c) 2015, 2016 Intel Corporation.
	3	+ *
	4	+ * This file is provided under a dual BSD/GPLv2 license. When using or
	5	+ * redistributing this file, you may do so under either license.
	6	+ *
	7	+ * GPL LICENSE SUMMARY
	8	+ *
	9	+ * This program is free software; you can redistribute it and/or modify
	10	+ * it under the terms of version 2 of the GNU General Public License as
	11	+ * published by the Free Software Foundation.
	12	+ *
	13	+ * This program is distributed in the hope that it will be useful, but
	14	+ * WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+ * General Public License for more details.
	17	+ *
	18	+ * BSD LICENSE
	19	+ *
	20	+ * Redistribution and use in source and binary forms, with or without
	21	+ * modification, are permitted provided that the following conditions
	22	+ * are met:
	23	+ *
	24	+ * - Redistributions of source code must retain the above copyright
	25	+ * notice, this list of conditions and the following disclaimer.
	26	+ * - Redistributions in binary form must reproduce the above copyright
	27	+ * notice, this list of conditions and the following disclaimer in
	28	+ * the documentation and/or other materials provided with the
	29	+ * distribution.
	30	+ * - Neither the name of Intel Corporation nor the names of its
	31	+ * contributors may be used to endorse or promote products derived
	32	+ * from this software without specific prior written permission.
	33	+ *
	34	+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	35	+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	36	+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	37	+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	38	+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	39	+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	40	+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	41	+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	42	+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	43	+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	44	+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	45	+ *
	46	+ */
	47	+#if !defined(__HFI1_TRACE_TX_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	48	+#define __HFI1_TRACE_TX_H
	49	+
	50	+#include <linux/tracepoint.h>
	51	+#include <linux/trace_seq.h>
	52	+
	53	+#include "hfi.h"
	54	+#include "mad.h"
	55	+#include "sdma.h"
	56	+
	57	+const char parse_sdma_flags(struct trace_seq p, u64 desc0, u64 desc1);
	58	+
	59	+#define __parse_sdma_flags(desc0, desc1) parse_sdma_flags(p, desc0, desc1)
	60	+
	61	+#undef TRACE_SYSTEM
	62	+#define TRACE_SYSTEM hfi1_tx
	63	+
	64	+TRACE_EVENT(hfi1_piofree,
	65	+ TP_PROTO(struct send_context *sc, int extra),
	66	+ TP_ARGS(sc, extra),
	67	+ TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
	68	+ __field(u32, sw_index)
	69	+ __field(u32, hw_context)
	70	+ __field(int, extra)
	71	+ ),
	72	+ TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
	73	+ __entry->sw_index = sc->sw_index;
	74	+ __entry->hw_context = sc->hw_context;
	75	+ __entry->extra = extra;
	76	+ ),
	77	+ TP_printk("[%s] ctxt %u(%u) extra %d",
	78	+ __get_str(dev),
	79	+ __entry->sw_index,
	80	+ __entry->hw_context,
	81	+ __entry->extra
	82	+ )
	83	+);
	84	+
	85	+TRACE_EVENT(hfi1_wantpiointr,
	86	+ TP_PROTO(struct send_context *sc, u32 needint, u64 credit_ctrl),
	87	+ TP_ARGS(sc, needint, credit_ctrl),
	88	+ TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
	89	+ __field(u32, sw_index)
	90	+ __field(u32, hw_context)
	91	+ __field(u32, needint)
	92	+ __field(u64, credit_ctrl)
	93	+ ),
	94	+ TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
	95	+ __entry->sw_index = sc->sw_index;
	96	+ __entry->hw_context = sc->hw_context;
	97	+ __entry->needint = needint;
	98	+ __entry->credit_ctrl = credit_ctrl;
	99	+ ),
	100	+ TP_printk("[%s] ctxt %u(%u) on %d credit_ctrl 0x%llx",
	101	+ __get_str(dev),
	102	+ __entry->sw_index,
	103	+ __entry->hw_context,
	104	+ __entry->needint,
	105	+ (unsigned long long)__entry->credit_ctrl
	106	+ )
	107	+);
	108	+
	109	+DECLARE_EVENT_CLASS(hfi1_qpsleepwakeup_template,
	110	+ TP_PROTO(struct rvt_qp *qp, u32 flags),
	111	+ TP_ARGS(qp, flags),
	112	+ TP_STRUCT__entry(
	113	+ DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
	114	+ __field(u32, qpn)
	115	+ __field(u32, flags)
	116	+ __field(u32, s_flags)
	117	+ ),
	118	+ TP_fast_assign(
	119	+ DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device))
	120	+ __entry->flags = flags;
	121	+ __entry->qpn = qp->ibqp.qp_num;
	122	+ __entry->s_flags = qp->s_flags;
	123	+ ),
	124	+ TP_printk(
	125	+ "[%s] qpn 0x%x flags 0x%x s_flags 0x%x",
	126	+ __get_str(dev),
	127	+ __entry->qpn,
	128	+ __entry->flags,
	129	+ __entry->s_flags
	130	+ )
	131	+);
	132	+
	133	+DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpwakeup,
	134	+ TP_PROTO(struct rvt_qp *qp, u32 flags),
	135	+ TP_ARGS(qp, flags));
	136	+
	137	+DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpsleep,
	138	+ TP_PROTO(struct rvt_qp *qp, u32 flags),
	139	+ TP_ARGS(qp, flags));
	140	+
	141	+TRACE_EVENT(hfi1_sdma_descriptor,
	142	+ TP_PROTO(struct sdma_engine *sde,
	143	+ u64 desc0,
	144	+ u64 desc1,
	145	+ u16 e,
	146	+ void *descp),
	147	+ TP_ARGS(sde, desc0, desc1, e, descp),
	148	+ TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	149	+ __field(void *, descp)
	150	+ __field(u64, desc0)
	151	+ __field(u64, desc1)
	152	+ __field(u16, e)
	153	+ __field(u8, idx)
	154	+ ),
	155	+ TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	156	+ __entry->desc0 = desc0;
	157	+ __entry->desc1 = desc1;
	158	+ __entry->idx = sde->this_idx;
	159	+ __entry->descp = descp;
	160	+ __entry->e = e;
	161	+ ),
	162	+ TP_printk(
	163	+ "[%s] SDE(%u) flags:%s addr:0x%016llx gen:%u len:%u d0:%016llx d1:%016llx to %p,%u",
	164	+ __get_str(dev),
	165	+ __entry->idx,
	166	+ __parse_sdma_flags(__entry->desc0, __entry->desc1),
	167	+ (__entry->desc0 >> SDMA_DESC0_PHY_ADDR_SHIFT) &
	168	+ SDMA_DESC0_PHY_ADDR_MASK,
	169	+ (u8)((__entry->desc1 >> SDMA_DESC1_GENERATION_SHIFT) &
	170	+ SDMA_DESC1_GENERATION_MASK),
	171	+ (u16)((__entry->desc0 >> SDMA_DESC0_BYTE_COUNT_SHIFT) &
	172	+ SDMA_DESC0_BYTE_COUNT_MASK),
	173	+ __entry->desc0,
	174	+ __entry->desc1,
	175	+ __entry->descp,
	176	+ __entry->e
	177	+ )
	178	+);
	179	+
	180	+TRACE_EVENT(hfi1_sdma_engine_select,
	181	+ TP_PROTO(struct hfi1_devdata *dd, u32 sel, u8 vl, u8 idx),
	182	+ TP_ARGS(dd, sel, vl, idx),
	183	+ TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	184	+ __field(u32, sel)
	185	+ __field(u8, vl)
	186	+ __field(u8, idx)
	187	+ ),
	188	+ TP_fast_assign(DD_DEV_ASSIGN(dd);
	189	+ __entry->sel = sel;
	190	+ __entry->vl = vl;
	191	+ __entry->idx = idx;
	192	+ ),
	193	+ TP_printk("[%s] selecting SDE %u sel 0x%x vl %u",
	194	+ __get_str(dev),
	195	+ __entry->idx,
	196	+ __entry->sel,
	197	+ __entry->vl
	198	+ )
	199	+);
	200	+
	201	+DECLARE_EVENT_CLASS(hfi1_sdma_engine_class,
	202	+ TP_PROTO(struct sdma_engine *sde, u64 status),
	203	+ TP_ARGS(sde, status),
	204	+ TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	205	+ __field(u64, status)
	206	+ __field(u8, idx)
	207	+ ),
	208	+ TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	209	+ __entry->status = status;
	210	+ __entry->idx = sde->this_idx;
	211	+ ),
	212	+ TP_printk("[%s] SDE(%u) status %llx",
	213	+ __get_str(dev),
	214	+ __entry->idx,
	215	+ (unsigned long long)__entry->status
	216	+ )
	217	+);
	218	+
	219	+DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_interrupt,
	220	+ TP_PROTO(struct sdma_engine *sde, u64 status),
	221	+ TP_ARGS(sde, status)
	222	+);
	223	+
	224	+DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_progress,
	225	+ TP_PROTO(struct sdma_engine *sde, u64 status),
	226	+ TP_ARGS(sde, status)
	227	+);
	228	+
	229	+DECLARE_EVENT_CLASS(hfi1_sdma_ahg_ad,
	230	+ TP_PROTO(struct sdma_engine *sde, int aidx),
	231	+ TP_ARGS(sde, aidx),
	232	+ TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	233	+ __field(int, aidx)
	234	+ __field(u8, idx)
	235	+ ),
	236	+ TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	237	+ __entry->idx = sde->this_idx;
	238	+ __entry->aidx = aidx;
	239	+ ),
	240	+ TP_printk("[%s] SDE(%u) aidx %d",
	241	+ __get_str(dev),
	242	+ __entry->idx,
	243	+ __entry->aidx
	244	+ )
	245	+);
	246	+
	247	+DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_allocate,
	248	+ TP_PROTO(struct sdma_engine *sde, int aidx),
	249	+ TP_ARGS(sde, aidx));
	250	+
	251	+DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_deallocate,
	252	+ TP_PROTO(struct sdma_engine *sde, int aidx),
	253	+ TP_ARGS(sde, aidx));
	254	+
	255	+#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
	256	+TRACE_EVENT(hfi1_sdma_progress,
	257	+ TP_PROTO(struct sdma_engine *sde,
	258	+ u16 hwhead,
	259	+ u16 swhead,
	260	+ struct sdma_txreq *txp
	261	+ ),
	262	+ TP_ARGS(sde, hwhead, swhead, txp),
	263	+ TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	264	+ __field(u64, sn)
	265	+ __field(u16, hwhead)
	266	+ __field(u16, swhead)
	267	+ __field(u16, txnext)
	268	+ __field(u16, tx_tail)
	269	+ __field(u16, tx_head)
	270	+ __field(u8, idx)
	271	+ ),
	272	+ TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	273	+ __entry->hwhead = hwhead;
	274	+ __entry->swhead = swhead;
	275	+ __entry->tx_tail = sde->tx_tail;
	276	+ __entry->tx_head = sde->tx_head;
	277	+ __entry->txnext = txp ? txp->next_descq_idx : ~0;
	278	+ __entry->idx = sde->this_idx;
	279	+ __entry->sn = txp ? txp->sn : ~0;
	280	+ ),
	281	+ TP_printk(
	282	+ "[%s] SDE(%u) sn %llu hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
	283	+ __get_str(dev),
	284	+ __entry->idx,
	285	+ __entry->sn,
	286	+ __entry->hwhead,
	287	+ __entry->swhead,
	288	+ __entry->txnext,
	289	+ __entry->tx_head,
	290	+ __entry->tx_tail
	291	+ )
	292	+);
	293	+#else
	294	+TRACE_EVENT(hfi1_sdma_progress,
	295	+ TP_PROTO(struct sdma_engine *sde,
	296	+ u16 hwhead, u16 swhead,
	297	+ struct sdma_txreq *txp
	298	+ ),
	299	+ TP_ARGS(sde, hwhead, swhead, txp),
	300	+ TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	301	+ __field(u16, hwhead)
	302	+ __field(u16, swhead)
	303	+ __field(u16, txnext)
	304	+ __field(u16, tx_tail)
	305	+ __field(u16, tx_head)
	306	+ __field(u8, idx)
	307	+ ),
	308	+ TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	309	+ __entry->hwhead = hwhead;
	310	+ __entry->swhead = swhead;
	311	+ __entry->tx_tail = sde->tx_tail;
	312	+ __entry->tx_head = sde->tx_head;
	313	+ __entry->txnext = txp ? txp->next_descq_idx : ~0;
	314	+ __entry->idx = sde->this_idx;
	315	+ ),
	316	+ TP_printk(
	317	+ "[%s] SDE(%u) hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
	318	+ __get_str(dev),
	319	+ __entry->idx,
	320	+ __entry->hwhead,
	321	+ __entry->swhead,
	322	+ __entry->txnext,
	323	+ __entry->tx_head,
	324	+ __entry->tx_tail
	325	+ )
	326	+);
	327	+#endif
	328	+
	329	+DECLARE_EVENT_CLASS(hfi1_sdma_sn,
	330	+ TP_PROTO(struct sdma_engine *sde, u64 sn),
	331	+ TP_ARGS(sde, sn),
	332	+ TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	333	+ __field(u64, sn)
	334	+ __field(u8, idx)
	335	+ ),
	336	+ TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	337	+ __entry->sn = sn;
	338	+ __entry->idx = sde->this_idx;
	339	+ ),
	340	+ TP_printk("[%s] SDE(%u) sn %llu",
	341	+ __get_str(dev),
	342	+ __entry->idx,
	343	+ __entry->sn
	344	+ )
	345	+);
	346	+
	347	+DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_out_sn,
	348	+ TP_PROTO(
	349	+ struct sdma_engine *sde,
	350	+ u64 sn
	351	+ ),
	352	+ TP_ARGS(sde, sn)
	353	+);
	354	+
	355	+DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_in_sn,
	356	+ TP_PROTO(struct sdma_engine *sde, u64 sn),
	357	+ TP_ARGS(sde, sn)
	358	+);
	359	+
	360	+#define USDMA_HDR_FORMAT \
	361	+ "[%s:%u:%u:%u] PBC=(0x%x 0x%x) LRH=(0x%x 0x%x) BTH=(0x%x 0x%x 0x%x) KDETH=(0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x) TIDVal=0x%x"
	362	+
	363	+TRACE_EVENT(hfi1_sdma_user_header,
	364	+ TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
	365	+ struct hfi1_pkt_header *hdr, u32 tidval),
	366	+ TP_ARGS(dd, ctxt, subctxt, req, hdr, tidval),
	367	+ TP_STRUCT__entry(
	368	+ DD_DEV_ENTRY(dd)
	369	+ __field(u16, ctxt)
	370	+ __field(u8, subctxt)
	371	+ __field(u16, req)
	372	+ __field(u32, pbc0)
	373	+ __field(u32, pbc1)
	374	+ __field(u32, lrh0)
	375	+ __field(u32, lrh1)
	376	+ __field(u32, bth0)
	377	+ __field(u32, bth1)
	378	+ __field(u32, bth2)
	379	+ __field(u32, kdeth0)
	380	+ __field(u32, kdeth1)
	381	+ __field(u32, kdeth2)
	382	+ __field(u32, kdeth3)
	383	+ __field(u32, kdeth4)
	384	+ __field(u32, kdeth5)
	385	+ __field(u32, kdeth6)
	386	+ __field(u32, kdeth7)
	387	+ __field(u32, kdeth8)
	388	+ __field(u32, tidval)
	389	+ ),
	390	+ TP_fast_assign(
	391	+ __le32 pbc = (__le32 )hdr->pbc;
	392	+ __be32 lrh = (__be32 )hdr->lrh;
	393	+ __be32 bth = (__be32 )hdr->bth;
	394	+ __le32 kdeth = (__le32 )&hdr->kdeth;
	395	+
	396	+ DD_DEV_ASSIGN(dd);
	397	+ __entry->ctxt = ctxt;
	398	+ __entry->subctxt = subctxt;
	399	+ __entry->req = req;
	400	+ __entry->pbc0 = le32_to_cpu(pbc[0]);
	401	+ __entry->pbc1 = le32_to_cpu(pbc[1]);
	402	+ __entry->lrh0 = be32_to_cpu(lrh[0]);
	403	+ __entry->lrh1 = be32_to_cpu(lrh[1]);
	404	+ __entry->bth0 = be32_to_cpu(bth[0]);
	405	+ __entry->bth1 = be32_to_cpu(bth[1]);
	406	+ __entry->bth2 = be32_to_cpu(bth[2]);
	407	+ __entry->kdeth0 = le32_to_cpu(kdeth[0]);
	408	+ __entry->kdeth1 = le32_to_cpu(kdeth[1]);
	409	+ __entry->kdeth2 = le32_to_cpu(kdeth[2]);
	410	+ __entry->kdeth3 = le32_to_cpu(kdeth[3]);
	411	+ __entry->kdeth4 = le32_to_cpu(kdeth[4]);
	412	+ __entry->kdeth5 = le32_to_cpu(kdeth[5]);
	413	+ __entry->kdeth6 = le32_to_cpu(kdeth[6]);
	414	+ __entry->kdeth7 = le32_to_cpu(kdeth[7]);
	415	+ __entry->kdeth8 = le32_to_cpu(kdeth[8]);
	416	+ __entry->tidval = tidval;
	417	+ ),
	418	+ TP_printk(USDMA_HDR_FORMAT,
	419	+ __get_str(dev),
	420	+ __entry->ctxt,
	421	+ __entry->subctxt,
	422	+ __entry->req,
	423	+ __entry->pbc1,
	424	+ __entry->pbc0,
	425	+ __entry->lrh0,
	426	+ __entry->lrh1,
	427	+ __entry->bth0,
	428	+ __entry->bth1,
	429	+ __entry->bth2,
	430	+ __entry->kdeth0,
	431	+ __entry->kdeth1,
	432	+ __entry->kdeth2,
	433	+ __entry->kdeth3,
	434	+ __entry->kdeth4,
	435	+ __entry->kdeth5,
	436	+ __entry->kdeth6,
	437	+ __entry->kdeth7,
	438	+ __entry->kdeth8,
	439	+ __entry->tidval
	440	+ )
	441	+);
	442	+
	443	+#define SDMA_UREQ_FMT \
	444	+ "[%s:%u:%u] ver/op=0x%x, iovcnt=%u, npkts=%u, frag=%u, idx=%u"
	445	+TRACE_EVENT(hfi1_sdma_user_reqinfo,
	446	+ TP_PROTO(struct hfi1_devdata dd, u16 ctxt, u8 subctxt, u16 i),
	447	+ TP_ARGS(dd, ctxt, subctxt, i),
	448	+ TP_STRUCT__entry(
	449	+ DD_DEV_ENTRY(dd);
	450	+ __field(u16, ctxt)
	451	+ __field(u8, subctxt)
	452	+ __field(u8, ver_opcode)
	453	+ __field(u8, iovcnt)
	454	+ __field(u16, npkts)
	455	+ __field(u16, fragsize)
	456	+ __field(u16, comp_idx)
	457	+ ),
	458	+ TP_fast_assign(
	459	+ DD_DEV_ASSIGN(dd);
	460	+ __entry->ctxt = ctxt;
	461	+ __entry->subctxt = subctxt;
	462	+ __entry->ver_opcode = i[0] & 0xff;
	463	+ __entry->iovcnt = (i[0] >> 8) & 0xff;
	464	+ __entry->npkts = i[1];
	465	+ __entry->fragsize = i[2];
	466	+ __entry->comp_idx = i[3];
	467	+ ),
	468	+ TP_printk(SDMA_UREQ_FMT,
	469	+ __get_str(dev),
	470	+ __entry->ctxt,
	471	+ __entry->subctxt,
	472	+ __entry->ver_opcode,
	473	+ __entry->iovcnt,
	474	+ __entry->npkts,
	475	+ __entry->fragsize,
	476	+ __entry->comp_idx
	477	+ )
	478	+);
	479	+
	480	+#define usdma_complete_name(st) { st, #st }
	481	+#define show_usdma_complete_state(st) \
	482	+ __print_symbolic(st, \
	483	+ usdma_complete_name(FREE), \
	484	+ usdma_complete_name(QUEUED), \
	485	+ usdma_complete_name(COMPLETE), \
	486	+ usdma_complete_name(ERROR))
	487	+
	488	+TRACE_EVENT(hfi1_sdma_user_completion,
	489	+ TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 idx,
	490	+ u8 state, int code),
	491	+ TP_ARGS(dd, ctxt, subctxt, idx, state, code),
	492	+ TP_STRUCT__entry(
	493	+ DD_DEV_ENTRY(dd)
	494	+ __field(u16, ctxt)
	495	+ __field(u8, subctxt)
	496	+ __field(u16, idx)
	497	+ __field(u8, state)
	498	+ __field(int, code)
	499	+ ),
	500	+ TP_fast_assign(
	501	+ DD_DEV_ASSIGN(dd);
	502	+ __entry->ctxt = ctxt;
	503	+ __entry->subctxt = subctxt;
	504	+ __entry->idx = idx;
	505	+ __entry->state = state;
	506	+ __entry->code = code;
	507	+ ),
	508	+ TP_printk("[%s:%u:%u:%u] SDMA completion state %s (%d)",
	509	+ __get_str(dev), __entry->ctxt, __entry->subctxt,
	510	+ __entry->idx, show_usdma_complete_state(__entry->state),
	511	+ __entry->code)
	512	+);
	513	+
	514	+const char print_u32_array(struct trace_seq , u32 *, int);
	515	+#define __print_u32_hex(arr, len) print_u32_array(p, arr, len)
	516	+
	517	+TRACE_EVENT(hfi1_sdma_user_header_ahg,
	518	+ TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
	519	+ u8 sde, u8 ahgidx, u32 *ahg, int len, u32 tidval),
	520	+ TP_ARGS(dd, ctxt, subctxt, req, sde, ahgidx, ahg, len, tidval),
	521	+ TP_STRUCT__entry(
	522	+ DD_DEV_ENTRY(dd)
	523	+ __field(u16, ctxt)
	524	+ __field(u8, subctxt)
	525	+ __field(u16, req)
	526	+ __field(u8, sde)
	527	+ __field(u8, idx)
	528	+ __field(int, len)
	529	+ __field(u32, tidval)
	530	+ __array(u32, ahg, 10)
	531	+ ),
	532	+ TP_fast_assign(
	533	+ DD_DEV_ASSIGN(dd);
	534	+ __entry->ctxt = ctxt;
	535	+ __entry->subctxt = subctxt;
	536	+ __entry->req = req;
	537	+ __entry->sde = sde;
	538	+ __entry->idx = ahgidx;
	539	+ __entry->len = len;
	540	+ __entry->tidval = tidval;
	541	+ memcpy(__entry->ahg, ahg, len * sizeof(u32));
	542	+ ),
	543	+ TP_printk("[%s:%u:%u:%u] (SDE%u/AHG%u) ahg[0-%d]=(%s) TIDVal=0x%x",
	544	+ __get_str(dev),
	545	+ __entry->ctxt,
	546	+ __entry->subctxt,
	547	+ __entry->req,
	548	+ __entry->sde,
	549	+ __entry->idx,
	550	+ __entry->len - 1,
	551	+ __print_u32_hex(__entry->ahg, __entry->len),
	552	+ __entry->tidval
	553	+ )
	554	+);
	555	+
	556	+TRACE_EVENT(hfi1_sdma_state,
	557	+ TP_PROTO(struct sdma_engine *sde,
	558	+ const char *cstate,
	559	+ const char *nstate
	560	+ ),
	561	+ TP_ARGS(sde, cstate, nstate),
	562	+ TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
	563	+ __string(curstate, cstate)
	564	+ __string(newstate, nstate)
	565	+ ),
	566	+ TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
	567	+ __assign_str(curstate, cstate);
	568	+ __assign_str(newstate, nstate);
	569	+ ),
	570	+ TP_printk("[%s] current state %s new state %s",
	571	+ __get_str(dev),
	572	+ __get_str(curstate),
	573	+ __get_str(newstate)
	574	+ )
	575	+);
	576	+
	577	+#define BCT_FORMAT \
	578	+ "shared_limit %x vls 0-7 [%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x] 15 [%x,%x]"
	579	+
	580	+#define BCT(field) \
	581	+ be16_to_cpu( \
	582	+ ((struct buffer_control *)__get_dynamic_array(bct))->field \
	583	+ )
	584	+
	585	+DECLARE_EVENT_CLASS(hfi1_bct_template,
	586	+ TP_PROTO(struct hfi1_devdata *dd,
	587	+ struct buffer_control *bc),
	588	+ TP_ARGS(dd, bc),
	589	+ TP_STRUCT__entry(DD_DEV_ENTRY(dd)
	590	+ __dynamic_array(u8, bct, sizeof(*bc))
	591	+ ),
	592	+ TP_fast_assign(DD_DEV_ASSIGN(dd);
	593	+ memcpy(__get_dynamic_array(bct), bc,
	594	+ sizeof(*bc));
	595	+ ),
	596	+ TP_printk(BCT_FORMAT,
	597	+ BCT(overall_shared_limit),
	598	+
	599	+ BCT(vl[0].dedicated),
	600	+ BCT(vl[0].shared),
	601	+
	602	+ BCT(vl[1].dedicated),
	603	+ BCT(vl[1].shared),
	604	+
	605	+ BCT(vl[2].dedicated),
	606	+ BCT(vl[2].shared),
	607	+
	608	+ BCT(vl[3].dedicated),
	609	+ BCT(vl[3].shared),
	610	+
	611	+ BCT(vl[4].dedicated),
	612	+ BCT(vl[4].shared),
	613	+
	614	+ BCT(vl[5].dedicated),
	615	+ BCT(vl[5].shared),
	616	+
	617	+ BCT(vl[6].dedicated),
	618	+ BCT(vl[6].shared),
	619	+
	620	+ BCT(vl[7].dedicated),
	621	+ BCT(vl[7].shared),
	622	+
	623	+ BCT(vl[15].dedicated),
	624	+ BCT(vl[15].shared)
	625	+ )
	626	+);
	627	+
	628	+DEFINE_EVENT(hfi1_bct_template, bct_set,
	629	+ TP_PROTO(struct hfi1_devdata dd, struct buffer_control bc),
	630	+ TP_ARGS(dd, bc));
	631	+
	632	+DEFINE_EVENT(hfi1_bct_template, bct_get,
	633	+ TP_PROTO(struct hfi1_devdata dd, struct buffer_control bc),
	634	+ TP_ARGS(dd, bc));
	635	+
	636	+#endif /* __HFI1_TRACE_TX_H */
	637	+
	638	+#undef TRACE_INCLUDE_PATH
	639	+#undef TRACE_INCLUDE_FILE
	640	+#define TRACE_INCLUDE_PATH .
	641	+#define TRACE_INCLUDE_FILE trace_tx
	642	+#include <trace/define_trace.h>

drivers/infiniband/hw/hfi1/twsi.c+0−489

@@ -1,489 +0,0 @@
1		−/*
2		− * Copyright(c) 2015, 2016 Intel Corporation.
3		− *
4		− * This file is provided under a dual BSD/GPLv2 license. When using or
5		− * redistributing this file, you may do so under either license.
6		− *
7		− * GPL LICENSE SUMMARY
8		− *
9		− * This program is free software; you can redistribute it and/or modify
10		− * it under the terms of version 2 of the GNU General Public License as
11		− * published by the Free Software Foundation.
12		− *
13		− * This program is distributed in the hope that it will be useful, but
14		− * WITHOUT ANY WARRANTY; without even the implied warranty of
15		− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16		− * General Public License for more details.
17		− *
18		− * BSD LICENSE
19		− *
20		− * Redistribution and use in source and binary forms, with or without
21		− * modification, are permitted provided that the following conditions
22		− * are met:
23		− *
24		− * - Redistributions of source code must retain the above copyright
25		− * notice, this list of conditions and the following disclaimer.
26		− * - Redistributions in binary form must reproduce the above copyright
27		− * notice, this list of conditions and the following disclaimer in
28		− * the documentation and/or other materials provided with the
29		− * distribution.
30		− * - Neither the name of Intel Corporation nor the names of its
31		− * contributors may be used to endorse or promote products derived
32		− * from this software without specific prior written permission.
33		− *
34		− * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35		− * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36		− * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37		− * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38		− * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39		− * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40		− * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41		− * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42		− * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43		− * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44		− * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45		− *
46		− */
47		−
48		−#include <linux/delay.h>
49		−#include <linux/pci.h>
50		−#include <linux/vmalloc.h>
51		−
52		−#include "hfi.h"
53		−#include "twsi.h"
54		−
55		−/*
56		− * "Two Wire Serial Interface" support.
57		− *
58		− * Originally written for a not-quite-i2c serial eeprom, which is
59		− * still used on some supported boards. Later boards have added a
60		− * variety of other uses, most board-specific, so the bit-boffing
61		− * part has been split off to this file, while the other parts
62		− * have been moved to chip-specific files.
63		− *
64		− * We have also dropped all pretense of fully generic (e.g. pretend
65		− * we don't know whether '1' is the higher voltage) interface, as
66		− * the restrictions of the generic i2c interface (e.g. no access from
67		− * driver itself) make it unsuitable for this use.
68		− */
69		−
70		−#define READ_CMD 1
71		−#define WRITE_CMD 0
72		−
73		−/**
74		− * i2c_wait_for_writes - wait for a write
75		− * @dd: the hfi1_ib device
76		− *
77		− * We use this instead of udelay directly, so we can make sure
78		− * that previous register writes have been flushed all the way
79		− * to the chip. Since we are delaying anyway, the cost doesn't
80		− * hurt, and makes the bit twiddling more regular
81		− */
82		−static void i2c_wait_for_writes(struct hfi1_devdata *dd, u32 target)
83		−{
84		− /*
85		− * implicit read of EXTStatus is as good as explicit
86		− * read of scratch, if all we want to do is flush
87		− * writes.
88		− */
89		− hfi1_gpio_mod(dd, target, 0, 0, 0);
90		− rmb(); /* inlined, so prevent compiler reordering */
91		−}
92		−
93		−/*
94		− * QSFP modules are allowed to hold SCL low for 500uSec. Allow twice that
95		− * for "almost compliant" modules
96		− */
97		−#define SCL_WAIT_USEC 1000
98		−
99		−/* BUF_WAIT is time bus must be free between STOP or ACK and to next START.
100		− * Should be 20, but some chips need more.
101		− */
102		−#define TWSI_BUF_WAIT_USEC 60
103		−
104		−static void scl_out(struct hfi1_devdata *dd, u32 target, u8 bit)
105		−{
106		− u32 mask;
107		−
108		− udelay(1);
109		−
110		− mask = QSFP_HFI0_I2CCLK;
111		−
112		− /* SCL is meant to be bare-drain, so never set "OUT", just DIR */
113		− hfi1_gpio_mod(dd, target, 0, bit ? 0 : mask, mask);
114		−
115		− /*
116		− * Allow for slow slaves by simple
117		− * delay for falling edge, sampling on rise.
118		− */
119		− if (!bit) {
120		− udelay(2);
121		− } else {
122		− int rise_usec;
123		−
124		− for (rise_usec = SCL_WAIT_USEC; rise_usec > 0; rise_usec -= 2) {
125		− if (mask & hfi1_gpio_mod(dd, target, 0, 0, 0))
126		− break;
127		− udelay(2);
128		− }
129		− if (rise_usec <= 0)
130		− dd_dev_err(dd, "SCL interface stuck low > %d uSec\n",
131		− SCL_WAIT_USEC);
132		− }
133		− i2c_wait_for_writes(dd, target);
134		−}
135		−
136		−static u8 scl_in(struct hfi1_devdata *dd, u32 target, int wait)
137		−{
138		− u32 read_val, mask;
139		−
140		− mask = QSFP_HFI0_I2CCLK;
141		− /* SCL is meant to be bare-drain, so never set "OUT", just DIR */
142		− hfi1_gpio_mod(dd, target, 0, 0, mask);
143		− read_val = hfi1_gpio_mod(dd, target, 0, 0, 0);
144		− if (wait)
145		− i2c_wait_for_writes(dd, target);
146		− return (read_val & mask) >> GPIO_SCL_NUM;
147		−}
148		−
149		−static void sda_out(struct hfi1_devdata *dd, u32 target, u8 bit)
150		−{
151		− u32 mask;
152		−
153		− mask = QSFP_HFI0_I2CDAT;
154		−
155		− /* SDA is meant to be bare-drain, so never set "OUT", just DIR */
156		− hfi1_gpio_mod(dd, target, 0, bit ? 0 : mask, mask);
157		−
158		− i2c_wait_for_writes(dd, target);
159		− udelay(2);
160		−}
161		−
162		−static u8 sda_in(struct hfi1_devdata *dd, u32 target, int wait)
163		−{
164		− u32 read_val, mask;
165		−
166		− mask = QSFP_HFI0_I2CDAT;
167		− /* SDA is meant to be bare-drain, so never set "OUT", just DIR */
168		− hfi1_gpio_mod(dd, target, 0, 0, mask);
169		− read_val = hfi1_gpio_mod(dd, target, 0, 0, 0);
170		− if (wait)
171		− i2c_wait_for_writes(dd, target);
172		− return (read_val & mask) >> GPIO_SDA_NUM;
173		−}
174		−
175		−/**
176		− * i2c_ackrcv - see if ack following write is true
177		− * @dd: the hfi1_ib device
178		− */
179		−static int i2c_ackrcv(struct hfi1_devdata *dd, u32 target)
180		−{
181		− u8 ack_received;
182		−
183		− /* AT ENTRY SCL = LOW */
184		− /* change direction, ignore data */
185		− ack_received = sda_in(dd, target, 1);
186		− scl_out(dd, target, 1);
187		− ack_received = sda_in(dd, target, 1) == 0;
188		− scl_out(dd, target, 0);
189		− return ack_received;
190		−}
191		−
192		−static void stop_cmd(struct hfi1_devdata *dd, u32 target);
193		−
194		−/**
195		− * rd_byte - read a byte, sending STOP on last, else ACK
196		− * @dd: the hfi1_ib device
197		− *
198		− * Returns byte shifted out of device
199		− */
200		−static int rd_byte(struct hfi1_devdata *dd, u32 target, int last)
201		−{
202		− int bit_cntr, data;
203		−
204		− data = 0;
205		−
206		− for (bit_cntr = 7; bit_cntr >= 0; --bit_cntr) {
207		− data <<= 1;
208		− scl_out(dd, target, 1);
209		− data \|= sda_in(dd, target, 0);
210		− scl_out(dd, target, 0);
211		− }
212		− if (last) {
213		− scl_out(dd, target, 1);
214		− stop_cmd(dd, target);
215		− } else {
216		− sda_out(dd, target, 0);
217		− scl_out(dd, target, 1);
218		− scl_out(dd, target, 0);
219		− sda_out(dd, target, 1);
220		− }
221		− return data;
222		−}
223		−
224		−/**
225		− * wr_byte - write a byte, one bit at a time
226		− * @dd: the hfi1_ib device
227		− * @data: the byte to write
228		− *
229		− * Returns 0 if we got the following ack, otherwise 1
230		− */
231		−static int wr_byte(struct hfi1_devdata *dd, u32 target, u8 data)
232		−{
233		− int bit_cntr;
234		− u8 bit;
235		−
236		− for (bit_cntr = 7; bit_cntr >= 0; bit_cntr--) {
237		− bit = (data >> bit_cntr) & 1;
238		− sda_out(dd, target, bit);
239		− scl_out(dd, target, 1);
240		− scl_out(dd, target, 0);
241		− }
242		− return (!i2c_ackrcv(dd, target)) ? 1 : 0;
243		−}
244		−
245		−/*
246		− * issue TWSI start sequence:
247		− * (both clock/data high, clock high, data low while clock is high)
248		− */
249		−static void start_seq(struct hfi1_devdata *dd, u32 target)
250		−{
251		− sda_out(dd, target, 1);
252		− scl_out(dd, target, 1);
253		− sda_out(dd, target, 0);
254		− udelay(1);
255		− scl_out(dd, target, 0);
256		−}
257		−
258		−/**
259		− * stop_seq - transmit the stop sequence
260		− * @dd: the hfi1_ib device
261		− *
262		− * (both clock/data low, clock high, data high while clock is high)
263		− */
264		−static void stop_seq(struct hfi1_devdata *dd, u32 target)
265		−{
266		− scl_out(dd, target, 0);
267		− sda_out(dd, target, 0);
268		− scl_out(dd, target, 1);
269		− sda_out(dd, target, 1);
270		−}
271		−
272		−/**
273		− * stop_cmd - transmit the stop condition
274		− * @dd: the hfi1_ib device
275		− *
276		− * (both clock/data low, clock high, data high while clock is high)
277		− */
278		−static void stop_cmd(struct hfi1_devdata *dd, u32 target)
279		−{
280		− stop_seq(dd, target);
281		− udelay(TWSI_BUF_WAIT_USEC);
282		−}
283		−
284		−/**
285		− * hfi1_twsi_reset - reset I2C communication
286		− * @dd: the hfi1_ib device
287		− * returns 0 if ok, -EIO on error
288		− */
289		−int hfi1_twsi_reset(struct hfi1_devdata *dd, u32 target)
290		−{
291		− int clock_cycles_left = 9;
292		− u32 mask;
293		−
294		− /* Both SCL and SDA should be high. If not, there
295		− * is something wrong.
296		− */
297		− mask = QSFP_HFI0_I2CCLK \| QSFP_HFI0_I2CDAT;
298		−
299		− /*
300		− * Force pins to desired innocuous state.
301		− * This is the default power-on state with out=0 and dir=0,
302		− * So tri-stated and should be floating high (barring HW problems)
303		− */
304		− hfi1_gpio_mod(dd, target, 0, 0, mask);
305		−
306		− /* Check if SCL is low, if it is low then we have a slave device
307		− * misbehaving and there is not much we can do.
308		− */
309		− if (!scl_in(dd, target, 0))
310		− return -EIO;
311		−
312		− /* Check if SDA is low, if it is low then we have to clock SDA
313		− * up to 9 times for the device to release the bus
314		− */
315		− while (clock_cycles_left--) {
316		− if (sda_in(dd, target, 0))
317		− return 0;
318		− scl_out(dd, target, 0);
319		− scl_out(dd, target, 1);
320		− }
321		−
322		− return -EIO;
323		−}
324		−
325		−#define HFI1_TWSI_START 0x100
326		−#define HFI1_TWSI_STOP 0x200
327		−
328		−/* Write byte to TWSI, optionally prefixed with START or suffixed with
329		− * STOP.
330		− * returns 0 if OK (ACK received), else != 0
331		− */
332		−static int twsi_wr(struct hfi1_devdata *dd, u32 target, int data, int flags)
333		−{
334		− int ret = 1;
335		−
336		− if (flags & HFI1_TWSI_START)
337		− start_seq(dd, target);
338		−
339		− /* Leaves SCL low (from i2c_ackrcv()) */
340		− ret = wr_byte(dd, target, data);
341		−
342		− if (flags & HFI1_TWSI_STOP)
343		− stop_cmd(dd, target);
344		− return ret;
345		−}
346		−
347		−/* Added functionality for IBA7220-based cards */
348		−#define HFI1_TEMP_DEV 0x98
349		−
350		−/*
351		− * hfi1_twsi_blk_rd
352		− * General interface for data transfer from twsi devices.
353		− * One vestige of its former role is that it recognizes a device
354		− * HFI1_TWSI_NO_DEV and does the correct operation for the legacy part,
355		− * which responded to all TWSI device codes, interpreting them as
356		− * address within device. On all other devices found on board handled by
357		− * this driver, the device is followed by a N-byte "address" which selects
358		− * the "register" or "offset" within the device from which data should
359		− * be read.
360		− */
361		−int hfi1_twsi_blk_rd(struct hfi1_devdata *dd, u32 target, int dev, int addr,
362		− void *buffer, int len)
363		−{
364		− u8 *bp = buffer;
365		− int ret = 1;
366		− int i;
367		− int offset_size;
368		−
369		− /* obtain the offset size, strip it from the device address */
370		− offset_size = (dev >> 8) & 0xff;
371		− dev &= 0xff;
372		−
373		− /* allow at most a 2 byte offset */
374		− if (offset_size > 2)
375		− goto bail;
376		−
377		− if (dev == HFI1_TWSI_NO_DEV) {
378		− /* legacy not-really-I2C */
379		− addr = (addr << 1) \| READ_CMD;
380		− ret = twsi_wr(dd, target, addr, HFI1_TWSI_START);
381		− } else {
382		− /* Actual I2C */
383		− if (offset_size) {
384		− ret = twsi_wr(dd, target,
385		− dev \| WRITE_CMD, HFI1_TWSI_START);
386		− if (ret) {
387		− stop_cmd(dd, target);
388		− goto bail;
389		− }
390		−
391		− for (i = 0; i < offset_size; i++) {
392		− ret = twsi_wr(dd, target,
393		− (addr >> (i * 8)) & 0xff, 0);
394		− udelay(TWSI_BUF_WAIT_USEC);
395		− if (ret) {
396		− dd_dev_err(dd, "Failed to write byte %d of offset 0x%04X\n",
397		− i, addr);
398		− goto bail;
399		− }
400		− }
401		− }
402		− ret = twsi_wr(dd, target, dev \| READ_CMD, HFI1_TWSI_START);
403		− }
404		− if (ret) {
405		− stop_cmd(dd, target);
406		− goto bail;
407		− }
408		−
409		− /*
410		− * block devices keeps clocking data out as long as we ack,
411		− * automatically incrementing the address. Some have "pages"
412		− * whose boundaries will not be crossed, but the handling
413		− * of these is left to the caller, who is in a better
414		− * position to know.
415		− */
416		− while (len-- > 0) {
417		− /*
418		− * Get and store data, sending ACK if length remaining,
419		− * else STOP
420		− */
421		− *bp++ = rd_byte(dd, target, !len);
422		− }
423		−
424		− ret = 0;
425		−
426		−bail:
427		− return ret;
428		−}
429		−
430		−/*
431		− * hfi1_twsi_blk_wr
432		− * General interface for data transfer to twsi devices.
433		− * One vestige of its former role is that it recognizes a device
434		− * HFI1_TWSI_NO_DEV and does the correct operation for the legacy part,
435		− * which responded to all TWSI device codes, interpreting them as
436		− * address within device. On all other devices found on board handled by
437		− * this driver, the device is followed by a N-byte "address" which selects
438		− * the "register" or "offset" within the device to which data should
439		− * be written.
440		− */
441		−int hfi1_twsi_blk_wr(struct hfi1_devdata *dd, u32 target, int dev, int addr,
442		− const void *buffer, int len)
443		−{
444		− const u8 *bp = buffer;
445		− int ret = 1;
446		− int i;
447		− int offset_size;
448		−
449		− /* obtain the offset size, strip it from the device address */
450		− offset_size = (dev >> 8) & 0xff;
451		− dev &= 0xff;
452		−
453		− /* allow at most a 2 byte offset */
454		− if (offset_size > 2)
455		− goto bail;
456		−
457		− if (dev == HFI1_TWSI_NO_DEV) {
458		− if (twsi_wr(dd, target, (addr << 1) \| WRITE_CMD,
459		− HFI1_TWSI_START)) {
460		− goto failed_write;
461		− }
462		− } else {
463		− /* Real I2C */
464		− if (twsi_wr(dd, target, dev \| WRITE_CMD, HFI1_TWSI_START))
465		− goto failed_write;
466		− }
467		−
468		− for (i = 0; i < offset_size; i++) {
469		− ret = twsi_wr(dd, target, (addr >> (i * 8)) & 0xff, 0);
470		− udelay(TWSI_BUF_WAIT_USEC);
471		− if (ret) {
472		− dd_dev_err(dd, "Failed to write byte %d of offset 0x%04X\n",
473		− i, addr);
474		− goto bail;
475		− }
476		− }
477		−
478		− for (i = 0; i < len; i++)
479		− if (twsi_wr(dd, target, *bp++, 0))
480		− goto failed_write;
481		−
482		− ret = 0;
483		−
484		−failed_write:
485		− stop_cmd(dd, target);
486		−
487		−bail:
488		− return ret;
489		−}

drivers/infiniband/hw/hfi1/twsi.h+0−65

@@ -1,65 +0,0 @@
1		−#ifndef _TWSI_H
2		−#define _TWSI_H
3		−/*
4		− * Copyright(c) 2015, 2016 Intel Corporation.
5		− *
6		− * This file is provided under a dual BSD/GPLv2 license. When using or
7		− * redistributing this file, you may do so under either license.
8		− *
9		− * GPL LICENSE SUMMARY
10		− *
11		− * This program is free software; you can redistribute it and/or modify
12		− * it under the terms of version 2 of the GNU General Public License as
13		− * published by the Free Software Foundation.
14		− *
15		− * This program is distributed in the hope that it will be useful, but
16		− * WITHOUT ANY WARRANTY; without even the implied warranty of
17		− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18		− * General Public License for more details.
19		− *
20		− * BSD LICENSE
21		− *
22		− * Redistribution and use in source and binary forms, with or without
23		− * modification, are permitted provided that the following conditions
24		− * are met:
25		− *
26		− * - Redistributions of source code must retain the above copyright
27		− * notice, this list of conditions and the following disclaimer.
28		− * - Redistributions in binary form must reproduce the above copyright
29		− * notice, this list of conditions and the following disclaimer in
30		− * the documentation and/or other materials provided with the
31		− * distribution.
32		− * - Neither the name of Intel Corporation nor the names of its
33		− * contributors may be used to endorse or promote products derived
34		− * from this software without specific prior written permission.
35		− *
36		− * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37		− * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38		− * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
39		− * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
40		− * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
41		− * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
42		− * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
43		− * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
44		− * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
45		− * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
46		− * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47		− *
48		− */
49		−
50		−#define HFI1_TWSI_NO_DEV 0xFF
51		−
52		−struct hfi1_devdata;
53		−
54		−/* Bit position of SDA/SCL pins in ASIC_QSFP* registers */
55		−#define GPIO_SDA_NUM 1
56		−#define GPIO_SCL_NUM 0
57		−
58		−/* these functions must be called with qsfp_lock held */
59		−int hfi1_twsi_reset(struct hfi1_devdata *dd, u32 target);
60		−int hfi1_twsi_blk_rd(struct hfi1_devdata *dd, u32 target, int dev, int addr,
61		− void *buffer, int len);
62		−int hfi1_twsi_blk_wr(struct hfi1_devdata *dd, u32 target, int dev, int addr,
63		− const void *buffer, int len);
64		−
65		−#endif /* _TWSI_H */

drivers/infiniband/hw/hfi1/uc.c+26−35

@@ -118,6 +118,31 @@ int hfi1_make_uc_req(struct rvt_qp qp, struct hfi1_pkt_state ps) · 117 unmodified lines
118	118	clear_ahg(qp);
119	119	goto bail;
120	120	}
	121	+ /*
	122	+ * Local operations are processed immediately
	123	+ * after all prior requests have completed.
	124	+ */
	125	+ if (wqe->wr.opcode == IB_WR_REG_MR \|\|
	126	+ wqe->wr.opcode == IB_WR_LOCAL_INV) {
	127	+ int local_ops = 0;
	128	+ int err = 0;
	129	+
	130	+ if (qp->s_last != qp->s_cur)
	131	+ goto bail;
	132	+ if (++qp->s_cur == qp->s_size)
	133	+ qp->s_cur = 0;
	134	+ if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
	135	+ err = rvt_invalidate_rkey(
	136	+ qp, wqe->wr.ex.invalidate_rkey);
	137	+ local_ops = 1;
	138	+ }
	139	+ hfi1_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR
	140	+ : IB_WC_SUCCESS);
	141	+ if (local_ops)
	142	+ atomic_dec(&qp->local_ops_pending);
	143	+ qp->s_hdrwords = 0;
	144	+ goto done_free_tx;
	145	+ }
121	146	/*
122	147	* Start a new request.
123	148	*/
@@ -294,46 +319,12 @@ void hfi1_uc_rcv(struct hfi1_packet *packet) · 170 unmodified lines
294	319	struct ib_reth *reth;
295	320	int has_grh = rcv_flags & HFI1_HAS_GRH;
296	321	int ret;
297		− u32 bth1;
298	322
299	323	bth0 = be32_to_cpu(ohdr->bth[0]);
300	324	if (hfi1_ruc_check_hdr(ibp, hdr, has_grh, qp, bth0))
301	325	return;
302	326
303		− bth1 = be32_to_cpu(ohdr->bth[1]);
304		− if (unlikely(bth1 & (HFI1_BECN_SMASK \| HFI1_FECN_SMASK))) {
305		− if (bth1 & HFI1_BECN_SMASK) {
306		− struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
307		− u32 rqpn, lqpn;
308		− u16 rlid = be16_to_cpu(hdr->lrh[3]);
309		− u8 sl, sc5;
310		−
311		− lqpn = bth1 & RVT_QPN_MASK;
312		− rqpn = qp->remote_qpn;
313		−
314		− sc5 = ibp->sl_to_sc[qp->remote_ah_attr.sl];
315		− sl = ibp->sc_to_sl[sc5];
316		−
317		− process_becn(ppd, sl, rlid, lqpn, rqpn,
318		− IB_CC_SVCTYPE_UC);
319		− }
320		−
321		− if (bth1 & HFI1_FECN_SMASK) {
322		− struct ib_grh *grh = NULL;
323		− u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);
324		− u16 slid = be16_to_cpu(hdr->lrh[3]);
325		− u16 dlid = be16_to_cpu(hdr->lrh[1]);
326		− u32 src_qp = qp->remote_qpn;
327		− u8 sc5;
328		−
329		− sc5 = ibp->sl_to_sc[qp->remote_ah_attr.sl];
330		− if (has_grh)
331		− grh = &hdr->u.l.grh;
332		−
333		− return_cnp(ibp, qp, src_qp, pkey, dlid, slid, sc5,
334		− grh);
335		− }
336		− }
	327	+ process_ecn(qp, packet, true);
337	328
338	329	psn = be32_to_cpu(ohdr->bth[2]);
339	330	opcode = (bth0 >> 24) & 0xff;

drivers/infiniband/hw/hfi1/ud.c+29−57

@@ -184,8 +184,12 @@ static void ud_loopback(struct rvt_qp sqp, struct rvt_swqe swqe) · 183 unmodified lines
184	184	}
185	185
186	186	if (ah_attr->ah_flags & IB_AH_GRH) {
187		− hfi1_copy_sge(&qp->r_sge, &ah_attr->grh,
188		− sizeof(struct ib_grh), 1, 0);
	187	+ struct ib_grh grh;
	188	+ struct ib_global_route grd = ah_attr->grh;
	189	+
	190	+ hfi1_make_grh(ibp, &grh, &grd, 0, 0);
	191	+ hfi1_copy_sge(&qp->r_sge, &grh,
	192	+ sizeof(grh), 1, 0);
189	193	wc.wc_flags \|= IB_WC_GRH;
190	194	} else {
191	195	hfi1_skip_sge(&qp->r_sge, sizeof(struct ib_grh), 1);
@@ -430,10 +434,9 @@ int hfi1_make_ud_req(struct rvt_qp qp, struct hfi1_pkt_state ps) · 238 unmodified lines
430	434	qp->qkey : wqe->ud_wr.remote_qkey);
431	435	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
432	436	/* disarm any ahg */
433		− priv->s_hdr->ahgcount = 0;
434		− priv->s_hdr->ahgidx = 0;
435		− priv->s_hdr->tx_flags = 0;
436		− priv->s_hdr->sde = NULL;
	437	+ priv->s_ahg->ahgcount = 0;
	438	+ priv->s_ahg->ahgidx = 0;
	439	+ priv->s_ahg->tx_flags = 0;
437	440	/* pbc */
438	441	ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
439	442
@@ -665,13 +668,13 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) · 225 unmodified lines
665	668	struct hfi1_other_headers *ohdr = packet->ohdr;
666	669	int opcode;
667	670	u32 hdrsize = packet->hlen;
668		− u32 pad;
669	671	struct ib_wc wc;
670	672	u32 qkey;
671	673	u32 src_qp;
672	674	u16 dlid, pkey;
673	675	int mgmt_pkey_idx = -1;
674	676	struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
	677	+ struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
675	678	struct hfi1_ib_header *hdr = packet->hdr;
676	679	u32 rcv_flags = packet->rcv_flags;
677	680	void *data = packet->ebuf;
@@ -680,52 +683,33 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) · 2 unmodified lines
680	683	bool has_grh = rcv_flags & HFI1_HAS_GRH;
681	684	u8 sc5 = hdr2sc((struct hfi1_message_header *)hdr, packet->rhf);
682	685	u32 bth1;
683		− int is_mcast;
684		− struct ib_grh *grh = NULL;
	686	+ u8 sl_from_sc, sl;
	687	+ u16 slid;
	688	+ u8 extra_bytes;
685	689
686	690	qkey = be32_to_cpu(ohdr->u.ud.deth[0]);
687	691	src_qp = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
688	692	dlid = be16_to_cpu(hdr->lrh[1]);
689		− is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
690		− (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
691	693	bth1 = be32_to_cpu(ohdr->bth[1]);
692		− if (unlikely(bth1 & HFI1_BECN_SMASK)) {
693		− /*
694		− * In pre-B0 h/w the CNP_OPCODE is handled via an
695		− * error path.
696		− */
697		− struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
698		− u32 lqpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
699		− u8 sl;
700		−
701		− sl = ibp->sc_to_sl[sc5];
702		−
703		− process_becn(ppd, sl, 0, lqpn, 0, IB_CC_SVCTYPE_UD);
704		− }
	694	+ slid = be16_to_cpu(hdr->lrh[3]);
	695	+ pkey = (u16)be32_to_cpu(ohdr->bth[0]);
	696	+ sl = (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
	697	+ extra_bytes = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
	698	+ extra_bytes += (SIZE_OF_CRC << 2);
	699	+ sl_from_sc = ibp->sc_to_sl[sc5];
705	700
706		− /*
707		− * The opcode is in the low byte when its in network order
708		− * (top byte when in host order).
709		− */
710	701	opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
711	702	opcode &= 0xff;
712	703
713		− pkey = (u16)be32_to_cpu(ohdr->bth[0]);
714		−
715		− if (!is_mcast && (opcode != IB_OPCODE_CNP) && bth1 & HFI1_FECN_SMASK) {
716		− u16 slid = be16_to_cpu(hdr->lrh[3]);
717		−
718		− return_cnp(ibp, qp, src_qp, pkey, dlid, slid, sc5, grh);
719		− }
	704	+ process_ecn(qp, packet, (opcode != IB_OPCODE_CNP));
720	705	/*
721	706	* Get the number of bytes the message was padded by
722	707	* and drop incomplete packets.
723	708	*/
724		− pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
725		− if (unlikely(tlen < (hdrsize + pad + 4)))
	709	+ if (unlikely(tlen < (hdrsize + extra_bytes)))
726	710	goto drop;
727	711
728		− tlen -= hdrsize + pad + 4;
	712	+ tlen -= hdrsize + extra_bytes;
729	713
730	714	/*
731	715	* Check that the permissive LID is only used on QP0
@@ -736,10 +720,6 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) · 4 unmodified lines
736	720	hdr->lrh[3] == IB_LID_PERMISSIVE))
737	721	goto drop;
738	722	if (qp->ibqp.qp_num > 1) {
739		− struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
740		− u16 slid;
741		−
742		− slid = be16_to_cpu(hdr->lrh[3]);
743	723	if (unlikely(rcv_pkey_check(ppd, pkey, sc5, slid))) {
744	724	/*
745	725	* Traps will not be sent for packets dropped
@@ -748,12 +728,9 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) · 2 unmodified lines
748	728	* IB spec (release 1.3, section 10.9.4)
749	729	*/
750	730	hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_P_KEY,
751		− pkey,
752		− (be16_to_cpu(hdr->lrh[0]) >> 4) &
753		− 0xF,
	731	+ pkey, sl,
754	732	src_qp, qp->ibqp.qp_num,
755		− be16_to_cpu(hdr->lrh[3]),
756		− be16_to_cpu(hdr->lrh[1]));
	733	+ slid, dlid);
757	734	return;
758	735	}
759	736	} else {
@@ -763,22 +740,18 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) · 3 unmodified lines
763	740	goto drop;
764	741	}
765	742	if (unlikely(qkey != qp->qkey)) {
766		− hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_Q_KEY, qkey,
767		− (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF,
	743	+ hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_Q_KEY, qkey, sl,
768	744	src_qp, qp->ibqp.qp_num,
769		− be16_to_cpu(hdr->lrh[3]),
770		− be16_to_cpu(hdr->lrh[1]));
	745	+ slid, dlid);
771	746	return;
772	747	}
773	748	/* Drop invalid MAD packets (see 13.5.3.1). */
774	749	if (unlikely(qp->ibqp.qp_num == 1 &&
775		− (tlen > 2048 \|\|
776		− (be16_to_cpu(hdr->lrh[0]) >> 12) == 15)))
	750	+ (tlen > 2048 \|\| (sc5 == 0xF))))
777	751	goto drop;
778	752	} else {
779	753	/* Received on QP0, and so by definition, this is an SMP */
780	754	struct opa_smp smp = (struct opa_smp )data;
781		− u16 slid = be16_to_cpu(hdr->lrh[3]);
782	755
783	756	if (opa_smp_check(ibp, pkey, sc5, qp, slid, smp))
784	757	goto drop;
@@ -861,7 +834,6 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) · 76 unmodified lines
861	834	qp->ibqp.qp_type == IB_QPT_SMI) {
862	835	if (mgmt_pkey_idx < 0) {
863	836	if (net_ratelimit()) {
864		− struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
865	837	struct hfi1_devdata *dd = ppd->dd;
866	838
867	839	dd_dev_err(dd, "QP type %d mgmt_pkey_idx < 0 and packet not dropped???\n",
@@ -874,8 +846,8 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) · 6 unmodified lines
874	846	wc.pkey_index = 0;
875	847	}
876	848
877		− wc.slid = be16_to_cpu(hdr->lrh[3]);
878		− wc.sl = ibp->sc_to_sl[sc5];
	849	+ wc.slid = slid;
	850	+ wc.sl = sl_from_sc;
879	851
880	852	/*
881	853	* Save the LMC lower bits if the destination LID is a unicast LID.

drivers/infiniband/hw/hfi1/user_exp_rcv.c+65−59

@@ -82,24 +82,25 @@ struct tid_pageset { · 81 unmodified lines
82	82	((unsigned long)vaddr & PAGE_MASK)) >> PAGE_SHIFT))
83	83
84	84	static void unlock_exp_tids(struct hfi1_ctxtdata , struct exp_tid_set ,
85		− struct rb_root *);
	85	+ struct hfi1_filedata *);
86	86	static u32 find_phys_blocks(struct page *, unsigned, struct tid_pageset );
87	87	static int set_rcvarray_entry(struct file *, unsigned long, u32,
88	88	struct tid_group , struct page *, unsigned);
89		−static int mmu_rb_insert(struct rb_root , struct mmu_rb_node );
90		−static void mmu_rb_remove(struct rb_root , struct mmu_rb_node ,
91		− struct mm_struct *);
92		−static int mmu_rb_invalidate(struct rb_root , struct mmu_rb_node );
	89	+static int tid_rb_insert(void , struct mmu_rb_node );
	90	+static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,
	91	+ struct tid_rb_node *tnode);
	92	+static void tid_rb_remove(void , struct mmu_rb_node );
	93	+static int tid_rb_invalidate(void , struct mmu_rb_node );
93	94	static int program_rcvarray(struct file , unsigned long, struct tid_group ,
94	95	struct tid_pageset , unsigned, u16, struct page *,
95	96	u32 , unsigned , unsigned *);
96	97	static int unprogram_rcvarray(struct file , u32, struct tid_group *);
97		−static void clear_tid_node(struct hfi1_filedata , u16, struct tid_rb_node );
	98	+static void clear_tid_node(struct hfi1_filedata fd, struct tid_rb_node node);
98	99
99	100	static struct mmu_rb_ops tid_rb_ops = {
100		− .insert = mmu_rb_insert,
101		− .remove = mmu_rb_remove,
102		− .invalidate = mmu_rb_invalidate
	101	+ .insert = tid_rb_insert,
	102	+ .remove = tid_rb_remove,
	103	+ .invalidate = tid_rb_invalidate
103	104	};
104	105
105	106	static inline u32 rcventry2tidinfo(u32 rcventry)
@@ -162,7 +163,6 @@ int hfi1_user_exp_rcv_init(struct file *fp) · 56 unmodified lines
162	163
163	164	spin_lock_init(&fd->tid_lock);
164	165	spin_lock_init(&fd->invalid_lock);
165		− fd->tid_rb_root = RB_ROOT;
166	166
167	167	if (!uctxt->subctxt_cnt \|\| !fd->subctxt) {
168	168	exp_tid_group_init(&uctxt->tid_group_list);
@@ -197,7 +197,7 @@ int hfi1_user_exp_rcv_init(struct file *fp) · 28 unmodified lines
197	197	if (!fd->entry_to_rb)
198	198	return -ENOMEM;
199	199
200		− if (!HFI1_CAP_IS_USET(TID_UNMAP)) {
	200	+ if (!HFI1_CAP_UGET_MASK(uctxt->flags, TID_UNMAP)) {
201	201	fd->invalid_tid_idx = 0;
202	202	fd->invalid_tids = kzalloc(uctxt->expected_count *
203	203	sizeof(u32), GFP_KERNEL);
@@ -208,15 +208,15 @@ int hfi1_user_exp_rcv_init(struct file *fp) · 4 unmodified lines
208	208
209	209	/*
210	210	* Register MMU notifier callbacks. If the registration
211		− * fails, continue but turn off the TID caching for
212		− * all user contexts.
	211	+ * fails, continue without TID caching for this context.
213	212	*/
214		− ret = hfi1_mmu_rb_register(&fd->tid_rb_root, &tid_rb_ops);
	213	+ ret = hfi1_mmu_rb_register(fd, fd->mm, &tid_rb_ops,
	214	+ dd->pport->hfi1_wq,
	215	+ &fd->handler);
215	216	if (ret) {
216	217	dd_dev_info(dd,
217	218	"Failed MMU notifier registration %d\n",
218	219	ret);
219		− HFI1_CAP_USET(TID_UNMAP);
220	220	ret = 0;
221	221	}
222	222	}
@@ -235,7 +235,7 @@ int hfi1_user_exp_rcv_init(struct file *fp) · 12 unmodified lines
235	235	* init.
236	236	*/
237	237	spin_lock(&fd->tid_lock);
238		− if (uctxt->subctxt_cnt && !HFI1_CAP_IS_USET(TID_UNMAP)) {
	238	+ if (uctxt->subctxt_cnt && fd->handler) {
239	239	u16 remainder;
240	240
241	241	fd->tid_limit = uctxt->expected_count / uctxt->subctxt_cnt;
@@ -261,18 +261,16 @@ int hfi1_user_exp_rcv_free(struct hfi1_filedata *fd) · 19 unmodified lines
261	261	* The notifier would have been removed when the process'es mm
262	262	* was freed.
263	263	*/
264		− if (!HFI1_CAP_IS_USET(TID_UNMAP))
265		− hfi1_mmu_rb_unregister(&fd->tid_rb_root);
	264	+ if (fd->handler)
	265	+ hfi1_mmu_rb_unregister(fd->handler);
266	266
267	267	kfree(fd->invalid_tids);
268	268
269	269	if (!uctxt->cnt) {
270	270	if (!EXP_TID_SET_EMPTY(uctxt->tid_full_list))
271		− unlock_exp_tids(uctxt, &uctxt->tid_full_list,
272		− &fd->tid_rb_root);
	271	+ unlock_exp_tids(uctxt, &uctxt->tid_full_list, fd);
273	272	if (!EXP_TID_SET_EMPTY(uctxt->tid_used_list))
274		− unlock_exp_tids(uctxt, &uctxt->tid_used_list,
275		− &fd->tid_rb_root);
	273	+ unlock_exp_tids(uctxt, &uctxt->tid_used_list, fd);
276	274	list_for_each_entry_safe(grp, gptr, &uctxt->tid_group_list.list,
277	275	list) {
278	276	list_del_init(&grp->list);
@@ -399,12 +397,12 @@ int hfi1_user_exp_rcv_setup(struct file fp, struct hfi1_tid_info tinfo) · 120 unmodified lines
399	397	* pages, accept the amount pinned so far and program only that.
400	398	* User space knows how to deal with partially programmed buffers.
401	399	*/
402		− if (!hfi1_can_pin_pages(dd, fd->tid_n_pinned, npages)) {
	400	+ if (!hfi1_can_pin_pages(dd, fd->mm, fd->tid_n_pinned, npages)) {
403	401	ret = -ENOMEM;
404	402	goto bail;
405	403	}
406	404
407		− pinned = hfi1_acquire_user_pages(vaddr, npages, true, pages);
	405	+ pinned = hfi1_acquire_user_pages(fd->mm, vaddr, npages, true, pages);
408	406	if (pinned <= 0) {
409	407	ret = pinned;
410	408	goto bail;
@@ -559,7 +557,7 @@ nomem: · 148 unmodified lines
559	557	* for example), unpin all unmapped pages so we can pin them nex time.
560	558	*/
561	559	if (mapped_pages != pinned) {
562		− hfi1_release_user_pages(current->mm, &pages[mapped_pages],
	560	+ hfi1_release_user_pages(fd->mm, &pages[mapped_pages],
563	561	pinned - mapped_pages,
564	562	false);
565	563	fd->tid_n_pinned -= pinned - mapped_pages;
@@ -829,7 +827,6 @@ static int set_rcvarray_entry(struct file *fp, unsigned long vaddr, · 263 unmodified lines
829	827	struct hfi1_ctxtdata *uctxt = fd->uctxt;
830	828	struct tid_rb_node *node;
831	829	struct hfi1_devdata *dd = uctxt->dd;
832		− struct rb_root *root = &fd->tid_rb_root;
833	830	dma_addr_t phys;
834	831
835	832	/*
@@ -861,10 +858,10 @@ static int set_rcvarray_entry(struct file *fp, unsigned long vaddr, · 25 unmodified lines
861	858	node->freed = false;
862	859	memcpy(node->pages, pages, sizeof(struct page ) npages);
863	860
864		− if (HFI1_CAP_IS_USET(TID_UNMAP))
865		− ret = mmu_rb_insert(root, &node->mmu);
	861	+ if (!fd->handler)
	862	+ ret = tid_rb_insert(fd, &node->mmu);
866	863	else
867		− ret = hfi1_mmu_rb_insert(root, &node->mmu);
	864	+ ret = hfi1_mmu_rb_insert(fd->handler, &node->mmu);
868	865
869	866	if (ret) {
870	867	hfi1_cdbg(TID, "Failed to insert RB node %u 0x%lx, 0x%lx %d",
@@ -904,19 +901,19 @@ static int unprogram_rcvarray(struct file *fp, u32 tidinfo, · 33 unmodified lines
904	901	node = fd->entry_to_rb[rcventry];
905	902	if (!node \|\| node->rcventry != (uctxt->expected_base + rcventry))
906	903	return -EBADF;
907		− if (HFI1_CAP_IS_USET(TID_UNMAP))
908		− mmu_rb_remove(&fd->tid_rb_root, &node->mmu, NULL);
909		− else
910		− hfi1_mmu_rb_remove(&fd->tid_rb_root, &node->mmu);
911	904
912	905	if (grp)
913	906	*grp = node->grp;
914		− clear_tid_node(fd, fd->subctxt, node);
	907	+
	908	+ if (!fd->handler)
	909	+ cacheless_tid_rb_remove(fd, node);
	910	+ else
	911	+ hfi1_mmu_rb_remove(fd->handler, &node->mmu);
	912	+
915	913	return 0;
916	914	}
917	915
918		−static void clear_tid_node(struct hfi1_filedata *fd, u16 subctxt,
919		− struct tid_rb_node *node)
	916	+static void clear_tid_node(struct hfi1_filedata fd, struct tid_rb_node node)
920	917	{
921	918	struct hfi1_ctxtdata *uctxt = fd->uctxt;
922	919	struct hfi1_devdata *dd = uctxt->dd;
@@ -934,7 +931,7 @@ static void clear_tid_node(struct hfi1_filedata *fd, u16 subctxt, · 11 unmodified lines
934	931
935	932	pci_unmap_single(dd->pcidev, node->dma_addr, node->mmu.len,
936	933	PCI_DMA_FROMDEVICE);
937		− hfi1_release_user_pages(current->mm, node->pages, node->npages, true);
	934	+ hfi1_release_user_pages(fd->mm, node->pages, node->npages, true);
938	935	fd->tid_n_pinned -= node->npages;
939	936
940	937	node->grp->used--;
@@ -949,12 +946,15 @@ static void clear_tid_node(struct hfi1_filedata *fd, u16 subctxt, · 8 unmodified lines
949	946	kfree(node);
950	947	}
951	948
	949	+/*
	950	+ * As a simple helper for hfi1_user_exp_rcv_free, this function deals with
	951	+ * clearing nodes in the non-cached case.
	952	+ */
952	953	static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,
953		− struct exp_tid_set set, struct rb_root root)
	954	+ struct exp_tid_set *set,
	955	+ struct hfi1_filedata *fd)
954	956	{
955	957	struct tid_group grp, ptr;
956		− struct hfi1_filedata *fd = container_of(root, struct hfi1_filedata,
957		− tid_rb_root);
958	958	int i;
959	959
960	960	list_for_each_entry_safe(grp, ptr, &set->list, list) {
@@ -969,22 +969,23 @@ static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt, · 8 unmodified lines
969	969	uctxt->expected_base];
970	970	if (!node \|\| node->rcventry != rcventry)
971	971	continue;
972		− if (HFI1_CAP_IS_USET(TID_UNMAP))
973		− mmu_rb_remove(&fd->tid_rb_root,
974		− &node->mmu, NULL);
975		− else
976		− hfi1_mmu_rb_remove(&fd->tid_rb_root,
977		− &node->mmu);
978		− clear_tid_node(fd, -1, node);
	972	+
	973	+ cacheless_tid_rb_remove(fd, node);
979	974	}
980	975	}
981	976	}
982	977	}
983	978
984		−static int mmu_rb_invalidate(struct rb_root root, struct mmu_rb_node mnode)
	979	+/*
	980	+ * Always return 0 from this function. A non-zero return indicates that the
	981	+ * remove operation will be called and that memory should be unpinned.
	982	+ * However, the driver cannot unpin out from under PSM. Instead, retain the
	983	+ * memory (by returning 0) and inform PSM that the memory is going away. PSM
	984	+ * will call back later when it has removed the memory from its list.
	985	+ */
	986	+static int tid_rb_invalidate(void arg, struct mmu_rb_node mnode)
985	987	{
986		− struct hfi1_filedata *fdata =
987		− container_of(root, struct hfi1_filedata, tid_rb_root);
	988	+ struct hfi1_filedata *fdata = arg;
988	989	struct hfi1_ctxtdata *uctxt = fdata->uctxt;
989	990	struct tid_rb_node *node =
990	991	container_of(mnode, struct tid_rb_node, mmu);
@@ -1025,10 +1026,9 @@ static int mmu_rb_invalidate(struct rb_root root, struct mmu_rb_node mnode) · 34 unmodified lines
1025	1026	return 0;
1026	1027	}
1027	1028
1028		−static int mmu_rb_insert(struct rb_root root, struct mmu_rb_node node)
	1029	+static int tid_rb_insert(void arg, struct mmu_rb_node node)
1029	1030	{
1030		− struct hfi1_filedata *fdata =
1031		− container_of(root, struct hfi1_filedata, tid_rb_root);
	1031	+ struct hfi1_filedata *fdata = arg;
1032	1032	struct tid_rb_node *tnode =
1033	1033	container_of(node, struct tid_rb_node, mmu);
1034	1034	u32 base = fdata->uctxt->expected_base;
@@ -1037,14 +1037,20 @@ static int mmu_rb_insert(struct rb_root root, struct mmu_rb_node node) · 2 unmodified lines
1037	1037	return 0;
1038	1038	}
1039	1039
1040		−static void mmu_rb_remove(struct rb_root root, struct mmu_rb_node node,
1041		− struct mm_struct *mm)
	1040	+static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,
	1041	+ struct tid_rb_node *tnode)
1042	1042	{
1043		− struct hfi1_filedata *fdata =
1044		− container_of(root, struct hfi1_filedata, tid_rb_root);
1045		− struct tid_rb_node *tnode =
1046		− container_of(node, struct tid_rb_node, mmu);
1047	1043	u32 base = fdata->uctxt->expected_base;
1048	1044
1049	1045	fdata->entry_to_rb[tnode->rcventry - base] = NULL;
	1046	+ clear_tid_node(fdata, tnode);
	1047	+}
	1048	+
	1049	+static void tid_rb_remove(void arg, struct mmu_rb_node node)
	1050	+{
	1051	+ struct hfi1_filedata *fdata = arg;
	1052	+ struct tid_rb_node *tnode =
	1053	+ container_of(node, struct tid_rb_node, mmu);
	1054	+
	1055	+ cacheless_tid_rb_remove(fdata, tnode);
1050	1056	}

drivers/infiniband/hw/hfi1/user_pages.c+10−9

@@ -68,7 +68,8 @@ MODULE_PARM_DESC(cache_size, "Send and receive side cache size limit (in MB)"); · 67 unmodified lines
68	68	* could keeping caching buffers.
69	69	*
70	70	*/
71		−bool hfi1_can_pin_pages(struct hfi1_devdata *dd, u32 nlocked, u32 npages)
	71	+bool hfi1_can_pin_pages(struct hfi1_devdata dd, struct mm_struct mm,
	72	+ u32 nlocked, u32 npages)
72	73	{
73	74	unsigned long ulimit = rlimit(RLIMIT_MEMLOCK), pinned, cache_limit,
74	75	size = (cache_size * (1UL << 20)); /* convert to bytes */
@@ -89,9 +90,9 @@ bool hfi1_can_pin_pages(struct hfi1_devdata *dd, u32 nlocked, u32 npages) · 14 unmodified lines
89	90	/* Convert to number of pages */
90	91	size = DIV_ROUND_UP(size, PAGE_SIZE);
91	92
92		− down_read(&current->mm->mmap_sem);
93		− pinned = current->mm->pinned_vm;
94		− up_read(&current->mm->mmap_sem);
	93	+ down_read(&mm->mmap_sem);
	94	+ pinned = mm->pinned_vm;
	95	+ up_read(&mm->mmap_sem);
95	96
96	97	/* First, check the absolute limit against all pinned pages. */
97	98	if (pinned + npages >= ulimit && !can_lock)
@@ -100,8 +101,8 @@ bool hfi1_can_pin_pages(struct hfi1_devdata *dd, u32 nlocked, u32 npages) · 2 unmodified lines
100	101	return ((nlocked + npages) <= size) \|\| can_lock;
101	102	}
102	103
103		−int hfi1_acquire_user_pages(unsigned long vaddr, size_t npages, bool writable,
104		− struct page **pages)
	104	+int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t npages,
	105	+ bool writable, struct page **pages)
105	106	{
106	107	int ret;
107	108
@@ -109,9 +110,9 @@ int hfi1_acquire_user_pages(unsigned long vaddr, size_t npages, bool writable, · 1 unmodified line
109	110	if (ret < 0)
110	111	return ret;
111	112
112		− down_write(&current->mm->mmap_sem);
113		− current->mm->pinned_vm += ret;
114		− up_write(&current->mm->mmap_sem);
	113	+ down_write(&mm->mmap_sem);
	114	+ mm->pinned_vm += ret;
	115	+ up_write(&mm->mmap_sem);
115	116
116	117	return ret;
117	118	}

drivers/infiniband/hw/hfi1/user_sdma.c+171−130

@@ -145,7 +145,7 @@ MODULE_PARM_DESC(sdma_comp_size, "Size of User SDMA completion ring. Default: 12 · 144 unmodified lines
145	145	/* Last packet in the request */
146	146	#define TXREQ_FLAGS_REQ_LAST_PKT BIT(0)
147	147
148		−#define SDMA_REQ_IN_USE 0
	148	+/* SDMA request flag bits */
149	149	#define SDMA_REQ_FOR_THREAD 1
150	150	#define SDMA_REQ_SEND_DONE 2
151	151	#define SDMA_REQ_HAVE_AHG 3
@@ -183,16 +183,18 @@ struct user_sdma_iovec { · 31 unmodified lines
183	183	struct sdma_mmu_node *node;
184	184	};
185	185
186		−#define SDMA_CACHE_NODE_EVICT 0
187		−
188	186	struct sdma_mmu_node {
189	187	struct mmu_rb_node rb;
190		− struct list_head list;
191	188	struct hfi1_user_sdma_pkt_q *pq;
192	189	atomic_t refcount;
193	190	struct page **pages;
194	191	unsigned npages;
195		− unsigned long flags;
	192	+};
	193	+
	194	+/* evict operation argument */
	195	+struct evict_data {
	196	+ u32 cleared; /* count evicted so far */
	197	+ u32 target; /* target count to evict */
196	198	};
197	199
198	200	struct user_sdma_request {
@@ -305,14 +307,16 @@ static int defer_packet_queue( · 106 unmodified lines
305	307	unsigned seq);
306	308	static void activate_packet_queue(struct iowait *, int);
307	309	static bool sdma_rb_filter(struct mmu_rb_node *, unsigned long, unsigned long);
308		−static int sdma_rb_insert(struct rb_root , struct mmu_rb_node );
309		−static void sdma_rb_remove(struct rb_root , struct mmu_rb_node ,
310		− struct mm_struct *);
311		−static int sdma_rb_invalidate(struct rb_root , struct mmu_rb_node );
	310	+static int sdma_rb_insert(void , struct mmu_rb_node );
	311	+static int sdma_rb_evict(void arg, struct mmu_rb_node mnode,
	312	+ void arg2, bool stop);
	313	+static void sdma_rb_remove(void , struct mmu_rb_node );
	314	+static int sdma_rb_invalidate(void , struct mmu_rb_node );
312	315
313	316	static struct mmu_rb_ops sdma_rb_ops = {
314	317	.filter = sdma_rb_filter,
315	318	.insert = sdma_rb_insert,
	319	+ .evict = sdma_rb_evict,
316	320	.remove = sdma_rb_remove,
317	321	.invalidate = sdma_rb_invalidate
318	322	};
@@ -397,6 +401,11 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata uctxt, struct file fp) · 78 unmodified lines
397	401	if (!pq->reqs)
398	402	goto pq_reqs_nomem;
399	403
	404	+ memsize = BITS_TO_LONGS(hfi1_sdma_comp_ring_size) * sizeof(long);
	405	+ pq->req_in_use = kzalloc(memsize, GFP_KERNEL);
	406	+ if (!pq->req_in_use)
	407	+ goto pq_reqs_no_in_use;
	408	+
400	409	INIT_LIST_HEAD(&pq->list);
401	410	pq->dd = dd;
402	411	pq->ctxt = uctxt->ctxt;
@@ -405,9 +414,8 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata uctxt, struct file fp) · 2 unmodified lines
405	414	pq->state = SDMA_PKT_Q_INACTIVE;
406	415	atomic_set(&pq->n_reqs, 0);
407	416	init_waitqueue_head(&pq->wait);
408		− pq->sdma_rb_root = RB_ROOT;
409		− INIT_LIST_HEAD(&pq->evict);
410		− spin_lock_init(&pq->evict_lock);
	417	+ atomic_set(&pq->n_locked, 0);
	418	+ pq->mm = fd->mm;
411	419
412	420	iowait_init(&pq->busy, 0, NULL, defer_packet_queue,
413	421	activate_packet_queue, NULL);
@@ -437,7 +445,8 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata uctxt, struct file fp) · 23 unmodified lines
437	445	cq->nentries = hfi1_sdma_comp_ring_size;
438	446	fd->cq = cq;
439	447
440		− ret = hfi1_mmu_rb_register(&pq->sdma_rb_root, &sdma_rb_ops);
	448	+ ret = hfi1_mmu_rb_register(pq, pq->mm, &sdma_rb_ops, dd->pport->hfi1_wq,
	449	+ &pq->handler);
441	450	if (ret) {
442	451	dd_dev_err(dd, "Failed to register with MMU %d", ret);
443	452	goto done;
@@ -453,6 +462,8 @@ cq_comps_nomem: · 9 unmodified lines
453	462	cq_nomem:
454	463	kmem_cache_destroy(pq->txreq_cache);
455	464	pq_txreq_nomem:
	465	+ kfree(pq->req_in_use);
	466	+pq_reqs_no_in_use:
456	467	kfree(pq->reqs);
457	468	pq_reqs_nomem:
458	469	kfree(pq);
@@ -472,8 +483,9 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd) · 13 unmodified lines
472	483	hfi1_cdbg(SDMA, "[%u:%u:%u] Freeing user SDMA queues", uctxt->dd->unit,
473	484	uctxt->ctxt, fd->subctxt);
474	485	pq = fd->pq;
475		− hfi1_mmu_rb_unregister(&pq->sdma_rb_root);
476	486	if (pq) {
	487	+ if (pq->handler)
	488	+ hfi1_mmu_rb_unregister(pq->handler);
477	489	spin_lock_irqsave(&uctxt->sdma_qlock, flags);
478	490	if (!list_empty(&pq->list))
479	491	list_del_init(&pq->list);
@@ -484,6 +496,7 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd) · 4 unmodified lines
484	496	pq->wait,
485	497	(ACCESS_ONCE(pq->state) == SDMA_PKT_Q_INACTIVE));
486	498	kfree(pq->reqs);
	499	+ kfree(pq->req_in_use);
487	500	kmem_cache_destroy(pq->txreq_cache);
488	501	kfree(pq);
489	502	fd->pq = NULL;
@@ -496,10 +509,31 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd) · 6 unmodified lines
496	509	return 0;
497	510	}
498	511
	512	+static u8 dlid_to_selector(u16 dlid)
	513	+{
	514	+ static u8 mapping[256];
	515	+ static int initialized;
	516	+ static u8 next;
	517	+ int hash;
	518	+
	519	+ if (!initialized) {
	520	+ memset(mapping, 0xFF, 256);
	521	+ initialized = 1;
	522	+ }
	523	+
	524	+ hash = ((dlid >> 8) ^ dlid) & 0xFF;
	525	+ if (mapping[hash] == 0xFF) {
	526	+ mapping[hash] = next;
	527	+ next = (next + 1) & 0x7F;
	528	+ }
	529	+
	530	+ return mapping[hash];
	531	+}
	532	+
499	533	int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec,
500	534	unsigned long dim, unsigned long *count)
501	535	{
502		− int ret = 0, i = 0;
	536	+ int ret = 0, i;
503	537	struct hfi1_filedata *fd = fp->private_data;
504	538	struct hfi1_ctxtdata *uctxt = fd->uctxt;
505	539	struct hfi1_user_sdma_pkt_q *pq = fd->pq;
@@ -511,6 +545,8 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 5 unmodified lines
511	545	struct user_sdma_request *req;
512	546	u8 opcode, sc, vl;
513	547	int req_queued = 0;
	548	+ u16 dlid;
	549	+ u8 selector;
514	550
515	551	if (iovec[idx].iov_len < sizeof(info) + sizeof(req->hdr)) {
516	552	hfi1_cdbg(
@@ -529,30 +565,48 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 12 unmodified lines
529	565
530	566	trace_hfi1_sdma_user_reqinfo(dd, uctxt->ctxt, fd->subctxt,
531	567	(u16 *)&info);
532		− if (cq->comps[info.comp_idx].status == QUEUED \|\|
533		− test_bit(SDMA_REQ_IN_USE, &pq->reqs[info.comp_idx].flags)) {
534		− hfi1_cdbg(SDMA, "[%u:%u:%u] Entry %u is in QUEUED state",
535		− dd->unit, uctxt->ctxt, fd->subctxt,
536		− info.comp_idx);
537		− return -EBADSLT;
	568	+
	569	+ if (info.comp_idx >= hfi1_sdma_comp_ring_size) {
	570	+ hfi1_cdbg(SDMA,
	571	+ "[%u:%u:%u:%u] Invalid comp index",
	572	+ dd->unit, uctxt->ctxt, fd->subctxt, info.comp_idx);
	573	+ return -EINVAL;
538	574	}
	575	+
	576	+ /*
	577	+ * Sanity check the header io vector count. Need at least 1 vector
	578	+ * (header) and cannot be larger than the actual io vector count.
	579	+ */
	580	+ if (req_iovcnt(info.ctrl) < 1 \|\| req_iovcnt(info.ctrl) > dim) {
	581	+ hfi1_cdbg(SDMA,
	582	+ "[%u:%u:%u:%u] Invalid iov count %d, dim %ld",
	583	+ dd->unit, uctxt->ctxt, fd->subctxt, info.comp_idx,
	584	+ req_iovcnt(info.ctrl), dim);
	585	+ return -EINVAL;
	586	+ }
	587	+
539	588	if (!info.fragsize) {
540	589	hfi1_cdbg(SDMA,
541	590	"[%u:%u:%u:%u] Request does not specify fragsize",
542	591	dd->unit, uctxt->ctxt, fd->subctxt, info.comp_idx);
543	592	return -EINVAL;
544	593	}
	594	+
	595	+ /* Try to claim the request. */
	596	+ if (test_and_set_bit(info.comp_idx, pq->req_in_use)) {
	597	+ hfi1_cdbg(SDMA, "[%u:%u:%u] Entry %u is in use",
	598	+ dd->unit, uctxt->ctxt, fd->subctxt,
	599	+ info.comp_idx);
	600	+ return -EBADSLT;
	601	+ }
545	602	/*
546		− * We've done all the safety checks that we can up to this point,
547		− * "allocate" the request entry.
	603	+ * All safety checks have been done and this request has been claimed.
548	604	*/
549	605	hfi1_cdbg(SDMA, "[%u:%u:%u] Using req/comp entry %u\n", dd->unit,
550	606	uctxt->ctxt, fd->subctxt, info.comp_idx);
551	607	req = pq->reqs + info.comp_idx;
552	608	memset(req, 0, sizeof(*req));
553		− /* Mark the request as IN_USE before we start filling it in. */
554		− set_bit(SDMA_REQ_IN_USE, &req->flags);
555		− req->data_iovs = req_iovcnt(info.ctrl) - 1;
	609	+ req->data_iovs = req_iovcnt(info.ctrl) - 1; /* subtract header vector */
556	610	req->pq = pq;
557	611	req->cq = cq;
558	612	req->status = -1;
@@ -560,13 +614,22 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 1 unmodified line
560	614
561	615	memcpy(&req->info, &info, sizeof(info));
562	616
563		− if (req_opcode(info.ctrl) == EXPECTED)
	617	+ if (req_opcode(info.ctrl) == EXPECTED) {
	618	+ /* expected must have a TID info and at least one data vector */
	619	+ if (req->data_iovs < 2) {
	620	+ SDMA_DBG(req,
	621	+ "Not enough vectors for expected request");
	622	+ ret = -EINVAL;
	623	+ goto free_req;
	624	+ }
564	625	req->data_iovs--;
	626	+ }
565	627
566	628	if (!info.npkts \|\| req->data_iovs > MAX_VECTORS_PER_REQ) {
567	629	SDMA_DBG(req, "Too many vectors (%u/%u)", req->data_iovs,
568	630	MAX_VECTORS_PER_REQ);
569		− return -EINVAL;
	631	+ ret = -EINVAL;
	632	+ goto free_req;
570	633	}
571	634	/* Copy the header from the user buffer */
572	635	ret = copy_from_user(&req->hdr, iovec[idx].iov_base + sizeof(info),
@@ -634,7 +697,7 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 61 unmodified lines
634	697	idx++;
635	698
636	699	/* Save all the IO vector structures */
637		− while (i < req->data_iovs) {
	700	+ for (i = 0; i < req->data_iovs; i++) {
638	701	INIT_LIST_HEAD(&req->iovs[i].list);
639	702	memcpy(&req->iovs[i].iov, iovec + idx++, sizeof(struct iovec));
640	703	ret = pin_vector_pages(req, &req->iovs[i]);
@@ -642,7 +705,7 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 1 unmodified line
642	705	req->status = ret;
643	706	goto free_req;
644	707	}
645		− req->data_len += req->iovs[i++].iov.iov_len;
	708	+ req->data_len += req->iovs[i].iov.iov_len;
646	709	}
647	710	SDMA_DBG(req, "total data length %u", req->data_len);
648	711
@@ -686,9 +749,13 @@ int hfi1_user_sdma_process_request(struct file fp, struct iovec iovec, · 37 unmodified lines
686	749	idx++;
687	750	}
688	751
	752	+ dlid = be16_to_cpu(req->hdr.lrh[1]);
	753	+ selector = dlid_to_selector(dlid);
	754	+
689	755	/* Have to select the engine */
690	756	req->sde = sdma_select_engine_vl(dd,
691		− (u32)(uctxt->ctxt + fd->subctxt),
	757	+ (u32)(uctxt->ctxt + fd->subctxt +
	758	+ selector),
692	759	vl);
693	760	if (!req->sde \|\| !sdma_running(req->sde)) {
694	761	ret = -ECOMM;
@@ -766,14 +833,21 @@ static inline u32 compute_data_length(struct user_sdma_request *req, · 71 unmodified lines
766	833	* The size of the data of the first packet is in the header
767	834	* template. However, it includes the header and ICRC, which need
768	835	* to be subtracted.
	836	+ * The minimum representable packet data length in a header is 4 bytes,
	837	+ * therefore, when the data length request is less than 4 bytes, there's
	838	+ * only one packet, and the packet data length is equal to that of the
	839	+ * request data length.
769	840	* The size of the remaining packets is the minimum of the frag
770	841	* size (MTU) or remaining data in the request.
771	842	*/
772	843	u32 len;
773	844
774	845	if (!req->seqnum) {
775		− len = ((be16_to_cpu(req->hdr.lrh[2]) << 2) -
776		− (sizeof(tx->hdr) - 4));
	846	+ if (req->data_len < sizeof(u32))
	847	+ len = req->data_len;
	848	+ else
	849	+ len = ((be16_to_cpu(req->hdr.lrh[2]) << 2) -
	850	+ (sizeof(tx->hdr) - 4));
777	851	} else if (req_opcode(req->info.ctrl) == EXPECTED) {
778	852	u32 tidlen = EXP_TID_GET(req->tids[req->tididx], LEN) *
779	853	PAGE_SIZE;
@@ -803,6 +877,13 @@ static inline u32 compute_data_length(struct user_sdma_request *req, · 23 unmodified lines
803	877	return len;
804	878	}
805	879
	880	+static inline u32 pad_len(u32 len)
	881	+{
	882	+ if (len & (sizeof(u32) - 1))
	883	+ len += sizeof(u32) - (len & (sizeof(u32) - 1));
	884	+ return len;
	885	+}
	886	+
806	887	static inline u32 get_lrh_len(struct hfi1_pkt_header hdr, u32 len)
807	888	{
808	889	/* (Size of complete header - size of PBC) + 4B ICRC + data length */
@@ -894,7 +975,8 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts) · 85 unmodified lines
894	975	if (test_bit(SDMA_REQ_HAVE_AHG, &req->flags)) {
895	976	if (!req->seqnum) {
896	977	u16 pbclen = le16_to_cpu(req->hdr.pbc[0]);
897		− u32 lrhlen = get_lrh_len(req->hdr, datalen);
	978	+ u32 lrhlen = get_lrh_len(req->hdr,
	979	+ pad_len(datalen));
898	980	/*
899	981	* Copy the request header into the tx header
900	982	* because the HW needs a cacheline-aligned
@@ -1048,39 +1130,24 @@ static inline int num_user_pages(const struct iovec *iov) · 147 unmodified lines
1048	1130
1049	1131	static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)
1050	1132	{
1051		− u32 cleared = 0;
1052		− struct sdma_mmu_node node, ptr;
1053		− struct list_head to_evict = LIST_HEAD_INIT(to_evict);
1054		−
1055		− spin_lock(&pq->evict_lock);
1056		− list_for_each_entry_safe_reverse(node, ptr, &pq->evict, list) {
1057		− /* Make sure that no one is still using the node. */
1058		− if (!atomic_read(&node->refcount)) {
1059		− set_bit(SDMA_CACHE_NODE_EVICT, &node->flags);
1060		− list_del_init(&node->list);
1061		− list_add(&node->list, &to_evict);
1062		− cleared += node->npages;
1063		− if (cleared >= npages)
1064		− break;
1065		− }
1066		− }
1067		− spin_unlock(&pq->evict_lock);
1068		−
1069		− list_for_each_entry_safe(node, ptr, &to_evict, list)
1070		− hfi1_mmu_rb_remove(&pq->sdma_rb_root, &node->rb);
	1133	+ struct evict_data evict_data;
1071	1134
1072		− return cleared;
	1135	+ evict_data.cleared = 0;
	1136	+ evict_data.target = npages;
	1137	+ hfi1_mmu_rb_evict(pq->handler, &evict_data);
	1138	+ return evict_data.cleared;
1073	1139	}
1074	1140
1075	1141	static int pin_vector_pages(struct user_sdma_request *req,
1076		− struct user_sdma_iovec *iovec) {
	1142	+ struct user_sdma_iovec *iovec)
	1143	+{
1077	1144	int ret = 0, pinned, npages, cleared;
1078	1145	struct page **pages;
1079	1146	struct hfi1_user_sdma_pkt_q *pq = req->pq;
1080	1147	struct sdma_mmu_node *node = NULL;
1081	1148	struct mmu_rb_node *rb_node;
1082	1149
1083		− rb_node = hfi1_mmu_rb_extract(&pq->sdma_rb_root,
	1150	+ rb_node = hfi1_mmu_rb_extract(pq->handler,
1084	1151	(unsigned long)iovec->iov.iov_base,
1085	1152	iovec->iov.iov_len);
1086	1153	if (rb_node && !IS_ERR(rb_node))
@@ -1096,7 +1163,6 @@ static int pin_vector_pages(struct user_sdma_request *req, · 9 unmodified lines
1096	1163	node->rb.addr = (unsigned long)iovec->iov.iov_base;
1097	1164	node->pq = pq;
1098	1165	atomic_set(&node->refcount, 0);
1099		− INIT_LIST_HEAD(&node->list);
1100	1166	}
1101	1167
1102	1168	npages = num_user_pages(&iovec->iov);
@@ -1111,28 +1177,14 @@ static int pin_vector_pages(struct user_sdma_request *req, · 8 unmodified lines
1111	1177
1112	1178	npages -= node->npages;
1113	1179
1114		− /*
1115		− * If rb_node is NULL, it means that this is brand new node
1116		− * and, therefore not on the eviction list.
1117		− * If, however, the rb_node is non-NULL, it means that the
1118		− * node is already in RB tree and, therefore on the eviction
1119		− * list (nodes are unconditionally inserted in the eviction
1120		− * list). In that case, we have to remove the node prior to
1121		− * calling the eviction function in order to prevent it from
1122		− * freeing this node.
1123		− */
1124		− if (rb_node) {
1125		− spin_lock(&pq->evict_lock);
1126		− list_del_init(&node->list);
1127		− spin_unlock(&pq->evict_lock);
1128		− }
1129	1180	retry:
1130		− if (!hfi1_can_pin_pages(pq->dd, pq->n_locked, npages)) {
	1181	+ if (!hfi1_can_pin_pages(pq->dd, pq->mm,
	1182	+ atomic_read(&pq->n_locked), npages)) {
1131	1183	cleared = sdma_cache_evict(pq, npages);
1132	1184	if (cleared >= npages)
1133	1185	goto retry;
1134	1186	}
1135		− pinned = hfi1_acquire_user_pages(
	1187	+ pinned = hfi1_acquire_user_pages(pq->mm,
1136	1188	((unsigned long)iovec->iov.iov_base +
1137	1189	(node->npages * PAGE_SIZE)), npages, 0,
1138	1190	pages + node->npages);
@@ -1142,7 +1194,7 @@ retry: · 3 unmodified lines
1142	1194	goto bail;
1143	1195	}
1144	1196	if (pinned != npages) {
1145		− unpin_vector_pages(current->mm, pages, node->npages,
	1197	+ unpin_vector_pages(pq->mm, pages, node->npages,
1146	1198	pinned);
1147	1199	ret = -EFAULT;
1148	1200	goto bail;
@@ -1152,28 +1204,22 @@ retry: · 3 unmodified lines
1152	1204	node->pages = pages;
1153	1205	node->npages += pinned;
1154	1206	npages = node->npages;
1155		− spin_lock(&pq->evict_lock);
1156		− list_add(&node->list, &pq->evict);
1157		− pq->n_locked += pinned;
1158		− spin_unlock(&pq->evict_lock);
	1207	+ atomic_add(pinned, &pq->n_locked);
1159	1208	}
1160	1209	iovec->pages = node->pages;
1161	1210	iovec->npages = npages;
1162	1211	iovec->node = node;
1163	1212
1164		− ret = hfi1_mmu_rb_insert(&req->pq->sdma_rb_root, &node->rb);
	1213	+ ret = hfi1_mmu_rb_insert(req->pq->handler, &node->rb);
1165	1214	if (ret) {
1166		− spin_lock(&pq->evict_lock);
1167		− if (!list_empty(&node->list))
1168		− list_del(&node->list);
1169		− pq->n_locked -= node->npages;
1170		− spin_unlock(&pq->evict_lock);
	1215	+ atomic_sub(node->npages, &pq->n_locked);
	1216	+ iovec->node = NULL;
1171	1217	goto bail;
1172	1218	}
1173	1219	return 0;
1174	1220	bail:
1175	1221	if (rb_node)
1176		− unpin_vector_pages(current->mm, node->pages, 0, node->npages);
	1222	+ unpin_vector_pages(pq->mm, node->pages, 0, node->npages);
1177	1223	kfree(node);
1178	1224	return ret;
1179	1225	}
@@ -1181,7 +1227,7 @@ bail: · 1 unmodified line
1181	1227	static void unpin_vector_pages(struct mm_struct mm, struct page *pages,
1182	1228	unsigned start, unsigned npages)
1183	1229	{
1184		− hfi1_release_user_pages(mm, pages + start, npages, 0);
	1230	+ hfi1_release_user_pages(mm, pages + start, npages, false);
1185	1231	kfree(pages);
1186	1232	}
1187	1233
@@ -1192,16 +1238,14 @@ static int check_header_template(struct user_sdma_request *req, · 4 unmodified lines
1192	1238	/*
1193	1239	* Perform safety checks for any type of packet:
1194	1240	* - transfer size is multiple of 64bytes
1195		− * - packet length is multiple of 4bytes
1196		− * - entire request length is multiple of 4bytes
	1241	+ * - packet length is multiple of 4 bytes
1197	1242	* - packet length is not larger than MTU size
1198	1243	*
1199	1244	* These checks are only done for the first packet of the
1200	1245	* transfer since the header is "given" to us by user space.
1201	1246	* For the remainder of the packets we compute the values.
1202	1247	*/
1203		− if (req->info.fragsize % PIO_BLOCK_SIZE \|\|
1204		− lrhlen & 0x3 \|\| req->data_len & 0x3 \|\|
	1248	+ if (req->info.fragsize % PIO_BLOCK_SIZE \|\| lrhlen & 0x3 \|\|
1205	1249	lrhlen > get_lrh_len(*hdr, req->info.fragsize))
1206	1250	return -EINVAL;
1207	1251
@@ -1263,7 +1307,7 @@ static int set_txreq_header(struct user_sdma_request *req, · 55 unmodified lines
1263	1307	struct hfi1_pkt_header *hdr = &tx->hdr;
1264	1308	u16 pbclen;
1265	1309	int ret;
1266		− u32 tidval = 0, lrhlen = get_lrh_len(*hdr, datalen);
	1310	+ u32 tidval = 0, lrhlen = get_lrh_len(*hdr, pad_len(datalen));
1267	1311
1268	1312	/* Copy the header template to the request before modification */
1269	1313	memcpy(hdr, &req->hdr, sizeof(*hdr));
@@ -1374,7 +1418,7 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, · 104 unmodified lines
1374	1418	struct hfi1_user_sdma_pkt_q *pq = req->pq;
1375	1419	struct hfi1_pkt_header *hdr = &req->hdr;
1376	1420	u16 pbclen = le16_to_cpu(hdr->pbc[0]);
1377		− u32 val32, tidval = 0, lrhlen = get_lrh_len(*hdr, len);
	1421	+ u32 val32, tidval = 0, lrhlen = get_lrh_len(*hdr, pad_len(len));
1378	1422
1379	1423	if (PBC2LRH(pbclen) != lrhlen) {
1380	1424	/* PBC.PbcLengthDWs */
@@ -1534,14 +1578,14 @@ static void user_sdma_free_request(struct user_sdma_request *req, bool unpin) · 153 unmodified lines
1534	1578	continue;
1535	1579
1536	1580	if (unpin)
1537		− hfi1_mmu_rb_remove(&req->pq->sdma_rb_root,
	1581	+ hfi1_mmu_rb_remove(req->pq->handler,
1538	1582	&node->rb);
1539	1583	else
1540	1584	atomic_dec(&node->refcount);
1541	1585	}
1542	1586	}
1543	1587	kfree(req->tids);
1544		− clear_bit(SDMA_REQ_IN_USE, &req->flags);
	1588	+ clear_bit(req->info.comp_idx, req->pq->req_in_use);
1545	1589	}
1546	1590
1547	1591	static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq,
@@ -1564,7 +1608,7 @@ static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr, · 16 unmodified lines
1564	1608	return (bool)(node->addr == addr);
1565	1609	}
1566	1610
1567		−static int sdma_rb_insert(struct rb_root root, struct mmu_rb_node mnode)
	1611	+static int sdma_rb_insert(void arg, struct mmu_rb_node mnode)
1568	1612	{
1569	1613	struct sdma_mmu_node *node =
1570	1614	container_of(mnode, struct sdma_mmu_node, rb);
@@ -1573,48 +1617,45 @@ static int sdma_rb_insert(struct rb_root root, struct mmu_rb_node mnode) · 2 unmodified lines
1573	1617	return 0;
1574	1618	}
1575	1619
1576		−static void sdma_rb_remove(struct rb_root root, struct mmu_rb_node mnode,
1577		− struct mm_struct *mm)
	1620	+/*
	1621	+ * Return 1 to remove the node from the rb tree and call the remove op.
	1622	+ *
	1623	+ * Called with the rb tree lock held.
	1624	+ */
	1625	+static int sdma_rb_evict(void arg, struct mmu_rb_node mnode,
	1626	+ void evict_arg, bool stop)
	1627	+{
	1628	+ struct sdma_mmu_node *node =
	1629	+ container_of(mnode, struct sdma_mmu_node, rb);
	1630	+ struct evict_data *evict_data = evict_arg;
	1631	+
	1632	+ /* is this node still being used? */
	1633	+ if (atomic_read(&node->refcount))
	1634	+ return 0; /* keep this node */
	1635	+
	1636	+ /* this node will be evicted, add its pages to our count */
	1637	+ evict_data->cleared += node->npages;
	1638	+
	1639	+ /* have enough pages been cleared? */
	1640	+ if (evict_data->cleared >= evict_data->target)
	1641	+ *stop = true;
	1642	+
	1643	+ return 1; /* remove this node */
	1644	+}
	1645	+
	1646	+static void sdma_rb_remove(void arg, struct mmu_rb_node mnode)
1578	1647	{
1579	1648	struct sdma_mmu_node *node =
1580	1649	container_of(mnode, struct sdma_mmu_node, rb);
1581	1650
1582		− spin_lock(&node->pq->evict_lock);
1583		− /*
1584		− * We've been called by the MMU notifier but this node has been
1585		− * scheduled for eviction. The eviction function will take care
1586		− * of freeing this node.
1587		− * We have to take the above lock first because we are racing
1588		− * against the setting of the bit in the eviction function.
1589		− */
1590		− if (mm && test_bit(SDMA_CACHE_NODE_EVICT, &node->flags)) {
1591		− spin_unlock(&node->pq->evict_lock);
1592		− return;
1593		− }
	1651	+ atomic_sub(node->npages, &node->pq->n_locked);
1594	1652
1595		− if (!list_empty(&node->list))
1596		− list_del(&node->list);
1597		− node->pq->n_locked -= node->npages;
1598		− spin_unlock(&node->pq->evict_lock);
	1653	+ unpin_vector_pages(node->pq->mm, node->pages, 0, node->npages);
1599	1654
1600		− /*
1601		− * If mm is set, we are being called by the MMU notifier and we
1602		− * should not pass a mm_struct to unpin_vector_page(). This is to
1603		− * prevent a deadlock when hfi1_release_user_pages() attempts to
1604		− * take the mmap_sem, which the MMU notifier has already taken.
1605		− */
1606		− unpin_vector_pages(mm ? NULL : current->mm, node->pages, 0,
1607		− node->npages);
1608		− /*
1609		− * If called by the MMU notifier, we have to adjust the pinned
1610		− * page count ourselves.
1611		− */
1612		− if (mm)
1613		− mm->pinned_vm -= node->npages;
1614	1655	kfree(node);
1615	1656	}
1616	1657
1617		−static int sdma_rb_invalidate(struct rb_root root, struct mmu_rb_node mnode)
	1658	+static int sdma_rb_invalidate(void arg, struct mmu_rb_node mnode)
1618	1659	{
1619	1660	struct sdma_mmu_node *node =
1620	1661	container_of(mnode, struct sdma_mmu_node, rb);

drivers/infiniband/hw/hfi1/user_sdma.h+4−4

@@ -63,14 +63,14 @@ struct hfi1_user_sdma_pkt_q { · 62 unmodified lines
63	63	struct hfi1_devdata *dd;
64	64	struct kmem_cache *txreq_cache;
65	65	struct user_sdma_request *reqs;
	66	+ unsigned long *req_in_use;
66	67	struct iowait busy;
67	68	unsigned state;
68	69	wait_queue_head_t wait;
69	70	unsigned long unpinned;
70		− struct rb_root sdma_rb_root;
71		− u32 n_locked;
72		− struct list_head evict;
73		− spinlock_t evict_lock; /* protect evict and n_locked */
	71	+ struct mmu_rb_handler *handler;
	72	+ atomic_t n_locked;
	73	+ struct mm_struct *mm;
74	74	};
75	75
76	76	struct hfi1_user_sdma_comp_q {

drivers/infiniband/hw/hfi1/verbs.c+42−32

@@ -306,7 +306,10 @@ const enum ib_wc_opcode ib_hfi1_wc_opcode[] = { · 305 unmodified lines
306	306	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
307	307	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
308	308	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
309		− [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD
	309	+ [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD,
	310	+ [IB_WR_SEND_WITH_INV] = IB_WC_SEND,
	311	+ [IB_WR_LOCAL_INV] = IB_WC_LOCAL_INV,
	312	+ [IB_WR_REG_MR] = IB_WC_REG_MR
310	313	};
311	314
312	315	/*
@@ -378,6 +381,8 @@ static const opcode_handler opcode_handler_tbl[256] = { · 65 unmodified lines
378	381	[IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = &hfi1_rc_rcv,
379	382	[IB_OPCODE_RC_COMPARE_SWAP] = &hfi1_rc_rcv,
380	383	[IB_OPCODE_RC_FETCH_ADD] = &hfi1_rc_rcv,
	384	+ [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = &hfi1_rc_rcv,
	385	+ [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = &hfi1_rc_rcv,
381	386	/* UC */
382	387	[IB_OPCODE_UC_SEND_FIRST] = &hfi1_uc_rcv,
383	388	[IB_OPCODE_UC_SEND_MIDDLE] = &hfi1_uc_rcv,
@@ -540,19 +545,15 @@ void hfi1_skip_sge(struct rvt_sge_state *ss, u32 length, int release) · 156 unmodified lines
540	545	/*
541	546	* Make sure the QP is ready and able to accept the given opcode.
542	547	*/
543		−static inline int qp_ok(int opcode, struct hfi1_packet *packet)
	548	+static inline opcode_handler qp_ok(int opcode, struct hfi1_packet *packet)
544	549	{
545		− struct hfi1_ibport *ibp;
546		−
547	550	if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))
548		− goto dropit;
	551	+ return NULL;
549	552	if (((opcode & RVT_OPCODE_QP_MASK) == packet->qp->allowed_ops) \|\|
550	553	(opcode == IB_OPCODE_CNP))
551		− return 1;
552		−dropit:
553		− ibp = &packet->rcd->ppd->ibport_data;
554		− ibp->rvp.n_pkt_drops++;
555		− return 0;
	554	+ return opcode_handler_tbl[opcode];
	555	+
	556	+ return NULL;
556	557	}
557	558
558	559	/**
@@ -571,6 +572,7 @@ void hfi1_ib_rcv(struct hfi1_packet *packet) · 12 unmodified lines
571	572	struct hfi1_pportdata *ppd = rcd->ppd;
572	573	struct hfi1_ibport *ibp = &ppd->ibport_data;
573	574	struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi;
	575	+ opcode_handler packet_handler;
574	576	unsigned long flags;
575	577	u32 qp_num;
576	578	int lnh;
@@ -616,8 +618,11 @@ void hfi1_ib_rcv(struct hfi1_packet *packet) · 39 unmodified lines
616	618	list_for_each_entry_rcu(p, &mcast->qp_list, list) {
617	619	packet->qp = p->qp;
618	620	spin_lock_irqsave(&packet->qp->r_lock, flags);
619		− if (likely((qp_ok(opcode, packet))))
620		− opcode_handler_tbl[opcode](packet);
	621	+ packet_handler = qp_ok(opcode, packet);
	622	+ if (likely(packet_handler))
	623	+ packet_handler(packet);
	624	+ else
	625	+ ibp->rvp.n_pkt_drops++;
621	626	spin_unlock_irqrestore(&packet->qp->r_lock, flags);
622	627	}
623	628	/*
@@ -634,8 +639,11 @@ void hfi1_ib_rcv(struct hfi1_packet *packet) · 10 unmodified lines
634	639	goto drop;
635	640	}
636	641	spin_lock_irqsave(&packet->qp->r_lock, flags);
637		− if (likely((qp_ok(opcode, packet))))
638		− opcode_handler_tbl[opcode](packet);
	642	+ packet_handler = qp_ok(opcode, packet);
	643	+ if (likely(packet_handler))
	644	+ packet_handler(packet);
	645	+ else
	646	+ ibp->rvp.n_pkt_drops++;
639	647	spin_unlock_irqrestore(&packet->qp->r_lock, flags);
640	648	rcu_read_unlock();
641	649	}
@@ -808,19 +816,19 @@ static int build_verbs_tx_desc( · 166 unmodified lines
808	816	struct rvt_sge_state *ss,
809	817	u32 length,
810	818	struct verbs_txreq *tx,
811		− struct ahg_ib_header *ahdr,
	819	+ struct hfi1_ahg_info *ahg_info,
812	820	u64 pbc)
813	821	{
814	822	int ret = 0;
815		− struct hfi1_pio_header *phdr = &tx->phdr;
	823	+ struct hfi1_sdma_header *phdr = &tx->phdr;
816	824	u16 hdrbytes = tx->hdr_dwords << 2;
817	825
818		− if (!ahdr->ahgcount) {
	826	+ if (!ahg_info->ahgcount) {
819	827	ret = sdma_txinit_ahg(
820	828	&tx->txreq,
821		− ahdr->tx_flags,
	829	+ ahg_info->tx_flags,
822	830	hdrbytes + length,
823		− ahdr->ahgidx,
	831	+ ahg_info->ahgidx,
824	832	0,
825	833	NULL,
826	834	0,
@@ -838,11 +846,11 @@ static int build_verbs_tx_desc( · 11 unmodified lines
838	846	} else {
839	847	ret = sdma_txinit_ahg(
840	848	&tx->txreq,
841		− ahdr->tx_flags,
	849	+ ahg_info->tx_flags,
842	850	length,
843		− ahdr->ahgidx,
844		− ahdr->ahgcount,
845		− ahdr->ahgdesc,
	851	+ ahg_info->ahgidx,
	852	+ ahg_info->ahgcount,
	853	+ ahg_info->ahgdesc,
846	854	hdrbytes,
847	855	verbs_sdma_complete);
848	856	if (ret)
@@ -860,7 +868,7 @@ int hfi1_verbs_send_dma(struct rvt_qp qp, struct hfi1_pkt_state ps, · 11 unmodified lines
860	868	u64 pbc)
861	869	{
862	870	struct hfi1_qp_priv *priv = qp->priv;
863		− struct ahg_ib_header *ahdr = priv->s_hdr;
	871	+ struct hfi1_ahg_info *ahg_info = priv->s_ahg;
864	872	u32 hdrwords = qp->s_hdrwords;
865	873	struct rvt_sge_state *ss = qp->s_cur_sge;
866	874	u32 len = qp->s_cur_size;
@@ -888,7 +896,7 @@ int hfi1_verbs_send_dma(struct rvt_qp qp, struct hfi1_pkt_state ps, · 21 unmodified lines
888	896	plen);
889	897	}
890	898	tx->wqe = qp->s_wqe;
891		− ret = build_verbs_tx_desc(tx->sde, ss, len, tx, ahdr, pbc);
	899	+ ret = build_verbs_tx_desc(tx->sde, ss, len, tx, ahg_info, pbc);
892	900	if (unlikely(ret))
893	901	goto bail_build;
894	902	}
@@ -1300,13 +1308,15 @@ static void hfi1_fill_device_attr(struct hfi1_devdata *dd) · 405 unmodified lines
1300	1308	rdi->dparms.props.device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR \|
1301	1309	IB_DEVICE_BAD_QKEY_CNTR \| IB_DEVICE_SHUTDOWN_PORT \|
1302	1310	IB_DEVICE_SYS_IMAGE_GUID \| IB_DEVICE_RC_RNR_NAK_GEN \|
1303		− IB_DEVICE_PORT_ACTIVE_EVENT \| IB_DEVICE_SRQ_RESIZE;
	1311	+ IB_DEVICE_PORT_ACTIVE_EVENT \| IB_DEVICE_SRQ_RESIZE \|
	1312	+ IB_DEVICE_MEM_MGT_EXTENSIONS;
1304	1313	rdi->dparms.props.page_size_cap = PAGE_SIZE;
1305	1314	rdi->dparms.props.vendor_id = dd->oui1 << 16 \| dd->oui2 << 8 \| dd->oui3;
1306	1315	rdi->dparms.props.vendor_part_id = dd->pcidev->device;
1307	1316	rdi->dparms.props.hw_ver = dd->minrev;
1308	1317	rdi->dparms.props.sys_image_guid = ib_hfi1_sys_image_guid;
1309		− rdi->dparms.props.max_mr_size = ~0ULL;
	1318	+ rdi->dparms.props.max_mr_size = U64_MAX;
	1319	+ rdi->dparms.props.max_fast_reg_page_list_len = UINT_MAX;
1310	1320	rdi->dparms.props.max_qp = hfi1_max_qps;
1311	1321	rdi->dparms.props.max_qp_wr = hfi1_max_qp_wrs;
1312	1322	rdi->dparms.props.max_sge = hfi1_max_sges;
@@ -1695,6 +1705,9 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) · 382 unmodified lines
1695	1705	dd->verbs_dev.rdi.dparms.nports = dd->num_pports;
1696	1706	dd->verbs_dev.rdi.dparms.npkeys = hfi1_get_npkeys(dd);
1697	1707
	1708	+ /* post send table */
	1709	+ dd->verbs_dev.rdi.post_parms = hfi1_post_parms;
	1710	+
1698	1711	ppd = dd->pport;
1699	1712	for (i = 0; i < dd->num_pports; i++, ppd++)
1700	1713	rvt_init_port(&dd->verbs_dev.rdi,
@@ -1745,8 +1758,7 @@ void hfi1_cnp_rcv(struct hfi1_packet *packet) · 44 unmodified lines
1745	1758	struct rvt_qp *qp = packet->qp;
1746	1759	u32 lqpn, rqpn = 0;
1747	1760	u16 rlid = 0;
1748		− u8 sl, sc5, sc4_bit, svc_type;
1749		− bool sc4_set = has_sc4_bit(packet);
	1761	+ u8 sl, sc5, svc_type;
1750	1762
1751	1763	switch (packet->qp->ibqp.qp_type) {
1752	1764	case IB_QPT_UC:
@@ -1769,9 +1781,7 @@ void hfi1_cnp_rcv(struct hfi1_packet *packet) · 16 unmodified lines
1769	1781	return;
1770	1782	}
1771	1783
1772		− sc4_bit = sc4_set << 4;
1773		− sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf;
1774		− sc5 \|= sc4_bit;
	1784	+ sc5 = hdr2sc((struct hfi1_message_header *)hdr, packet->rhf);
1775	1785	sl = ibp->sc_to_sl[sc5];
1776	1786	lqpn = qp->ibqp.qp_num;
1777	1787

drivers/infiniband/hw/hfi1/verbs.h+3−5

@@ -178,16 +178,14 @@ struct hfi1_ib_header { · 177 unmodified lines
178	178	} u;
179	179	} __packed;
180	180
181		−struct ahg_ib_header {
182		− struct sdma_engine *sde;
	181	+struct hfi1_ahg_info {
183	182	u32 ahgdesc[2];
184	183	u16 tx_flags;
185	184	u8 ahgcount;
186	185	u8 ahgidx;
187		− struct hfi1_ib_header ibh;
188	186	};
189	187
190		−struct hfi1_pio_header {
	188	+struct hfi1_sdma_header {
191	189	__le64 pbc;
192	190	struct hfi1_ib_header hdr;
193	191	} __packed;
@@ -197,7 +195,7 @@ struct hfi1_pio_header { · 3 unmodified lines
197	195	* pair is made common
198	196	*/
199	197	struct hfi1_qp_priv {
200		− struct ahg_ib_header s_hdr; / next header to send */
	198	+ struct hfi1_ahg_info s_ahg; / ahg info for next header */
201	199	struct sdma_engine s_sde; / current sde */
202	200	struct send_context s_sendcontext; / current sendcontext */
203	201	u8 s_sc; /* SC[0..4] for next packet */

drivers/infiniband/hw/hfi1/verbs_txreq.h+1−1

@@ -56,7 +56,7 @@ · 55 unmodified lines
56	56	#include "iowait.h"
57	57
58	58	struct verbs_txreq {
59		− struct hfi1_pio_header phdr;
	59	+ struct hfi1_sdma_header phdr;
60	60	struct sdma_txreq txreq;
61	61	struct rvt_qp *qp;
62	62	struct rvt_swqe *wqe;

drivers/infiniband/hw/qib/qib_qp.c+43−0

@@ -106,6 +106,49 @@ static u32 credit_table[31] = { · 105 unmodified lines
106	106	32768 /* 1E */
107	107	};
108	108
	109	+const struct rvt_operation_params qib_post_parms[RVT_OPERATION_MAX] = {
	110	+[IB_WR_RDMA_WRITE] = {
	111	+ .length = sizeof(struct ib_rdma_wr),
	112	+ .qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	113	+},
	114	+
	115	+[IB_WR_RDMA_READ] = {
	116	+ .length = sizeof(struct ib_rdma_wr),
	117	+ .qpt_support = BIT(IB_QPT_RC),
	118	+ .flags = RVT_OPERATION_ATOMIC,
	119	+},
	120	+
	121	+[IB_WR_ATOMIC_CMP_AND_SWP] = {
	122	+ .length = sizeof(struct ib_atomic_wr),
	123	+ .qpt_support = BIT(IB_QPT_RC),
	124	+ .flags = RVT_OPERATION_ATOMIC \| RVT_OPERATION_ATOMIC_SGE,
	125	+},
	126	+
	127	+[IB_WR_ATOMIC_FETCH_AND_ADD] = {
	128	+ .length = sizeof(struct ib_atomic_wr),
	129	+ .qpt_support = BIT(IB_QPT_RC),
	130	+ .flags = RVT_OPERATION_ATOMIC \| RVT_OPERATION_ATOMIC_SGE,
	131	+},
	132	+
	133	+[IB_WR_RDMA_WRITE_WITH_IMM] = {
	134	+ .length = sizeof(struct ib_rdma_wr),
	135	+ .qpt_support = BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	136	+},
	137	+
	138	+[IB_WR_SEND] = {
	139	+ .length = sizeof(struct ib_send_wr),
	140	+ .qpt_support = BIT(IB_QPT_UD) \| BIT(IB_QPT_SMI) \| BIT(IB_QPT_GSI) \|
	141	+ BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	142	+},
	143	+
	144	+[IB_WR_SEND_WITH_IMM] = {
	145	+ .length = sizeof(struct ib_send_wr),
	146	+ .qpt_support = BIT(IB_QPT_UD) \| BIT(IB_QPT_SMI) \| BIT(IB_QPT_GSI) \|
	147	+ BIT(IB_QPT_UC) \| BIT(IB_QPT_RC),
	148	+},
	149	+
	150	+};
	151	+
109	152	static void get_map_page(struct rvt_qpn_table qpt, struct rvt_qpn_map map,
110	153	gfp_t gfp)
111	154	{

drivers/infiniband/hw/qib/qib_ud.c+6−2

@@ -169,8 +169,12 @@ static void qib_ud_loopback(struct rvt_qp sqp, struct rvt_swqe swqe) · 168 unmodified lines
169	169	}
170	170
171	171	if (ah_attr->ah_flags & IB_AH_GRH) {
172		− qib_copy_sge(&qp->r_sge, &ah_attr->grh,
173		− sizeof(struct ib_grh), 1);
	172	+ struct ib_grh grh;
	173	+ struct ib_global_route grd = ah_attr->grh;
	174	+
	175	+ qib_make_grh(ibp, &grh, &grd, 0, 0);
	176	+ qib_copy_sge(&qp->r_sge, &grh,
	177	+ sizeof(grh), 1);
174	178	wc.wc_flags \|= IB_WC_GRH;
175	179	} else
176	180	qib_skip_sge(&qp->r_sge, sizeof(struct ib_grh), 1);

drivers/infiniband/hw/qib/qib_verbs.c+2−0

@@ -1582,6 +1582,8 @@ static void qib_fill_device_attr(struct qib_devdata *dd) · 1581 unmodified lines
1582	1582	rdi->dparms.props.max_total_mcast_qp_attach =
1583	1583	rdi->dparms.props.max_mcast_qp_attach *
1584	1584	rdi->dparms.props.max_mcast_grp;
	1585	+ /* post send table */
	1586	+ dd->verbs_dev.rdi.post_parms = qib_post_parms;
1585	1587	}
1586	1588
1587	1589	/**

drivers/infiniband/hw/qib/qib_verbs.h+2−0

@@ -497,4 +497,6 @@ extern unsigned int ib_qib_max_srq_wrs; · 496 unmodified lines
497	497
498	498	extern const u32 ib_qib_rnr_table[];
499	499
	500	+extern const struct rvt_operation_params qib_post_parms[];
	501	+
500	502	#endif /* QIB_VERBS_H */

drivers/infiniband/sw/rdmavt/cq.c+1−0

@@ -510,6 +510,7 @@ int rvt_driver_cq_init(struct rvt_dev_info *rdi) · 509 unmodified lines
510	510
511	511	if (rdi->worker)
512	512	return 0;
	513	+ spin_lock_init(&rdi->n_cqs_lock);
513	514	rdi->worker = kzalloc(sizeof(*rdi->worker), GFP_KERNEL);
514	515	if (!rdi->worker)
515	516	return -ENOMEM;

drivers/infiniband/sw/rdmavt/mr.c+122−2

@@ -140,6 +140,7 @@ static int rvt_init_mregion(struct rvt_mregion mr, struct ib_pd pd, · 139 unmodified lines
140	140	init_completion(&mr->comp);
141	141	/* count returning the ptr to user */
142	142	atomic_set(&mr->refcount, 1);
	143	+ atomic_set(&mr->lkey_invalid, 0);
143	144	mr->pd = pd;
144	145	mr->max_segs = count;
145	146	return 0;
@@ -479,6 +480,123 @@ struct ib_mr rvt_alloc_mr(struct ib_pd pd, · 333 unmodified lines
479	480	return &mr->ibmr;
480	481	}
481	482
	483	+/**
	484	+ * rvt_set_page - page assignment function called by ib_sg_to_pages
	485	+ * @ibmr: memory region
	486	+ * @addr: dma address of mapped page
	487	+ *
	488	+ * Return: 0 on success
	489	+ */
	490	+static int rvt_set_page(struct ib_mr *ibmr, u64 addr)
	491	+{
	492	+ struct rvt_mr *mr = to_imr(ibmr);
	493	+ u32 ps = 1 << mr->mr.page_shift;
	494	+ u32 mapped_segs = mr->mr.length >> mr->mr.page_shift;
	495	+ int m, n;
	496	+
	497	+ if (unlikely(mapped_segs == mr->mr.max_segs))
	498	+ return -ENOMEM;
	499	+
	500	+ if (mr->mr.length == 0) {
	501	+ mr->mr.user_base = addr;
	502	+ mr->mr.iova = addr;
	503	+ }
	504	+
	505	+ m = mapped_segs / RVT_SEGSZ;
	506	+ n = mapped_segs % RVT_SEGSZ;
	507	+ mr->mr.map[m]->segs[n].vaddr = (void *)addr;
	508	+ mr->mr.map[m]->segs[n].length = ps;
	509	+ mr->mr.length += ps;
	510	+
	511	+ return 0;
	512	+}
	513	+
	514	+/**
	515	+ * rvt_map_mr_sg - map sg list and set it the memory region
	516	+ * @ibmr: memory region
	517	+ * @sg: dma mapped scatterlist
	518	+ * @sg_nents: number of entries in sg
	519	+ * @sg_offset: offset in bytes into sg
	520	+ *
	521	+ * Return: number of sg elements mapped to the memory region
	522	+ */
	523	+int rvt_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg,
	524	+ int sg_nents, unsigned int *sg_offset)
	525	+{
	526	+ struct rvt_mr *mr = to_imr(ibmr);
	527	+
	528	+ mr->mr.length = 0;
	529	+ mr->mr.page_shift = PAGE_SHIFT;
	530	+ return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
	531	+ rvt_set_page);
	532	+}
	533	+
	534	+/**
	535	+ * rvt_fast_reg_mr - fast register physical MR
	536	+ * @qp: the queue pair where the work request comes from
	537	+ * @ibmr: the memory region to be registered
	538	+ * @key: updated key for this memory region
	539	+ * @access: access flags for this memory region
	540	+ *
	541	+ * Returns 0 on success.
	542	+ */
	543	+int rvt_fast_reg_mr(struct rvt_qp qp, struct ib_mr ibmr, u32 key,
	544	+ int access)
	545	+{
	546	+ struct rvt_mr *mr = to_imr(ibmr);
	547	+
	548	+ if (qp->ibqp.pd != mr->mr.pd)
	549	+ return -EACCES;
	550	+
	551	+ /* not applicable to dma MR or user MR */
	552	+ if (!mr->mr.lkey \|\| mr->umem)
	553	+ return -EINVAL;
	554	+
	555	+ if ((key & 0xFFFFFF00) != (mr->mr.lkey & 0xFFFFFF00))
	556	+ return -EINVAL;
	557	+
	558	+ ibmr->lkey = key;
	559	+ ibmr->rkey = key;
	560	+ mr->mr.lkey = key;
	561	+ mr->mr.access_flags = access;
	562	+ atomic_set(&mr->mr.lkey_invalid, 0);
	563	+
	564	+ return 0;
	565	+}
	566	+EXPORT_SYMBOL(rvt_fast_reg_mr);
	567	+
	568	+/**
	569	+ * rvt_invalidate_rkey - invalidate an MR rkey
	570	+ * @qp: queue pair associated with the invalidate op
	571	+ * @rkey: rkey to invalidate
	572	+ *
	573	+ * Returns 0 on success.
	574	+ */
	575	+int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey)
	576	+{
	577	+ struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device);
	578	+ struct rvt_lkey_table *rkt = &dev->lkey_table;
	579	+ struct rvt_mregion *mr;
	580	+
	581	+ if (rkey == 0)
	582	+ return -EINVAL;
	583	+
	584	+ rcu_read_lock();
	585	+ mr = rcu_dereference(
	586	+ rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]);
	587	+ if (unlikely(!mr \|\| mr->lkey != rkey \|\| qp->ibqp.pd != mr->pd))
	588	+ goto bail;
	589	+
	590	+ atomic_set(&mr->lkey_invalid, 1);
	591	+ rcu_read_unlock();
	592	+ return 0;
	593	+
	594	+bail:
	595	+ rcu_read_unlock();
	596	+ return -EINVAL;
	597	+}
	598	+EXPORT_SYMBOL(rvt_invalidate_rkey);
	599	+
482	600	/**
483	601	* rvt_alloc_fmr - allocate a fast memory region
484	602	* @pd: the protection domain for this memory region
@@ -682,7 +800,8 @@ int rvt_lkey_ok(struct rvt_lkey_table rkt, struct rvt_pd pd, · 197 unmodified lines
682	800	}
683	801	mr = rcu_dereference(
684	802	rkt->table[(sge->lkey >> (32 - dev->dparms.lkey_table_size))]);
685		− if (unlikely(!mr \|\| mr->lkey != sge->lkey \|\| mr->pd != &pd->ibpd))
	803	+ if (unlikely(!mr \|\| atomic_read(&mr->lkey_invalid) \|\|
	804	+ mr->lkey != sge->lkey \|\| mr->pd != &pd->ibpd))
686	805	goto bail;
687	806
688	807	off = sge->addr - mr->user_base;
@@ -782,7 +901,8 @@ int rvt_rkey_ok(struct rvt_qp qp, struct rvt_sge sge, · 93 unmodified lines
782	901
783	902	mr = rcu_dereference(
784	903	rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]);
785		− if (unlikely(!mr \|\| mr->lkey != rkey \|\| qp->ibqp.pd != mr->pd))
	904	+ if (unlikely(!mr \|\| atomic_read(&mr->lkey_invalid) \|\|
	905	+ mr->lkey != rkey \|\| qp->ibqp.pd != mr->pd))
786	906	goto bail;
787	907
788	908	off = vaddr - mr->iova;

drivers/infiniband/sw/rdmavt/mr.h+2−0

@@ -82,6 +82,8 @@ int rvt_dereg_mr(struct ib_mr *ibmr); · 81 unmodified lines
82	82	struct ib_mr rvt_alloc_mr(struct ib_pd pd,
83	83	enum ib_mr_type mr_type,
84	84	u32 max_num_sg);
	85	+int rvt_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg,
	86	+ int sg_nents, unsigned int *sg_offset);
85	87	struct ib_fmr rvt_alloc_fmr(struct ib_pd pd, int mr_access_flags,
86	88	struct ib_fmr_attr *fmr_attr);
87	89	int rvt_map_phys_fmr(struct ib_fmr ibfmr, u64 page_list,

drivers/infiniband/sw/rdmavt/qp.c+181−65

@@ -435,8 +435,7 @@ static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends) · 434 unmodified lines
435	435	for (n = 0; n < rvt_max_atomic(rdi); n++) {
436	436	struct rvt_ack_entry *e = &qp->s_ack_queue[n];
437	437
438		− if (e->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST &&
439		− e->rdma_sge.mr) {
	438	+ if (e->rdma_sge.mr) {
440	439	rvt_put_mr(e->rdma_sge.mr);
441	440	e->rdma_sge.mr = NULL;
442	441	}
@@ -584,6 +583,7 @@ static void rvt_reset_qp(struct rvt_dev_info rdi, struct rvt_qp qp, · 141 unmodified lines
584	583	qp->r_rq.wq->tail = 0;
585	584	}
586	585	qp->r_sge.num_sge = 0;
	586	+ atomic_set(&qp->s_reserved_used, 0);
587	587	}
588	588
589	589	/**
@@ -613,6 +613,7 @@ struct ib_qp rvt_create_qp(struct ib_pd ibpd, · 23 unmodified lines
613	613	struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device);
614	614	void *priv = NULL;
615	615	gfp_t gfp;
	616	+ size_t sqsize;
616	617
617	618	if (!rdi)
618	619	return ERR_PTR(-EINVAL);
@@ -643,7 +644,9 @@ struct ib_qp rvt_create_qp(struct ib_pd ibpd, · 24 unmodified lines
643	644	init_attr->cap.max_recv_wr == 0)
644	645	return ERR_PTR(-EINVAL);
645	646	}
646		−
	647	+ sqsize =
	648	+ init_attr->cap.max_send_wr + 1 +
	649	+ rdi->dparms.reserved_operations;
647	650	switch (init_attr->qp_type) {
648	651	case IB_QPT_SMI:
649	652	case IB_QPT_GSI:
@@ -658,11 +661,11 @@ struct ib_qp rvt_create_qp(struct ib_pd ibpd, · 8 unmodified lines
658	661	sizeof(struct rvt_swqe);
659	662	if (gfp == GFP_NOIO)
660	663	swq = __vmalloc(
661		− (init_attr->cap.max_send_wr + 1) * sz,
	664	+ sqsize * sz,
662	665	gfp \| __GFP_ZERO, PAGE_KERNEL);
663	666	else
664	667	swq = vzalloc_node(
665		− (init_attr->cap.max_send_wr + 1) * sz,
	668	+ sqsize * sz,
666	669	rdi->dparms.node);
667	670	if (!swq)
668	671	return ERR_PTR(-ENOMEM);
@@ -741,13 +744,14 @@ struct ib_qp rvt_create_qp(struct ib_pd ibpd, · 72 unmodified lines
741	744	spin_lock_init(&qp->s_lock);
742	745	spin_lock_init(&qp->r_rq.lock);
743	746	atomic_set(&qp->refcount, 0);
	747	+ atomic_set(&qp->local_ops_pending, 0);
744	748	init_waitqueue_head(&qp->wait);
745	749	init_timer(&qp->s_timer);
746	750	qp->s_timer.data = (unsigned long)qp;
747	751	INIT_LIST_HEAD(&qp->rspwait);
748	752	qp->state = IB_QPS_RESET;
749	753	qp->s_wq = swq;
750		− qp->s_size = init_attr->cap.max_send_wr + 1;
	754	+ qp->s_size = sqsize;
751	755	qp->s_avail = init_attr->cap.max_send_wr;
752	756	qp->s_max_sge = init_attr->cap.max_send_sge;
753	757	if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR)
@@ -1332,7 +1336,8 @@ int rvt_query_qp(struct ib_qp ibqp, struct ib_qp_attr attr, · 578 unmodified lines
1332	1336	attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask;
1333	1337	attr->dest_qp_num = qp->remote_qpn;
1334	1338	attr->qp_access_flags = qp->qp_access_flags;
1335		− attr->cap.max_send_wr = qp->s_size - 1;
	1339	+ attr->cap.max_send_wr = qp->s_size - 1 -
	1340	+ rdi->dparms.reserved_operations;
1336	1341	attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1;
1337	1342	attr->cap.max_send_sge = qp->s_max_sge;
1338	1343	attr->cap.max_recv_sge = qp->r_rq.max_sge;
@@ -1440,25 +1445,116 @@ int rvt_post_recv(struct ib_qp ibqp, struct ib_recv_wr wr, · 101 unmodified lines
1440	1445	}
1441	1446
1442	1447	/**
1443		− * qp_get_savail - return number of avail send entries
	1448	+ * rvt_qp_valid_operation - validate post send wr request
	1449	+ * @qp - the qp
	1450	+ * @post-parms - the post send table for the driver
	1451	+ * @wr - the work request
	1452	+ *
	1453	+ * The routine validates the operation based on the
	1454	+ * validation table an returns the length of the operation
	1455	+ * which can extend beyond the ib_send_bw. Operation
	1456	+ * dependent flags key atomic operation validation.
1444	1457	*
	1458	+ * There is an exception for UD qps that validates the pd and
	1459	+ * overrides the length to include the additional UD specific
	1460	+ * length.
	1461	+ *
	1462	+ * Returns a negative error or the length of the work request
	1463	+ * for building the swqe.
	1464	+ */
	1465	+static inline int rvt_qp_valid_operation(
	1466	+ struct rvt_qp *qp,
	1467	+ const struct rvt_operation_params *post_parms,
	1468	+ struct ib_send_wr *wr)
	1469	+{
	1470	+ int len;
	1471	+
	1472	+ if (wr->opcode >= RVT_OPERATION_MAX \|\| !post_parms[wr->opcode].length)
	1473	+ return -EINVAL;
	1474	+ if (!(post_parms[wr->opcode].qpt_support & BIT(qp->ibqp.qp_type)))
	1475	+ return -EINVAL;
	1476	+ if ((post_parms[wr->opcode].flags & RVT_OPERATION_PRIV) &&
	1477	+ ibpd_to_rvtpd(qp->ibqp.pd)->user)
	1478	+ return -EINVAL;
	1479	+ if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC_SGE &&
	1480	+ (wr->num_sge == 0 \|\|
	1481	+ wr->sg_list[0].length < sizeof(u64) \|\|
	1482	+ wr->sg_list[0].addr & (sizeof(u64) - 1)))
	1483	+ return -EINVAL;
	1484	+ if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC &&
	1485	+ !qp->s_max_rd_atomic)
	1486	+ return -EINVAL;
	1487	+ len = post_parms[wr->opcode].length;
	1488	+ /* UD specific */
	1489	+ if (qp->ibqp.qp_type != IB_QPT_UC &&
	1490	+ qp->ibqp.qp_type != IB_QPT_RC) {
	1491	+ if (qp->ibqp.pd != ud_wr(wr)->ah->pd)
	1492	+ return -EINVAL;
	1493	+ len = sizeof(struct ib_ud_wr);
	1494	+ }
	1495	+ return len;
	1496	+}
	1497	+
	1498	+/**
	1499	+ * rvt_qp_is_avail - determine queue capacity
1445	1500	* @qp - the qp
	1501	+ * @rdi - the rdmavt device
	1502	+ * @reserved_op - is reserved operation
1446	1503	*
1447	1504	* This assumes the s_hlock is held but the s_last
1448	1505	* qp variable is uncontrolled.
	1506	+ *
	1507	+ * For non reserved operations, the qp->s_avail
	1508	+ * may be changed.
	1509	+ *
	1510	+ * The return value is zero or a -ENOMEM.
1449	1511	*/
1450		−static inline u32 qp_get_savail(struct rvt_qp *qp)
	1512	+static inline int rvt_qp_is_avail(
	1513	+ struct rvt_qp *qp,
	1514	+ struct rvt_dev_info *rdi,
	1515	+ bool reserved_op)
1451	1516	{
1452	1517	u32 slast;
1453		− u32 ret;
1454		−
	1518	+ u32 avail;
	1519	+ u32 reserved_used;
	1520	+
	1521	+ /* see rvt_qp_wqe_unreserve() */
	1522	+ smp_mb__before_atomic();
	1523	+ reserved_used = atomic_read(&qp->s_reserved_used);
	1524	+ if (unlikely(reserved_op)) {
	1525	+ /* see rvt_qp_wqe_unreserve() */
	1526	+ smp_mb__before_atomic();
	1527	+ if (reserved_used >= rdi->dparms.reserved_operations)
	1528	+ return -ENOMEM;
	1529	+ return 0;
	1530	+ }
	1531	+ /* non-reserved operations */
	1532	+ if (likely(qp->s_avail))
	1533	+ return 0;
1455	1534	smp_read_barrier_depends(); /* see rc.c */
1456	1535	slast = ACCESS_ONCE(qp->s_last);
1457	1536	if (qp->s_head >= slast)
1458		− ret = qp->s_size - (qp->s_head - slast);
	1537	+ avail = qp->s_size - (qp->s_head - slast);
1459	1538	else
1460		− ret = slast - qp->s_head;
1461		− return ret - 1;
	1539	+ avail = slast - qp->s_head;
	1540	+
	1541	+ /* see rvt_qp_wqe_unreserve() */
	1542	+ smp_mb__before_atomic();
	1543	+ reserved_used = atomic_read(&qp->s_reserved_used);
	1544	+ avail = avail - 1 -
	1545	+ (rdi->dparms.reserved_operations - reserved_used);
	1546	+ /* insure we don't assign a negative s_avail */
	1547	+ if ((s32)avail <= 0)
	1548	+ return -ENOMEM;
	1549	+ qp->s_avail = avail;
	1550	+ if (WARN_ON(qp->s_avail >
	1551	+ (qp->s_size - 1 - rdi->dparms.reserved_operations)))
	1552	+ rvt_pr_err(rdi,
	1553	+ "More avail entries than QP RB size.\nQP: %u, size: %u, avail: %u\nhead: %u, tail: %u, cur: %u, acked: %u, last: %u",
	1554	+ qp->ibqp.qp_num, qp->s_size, qp->s_avail,
	1555	+ qp->s_head, qp->s_tail, qp->s_cur,
	1556	+ qp->s_acked, qp->s_last);
	1557	+ return 0;
1462	1558	}
1463	1559
1464	1560	/**
@@ -1480,49 +1576,64 @@ static int rvt_post_one_wr(struct rvt_qp *qp, · 15 unmodified lines
1480	1576	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
1481	1577	u8 log_pmtu;
1482	1578	int ret;
	1579	+ size_t cplen;
	1580	+ bool reserved_op;
	1581	+ int local_ops_delayed = 0;
	1582	+
	1583	+ BUILD_BUG_ON(IB_QPT_MAX >= (sizeof(u32) * BITS_PER_BYTE));
1483	1584
1484	1585	/* IB spec says that num_sge == 0 is OK. */
1485	1586	if (unlikely(wr->num_sge > qp->s_max_sge))
1486	1587	return -EINVAL;
1487	1588
	1589	+ ret = rvt_qp_valid_operation(qp, rdi->post_parms, wr);
	1590	+ if (ret < 0)
	1591	+ return ret;
	1592	+ cplen = ret;
	1593	+
1488	1594	/*
1489		− * Don't allow RDMA reads or atomic operations on UC or
1490		− * undefined operations.
1491		− * Make sure buffer is large enough to hold the result for atomics.
	1595	+ * Local operations include fast register and local invalidate.
	1596	+ * Fast register needs to be processed immediately because the
	1597	+ * registered lkey may be used by following work requests and the
	1598	+ * lkey needs to be valid at the time those requests are posted.
	1599	+ * Local invalidate can be processed immediately if fencing is
	1600	+ * not required and no previous local invalidate ops are pending.
	1601	+ * Signaled local operations that have been processed immediately
	1602	+ * need to have requests with "completion only" flags set posted
	1603	+ * to the send queue in order to generate completions.
1492	1604	*/
1493		− if (qp->ibqp.qp_type == IB_QPT_UC) {
1494		− if ((unsigned)wr->opcode >= IB_WR_RDMA_READ)
1495		− return -EINVAL;
1496		− } else if (qp->ibqp.qp_type != IB_QPT_RC) {
1497		− /* Check IB_QPT_SMI, IB_QPT_GSI, IB_QPT_UD opcode */
1498		− if (wr->opcode != IB_WR_SEND &&
1499		− wr->opcode != IB_WR_SEND_WITH_IMM)
1500		− return -EINVAL;
1501		− /* Check UD destination address PD */
1502		− if (qp->ibqp.pd != ud_wr(wr)->ah->pd)
	1605	+ if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) {
	1606	+ switch (wr->opcode) {
	1607	+ case IB_WR_REG_MR:
	1608	+ ret = rvt_fast_reg_mr(qp,
	1609	+ reg_wr(wr)->mr,
	1610	+ reg_wr(wr)->key,
	1611	+ reg_wr(wr)->access);
	1612	+ if (ret \|\| !(wr->send_flags & IB_SEND_SIGNALED))
	1613	+ return ret;
	1614	+ break;
	1615	+ case IB_WR_LOCAL_INV:
	1616	+ if ((wr->send_flags & IB_SEND_FENCE) \|\|
	1617	+ atomic_read(&qp->local_ops_pending)) {
	1618	+ local_ops_delayed = 1;
	1619	+ } else {
	1620	+ ret = rvt_invalidate_rkey(
	1621	+ qp, wr->ex.invalidate_rkey);
	1622	+ if (ret \|\| !(wr->send_flags & IB_SEND_SIGNALED))
	1623	+ return ret;
	1624	+ }
	1625	+ break;
	1626	+ default:
1503	1627	return -EINVAL;
1504		− } else if ((unsigned)wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) {
1505		− return -EINVAL;
1506		− } else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP &&
1507		− (wr->num_sge == 0 \|\|
1508		− wr->sg_list[0].length < sizeof(u64) \|\|
1509		− wr->sg_list[0].addr & (sizeof(u64) - 1))) {
1510		− return -EINVAL;
1511		− } else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic) {
1512		− return -EINVAL;
	1628	+ }
1513	1629	}
	1630	+
	1631	+ reserved_op = rdi->post_parms[wr->opcode].flags &
	1632	+ RVT_OPERATION_USE_RESERVE;
1514	1633	/* check for avail */
1515		− if (unlikely(!qp->s_avail)) {
1516		− qp->s_avail = qp_get_savail(qp);
1517		− if (WARN_ON(qp->s_avail > (qp->s_size - 1)))
1518		− rvt_pr_err(rdi,
1519		− "More avail entries than QP RB size.\nQP: %u, size: %u, avail: %u\nhead: %u, tail: %u, cur: %u, acked: %u, last: %u",
1520		− qp->ibqp.qp_num, qp->s_size, qp->s_avail,
1521		− qp->s_head, qp->s_tail, qp->s_cur,
1522		− qp->s_acked, qp->s_last);
1523		− if (!qp->s_avail)
1524		− return -ENOMEM;
1525		− }
	1634	+ ret = rvt_qp_is_avail(qp, rdi, reserved_op);
	1635	+ if (ret)
	1636	+ return ret;
1526	1637	next = qp->s_head + 1;
1527	1638	if (next >= qp->s_size)
1528	1639	next = 0;
@@ -1531,18 +1642,8 @@ static int rvt_post_one_wr(struct rvt_qp *qp, · 2 unmodified lines
1531	1642	pd = ibpd_to_rvtpd(qp->ibqp.pd);
1532	1643	wqe = rvt_get_swqe_ptr(qp, qp->s_head);
1533	1644
1534		− if (qp->ibqp.qp_type != IB_QPT_UC &&
1535		− qp->ibqp.qp_type != IB_QPT_RC)
1536		− memcpy(&wqe->ud_wr, ud_wr(wr), sizeof(wqe->ud_wr));
1537		− else if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM \|\|
1538		− wr->opcode == IB_WR_RDMA_WRITE \|\|
1539		− wr->opcode == IB_WR_RDMA_READ)
1540		− memcpy(&wqe->rdma_wr, rdma_wr(wr), sizeof(wqe->rdma_wr));
1541		− else if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP \|\|
1542		− wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
1543		− memcpy(&wqe->atomic_wr, atomic_wr(wr), sizeof(wqe->atomic_wr));
1544		− else
1545		− memcpy(&wqe->wr, wr, sizeof(wqe->wr));
	1645	+ /* cplen has length from above */
	1646	+ memcpy(&wqe->wr, wr, cplen);
1546	1647
1547	1648	wqe->length = 0;
1548	1649	j = 0;
@@ -1585,14 +1686,29 @@ static int rvt_post_one_wr(struct rvt_qp *qp, · 36 unmodified lines
1585	1686	atomic_inc(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount);
1586	1687	}
1587	1688
1588		− wqe->ssn = qp->s_ssn++;
1589		− wqe->psn = qp->s_next_psn;
1590		− wqe->lpsn = wqe->psn +
1591		− (wqe->length ? ((wqe->length - 1) >> log_pmtu) : 0);
1592		− qp->s_next_psn = wqe->lpsn + 1;
	1689	+ if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) {
	1690	+ if (local_ops_delayed)
	1691	+ atomic_inc(&qp->local_ops_pending);
	1692	+ else
	1693	+ wqe->wr.send_flags \|= RVT_SEND_COMPLETION_ONLY;
	1694	+ wqe->ssn = 0;
	1695	+ wqe->psn = 0;
	1696	+ wqe->lpsn = 0;
	1697	+ } else {
	1698	+ wqe->ssn = qp->s_ssn++;
	1699	+ wqe->psn = qp->s_next_psn;
	1700	+ wqe->lpsn = wqe->psn +
	1701	+ (wqe->length ?
	1702	+ ((wqe->length - 1) >> log_pmtu) :
	1703	+ 0);
	1704	+ qp->s_next_psn = wqe->lpsn + 1;
	1705	+ }
1593	1706	trace_rvt_post_one_wr(qp, wqe);
	1707	+ if (unlikely(reserved_op))
	1708	+ rvt_qp_wqe_reserve(qp, wqe);
	1709	+ else
	1710	+ qp->s_avail--;
1594	1711	smp_wmb(); /* see request builders */
1595		− qp->s_avail--;
1596	1712	qp->s_head = next;
1597	1713
1598	1714	return 0;

drivers/infiniband/sw/rdmavt/vt.c+9−1

@@ -370,6 +370,7 @@ enum { · 369 unmodified lines
370	370	REG_USER_MR,
371	371	DEREG_MR,
372	372	ALLOC_MR,
	373	+ MAP_MR_SG,
373	374	ALLOC_FMR,
374	375	MAP_PHYS_FMR,
375	376	UNMAP_FMR,
@@ -528,7 +529,8 @@ static noinline int check_support(struct rvt_dev_info *rdi, int verb) · 152 unmodified lines
528	529	post_send),
529	530	rvt_post_send))
530	531	if (!rdi->driver_f.schedule_send \|\|
531		− !rdi->driver_f.do_send)
	532	+ !rdi->driver_f.do_send \|\|
	533	+ !rdi->post_parms)
532	534	return -EINVAL;
533	535	break;
534	536
@@ -633,6 +635,12 @@ static noinline int check_support(struct rvt_dev_info *rdi, int verb) · 98 unmodified lines
633	635	rvt_alloc_mr);
634	636	break;
635	637
	638	+ case MAP_MR_SG:
	639	+ check_driver_override(rdi, offsetof(struct ib_device,
	640	+ map_mr_sg),
	641	+ rvt_map_mr_sg);
	642	+ break;
	643	+
636	644	case MAP_PHYS_FMR:
637	645	check_driver_override(rdi, offsetof(struct ib_device,
638	646	map_phys_fmr),

drivers/infiniband/ulp/isert/ib_isert.c+0−2

@@ -137,8 +137,6 @@ isert_create_qp(struct isert_conn *isert_conn, · 136 unmodified lines
137	137	attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
138	138	attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
139	139	attr.cap.max_send_sge = device->ib_device->attrs.max_sge;
140		− isert_conn->max_sge = min(device->ib_device->attrs.max_sge,
141		− device->ib_device->attrs.max_sge_rd);
142	140	attr.cap.max_recv_sge = 1;
143	141	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
144	142	attr.qp_type = IB_QPT_RC;

drivers/infiniband/ulp/isert/ib_isert.h+0−1

@@ -138,7 +138,6 @@ struct isert_conn { · 137 unmodified lines
138	138	u32 responder_resources;
139	139	u32 initiator_depth;
140	140	bool pi_support;
141		− u32 max_sge;
142	141	struct iser_rx_desc *login_req_buf;
143	142	char *login_rsp_buf;
144	143	u64 login_req_dma;

drivers/infiniband/ulp/srpt/ib_srpt.c+4−6

@@ -1601,6 +1601,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) · 1600 unmodified lines
1601	1601	struct ib_qp_init_attr *qp_init;
1602	1602	struct srpt_port *sport = ch->sport;
1603	1603	struct srpt_device *sdev = sport->sdev;
	1604	+ const struct ib_device_attr *attrs = &sdev->device->attrs;
1604	1605	u32 srp_sq_size = sport->port_attrib.srp_sq_size;
1605	1606	int ret;
1606	1607
@@ -1638,7 +1639,7 @@ retry: · 31 unmodified lines
1638	1639	*/
1639	1640	qp_init->cap.max_send_wr = srp_sq_size / 2;
1640	1641	qp_init->cap.max_rdma_ctxs = srp_sq_size / 2;
1641		− qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
	1642	+ qp_init->cap.max_send_sge = min(attrs->max_sge, SRPT_MAX_SG_PER_WQE);
1642	1643	qp_init->port_num = ch->sport->port;
1643	1644
1644	1645	ch->qp = ib_create_qp(sdev->pd, qp_init);
@@ -2261,7 +2262,7 @@ static void srpt_queue_response(struct se_cmd *cmd) · 616 unmodified lines
2261	2262	container_of(cmd, struct srpt_send_ioctx, cmd);
2262	2263	struct srpt_rdma_ch *ch = ioctx->ch;
2263	2264	struct srpt_device *sdev = ch->sport->sdev;
2264		− struct ib_send_wr send_wr, first_wr = NULL, bad_wr;
	2265	+ struct ib_send_wr send_wr, first_wr = &send_wr, bad_wr;
2265	2266	struct ib_sge sge;
2266	2267	enum srpt_command_state state;
2267	2268	unsigned long flags;
@@ -2302,11 +2303,8 @@ static void srpt_queue_response(struct se_cmd *cmd) · 34 unmodified lines
2302	2303	struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2303	2304
2304	2305	first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp,
2305		− ch->sport->port, NULL,
2306		− first_wr ? first_wr : &send_wr);
	2306	+ ch->sport->port, NULL, first_wr);
2307	2307	}
2308		− } else {
2309		− first_wr = &send_wr;
2310	2308	}
2311	2309
2312	2310	if (state != SRPT_STATE_MGMT)

drivers/infiniband/ulp/srpt/ib_srpt.h+5−1

@@ -106,7 +106,11 @@ enum { · 105 unmodified lines
106	106	SRP_LOGIN_RSP_MULTICHAN_MAINTAINED = 0x2,
107	107
108	108	SRPT_DEF_SG_TABLESIZE = 128,
109		− SRPT_DEF_SG_PER_WQE = 16,
	109	+ /*
	110	+ * An experimentally determined value that avoids that QP creation
	111	+ * fails due to "swiotlb buffer is full" on systems using the swiotlb.
	112	+ */
	113	+ SRPT_MAX_SG_PER_WQE = 16,
110	114
111	115	MIN_SRPT_SQ_SIZE = 16,
112	116	DEF_SRPT_SQ_SIZE = 4096,

include/rdma/ib_verbs.h+6−0

@@ -1490,6 +1490,10 @@ struct ib_rwq_ind_table_init_attr { · 1489 unmodified lines
1490	1490	struct ib_wq **ind_tbl;
1491	1491	};
1492	1492
	1493	+/*
	1494	+ * @max_write_sge: Maximum SGE elements per RDMA WRITE request.
	1495	+ * @max_read_sge: Maximum SGE elements per RDMA READ request.
	1496	+ */
1493	1497	struct ib_qp {
1494	1498	struct ib_device *device;
1495	1499	struct ib_pd *pd;
@@ -1511,6 +1515,8 @@ struct ib_qp { · 15 unmodified lines
1511	1515	void (event_handler)(struct ib_event , void *);
1512	1516	void *qp_context;
1513	1517	u32 qp_num;
	1518	+ u32 max_write_sge;
	1519	+ u32 max_read_sge;
1514	1520	enum ib_qp_type qp_type;
1515	1521	struct ib_rwq_ind_table *rwq_ind_tbl;
1516	1522	};

include/rdma/opa_port_info.h+0−16

@@ -33,11 +33,6 @@ · 32 unmodified lines
33	33	#if !defined(OPA_PORT_INFO_H)
34	34	#define OPA_PORT_INFO_H
35	35
36		−/* Temporary until HFI driver is updated */
37		−#ifndef USE_PI_LED_ENABLE
38		−#define USE_PI_LED_ENABLE 0
39		−#endif
40		−
41	36	#define OPA_PORT_LINK_MODE_NOP 0 /* No change */
42	37	#define OPA_PORT_LINK_MODE_OPA 4 /* Port mode is OPA */
43	38
@@ -274,23 +269,12 @@ enum port_info_field_masks { · 230 unmodified lines
274	269	OPA_PI_MASK_MTU_CAP = 0x0F,
275	270	};
276	271
277		−#if USE_PI_LED_ENABLE
278	272	struct opa_port_states {
279	273	u8 reserved;
280	274	u8 ledenable_offlinereason; /* 1 res, 1 bit, 6 bits */
281	275	u8 reserved2;
282	276	u8 portphysstate_portstate; /* 4 bits, 4 bits */
283	277	};
284		−#define PI_LED_ENABLE_SUP 1
285		−#else
286		−struct opa_port_states {
287		− u8 reserved;
288		− u8 offline_reason; /* 2 res, 6 bits */
289		− u8 reserved2;
290		− u8 portphysstate_portstate; /* 4 bits, 4 bits */
291		−};
292		−#define PI_LED_ENABLE_SUP 0
293		−#endif
294	278
295	279	struct opa_port_state_info {
296	280	struct opa_port_states port_states;

include/rdma/rdma_vt.h+7−0

@@ -158,6 +158,7 @@ struct rvt_driver_params { · 157 unmodified lines
158	158	u32 max_mad_size;
159	159	u8 qos_shift;
160	160	u8 max_rdma_atomic;
	161	+ u8 reserved_operations;
161	162	};
162	163
163	164	/* Protection domain */
@@ -351,6 +352,9 @@ struct rvt_dev_info { · 187 unmodified lines
351	352	/* Driver specific properties */
352	353	struct rvt_driver_params dparms;
353	354
	355	+ /* post send table */
	356	+ const struct rvt_operation_params *post_parms;
	357	+
354	358	struct rvt_mregion __rcu *dma_mr;
355	359	struct rvt_lkey_table lkey_table;
356	360
@@ -484,6 +488,9 @@ void rvt_unregister_device(struct rvt_dev_info *rvd); · 127 unmodified lines
484	488	int rvt_check_ah(struct ib_device ibdev, struct ib_ah_attr ah_attr);
485	489	int rvt_init_port(struct rvt_dev_info rdi, struct rvt_ibport port,
486	490	int port_index, u16 *pkey_table);
	491	+int rvt_fast_reg_mr(struct rvt_qp qp, struct ib_mr ibmr, u32 key,
	492	+ int access);
	493	+int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey);
487	494	int rvt_rkey_ok(struct rvt_qp qp, struct rvt_sge sge,
488	495	u32 len, u64 vaddr, u32 rkey, int acc);
489	496	int rvt_lkey_ok(struct rvt_lkey_table rkt, struct rvt_pd pd,

include/rdma/rdmavt_mr.h+1−0

@@ -81,6 +81,7 @@ struct rvt_mregion { · 80 unmodified lines
81	81	u32 mapsz; /* size of the map array */
82	82	u8 page_shift; /* 0 - non unform/non powerof2 sizes */
83	83	u8 lkey_published; /* in global table */
	84	+ atomic_t lkey_invalid; /* true if current lkey is invalid */
84	85	struct completion comp; /* complete when refcount goes to zero */
85	86	atomic_t refcount;
86	87	struct rvt_segarray map[0]; / the segments */

include/rdma/rdmavt_qp.h+82−10

@@ -144,6 +144,12 @@ · 143 unmodified lines
144	144	#define RVT_PROCESS_OR_FLUSH_SEND \
145	145	(RVT_PROCESS_SEND_OK \| RVT_FLUSH_SEND)
146	146
	147	+/*
	148	+ * Internal send flags
	149	+ */
	150	+#define RVT_SEND_RESERVE_USED IB_SEND_RESERVED_START
	151	+#define RVT_SEND_COMPLETION_ONLY (IB_SEND_RESERVED_START << 1)
	152	+
147	153	/*
148	154	* Send work request queue entry.
149	155	* The size of the sg_list is determined when the QP is created and stored
@@ -216,23 +222,43 @@ struct rvt_mmap_info { · 66 unmodified lines
216	222	* to send a RDMA read response or atomic operation.
217	223	*/
218	224	struct rvt_ack_entry {
219		− u8 opcode;
220		− u8 sent;
	225	+ struct rvt_sge rdma_sge;
	226	+ u64 atomic_data;
221	227	u32 psn;
222	228	u32 lpsn;
223		− union {
224		− struct rvt_sge rdma_sge;
225		− u64 atomic_data;
226		− };
	229	+ u8 opcode;
	230	+ u8 sent;
227	231	};
228	232
229	233	#define RC_QP_SCALING_INTERVAL 5
230	234
231		−/*
232		− * Variables prefixed with s_ are for the requester (sender).
233		− * Variables prefixed with r_ are for the responder (receiver).
234		− * Variables prefixed with ack_ are for responder replies.
	235	+#define RVT_OPERATION_PRIV 0x00000001
	236	+#define RVT_OPERATION_ATOMIC 0x00000002
	237	+#define RVT_OPERATION_ATOMIC_SGE 0x00000004
	238	+#define RVT_OPERATION_LOCAL 0x00000008
	239	+#define RVT_OPERATION_USE_RESERVE 0x00000010
	240	+
	241	+#define RVT_OPERATION_MAX (IB_WR_RESERVED10 + 1)
	242	+
	243	+/**
	244	+ * rvt_operation_params - op table entry
	245	+ * @length - the length to copy into the swqe entry
	246	+ * @qpt_support - a bit mask indicating QP type support
	247	+ * @flags - RVT_OPERATION flags (see above)
235	248	*
	249	+ * This supports table driven post send so that
	250	+ * the driver can have differing an potentially
	251	+ * different sets of operations.
	252	+ *
	253	+ **/
	254	+
	255	+struct rvt_operation_params {
	256	+ size_t length;
	257	+ u32 qpt_support;
	258	+ u32 flags;
	259	+};
	260	+
	261	+/*
236	262	* Common variables are protected by both r_rq.lock and s_lock in that order
237	263	* which only happens in modify_qp() or changing the QP 'state'.
238	264	*/
@@ -307,6 +333,7 @@ struct rvt_qp { · 68 unmodified lines
307	333	u32 s_next_psn; /* PSN for next request */
308	334	u32 s_avail; /* number of entries avail */
309	335	u32 s_ssn; /* SSN of tail entry */
	336	+ atomic_t s_reserved_used; /* reserved entries in use */
310	337
311	338	spinlock_t s_lock ____cacheline_aligned_in_smp;
312	339	u32 s_flags;
@@ -343,6 +370,8 @@ struct rvt_qp { · 30 unmodified lines
343	370	struct rvt_sge_state s_ack_rdma_sge;
344	371	struct timer_list s_timer;
345	372
	373	+ atomic_t local_ops_pending; /* number of fast_reg/local_inv reqs */
	374	+
346	375	/*
347	376	* This sge list MUST be last. Do not add anything below here.
348	377	*/
@@ -436,6 +465,49 @@ static inline struct rvt_rwqe rvt_get_rwqe_ptr(struct rvt_rq rq, unsigned n) · 87 unmodified lines
436	465	rq->max_sge * sizeof(struct ib_sge)) * n);
437	466	}
438	467
	468	+/**
	469	+ * rvt_qp_wqe_reserve - reserve operation
	470	+ * @qp - the rvt qp
	471	+ * @wqe - the send wqe
	472	+ *
	473	+ * This routine used in post send to record
	474	+ * a wqe relative reserved operation use.
	475	+ */
	476	+static inline void rvt_qp_wqe_reserve(
	477	+ struct rvt_qp *qp,
	478	+ struct rvt_swqe *wqe)
	479	+{
	480	+ wqe->wr.send_flags \|= RVT_SEND_RESERVE_USED;
	481	+ atomic_inc(&qp->s_reserved_used);
	482	+}
	483	+
	484	+/**
	485	+ * rvt_qp_wqe_unreserve - clean reserved operation
	486	+ * @qp - the rvt qp
	487	+ * @wqe - the send wqe
	488	+ *
	489	+ * This decrements the reserve use count.
	490	+ *
	491	+ * This call MUST precede the change to
	492	+ * s_last to insure that post send sees a stable
	493	+ * s_avail.
	494	+ *
	495	+ * An smp_mp__after_atomic() is used to insure
	496	+ * the compiler does not juggle the order of the s_last
	497	+ * ring index and the decrementing of s_reserved_used.
	498	+ */
	499	+static inline void rvt_qp_wqe_unreserve(
	500	+ struct rvt_qp *qp,
	501	+ struct rvt_swqe *wqe)
	502	+{
	503	+ if (unlikely(wqe->wr.send_flags & RVT_SEND_RESERVE_USED)) {
	504	+ wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED;
	505	+ atomic_dec(&qp->s_reserved_used);
	506	+ /* insure no compiler re-order up to s_last change */
	507	+ smp_mb__after_atomic();
	508	+ }
	509	+}
	510	+
439	511	extern const int ib_rvt_state_ops[];
440	512
441	513	struct rvt_dev_info;

include/uapi/rdma/hfi/hfi1_user.h+1−1

@@ -75,7 +75,7 @@ · 74 unmodified lines
75	75	* may not be implemented; the user code must deal with this if it
76	76	* cares, or it must abort after initialization reports the difference.
77	77	*/
78		−#define HFI1_USER_SWMINOR 1
	78	+#define HFI1_USER_SWMINOR 2
79	79
80	80	/*
81	81	* We will encode the major/minor inside a single 32bit version number.

Merge tag 'for-linus-2' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code