Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull rdma updates from Jason Gunthorpe: "Usual size of updates, a new driver, and most of the bulk focusing on rxe: - Usual typos, style, and language updates - Driver updates for mlx5, irdma, siw, rts, srp, hfi1, hns, erdma, mlx4, srp - Lots of RXE updates: * Improve reply error handling for bad MR operations * Code tidying * Debug printing uses common loggers * Remove half implemented RD related stuff * Support IBA's recently defined Atomic Write and Flush operations - erdma support for atomic operations - New driver 'mana' for Ethernet HW available in Azure VMs. This driver only supports DPDK" * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (122 commits) IB/IPoIB: Fix queue count inconsistency for PKEY child interfaces RDMA: Add missed netdev_put() for the netdevice_tracker RDMA/rxe: Enable RDMA FLUSH capability for rxe device RDMA/cm: Make QP FLUSHABLE for supported device RDMA/rxe: Implement flush completion RDMA/rxe: Implement flush execution in responder side RDMA/rxe: Implement RC RDMA FLUSH service in requester side RDMA/rxe: Extend rxe packet format to support flush RDMA/rxe: Allow registering persistent flag for pmem MR only RDMA/rxe: Extend rxe user ABI to support flush RDMA: Extend RDMA kernel verbs ABI to support flush RDMA: Extend RDMA user ABI to support flush RDMA/rxe: Fix incorrect responder length checking RDMA/rxe: Fix oops with zero length reads RDMA/mlx5: Remove not-used IB_FLOW_SPEC_IB define RDMA/hns: Fix XRC caps on HIP08 RDMA/hns: Fix error code of CMD RDMA/hns: Fix page size cap from firmware RDMA/hns: Fix PBL page MTR find RDMA/hns: Fix AH attr queried by query_qp ...

Linus Torvalds committed 3 years ago

Commit ab425fe2 parents 08cdc21 dbc94a099 files · +3621 −897Browse files

Unified · Split

99 files

MAINTAINERS +9 −0
drivers/infiniband/Kconfig +3 −0
drivers/infiniband/core/cache.c +1 −1
drivers/infiniband/core/cm.c +11 −2
drivers/infiniband/core/cma.c +1 −1
drivers/infiniband/core/device.c +6 −4
drivers/infiniband/core/mad.c +0 −5
drivers/infiniband/core/nldev.c +40 −10
drivers/infiniband/core/restrack.c +0 −2
drivers/infiniband/core/sysfs.c +11 −6
drivers/infiniband/core/uverbs_std_types_qp.c +1 −1
drivers/infiniband/hw/Makefile +1 −0
drivers/infiniband/hw/erdma/erdma.h +3 −1
drivers/infiniband/hw/erdma/erdma_cq.c +2 −0
drivers/infiniband/hw/erdma/erdma_hw.h +30 −7
drivers/infiniband/hw/erdma/erdma_main.c +13 −2
drivers/infiniband/hw/erdma/erdma_qp.c +57 −16
drivers/infiniband/hw/erdma/erdma_verbs.c +23 −2
drivers/infiniband/hw/erdma/erdma_verbs.h +14 −5
drivers/infiniband/hw/hfi1/affinity.c +2 −0
drivers/infiniband/hw/hfi1/driver.c +1 −1
drivers/infiniband/hw/hfi1/firmware.c +6 −0
drivers/infiniband/hw/hfi1/mad.c +11 −11
drivers/infiniband/hw/hfi1/netdev_rx.c +1 −1
drivers/infiniband/hw/hns/hns_roce_device.h +3 −0
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +168 −49
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +10 −3
drivers/infiniband/hw/hns/hns_roce_main.c +16 −2
drivers/infiniband/hw/hns/hns_roce_mr.c +2 −2
drivers/infiniband/hw/hns/hns_roce_qp.c +90 −17
drivers/infiniband/hw/irdma/uk.c +107 −63
drivers/infiniband/hw/irdma/user.h +3 −17
drivers/infiniband/hw/irdma/utils.c +2 −0
drivers/infiniband/hw/irdma/verbs.c +34 −111
drivers/infiniband/hw/irdma/verbs.h +53 −0
drivers/infiniband/hw/mana/Kconfig +10 −0
drivers/infiniband/hw/mana/Makefile +4 −0
drivers/infiniband/hw/mana/cq.c +79 −0
drivers/infiniband/hw/mana/device.c +117 −0
drivers/infiniband/hw/mana/main.c +521 −0
drivers/infiniband/hw/mana/mana_ib.h +162 −0
drivers/infiniband/hw/mana/mr.c +197 −0
drivers/infiniband/hw/mana/qp.c +506 −0
drivers/infiniband/hw/mana/wq.c +115 −0
drivers/infiniband/hw/mlx4/main.c +4 −8
drivers/infiniband/hw/mlx5/cq.c +15 −12
drivers/infiniband/hw/mlx5/fs.c +0 −1
drivers/infiniband/hw/mlx5/mlx5_ib.h +4 −0
drivers/infiniband/hw/mlx5/mr.c +2 −4
drivers/infiniband/hw/qedr/main.c +1 −1
drivers/infiniband/hw/qib/qib_iba6120.c +2 −5
drivers/infiniband/hw/qib/qib_tx.c +1 −4
drivers/infiniband/hw/qib/qib_user_sdma.c +1 −1
drivers/infiniband/sw/rxe/rxe.c +2 −2
drivers/infiniband/sw/rxe/rxe.h +19 −0
drivers/infiniband/sw/rxe/rxe_av.c +36 −7
drivers/infiniband/sw/rxe/rxe_comp.c +31 −16
drivers/infiniband/sw/rxe/rxe_cq.c +4 −4
drivers/infiniband/sw/rxe/rxe_hdr.h +47 −1
drivers/infiniband/sw/rxe/rxe_icrc.c +2 −2
drivers/infiniband/sw/rxe/rxe_loc.h +3 −6
drivers/infiniband/sw/rxe/rxe_mmap.c +3 −3
drivers/infiniband/sw/rxe/rxe_mr.c +82 −40
drivers/infiniband/sw/rxe/rxe_mw.c +11 −12
drivers/infiniband/sw/rxe/rxe_net.c +23 −19
drivers/infiniband/sw/rxe/rxe_opcode.c +35 −0
drivers/infiniband/sw/rxe/rxe_opcode.h +12 −5
drivers/infiniband/sw/rxe/rxe_param.h +7 −0
drivers/infiniband/sw/rxe/rxe_qp.c +43 −55
drivers/infiniband/sw/rxe/rxe_req.c +37 −13
drivers/infiniband/sw/rxe/rxe_resp.c +279 −50
drivers/infiniband/sw/rxe/rxe_srq.c +10 −10
drivers/infiniband/sw/rxe/rxe_task.c +26 −26
drivers/infiniband/sw/rxe/rxe_task.h +4 −15
drivers/infiniband/sw/rxe/rxe_verbs.c +38 −68
drivers/infiniband/sw/rxe/rxe_verbs.h +6 −1
drivers/infiniband/sw/siw/siw_cq.c +22 −2
drivers/infiniband/sw/siw/siw_verbs.c +36 −4
drivers/infiniband/ulp/ipoib/ipoib_netlink.c +7 −0
drivers/infiniband/ulp/iser/iser_verbs.c +29 −38
drivers/infiniband/ulp/isert/ib_isert.c +2 −3
drivers/infiniband/ulp/rtrs/rtrs-clt.c +2 −4
drivers/infiniband/ulp/rtrs/rtrs-pri.h +0 −3
drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c +9 −4
drivers/infiniband/ulp/rtrs/rtrs-srv.c +29 −43
drivers/infiniband/ulp/rtrs/rtrs.c +4 −18
drivers/infiniband/ulp/srp/ib_srp.c +82 −14
drivers/net/ethernet/microsoft/mana/gdma_main.c +1 −2
include/net/mana/gdma.h +14 −17
include/net/mana/mana.h +3 −0
include/rdma/ib_pack.h +5 −0
include/rdma/ib_verbs.h +22 −2
include/rdma/opa_vnic.h +1 −1
include/trace/events/ib_mad.h +4 −9
include/uapi/rdma/hns-abi.h +15 −0
include/uapi/rdma/ib_user_ioctl_verbs.h +3 −0
include/uapi/rdma/ib_user_verbs.h +21 −0
include/uapi/rdma/mana-abi.h +66 −0
include/uapi/rdma/rdma_user_rxe.h +8 −0

MAINTAINERS+9−0

@@ -13806,6 +13806,15 @@ F: drivers/scsi/smartpqi/smartpqi*.[ch] · 13805 unmodified lines
13806	13806	F: include/linux/cciss*.h
13807	13807	F: include/uapi/linux/cciss*.h
13808	13808
	13809	+MICROSOFT MANA RDMA DRIVER
	13810	+M: Long Li <longli@microsoft.com>
	13811	+M: Ajay Sharma <sharmaajay@microsoft.com>
	13812	+L: linux-rdma@vger.kernel.org
	13813	+S: Supported
	13814	+F: drivers/infiniband/hw/mana/
	13815	+F: include/net/mana
	13816	+F: include/uapi/rdma/mana-abi.h
	13817	+
13809	13818	MICROSOFT SURFACE AGGREGATOR TABLET-MODE SWITCH
13810	13819	M: Maximilian Luz <luzmaximilian@gmail.com>
13811	13820	L: platform-driver-x86@vger.kernel.org

drivers/infiniband/Kconfig+3−0

@@ -78,6 +78,7 @@ config INFINIBAND_VIRT_DMA · 77 unmodified lines
78	78	def_bool !HIGHMEM
79	79
80	80	if INFINIBAND_USER_ACCESS \|\| !INFINIBAND_USER_ACCESS
	81	+if !UML
81	82	source "drivers/infiniband/hw/bnxt_re/Kconfig"
82	83	source "drivers/infiniband/hw/cxgb4/Kconfig"
83	84	source "drivers/infiniband/hw/efa/Kconfig"
@@ -85,6 +86,7 @@ source "drivers/infiniband/hw/erdma/Kconfig" · 1 unmodified line
85	86	source "drivers/infiniband/hw/hfi1/Kconfig"
86	87	source "drivers/infiniband/hw/hns/Kconfig"
87	88	source "drivers/infiniband/hw/irdma/Kconfig"
	89	+source "drivers/infiniband/hw/mana/Kconfig"
88	90	source "drivers/infiniband/hw/mlx4/Kconfig"
89	91	source "drivers/infiniband/hw/mlx5/Kconfig"
90	92	source "drivers/infiniband/hw/mthca/Kconfig"
@@ -94,6 +96,7 @@ source "drivers/infiniband/hw/qib/Kconfig" · 3 unmodified lines
94	96	source "drivers/infiniband/hw/usnic/Kconfig"
95	97	source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"
96	98	source "drivers/infiniband/sw/rdmavt/Kconfig"
	99	+endif # !UML
97	100	source "drivers/infiniband/sw/rxe/Kconfig"
98	101	source "drivers/infiniband/sw/siw/Kconfig"
99	102	endif

drivers/infiniband/core/cache.c+1−1

@@ -1422,7 +1422,7 @@ int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr, · 1421 unmodified lines
1422	1422	*vlan_id = vlan_dev_vlan_id(ndev);
1423	1423	} else {
1424	1424	/* If the netdev is upper device and if it's lower
1425		− * device is vlan device, consider vlan id of the
	1425	+ * device is vlan device, consider vlan id of
1426	1426	* the lower vlan device for this gid entry.
1427	1427	*/
1428	1428	netdev_walk_all_lower_dev_rcu(attr->ndev,

drivers/infiniband/core/cm.c+11−2

@@ -4094,9 +4094,18 @@ static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv, · 4093 unmodified lines
4094	4094	*qp_attr_mask = IB_QP_STATE \| IB_QP_ACCESS_FLAGS \|
4095	4095	IB_QP_PKEY_INDEX \| IB_QP_PORT;
4096	4096	qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4097		− if (cm_id_priv->responder_resources)
	4097	+ if (cm_id_priv->responder_resources) {
	4098	+ struct ib_device *ib_dev = cm_id_priv->id.device;
	4099	+ u64 support_flush = ib_dev->attrs.device_cap_flags &
	4100	+ (IB_DEVICE_FLUSH_GLOBAL \| IB_DEVICE_FLUSH_PERSISTENT);
	4101	+ u32 flushable = support_flush ?
	4102	+ (IB_ACCESS_FLUSH_GLOBAL \|
	4103	+ IB_ACCESS_FLUSH_PERSISTENT) : 0;
	4104	+
4098	4105	qp_attr->qp_access_flags \|= IB_ACCESS_REMOTE_READ \|
4099		− IB_ACCESS_REMOTE_ATOMIC;
	4106	+ IB_ACCESS_REMOTE_ATOMIC \|
	4107	+ flushable;
	4108	+ }
4100	4109	qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4101	4110	if (cm_id_priv->av.port)
4102	4111	qp_attr->port_num = cm_id_priv->av.port->port_num;

drivers/infiniband/core/cma.c+1−1

@@ -47,7 +47,7 @@ MODULE_LICENSE("Dual BSD/GPL"); · 46 unmodified lines
47	47	#define CMA_CM_RESPONSE_TIMEOUT 20
48	48	#define CMA_MAX_CM_RETRIES 15
49	49	#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY \| 24)
50		−#define CMA_IBOE_PACKET_LIFETIME 18
	50	+#define CMA_IBOE_PACKET_LIFETIME 16
51	51	#define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
52	52
53	53	static const char * const cma_events[] = {

drivers/infiniband/core/device.c+6−4

@@ -2159,14 +2159,16 @@ int ib_device_set_netdev(struct ib_device ib_dev, struct net_device ndev, · 2158 unmodified lines
2159	2159	return 0;
2160	2160	}
2161	2161
	2162	+ if (old_ndev)
	2163	+ netdev_tracker_free(ndev, &pdata->netdev_tracker);
2162	2164	if (ndev)
2163		− dev_hold(ndev);
	2165	+ netdev_hold(ndev, &pdata->netdev_tracker, GFP_ATOMIC);
2164	2166	rcu_assign_pointer(pdata->netdev, ndev);
2165	2167	spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2166	2168
2167	2169	add_ndev_hash(pdata);
2168	2170	if (old_ndev)
2169		− dev_put(old_ndev);
	2171	+ __dev_put(old_ndev);
2170	2172
2171	2173	return 0;
2172	2174	}
@@ -2199,7 +2201,7 @@ static void free_netdevs(struct ib_device *ib_dev) · 26 unmodified lines
2199	2201	* comparisons after the put
2200	2202	*/
2201	2203	rcu_assign_pointer(pdata->netdev, NULL);
2202		− dev_put(ndev);
	2204	+ netdev_put(ndev, &pdata->netdev_tracker);
2203	2205	}
2204	2206	spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2205	2207	}
@@ -2851,8 +2853,8 @@ err: · 645 unmodified lines
2851	2853	static void __exit ib_core_cleanup(void)
2852	2854	{
2853	2855	roce_gid_mgmt_cleanup();
2854		− nldev_exit();
2855	2856	rdma_nl_unregister(RDMA_NL_LS);
	2857	+ nldev_exit();
2856	2858	unregister_pernet_device(&rdma_dev_net_ops);
2857	2859	unregister_blocking_lsm_notifier(&ibdev_lsm_nb);
2858	2860	ib_sa_cleanup();

drivers/infiniband/core/mad.c+0−5

@@ -59,9 +59,6 @@ static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, · 58 unmodified lines
59	59	struct ib_mad_qp_info *qp_info,
60	60	struct trace_event_raw_ib_mad_send_template *entry)
61	61	{
62		− u16 pkey;
63		− struct ib_device *dev = qp_info->port_priv->device;
64		− u32 pnum = qp_info->port_priv->port_num;
65	62	struct ib_ud_wr *wr = &mad_send_wr->send_wr;
66	63	struct rdma_ah_attr attr = {};
67	64
@@ -69,8 +66,6 @@ static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, · 1 unmodified line
69	66
70	67	/* These are common */
71	68	entry->sl = attr.sl;
72		− ib_query_pkey(dev, pnum, wr->pkey_index, &pkey);
73		− entry->pkey = pkey;
74	69	entry->rqpn = wr->remote_qpn;
75	70	entry->rqkey = wr->remote_qkey;
76	71	entry->dlid = rdma_ah_get_dlid(&attr);

drivers/infiniband/core/nldev.c+40−10

@@ -513,7 +513,7 @@ static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin, · 512 unmodified lines
513	513
514	514	/* In create_qp() port is not set yet */
515	515	if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port))
516		− return -EINVAL;
	516	+ return -EMSGSIZE;
517	517
518	518	ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num);
519	519	if (ret)
@@ -552,7 +552,7 @@ static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin, · 32 unmodified lines
552	552	struct rdma_cm_id *cm_id = &id_priv->id;
553	553
554	554	if (port && port != cm_id->port_num)
555		− return 0;
	555	+ return -EAGAIN;
556	556
557	557	if (cm_id->port_num &&
558	558	nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id->port_num))
@@ -894,6 +894,8 @@ static int fill_stat_counter_qps(struct sk_buff *msg, · 335 unmodified lines
894	894	int ret = 0;
895	895
896	896	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);
	897	+ if (!table_attr)
	898	+ return -EMSGSIZE;
897	899
898	900	rt = &counter->device->res[RDMA_RESTRACK_QP];
899	901	xa_lock(&rt->xa);
@@ -1041,6 +1043,10 @@ static int nldev_get_doit(struct sk_buff skb, struct nlmsghdr nlh, · 141 unmodified lines
1041	1043	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1042	1044	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1043	1045	0, 0);
	1046	+ if (!nlh) {
	1047	+ err = -EMSGSIZE;
	1048	+ goto err_free;
	1049	+ }
1044	1050
1045	1051	err = fill_dev_info(msg, device);
1046	1052	if (err)
@@ -1126,7 +1132,7 @@ static int _nldev_get_dumpit(struct ib_device *device, · 79 unmodified lines
1126	1132	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1127	1133	0, NLM_F_MULTI);
1128	1134
1129		− if (fill_dev_info(skb, device)) {
	1135	+ if (!nlh \|\| fill_dev_info(skb, device)) {
1130	1136	nlmsg_cancel(skb, nlh);
1131	1137	goto out;
1132	1138	}
@@ -1185,6 +1191,10 @@ static int nldev_port_get_doit(struct sk_buff skb, struct nlmsghdr nlh, · 52 unmodified lines
1185	1191	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1186	1192	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1187	1193	0, 0);
	1194	+ if (!nlh) {
	1195	+ err = -EMSGSIZE;
	1196	+ goto err_free;
	1197	+ }
1188	1198
1189	1199	err = fill_port_info(msg, device, port, sock_net(skb->sk));
1190	1200	if (err)
@@ -1246,7 +1256,7 @@ static int nldev_port_get_dumpit(struct sk_buff *skb, · 55 unmodified lines
1246	1256	RDMA_NLDEV_CMD_PORT_GET),
1247	1257	0, NLM_F_MULTI);
1248	1258
1249		− if (fill_port_info(skb, device, p, sock_net(skb->sk))) {
	1259	+ if (!nlh \|\| fill_port_info(skb, device, p, sock_net(skb->sk))) {
1250	1260	nlmsg_cancel(skb, nlh);
1251	1261	goto out;
1252	1262	}
@@ -1288,6 +1298,10 @@ static int nldev_res_get_doit(struct sk_buff skb, struct nlmsghdr nlh, · 35 unmodified lines
1288	1298	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1289	1299	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
1290	1300	0, 0);
	1301	+ if (!nlh) {
	1302	+ ret = -EMSGSIZE;
	1303	+ goto err_free;
	1304	+ }
1291	1305
1292	1306	ret = fill_res_info(msg, device);
1293	1307	if (ret)
@@ -1319,7 +1333,7 @@ static int _nldev_res_get_dumpit(struct ib_device *device, · 25 unmodified lines
1319	1333	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
1320	1334	0, NLM_F_MULTI);
1321	1335
1322		− if (fill_res_info(skb, device)) {
	1336	+ if (!nlh \|\| fill_res_info(skb, device)) {
1323	1337	nlmsg_cancel(skb, nlh);
1324	1338	goto out;
1325	1339	}
@@ -1454,7 +1468,7 @@ static int res_get_common_doit(struct sk_buff skb, struct nlmsghdr nlh, · 128 unmodified lines
1454	1468	RDMA_NL_GET_OP(nlh->nlmsg_type)),
1455	1469	0, 0);
1456	1470
1457		− if (fill_nldev_handle(msg, device)) {
	1471	+ if (!nlh \|\| fill_nldev_handle(msg, device)) {
1458	1472	ret = -EMSGSIZE;
1459	1473	goto err_free;
1460	1474	}
@@ -1533,7 +1547,7 @@ static int res_get_common_dumpit(struct sk_buff *skb, · 72 unmodified lines
1533	1547	RDMA_NL_GET_OP(cb->nlh->nlmsg_type)),
1534	1548	0, NLM_F_MULTI);
1535	1549
1536		− if (fill_nldev_handle(skb, device)) {
	1550	+ if (!nlh \|\| fill_nldev_handle(skb, device)) {
1537	1551	ret = -EMSGSIZE;
1538	1552	goto err;
1539	1553	}
@@ -1795,6 +1809,10 @@ static int nldev_get_chardev(struct sk_buff skb, struct nlmsghdr nlh, · 255 unmodified lines
1795	1809	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1796	1810	RDMA_NLDEV_CMD_GET_CHARDEV),
1797	1811	0, 0);
	1812	+ if (!nlh) {
	1813	+ err = -EMSGSIZE;
	1814	+ goto out_nlmsg;
	1815	+ }
1798	1816
1799	1817	data.nl_msg = msg;
1800	1818	err = ib_get_client_nl_info(ibdev, client_name, &data);
@@ -1852,6 +1870,10 @@ static int nldev_sys_get_doit(struct sk_buff skb, struct nlmsghdr nlh, · 51 unmodified lines
1852	1870	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1853	1871	RDMA_NLDEV_CMD_SYS_GET),
1854	1872	0, 0);
	1873	+ if (!nlh) {
	1874	+ nlmsg_free(msg);
	1875	+ return -EMSGSIZE;
	1876	+ }
1855	1877
1856	1878	err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_NETNS_MODE,
1857	1879	(u8)ib_devices_shared_netns);
@@ -2032,7 +2054,7 @@ static int nldev_stat_set_doit(struct sk_buff skb, struct nlmsghdr nlh, · 174 unmodified lines
2032	2054	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2033	2055	RDMA_NLDEV_CMD_STAT_SET),
2034	2056	0, 0);
2035		− if (fill_nldev_handle(msg, device) \|\|
	2057	+ if (!nlh \|\| fill_nldev_handle(msg, device) \|\|
2036	2058	nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
2037	2059	ret = -EMSGSIZE;
2038	2060	goto err_free_msg;
@@ -2101,6 +2123,10 @@ static int nldev_stat_del_doit(struct sk_buff skb, struct nlmsghdr nlh, · 62 unmodified lines
2101	2123	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2102	2124	RDMA_NLDEV_CMD_STAT_SET),
2103	2125	0, 0);
	2126	+ if (!nlh) {
	2127	+ ret = -EMSGSIZE;
	2128	+ goto err_fill;
	2129	+ }
2104	2130
2105	2131	cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);
2106	2132	qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);
@@ -2171,7 +2197,7 @@ static int stat_get_doit_default_counter(struct sk_buff *skb, · 64 unmodified lines
2171	2197	RDMA_NLDEV_CMD_STAT_GET),
2172	2198	0, 0);
2173	2199
2174		− if (fill_nldev_handle(msg, device) \|\|
	2200	+ if (!nlh \|\| fill_nldev_handle(msg, device) \|\|
2175	2201	nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
2176	2202	ret = -EMSGSIZE;
2177	2203	goto err_msg;
@@ -2259,6 +2285,10 @@ static int stat_get_doit_qp(struct sk_buff skb, struct nlmsghdr nlh, · 81 unmodified lines
2259	2285	RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2260	2286	RDMA_NLDEV_CMD_STAT_GET),
2261	2287	0, 0);
	2288	+ if (!nlh) {
	2289	+ ret = -EMSGSIZE;
	2290	+ goto err_msg;
	2291	+ }
2262	2292
2263	2293	ret = rdma_counter_get_mode(device, port, &mode, &mask);
2264	2294	if (ret)
@@ -2391,7 +2421,7 @@ static int nldev_stat_get_counter_status_doit(struct sk_buff *skb, · 126 unmodified lines
2391	2421	0, 0);
2392	2422
2393	2423	ret = -EMSGSIZE;
2394		− if (fill_nldev_handle(msg, device) \|\|
	2424	+ if (!nlh \|\| fill_nldev_handle(msg, device) \|\|
2395	2425	nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
2396	2426	goto err_msg;
2397	2427

drivers/infiniband/core/restrack.c+0−2

@@ -343,8 +343,6 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) · 342 unmodified lines
343	343	rt = &dev->res[res->type];
344	344
345	345	old = xa_erase(&rt->xa, res->id);
346		− if (res->type == RDMA_RESTRACK_MR)
347		− return;
348	346	WARN_ON(old != res);
349	347
350	348	out:

drivers/infiniband/core/sysfs.c+11−6

@@ -1213,6 +1213,9 @@ static struct ib_port setup_port(struct ib_core_device coredev, int port_num, · 1212 unmodified lines
1213	1213	p->port_num = port_num;
1214	1214	kobject_init(&p->kobj, &port_type);
1215	1215
	1216	+ if (device->port_data && is_full_dev)
	1217	+ device->port_data[port_num].sysfs = p;
	1218	+
1216	1219	cur_group = p->groups_list;
1217	1220	ret = alloc_port_table_group("gids", &p->groups[0], p->attrs_list,
1218	1221	attr->gid_tbl_len, show_port_gid);
@@ -1258,9 +1261,6 @@ static struct ib_port setup_port(struct ib_core_device coredev, int port_num, · 39 unmodified lines
1258	1261	}
1259	1262
1260	1263	list_add_tail(&p->kobj.entry, &coredev->port_list);
1261		− if (device->port_data && is_full_dev)
1262		− device->port_data[port_num].sysfs = p;
1263		−
1264	1264	return p;
1265	1265
1266	1266	err_groups:
@@ -1268,6 +1268,8 @@ err_groups: · 1 unmodified line
1268	1268	err_del:
1269	1269	kobject_del(&p->kobj);
1270	1270	err_put:
	1271	+ if (device->port_data && is_full_dev)
	1272	+ device->port_data[port_num].sysfs = NULL;
1271	1273	kobject_put(&p->kobj);
1272	1274	return ERR_PTR(ret);
1273	1275	}
@@ -1276,14 +1278,17 @@ static void destroy_port(struct ib_core_device coredev, struct ib_port port) · 2 unmodified lines
1276	1278	{
1277	1279	bool is_full_dev = &port->ibdev->coredev == coredev;
1278	1280
1279		− if (port->ibdev->port_data &&
1280		− port->ibdev->port_data[port->port_num].sysfs == port)
1281		− port->ibdev->port_data[port->port_num].sysfs = NULL;
1282	1281	list_del(&port->kobj.entry);
1283	1282	if (is_full_dev)
1284	1283	sysfs_remove_groups(&port->kobj, port->ibdev->ops.port_groups);
	1284	+
1285	1285	sysfs_remove_groups(&port->kobj, port->groups_list);
1286	1286	kobject_del(&port->kobj);
	1287	+
	1288	+ if (port->ibdev->port_data &&
	1289	+ port->ibdev->port_data[port->port_num].sysfs == port)
	1290	+ port->ibdev->port_data[port->port_num].sysfs = NULL;
	1291	+
1287	1292	kobject_put(&port->kobj);
1288	1293	}
1289	1294

drivers/infiniband/core/uverbs_std_types_qp.c+1−1

@@ -163,7 +163,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( · 162 unmodified lines
163	163	UVERBS_ATTR_CREATE_QP_SRQ_HANDLE))
164	164	return -EINVAL;
165	165
166		− /* send_cq is optinal */
	166	+ /* send_cq is optional */
167	167	if (cap.max_send_wr) {
168	168	send_cq = uverbs_attr_get_obj(attrs,
169	169	UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE);

drivers/infiniband/hw/Makefile+1−0

@@ -4,6 +4,7 @@ obj-$(CONFIG_INFINIBAND_QIB) += qib/ · 3 unmodified lines
4	4	obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/
5	5	obj-$(CONFIG_INFINIBAND_EFA) += efa/
6	6	obj-$(CONFIG_INFINIBAND_IRDMA) += irdma/
	7	+obj-$(CONFIG_MANA_INFINIBAND) += mana/
7	8	obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/
8	9	obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/
9	10	obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/

drivers/infiniband/hw/erdma/erdma.h+3−1

@@ -124,6 +124,7 @@ struct erdma_devattr { · 123 unmodified lines
124	124	u32 fw_version;
125	125
126	126	unsigned char peer_addr[ETH_ALEN];
	127	+ unsigned long cap_flags;
127	128
128	129	int numa_node;
129	130	enum erdma_cc_alg cc;
@@ -189,6 +190,7 @@ struct erdma_dev { · 59 unmodified lines
189	190	struct net_device *netdev;
190	191	struct pci_dev *pdev;
191	192	struct notifier_block netdev_nb;
	193	+ struct workqueue_struct *reflush_wq;
192	194
193	195	resource_size_t func_bar_addr;
194	196	resource_size_t func_bar_len;
@@ -218,7 +220,7 @@ struct erdma_dev { · 23 unmodified lines
218	220	DECLARE_BITMAP(sdb_page, ERDMA_DWQE_TYPE0_CNT);
219	221	/*
220	222	* We provide max 496 uContexts that each has one SQ normal Db,
221		− * and one directWQE db。
	223	+ * and one directWQE db.
222	224	*/
223	225	DECLARE_BITMAP(sdb_entry, ERDMA_DWQE_TYPE1_CNT);
224	226

drivers/infiniband/hw/erdma/erdma_cq.c+2−0

@@ -64,6 +64,8 @@ static const enum ib_wc_opcode wc_mapping_table[ERDMA_NUM_OPCODES] = { · 63 unmodified lines
64	64	[ERDMA_OP_REG_MR] = IB_WC_REG_MR,
65	65	[ERDMA_OP_LOCAL_INV] = IB_WC_LOCAL_INV,
66	66	[ERDMA_OP_READ_WITH_INV] = IB_WC_RDMA_READ,
	67	+ [ERDMA_OP_ATOMIC_CAS] = IB_WC_COMP_SWAP,
	68	+ [ERDMA_OP_ATOMIC_FAD] = IB_WC_FETCH_ADD,
67	69	};
68	70
69	71	static const struct {

drivers/infiniband/hw/erdma/erdma_hw.h+30−7

@@ -145,6 +145,7 @@ enum CMDQ_RDMA_OPCODE { · 144 unmodified lines
145	145	CMDQ_OPCODE_MODIFY_QP = 3,
146	146	CMDQ_OPCODE_CREATE_CQ = 4,
147	147	CMDQ_OPCODE_DESTROY_CQ = 5,
	148	+ CMDQ_OPCODE_REFLUSH = 6,
148	149	CMDQ_OPCODE_REG_MR = 8,
149	150	CMDQ_OPCODE_DEREG_MR = 9
150	151	};
@@ -224,8 +225,7 @@ struct erdma_cmdq_create_cq_req { · 73 unmodified lines
224	225	/* regmr cfg1 */
225	226	#define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12)
226	227	#define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6)
227		−#define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 2)
228		−#define ERDMA_CMD_REGMR_ACC_MODE_MASK GENMASK(1, 0)
	228	+#define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1)
229	229
230	230	/* regmr cfg2 */
231	231	#define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)
@@ -302,8 +302,16 @@ struct erdma_cmdq_destroy_qp_req { · 70 unmodified lines
302	302	u32 qpn;
303	303	};
304	304
	305	+struct erdma_cmdq_reflush_req {
	306	+ u64 hdr;
	307	+ u32 qpn;
	308	+ u32 sq_pi;
	309	+ u32 rq_pi;
	310	+};
	311	+
305	312	/* cap qword 0 definition */
306	313	#define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)
	314	+#define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
307	315	#define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16)
308	316	#define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0)
309	317
@@ -315,6 +323,10 @@ struct erdma_cmdq_destroy_qp_req { · 5 unmodified lines
315	323
316	324	#define ERDMA_NQP_PER_QBLOCK 1024
317	325
	326	+enum {
	327	+ ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 << 7,
	328	+};
	329	+
318	330	#define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0)
319	331
320	332	/* CQE hdr */
@@ -340,9 +352,9 @@ struct erdma_cqe { · 19 unmodified lines
340	352	};
341	353
342	354	struct erdma_sge {
343		− __aligned_le64 laddr;
	355	+ __aligned_le64 addr;
344	356	__le32 length;
345		− __le32 lkey;
	357	+ __le32 key;
346	358	};
347	359
348	360	/* Receive Queue Element */
@@ -370,8 +382,7 @@ struct erdma_rqe { · 21 unmodified lines
370	382	#define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
371	383
372	384	/* REG MR attrs */
373		−#define ERDMA_SQE_MR_MODE_MASK GENMASK(1, 0)
374		−#define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 2)
	385	+#define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1)
375	386	#define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6)
376	387	#define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12)
377	388
@@ -410,6 +421,16 @@ struct erdma_readreq_sqe { · 32 unmodified lines
410	421	__le32 rsvd;
411	422	};
412	423
	424	+struct erdma_atomic_sqe {
	425	+ __le64 hdr;
	426	+ __le64 rsvd;
	427	+ __le64 fetchadd_swap_data;
	428	+ __le64 cmp_data;
	429	+
	430	+ struct erdma_sge remote;
	431	+ struct erdma_sge sgl;
	432	+};
	433	+
413	434	struct erdma_reg_mr_sqe {
414	435	__le64 hdr;
415	436	__le64 addr;
@@ -469,7 +490,9 @@ enum erdma_opcode { · 53 unmodified lines
469	490	ERDMA_OP_REG_MR = 14,
470	491	ERDMA_OP_LOCAL_INV = 15,
471	492	ERDMA_OP_READ_WITH_INV = 16,
472		− ERDMA_NUM_OPCODES = 17,
	493	+ ERDMA_OP_ATOMIC_CAS = 17,
	494	+ ERDMA_OP_ATOMIC_FAD = 18,
	495	+ ERDMA_NUM_OPCODES = 19,
473	496	ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
474	497	};
475	498

drivers/infiniband/hw/erdma/erdma_main.c+13−2

@@ -374,6 +374,7 @@ static int erdma_dev_attrs_init(struct erdma_dev *dev) · 373 unmodified lines
374	374	dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);
375	375	dev->attrs.max_mr = dev->attrs.max_qp << 1;
376	376	dev->attrs.max_cq = dev->attrs.max_qp << 1;
	377	+ dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);
377	378
378	379	dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR;
379	380	dev->attrs.max_ord = ERDMA_MAX_ORD;
@@ -520,13 +521,22 @@ static int erdma_ib_device_add(struct pci_dev *pdev) · 140 unmodified lines
520	521
521	522	u64_to_ether_addr(mac, dev->attrs.peer_addr);
522	523
	524	+ dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,
	525	+ WQ_UNBOUND_MAX_ACTIVE);
	526	+ if (!dev->reflush_wq) {
	527	+ ret = -ENOMEM;
	528	+ goto err_alloc_workqueue;
	529	+ }
	530	+
523	531	ret = erdma_device_register(dev);
524	532	if (ret)
525		− goto err_out;
	533	+ goto err_register;
526	534
527	535	return 0;
528	536
529		−err_out:
	537	+err_register:
	538	+ destroy_workqueue(dev->reflush_wq);
	539	+err_alloc_workqueue:
530	540	xa_destroy(&dev->qp_xa);
531	541	xa_destroy(&dev->cq_xa);
532	542
@@ -542,6 +552,7 @@ static void erdma_ib_device_remove(struct pci_dev *pdev) · 9 unmodified lines
542	552	unregister_netdevice_notifier(&dev->netdev_nb);
543	553	ib_unregister_device(&dev->ibdev);
544	554
	555	+ destroy_workqueue(dev->reflush_wq);
545	556	erdma_res_cb_free(dev);
546	557	xa_destroy(&dev->qp_xa);
547	558	xa_destroy(&dev->cq_xa);

drivers/infiniband/hw/erdma/erdma_qp.c+57−16

@@ -120,6 +120,7 @@ static int erdma_modify_qp_state_to_stop(struct erdma_qp *qp, · 119 unmodified lines
120	120	int erdma_modify_qp_internal(struct erdma_qp qp, struct erdma_qp_attrs attrs,
121	121	enum erdma_qp_attr_mask mask)
122	122	{
	123	+ bool need_reflush = false;
123	124	int drop_conn, ret = 0;
124	125
125	126	if (!mask)
@@ -135,6 +136,7 @@ int erdma_modify_qp_internal(struct erdma_qp qp, struct erdma_qp_attrs attrs, · 9 unmodified lines
135	136	ret = erdma_modify_qp_state_to_rts(qp, attrs, mask);
136	137	} else if (attrs->state == ERDMA_QP_STATE_ERROR) {
137	138	qp->attrs.state = ERDMA_QP_STATE_ERROR;
	139	+ need_reflush = true;
138	140	if (qp->cep) {
139	141	erdma_cep_put(qp->cep);
140	142	qp->cep = NULL;
@@ -145,17 +147,12 @@ int erdma_modify_qp_internal(struct erdma_qp qp, struct erdma_qp_attrs attrs, · 4 unmodified lines
145	147	case ERDMA_QP_STATE_RTS:
146	148	drop_conn = 0;
147	149
148		− if (attrs->state == ERDMA_QP_STATE_CLOSING) {
	150	+ if (attrs->state == ERDMA_QP_STATE_CLOSING \|\|
	151	+ attrs->state == ERDMA_QP_STATE_TERMINATE \|\|
	152	+ attrs->state == ERDMA_QP_STATE_ERROR) {
149	153	ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
150	154	drop_conn = 1;
151		− } else if (attrs->state == ERDMA_QP_STATE_TERMINATE) {
152		− qp->attrs.state = ERDMA_QP_STATE_TERMINATE;
153		− ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
154		− drop_conn = 1;
155		− } else if (attrs->state == ERDMA_QP_STATE_ERROR) {
156		− ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
157		− qp->attrs.state = ERDMA_QP_STATE_ERROR;
158		− drop_conn = 1;
	155	+ need_reflush = true;
159	156	}
160	157
161	158	if (drop_conn)
@@ -180,6 +177,12 @@ int erdma_modify_qp_internal(struct erdma_qp qp, struct erdma_qp_attrs attrs, · 18 unmodified lines
180	177	break;
181	178	}
182	179
	180	+ if (need_reflush && !ret && rdma_is_kernel_res(&qp->ibqp.res)) {
	181	+ qp->flags \|= ERDMA_QP_IN_FLUSHING;
	182	+ mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
	183	+ usecs_to_jiffies(100));
	184	+ }
	185	+
183	186	return ret;
184	187	}
185	188
@@ -285,15 +288,16 @@ static int erdma_push_one_sqe(struct erdma_qp qp, u16 pi, · 99 unmodified lines
285	288	u32 wqe_size, wqebb_cnt, hw_op, flags, sgl_offset;
286	289	u32 idx = *pi & (qp->attrs.sq_size - 1);
287	290	enum ib_wr_opcode op = send_wr->opcode;
	291	+ struct erdma_atomic_sqe *atomic_sqe;
288	292	struct erdma_readreq_sqe *read_sqe;
289	293	struct erdma_reg_mr_sqe *regmr_sge;
290	294	struct erdma_write_sqe *write_sqe;
291	295	struct erdma_send_sqe *send_sqe;
292	296	struct ib_rdma_wr *rdma_wr;
293		− struct erdma_mr *mr;
	297	+ struct erdma_sge *sge;
294	298	__le32 *length_field;
	299	+ struct erdma_mr *mr;
295	300	u64 wqe_hdr, *entry;
296		− struct ib_sge *sge;
297	301	u32 attrs;
298	302	int ret;
299	303
@@ -360,9 +364,9 @@ static int erdma_push_one_sqe(struct erdma_qp qp, u16 pi, · 60 unmodified lines
360	364
361	365	sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
362	366	qp->attrs.sq_size, SQEBB_SHIFT);
363		− sge->addr = rdma_wr->remote_addr;
364		− sge->lkey = rdma_wr->rkey;
365		− sge->length = send_wr->sg_list[0].length;
	367	+ sge->addr = cpu_to_le64(rdma_wr->remote_addr);
	368	+ sge->key = cpu_to_le32(rdma_wr->rkey);
	369	+ sge->length = cpu_to_le32(send_wr->sg_list[0].length);
366	370	wqe_size = sizeof(struct erdma_readreq_sqe) +
367	371	send_wr->num_sge * sizeof(struct ib_sge);
368	372
@@ -397,8 +401,7 @@ static int erdma_push_one_sqe(struct erdma_qp qp, u16 pi, · 28 unmodified lines
397	401	regmr_sge->addr = cpu_to_le64(mr->ibmr.iova);
398	402	regmr_sge->length = cpu_to_le32(mr->ibmr.length);
399	403	regmr_sge->stag = cpu_to_le32(reg_wr(send_wr)->key);
400		− attrs = FIELD_PREP(ERDMA_SQE_MR_MODE_MASK, 0) \|
401		− FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr->access) \|
	404	+ attrs = FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr->access) \|
402	405	FIELD_PREP(ERDMA_SQE_MR_MTT_CNT_MASK,
403	406	mr->mem.mtt_nents);
404	407
@@ -424,6 +427,35 @@ static int erdma_push_one_sqe(struct erdma_qp qp, u16 pi, · 19 unmodified lines
424	427	regmr_sge->stag = cpu_to_le32(send_wr->ex.invalidate_rkey);
425	428	wqe_size = sizeof(struct erdma_reg_mr_sqe);
426	429	goto out;
	430	+ case IB_WR_ATOMIC_CMP_AND_SWP:
	431	+ case IB_WR_ATOMIC_FETCH_AND_ADD:
	432	+ atomic_sqe = (struct erdma_atomic_sqe *)entry;
	433	+ if (op == IB_WR_ATOMIC_CMP_AND_SWP) {
	434	+ wqe_hdr \|= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
	435	+ ERDMA_OP_ATOMIC_CAS);
	436	+ atomic_sqe->fetchadd_swap_data =
	437	+ cpu_to_le64(atomic_wr(send_wr)->swap);
	438	+ atomic_sqe->cmp_data =
	439	+ cpu_to_le64(atomic_wr(send_wr)->compare_add);
	440	+ } else {
	441	+ wqe_hdr \|= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
	442	+ ERDMA_OP_ATOMIC_FAD);
	443	+ atomic_sqe->fetchadd_swap_data =
	444	+ cpu_to_le64(atomic_wr(send_wr)->compare_add);
	445	+ }
	446	+
	447	+ sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
	448	+ qp->attrs.sq_size, SQEBB_SHIFT);
	449	+ sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr);
	450	+ sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey);
	451	+ sge++;
	452	+
	453	+ sge->addr = cpu_to_le64(send_wr->sg_list[0].addr);
	454	+ sge->key = cpu_to_le32(send_wr->sg_list[0].lkey);
	455	+ sge->length = cpu_to_le32(send_wr->sg_list[0].length);
	456	+
	457	+ wqe_size = sizeof(*atomic_sqe);
	458	+ goto out;
427	459	default:
428	460	return -EOPNOTSUPP;
429	461	}
@@ -498,6 +530,10 @@ int erdma_post_send(struct ib_qp ibqp, const struct ib_send_wr send_wr, · 68 unmodified lines
498	530	}
499	531	spin_unlock_irqrestore(&qp->lock, flags);
500	532
	533	+ if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING))
	534	+ mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
	535	+ usecs_to_jiffies(100));
	536	+
501	537	return ret;
502	538	}
503	539
@@ -551,5 +587,10 @@ int erdma_post_recv(struct ib_qp ibqp, const struct ib_recv_wr recv_wr, · 47 unmodified lines
551	587	}
552	588
553	589	spin_unlock_irqrestore(&qp->lock, flags);
	590	+
	591	+ if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING))
	592	+ mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
	593	+ usecs_to_jiffies(100));
	594	+
554	595	return ret;
555	596	}

drivers/infiniband/hw/erdma/erdma_verbs.c+23−2

@@ -118,8 +118,7 @@ static int regmr_cmd(struct erdma_dev dev, struct erdma_mr mr) · 117 unmodified lines
118	118	FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
119	119	req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) \|
120	120	FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) \|
121		− FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access) \|
122		− FIELD_PREP(ERDMA_CMD_REGMR_ACC_MODE_MASK, 0);
	121	+ FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
123	122	req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
124	123	ilog2(mr->mem.page_size)) \|
125	124	FIELD_PREP(ERDMA_CMD_REGMR_MTT_TYPE_MASK, mr->mem.mtt_type) \|
@@ -289,6 +288,10 @@ int erdma_query_device(struct ib_device ibdev, struct ib_device_attr attr, · 163 unmodified lines
289	288	attr->max_mw = dev->attrs.max_mw;
290	289	attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
291	290	attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
	291	+
	292	+ if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
	293	+ attr->atomic_cap = IB_ATOMIC_GLOB;
	294	+
292	295	attr->fw_ver = dev->attrs.fw_version;
293	296
294	297	if (dev->netdev)
@@ -376,6 +379,21 @@ int erdma_dealloc_pd(struct ib_pd ibpd, struct ib_udata udata) · 81 unmodified lines
376	379	return 0;
377	380	}
378	381
	382	+static void erdma_flush_worker(struct work_struct *work)
	383	+{
	384	+ struct delayed_work *dwork = to_delayed_work(work);
	385	+ struct erdma_qp *qp =
	386	+ container_of(dwork, struct erdma_qp, reflush_dwork);
	387	+ struct erdma_cmdq_reflush_req req;
	388	+
	389	+ erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
	390	+ CMDQ_OPCODE_REFLUSH);
	391	+ req.qpn = QP_ID(qp);
	392	+ req.sq_pi = qp->kern_qp.sq_pi;
	393	+ req.rq_pi = qp->kern_qp.rq_pi;
	394	+ erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL);
	395	+}
	396	+
379	397	static int erdma_qp_validate_cap(struct erdma_dev *dev,
380	398	struct ib_qp_init_attr *attrs)
381	399	{
@@ -732,6 +750,7 @@ int erdma_create_qp(struct ib_qp ibqp, struct ib_qp_init_attr attrs, · 350 unmodified lines
732	750	qp->attrs.max_send_sge = attrs->cap.max_send_sge;
733	751	qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
734	752	qp->attrs.state = ERDMA_QP_STATE_IDLE;
	753	+ INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
735	754
736	755	ret = create_qp_cmd(dev, qp);
737	756	if (ret)
@@ -1025,6 +1044,8 @@ int erdma_destroy_qp(struct ib_qp ibqp, struct ib_udata udata) · 287 unmodified lines
1025	1044	erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
1026	1045	up_write(&qp->state_lock);
1027	1046
	1047	+ cancel_delayed_work_sync(&qp->reflush_dwork);
	1048	+
1028	1049	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1029	1050	CMDQ_OPCODE_DESTROY_QP);
1030	1051	req.qpn = QP_ID(qp);

drivers/infiniband/hw/erdma/erdma_verbs.h+14−5

@@ -71,16 +71,18 @@ struct erdma_pd { · 70 unmodified lines
71	71	#define ERDMA_MR_INLINE_MTT 0
72	72	#define ERDMA_MR_INDIRECT_MTT 1
73	73
74		−#define ERDMA_MR_ACC_LR BIT(0)
75		−#define ERDMA_MR_ACC_LW BIT(1)
76		−#define ERDMA_MR_ACC_RR BIT(2)
77		−#define ERDMA_MR_ACC_RW BIT(3)
	74	+#define ERDMA_MR_ACC_RA BIT(0)
	75	+#define ERDMA_MR_ACC_LR BIT(1)
	76	+#define ERDMA_MR_ACC_LW BIT(2)
	77	+#define ERDMA_MR_ACC_RR BIT(3)
	78	+#define ERDMA_MR_ACC_RW BIT(4)
78	79
79	80	static inline u8 to_erdma_access_flags(int access)
80	81	{
81	82	return (access & IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) \|
82	83	(access & IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) \|
83		− (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0);
	84	+ (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0) \|
	85	+ (access & IB_ACCESS_REMOTE_ATOMIC ? ERDMA_MR_ACC_RA : 0);
84	86	}
85	87
86	88	struct erdma_mem {
@@ -171,6 +173,10 @@ enum erdma_qp_attr_mask { · 84 unmodified lines
171	173	ERDMA_QP_ATTR_MPA = (1 << 7)
172	174	};
173	175
	176	+enum erdma_qp_flags {
	177	+ ERDMA_QP_IN_FLUSHING = (1 << 0),
	178	+};
	179	+
174	180	struct erdma_qp_attrs {
175	181	enum erdma_qp_state state;
176	182	enum erdma_cc_alg cc; /* Congestion control algorithm */
@@ -195,6 +201,9 @@ struct erdma_qp { · 18 unmodified lines
195	201	struct erdma_cep *cep;
196	202	struct rw_semaphore state_lock;
197	203
	204	+ unsigned long flags;
	205	+ struct delayed_work reflush_dwork;
	206	+
198	207	union {
199	208	struct erdma_kqp kern_qp;
200	209	struct erdma_uqp user_qp;

drivers/infiniband/hw/hfi1/affinity.c+2−0

@@ -177,6 +177,8 @@ out: · 176 unmodified lines
177	177	for (node = 0; node < node_affinity.num_possible_nodes; node++)
178	178	hfi1_per_node_cntr[node] = 1;
179	179
	180	+ pci_dev_put(dev);
	181	+
180	182	return 0;
181	183	}
182	184

drivers/infiniband/hw/hfi1/driver.c+1−1

@@ -112,7 +112,7 @@ static int hfi1_caps_get(char buffer, const struct kernel_param kp) · 111 unmodified lines
112	112	cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
113	113	cap_mask \|= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);
114	114
115		− return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
	115	+ return sysfs_emit(buffer, "0x%lx\n", cap_mask);
116	116	}
117	117
118	118	struct pci_dev get_pci_dev(struct rvt_dev_info rdi)

drivers/infiniband/hw/hfi1/firmware.c+6−0

@@ -1743,6 +1743,7 @@ int parse_platform_config(struct hfi1_devdata *dd) · 1742 unmodified lines
1743	1743
1744	1744	if (!dd->platform_config.data) {
1745	1745	dd_dev_err(dd, "%s: Missing config file\n", __func__);
	1746	+ ret = -EINVAL;
1746	1747	goto bail;
1747	1748	}
1748	1749	ptr = (u32 *)dd->platform_config.data;
@@ -1751,6 +1752,7 @@ int parse_platform_config(struct hfi1_devdata *dd) · 2 unmodified lines
1751	1752	ptr++;
1752	1753	if (magic_num != PLATFORM_CONFIG_MAGIC_NUM) {
1753	1754	dd_dev_err(dd, "%s: Bad config file\n", __func__);
	1755	+ ret = -EINVAL;
1754	1756	goto bail;
1755	1757	}
1756	1758
@@ -1774,6 +1776,7 @@ int parse_platform_config(struct hfi1_devdata *dd) · 17 unmodified lines
1774	1776	if (file_length > dd->platform_config.size) {
1775	1777	dd_dev_info(dd, "%s:File claims to be larger than read size\n",
1776	1778	__func__);
	1779	+ ret = -EINVAL;
1777	1780	goto bail;
1778	1781	} else if (file_length < dd->platform_config.size) {
1779	1782	dd_dev_info(dd,
@@ -1794,6 +1797,7 @@ int parse_platform_config(struct hfi1_devdata *dd) · 14 unmodified lines
1794	1797	dd_dev_err(dd, "%s: Failed validation at offset %ld\n",
1795	1798	__func__, (ptr - (u32 *)
1796	1799	dd->platform_config.data));
	1800	+ ret = -EINVAL;
1797	1801	goto bail;
1798	1802	}
1799	1803
@@ -1837,6 +1841,7 @@ int parse_platform_config(struct hfi1_devdata *dd) · 37 unmodified lines
1837	1841	__func__, table_type,
1838	1842	(ptr - (u32 *)
1839	1843	dd->platform_config.data));
	1844	+ ret = -EINVAL;
1840	1845	goto bail; /* We don't trust this file now */
1841	1846	}
1842	1847	pcfgcache->config_tables[table_type].table = ptr;
@@ -1856,6 +1861,7 @@ int parse_platform_config(struct hfi1_devdata *dd) · 13 unmodified lines
1856	1861	__func__, table_type,
1857	1862	(ptr -
1858	1863	(u32 *)dd->platform_config.data));
	1864	+ ret = -EINVAL;
1859	1865	goto bail; /* We don't trust this file now */
1860	1866	}
1861	1867	pcfgcache->config_tables[table_type].table_metadata =

drivers/infiniband/hw/hfi1/mad.c+11−11

@@ -2437,9 +2437,9 @@ struct opa_port_data_counters_msg { · 2436 unmodified lines
2437	2437	__be64 port_vl_xmit_wait_data;
2438	2438	__be64 port_vl_rcv_bubble;
2439	2439	__be64 port_vl_mark_fecn;
2440		− } vls[0];
	2440	+ } vls[];
2441	2441	/* array size defined by #bits set in vl_select_mask*/
2442		− } port[1]; /* array size defined by #ports in attribute modifier */
	2442	+ } port;
2443	2443	};
2444	2444
2445	2445	struct opa_port_error_counters64_msg {
@@ -2470,9 +2470,9 @@ struct opa_port_error_counters64_msg { · 24 unmodified lines
2470	2470	u8 reserved3[7];
2471	2471	struct _vls_ectrs {
2472	2472	__be64 port_vl_xmit_discards;
2473		− } vls[0];
	2473	+ } vls[];
2474	2474	/* array size defined by #bits set in vl_select_mask */
2475		− } port[1]; /* array size defined by #ports in attribute modifier */
	2475	+ } port;
2476	2476	};
2477	2477
2478	2478	struct opa_port_error_info_msg {
@@ -2543,7 +2543,7 @@ struct opa_port_error_info_msg { · 64 unmodified lines
2543	2543	u8 error_info;
2544	2544	} __packed fm_config_ei;
2545	2545	__u32 reserved9;
2546		− } port[1]; /* actual array size defined by #ports in attr modifier */
	2546	+ } port;
2547	2547	};
2548	2548
2549	2549	/* opa_port_error_info_msg error_info_select_mask bit definitions */
@@ -2966,7 +2966,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, · 416 unmodified lines
2966	2966	}
2967	2967
2968	2968	/* Sanity check */
2969		− response_data_size = struct_size(req, port[0].vls, num_vls);
	2969	+ response_data_size = struct_size(req, port.vls, num_vls);
2970	2970
2971	2971	if (response_data_size > sizeof(pmp->data)) {
2972	2972	pmp->mad_hdr.status \|= IB_SMP_INVALID_FIELD;
@@ -2986,7 +2986,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, · 13 unmodified lines
2986	2986	return reply((struct ib_mad_hdr *)pmp);
2987	2987	}
2988	2988
2989		− rsp = &req->port[0];
	2989	+ rsp = &req->port;
2990	2990	memset(rsp, 0, sizeof(*rsp));
2991	2991
2992	2992	rsp->port_number = port;
@@ -3182,7 +3182,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, · 189 unmodified lines
3182	3182	return reply((struct ib_mad_hdr *)pmp);
3183	3183	}
3184	3184
3185		− response_data_size = struct_size(req, port[0].vls, num_vls);
	3185	+ response_data_size = struct_size(req, port.vls, num_vls);
3186	3186
3187	3187	if (response_data_size > sizeof(pmp->data)) {
3188	3188	pmp->mad_hdr.status \|= IB_SMP_INVALID_FIELD;
@@ -3201,7 +3201,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, · 12 unmodified lines
3201	3201	return reply((struct ib_mad_hdr *)pmp);
3202	3202	}
3203	3203
3204		− rsp = &req->port[0];
	3204	+ rsp = &req->port;
3205	3205
3206	3206	ibp = to_iport(ibdev, port_num);
3207	3207	ppd = ppd_from_ibp(ibp);
@@ -3340,7 +3340,7 @@ static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp, · 132 unmodified lines
3340	3340	u64 reg;
3341	3341
3342	3342	req = (struct opa_port_error_info_msg *)pmp->data;
3343		− rsp = &req->port[0];
	3343	+ rsp = &req->port;
3344	3344
3345	3345	num_ports = OPA_AM_NPORT(be32_to_cpu(pmp->mad_hdr.attr_mod));
3346	3346	num_pslm = hweight64(be64_to_cpu(req->port_select_mask[3]));
@@ -3590,7 +3590,7 @@ static int pma_set_opa_errorinfo(struct opa_pma_mad *pmp, · 243 unmodified lines
3590	3590	u32 error_info_select;
3591	3591
3592	3592	req = (struct opa_port_error_info_msg *)pmp->data;
3593		− rsp = &req->port[0];
	3593	+ rsp = &req->port;
3594	3594
3595	3595	num_ports = OPA_AM_NPORT(be32_to_cpu(pmp->mad_hdr.attr_mod));
3596	3596	num_pslm = hweight64(be64_to_cpu(req->port_select_mask[3]));

drivers/infiniband/hw/hfi1/netdev_rx.c+1−1

@@ -216,7 +216,7 @@ static int hfi1_netdev_rxq_init(struct hfi1_netdev_rx *rx) · 215 unmodified lines
216	216	* right now.
217	217	*/
218	218	set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state);
219		− netif_napi_add_weight(dev, &rxq->napi, hfi1_netdev_rx_napi, 64);
	219	+ netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi);
220	220	rc = msix_netdev_request_rcd_irq(rxq->rcd);
221	221	if (rc)
222	222	goto bail_context_irq_failure;

drivers/infiniband/hw/hns/hns_roce_device.h+3−0

@@ -202,6 +202,7 @@ struct hns_roce_ucontext { · 201 unmodified lines
202	202	struct list_head page_list;
203	203	struct mutex page_mutex;
204	204	struct hns_user_mmap_entry *db_mmap_entry;
	205	+ u32 config;
205	206	};
206	207
207	208	struct hns_roce_pd {
@@ -334,6 +335,7 @@ struct hns_roce_wq { · 126 unmodified lines
334	335	u32 head;
335	336	u32 tail;
336	337	void __iomem *db_reg;
	338	+ u32 ext_sge_cnt;
337	339	};
338	340
339	341	struct hns_roce_sge {
@@ -635,6 +637,7 @@ struct hns_roce_qp { · 295 unmodified lines
635	637	struct list_head rq_node; /* all recv qps are on a list */
636	638	struct list_head sq_node; /* all send qps are on a list */
637	639	struct hns_user_mmap_entry *dwqe_mmap_entry;
	640	+ u32 config;
638	641	};
639	642
640	643	struct hns_roce_ib_iboe {

drivers/infiniband/hw/hns/hns_roce_hw_v2.c+168−49

@@ -192,7 +192,6 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, · 191 unmodified lines
192	192	unsigned int *sge_idx, u32 msg_len)
193	193	{
194	194	struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;
195		− unsigned int ext_sge_sz = qp->sq.max_gs * HNS_ROCE_SGE_SIZE;
196	195	unsigned int left_len_in_pg;
197	196	unsigned int idx = *sge_idx;
198	197	unsigned int i = 0;
@@ -200,7 +199,7 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, · 1 unmodified line
200	199	void *addr;
201	200	void *dseg;
202	201
203		− if (msg_len > ext_sge_sz) {
	202	+ if (msg_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) {
204	203	ibdev_err(ibdev,
205	204	"no enough extended sge space for inline data.\n");
206	205	return -EINVAL;
@@ -1274,6 +1273,30 @@ static void update_cmdq_status(struct hns_roce_dev *hr_dev) · 1067 unmodified lines
1274	1273	hr_dev->cmd.state = HNS_ROCE_CMDQ_STATE_FATAL_ERR;
1275	1274	}
1276	1275
	1276	+static int hns_roce_cmd_err_convert_errno(u16 desc_ret)
	1277	+{
	1278	+ struct hns_roce_cmd_errcode errcode_table[] = {
	1279	+ {CMD_EXEC_SUCCESS, 0},
	1280	+ {CMD_NO_AUTH, -EPERM},
	1281	+ {CMD_NOT_EXIST, -EOPNOTSUPP},
	1282	+ {CMD_CRQ_FULL, -EXFULL},
	1283	+ {CMD_NEXT_ERR, -ENOSR},
	1284	+ {CMD_NOT_EXEC, -ENOTBLK},
	1285	+ {CMD_PARA_ERR, -EINVAL},
	1286	+ {CMD_RESULT_ERR, -ERANGE},
	1287	+ {CMD_TIMEOUT, -ETIME},
	1288	+ {CMD_HILINK_ERR, -ENOLINK},
	1289	+ {CMD_INFO_ILLEGAL, -ENXIO},
	1290	+ {CMD_INVALID, -EBADR},
	1291	+ };
	1292	+ u16 i;
	1293	+
	1294	+ for (i = 0; i < ARRAY_SIZE(errcode_table); i++)
	1295	+ if (desc_ret == errcode_table[i].return_status)
	1296	+ return errcode_table[i].errno;
	1297	+ return -EIO;
	1298	+}
	1299	+
1277	1300	static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1278	1301	struct hns_roce_cmq_desc *desc, int num)
1279	1302	{
@@ -1319,7 +1342,7 @@ static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, · 39 unmodified lines
1319	1342	dev_err_ratelimited(hr_dev->dev,
1320	1343	"Cmdq IO error, opcode = 0x%x, return = 0x%x.\n",
1321	1344	desc->opcode, desc_ret);
1322		− ret = -EIO;
	1345	+ ret = hns_roce_cmd_err_convert_errno(desc_ret);
1323	1346	}
1324	1347	} else {
1325	1348	/* FW/HW reset or incorrect number of desc */
@@ -2024,13 +2047,14 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) · 698 unmodified lines
2024	2047
2025	2048	caps->flags \|= HNS_ROCE_CAP_FLAG_ATOMIC \| HNS_ROCE_CAP_FLAG_MW \|
2026	2049	HNS_ROCE_CAP_FLAG_SRQ \| HNS_ROCE_CAP_FLAG_FRMR \|
2027		− HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL \| HNS_ROCE_CAP_FLAG_XRC;
	2050	+ HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
2028	2051
2029	2052	caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
2030	2053
2031	2054	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2032	2055	caps->flags \|= HNS_ROCE_CAP_FLAG_STASH \|
2033		− HNS_ROCE_CAP_FLAG_DIRECT_WQE;
	2056	+ HNS_ROCE_CAP_FLAG_DIRECT_WQE \|
	2057	+ HNS_ROCE_CAP_FLAG_XRC;
2034	2058	caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE;
2035	2059	} else {
2036	2060	caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
@@ -2342,6 +2366,9 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) · 305 unmodified lines
2342	2366	caps->wqe_sge_hop_num = hr_reg_read(resp_d, PF_CAPS_D_EX_SGE_HOP_NUM);
2343	2367	caps->wqe_rq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_RQWQE_HOP_NUM);
2344	2368
	2369	+ if (!(caps->page_size_cap & PAGE_SIZE))
	2370	+ caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
	2371	+
2345	2372	return 0;
2346	2373	}
2347	2374
@@ -2631,31 +2658,124 @@ static void free_dip_list(struct hns_roce_dev *hr_dev) · 283 unmodified lines
2631	2658	spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags);
2632	2659	}
2633	2660
2634		−static void free_mr_exit(struct hns_roce_dev *hr_dev)
	2661	+static struct ib_pd free_mr_init_pd(struct hns_roce_dev hr_dev)
	2662	+{
	2663	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	2664	+ struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
	2665	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	2666	+ struct hns_roce_pd *hr_pd;
	2667	+ struct ib_pd *pd;
	2668	+
	2669	+ hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL);
	2670	+ if (ZERO_OR_NULL_PTR(hr_pd))
	2671	+ return NULL;
	2672	+ pd = &hr_pd->ibpd;
	2673	+ pd->device = ibdev;
	2674	+
	2675	+ if (hns_roce_alloc_pd(pd, NULL)) {
	2676	+ ibdev_err(ibdev, "failed to create pd for free mr.\n");
	2677	+ kfree(hr_pd);
	2678	+ return NULL;
	2679	+ }
	2680	+ free_mr->rsv_pd = to_hr_pd(pd);
	2681	+ free_mr->rsv_pd->ibpd.device = &hr_dev->ib_dev;
	2682	+ free_mr->rsv_pd->ibpd.uobject = NULL;
	2683	+ free_mr->rsv_pd->ibpd.__internal_mr = NULL;
	2684	+ atomic_set(&free_mr->rsv_pd->ibpd.usecnt, 0);
	2685	+
	2686	+ return pd;
	2687	+}
	2688	+
	2689	+static struct ib_cq free_mr_init_cq(struct hns_roce_dev hr_dev)
2635	2690	{
2636	2691	struct hns_roce_v2_priv *priv = hr_dev->priv;
2637	2692	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
	2693	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	2694	+ struct ib_cq_init_attr cq_init_attr = {};
	2695	+ struct hns_roce_cq *hr_cq;
	2696	+ struct ib_cq *cq;
	2697	+
	2698	+ cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;
	2699	+
	2700	+ hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);
	2701	+ if (ZERO_OR_NULL_PTR(hr_cq))
	2702	+ return NULL;
	2703	+
	2704	+ cq = &hr_cq->ib_cq;
	2705	+ cq->device = ibdev;
	2706	+
	2707	+ if (hns_roce_create_cq(cq, &cq_init_attr, NULL)) {
	2708	+ ibdev_err(ibdev, "failed to create cq for free mr.\n");
	2709	+ kfree(hr_cq);
	2710	+ return NULL;
	2711	+ }
	2712	+ free_mr->rsv_cq = to_hr_cq(cq);
	2713	+ free_mr->rsv_cq->ib_cq.device = &hr_dev->ib_dev;
	2714	+ free_mr->rsv_cq->ib_cq.uobject = NULL;
	2715	+ free_mr->rsv_cq->ib_cq.comp_handler = NULL;
	2716	+ free_mr->rsv_cq->ib_cq.event_handler = NULL;
	2717	+ free_mr->rsv_cq->ib_cq.cq_context = NULL;
	2718	+ atomic_set(&free_mr->rsv_cq->ib_cq.usecnt, 0);
	2719	+
	2720	+ return cq;
	2721	+}
	2722	+
	2723	+static int free_mr_init_qp(struct hns_roce_dev hr_dev, struct ib_cq cq,
	2724	+ struct ib_qp_init_attr *init_attr, int i)
	2725	+{
	2726	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	2727	+ struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
	2728	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	2729	+ struct hns_roce_qp *hr_qp;
	2730	+ struct ib_qp *qp;
2638	2731	int ret;
	2732	+
	2733	+ hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
	2734	+ if (ZERO_OR_NULL_PTR(hr_qp))
	2735	+ return -ENOMEM;
	2736	+
	2737	+ qp = &hr_qp->ibqp;
	2738	+ qp->device = ibdev;
	2739	+
	2740	+ ret = hns_roce_create_qp(qp, init_attr, NULL);
	2741	+ if (ret) {
	2742	+ ibdev_err(ibdev, "failed to create qp for free mr.\n");
	2743	+ kfree(hr_qp);
	2744	+ return ret;
	2745	+ }
	2746	+
	2747	+ free_mr->rsv_qp[i] = hr_qp;
	2748	+ free_mr->rsv_qp[i]->ibqp.recv_cq = cq;
	2749	+ free_mr->rsv_qp[i]->ibqp.send_cq = cq;
	2750	+
	2751	+ return 0;
	2752	+}
	2753	+
	2754	+static void free_mr_exit(struct hns_roce_dev *hr_dev)
	2755	+{
	2756	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	2757	+ struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
	2758	+ struct ib_qp *qp;
2639	2759	int i;
2640	2760
2641	2761	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
2642	2762	if (free_mr->rsv_qp[i]) {
2643		− ret = ib_destroy_qp(free_mr->rsv_qp[i]);
2644		− if (ret)
2645		− ibdev_err(&hr_dev->ib_dev,
2646		− "failed to destroy qp in free mr.\n");
2647		−
	2763	+ qp = &free_mr->rsv_qp[i]->ibqp;
	2764	+ hns_roce_v2_destroy_qp(qp, NULL);
	2765	+ kfree(free_mr->rsv_qp[i]);
2648	2766	free_mr->rsv_qp[i] = NULL;
2649	2767	}
2650	2768	}
2651	2769
2652	2770	if (free_mr->rsv_cq) {
2653		− ib_destroy_cq(free_mr->rsv_cq);
	2771	+ hns_roce_destroy_cq(&free_mr->rsv_cq->ib_cq, NULL);
	2772	+ kfree(free_mr->rsv_cq);
2654	2773	free_mr->rsv_cq = NULL;
2655	2774	}
2656	2775
2657	2776	if (free_mr->rsv_pd) {
2658		− ib_dealloc_pd(free_mr->rsv_pd);
	2777	+ hns_roce_dealloc_pd(&free_mr->rsv_pd->ibpd, NULL);
	2778	+ kfree(free_mr->rsv_pd);
2659	2779	free_mr->rsv_pd = NULL;
2660	2780	}
2661	2781	}
@@ -2664,55 +2784,46 @@ static int free_mr_alloc_res(struct hns_roce_dev *hr_dev) · 2 unmodified lines
2664	2784	{
2665	2785	struct hns_roce_v2_priv *priv = hr_dev->priv;
2666	2786	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2667		− struct ib_device *ibdev = &hr_dev->ib_dev;
2668		− struct ib_cq_init_attr cq_init_attr = {};
2669	2787	struct ib_qp_init_attr qp_init_attr = {};
2670	2788	struct ib_pd *pd;
2671	2789	struct ib_cq *cq;
2672		− struct ib_qp *qp;
2673	2790	int ret;
2674	2791	int i;
2675	2792
2676		− pd = ib_alloc_pd(ibdev, 0);
2677		− if (IS_ERR(pd)) {
2678		− ibdev_err(ibdev, "failed to create pd for free mr.\n");
2679		− return PTR_ERR(pd);
2680		− }
2681		− free_mr->rsv_pd = pd;
	2793	+ pd = free_mr_init_pd(hr_dev);
	2794	+ if (!pd)
	2795	+ return -ENOMEM;
2682	2796
2683		− cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;
2684		− cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_init_attr);
2685		− if (IS_ERR(cq)) {
2686		− ibdev_err(ibdev, "failed to create cq for free mr.\n");
2687		− ret = PTR_ERR(cq);
2688		− goto create_failed;
	2797	+ cq = free_mr_init_cq(hr_dev);
	2798	+ if (!cq) {
	2799	+ ret = -ENOMEM;
	2800	+ goto create_failed_cq;
2689	2801	}
2690		− free_mr->rsv_cq = cq;
2691	2802
2692	2803	qp_init_attr.qp_type = IB_QPT_RC;
2693	2804	qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
2694		− qp_init_attr.send_cq = free_mr->rsv_cq;
2695		− qp_init_attr.recv_cq = free_mr->rsv_cq;
	2805	+ qp_init_attr.send_cq = cq;
	2806	+ qp_init_attr.recv_cq = cq;
2696	2807	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
2697	2808	qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM;
2698	2809	qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM;
2699	2810	qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM;
2700	2811	qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM;
2701	2812
2702		− qp = ib_create_qp(free_mr->rsv_pd, &qp_init_attr);
2703		− if (IS_ERR(qp)) {
2704		− ibdev_err(ibdev, "failed to create qp for free mr.\n");
2705		− ret = PTR_ERR(qp);
2706		− goto create_failed;
2707		− }
2708		−
2709		− free_mr->rsv_qp[i] = qp;
	2813	+ ret = free_mr_init_qp(hr_dev, cq, &qp_init_attr, i);
	2814	+ if (ret)
	2815	+ goto create_failed_qp;
2710	2816	}
2711	2817
2712	2818	return 0;
2713	2819
2714		−create_failed:
2715		− free_mr_exit(hr_dev);
	2820	+create_failed_qp:
	2821	+ hns_roce_destroy_cq(cq, NULL);
	2822	+ kfree(cq);
	2823	+
	2824	+create_failed_cq:
	2825	+ hns_roce_dealloc_pd(pd, NULL);
	2826	+ kfree(pd);
2716	2827
2717	2828	return ret;
2718	2829	}
@@ -2728,14 +2839,17 @@ static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev, · 9 unmodified lines
2728	2839	int mask;
2729	2840	int ret;
2730	2841
2731		− hr_qp = to_hr_qp(free_mr->rsv_qp[sl_num]);
	2842	+ hr_qp = to_hr_qp(&free_mr->rsv_qp[sl_num]->ibqp);
2732	2843	hr_qp->free_mr_en = 1;
	2844	+ hr_qp->ibqp.device = ibdev;
	2845	+ hr_qp->ibqp.qp_type = IB_QPT_RC;
2733	2846
2734	2847	mask = IB_QP_STATE \| IB_QP_PKEY_INDEX \| IB_QP_PORT \| IB_QP_ACCESS_FLAGS;
2735	2848	attr->qp_state = IB_QPS_INIT;
2736	2849	attr->port_num = 1;
2737	2850	attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
2738		− ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
	2851	+ ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
	2852	+ IB_QPS_INIT);
2739	2853	if (ret) {
2740	2854	ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n",
2741	2855	ret);
@@ -2756,7 +2870,8 @@ static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev, · 14 unmodified lines
2756	2870
2757	2871	rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num);
2758	2872
2759		− ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
	2873	+ ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
	2874	+ IB_QPS_RTR);
2760	2875	hr_dev->loop_idc = loopback;
2761	2876	if (ret) {
2762	2877	ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",
@@ -2770,7 +2885,8 @@ static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev, · 7 unmodified lines
2770	2885	attr->sq_psn = HNS_ROCE_FREE_MR_USED_PSN;
2771	2886	attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;
2772	2887	attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;
2773		− ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
	2888	+ ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR,
	2889	+ IB_QPS_RTS);
2774	2890	if (ret)
2775	2891	ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",
2776	2892	ret);
@@ -3186,7 +3302,8 @@ static int set_mtpt_pbl(struct hns_roce_dev *hr_dev, · 409 unmodified lines
3186	3302	int i, count;
3187	3303
3188	3304	count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
3189		− ARRAY_SIZE(pages), &pbl_ba);
	3305	+ min_t(int, ARRAY_SIZE(pages), mr->npages),
	3306	+ &pbl_ba);
3190	3307	if (count < 1) {
3191	3308	ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
3192	3309	count);
@@ -3414,7 +3531,7 @@ static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev) · 221 unmodified lines
3414	3531	mutex_lock(&free_mr->mutex);
3415	3532
3416	3533	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
3417		− hr_qp = to_hr_qp(free_mr->rsv_qp[i]);
	3534	+ hr_qp = free_mr->rsv_qp[i];
3418	3535
3419	3536	ret = free_mr_post_send_lp_wqe(hr_qp);
3420	3537	if (ret) {
@@ -3429,7 +3546,7 @@ static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev) · 8 unmodified lines
3429	3546
3430	3547	end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies;
3431	3548	while (cqe_cnt) {
3432		− npolled = hns_roce_v2_poll_cq(free_mr->rsv_cq, cqe_cnt, wc);
	3549	+ npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc);
3433	3550	if (npolled < 0) {
3434	3551	ibdev_err(ibdev,
3435	3552	"failed to poll cqe for free mr, remain %d cqe.\n",
@@ -5375,6 +5492,8 @@ static int hns_roce_v2_query_qp(struct ib_qp ibqp, struct ib_qp_attr qp_attr, · 1939 unmodified lines
5375	5492
5376	5493	rdma_ah_set_sl(&qp_attr->ah_attr,
5377	5494	hr_reg_read(&context, QPC_SL));
	5495	+ rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1);
	5496	+ rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH);
5378	5497	grh->flow_label = hr_reg_read(&context, QPC_FL);
5379	5498	grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX);
5380	5499	grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT);
@@ -5468,7 +5587,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, · 87 unmodified lines
5468	5587	return ret;
5469	5588	}
5470	5589
5471		−static int hns_roce_v2_destroy_qp(struct ib_qp ibqp, struct ib_udata udata)
	5590	+int hns_roce_v2_destroy_qp(struct ib_qp ibqp, struct ib_udata udata)
5472	5591	{
5473	5592	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5474	5593	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);

drivers/infiniband/hw/hns/hns_roce_hw_v2.h+10−3

@@ -272,6 +272,11 @@ enum hns_roce_cmd_return_status { · 271 unmodified lines
272	272	CMD_OTHER_ERR = 0xff
273	273	};
274	274
	275	+struct hns_roce_cmd_errcode {
	276	+ enum hns_roce_cmd_return_status return_status;
	277	+ int errno;
	278	+};
	279	+
275	280	enum hns_roce_sgid_type {
276	281	GID_TYPE_FLAG_ROCE_V1 = 0,
277	282	GID_TYPE_FLAG_ROCE_V2_IPV4,
@@ -1327,9 +1332,9 @@ struct hns_roce_link_table { · 1049 unmodified lines
1327	1332	#define HNS_ROCE_EXT_LLM_MIN_PAGES(que_num) ((que_num) * 4 + 2)
1328	1333
1329	1334	struct hns_roce_v2_free_mr {
1330		− struct ib_qp *rsv_qp[HNS_ROCE_FREE_MR_USED_QP_NUM];
1331		− struct ib_cq *rsv_cq;
1332		− struct ib_pd *rsv_pd;
	1335	+ struct hns_roce_qp *rsv_qp[HNS_ROCE_FREE_MR_USED_QP_NUM];
	1336	+ struct hns_roce_cq *rsv_cq;
	1337	+ struct hns_roce_pd *rsv_pd;
1333	1338	struct mutex mutex;
1334	1339	};
1335	1340
@@ -1459,6 +1464,8 @@ struct hns_roce_sccc_clr_done { · 123 unmodified lines
1459	1464	__le32 rsv[5];
1460	1465	};
1461	1466
	1467	+int hns_roce_v2_destroy_qp(struct ib_qp ibqp, struct ib_udata udata);
	1468	+
1462	1469	static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],
1463	1470	void __iomem *dest)
1464	1471	{

drivers/infiniband/hw/hns/hns_roce_main.c+16−2

@@ -354,10 +354,11 @@ static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx) · 353 unmodified lines
354	354	static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
355	355	struct ib_udata *udata)
356	356	{
357		− int ret;
358	357	struct hns_roce_ucontext *context = to_hr_ucontext(uctx);
359		− struct hns_roce_ib_alloc_ucontext_resp resp = {};
360	358	struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device);
	359	+ struct hns_roce_ib_alloc_ucontext_resp resp = {};
	360	+ struct hns_roce_ib_alloc_ucontext ucmd = {};
	361	+ int ret;
361	362
362	363	if (!hr_dev->active)
363	364	return -EAGAIN;
@@ -365,6 +366,19 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, · 1 unmodified line
365	366	resp.qp_tab_size = hr_dev->caps.num_qps;
366	367	resp.srq_tab_size = hr_dev->caps.num_srqs;
367	368
	369	+ ret = ib_copy_from_udata(&ucmd, udata,
	370	+ min(udata->inlen, sizeof(ucmd)));
	371	+ if (ret)
	372	+ return ret;
	373	+
	374	+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
	375	+ context->config = ucmd.config & HNS_ROCE_EXSGE_FLAGS;
	376	+
	377	+ if (context->config & HNS_ROCE_EXSGE_FLAGS) {
	378	+ resp.config \|= HNS_ROCE_RSP_EXSGE_FLAGS;
	379	+ resp.max_inline_data = hr_dev->caps.max_sq_inline;
	380	+ }
	381	+
368	382	ret = hns_roce_uar_alloc(hr_dev, &context->uar);
369	383	if (ret)
370	384	goto error_fail_uar_alloc;

drivers/infiniband/hw/hns/hns_roce_mr.c+2−2

@@ -392,10 +392,10 @@ struct ib_mr hns_roce_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type, · 391 unmodified lines
392	392
393	393	return &mr->ibmr;
394	394
395		−err_key:
396		− free_mr_key(hr_dev, mr);
397	395	err_pbl:
398	396	free_mr_pbl(hr_dev, mr);
	397	+err_key:
	398	+ free_mr_key(hr_dev, mr);
399	399	err_free:
400	400	kfree(mr);
401	401	return ERR_PTR(ret);

drivers/infiniband/hw/hns/hns_roce_qp.c+90−17

@@ -476,38 +476,109 @@ static int set_rq_size(struct hns_roce_dev hr_dev, struct ib_qp_cap cap, · 475 unmodified lines
476	476	return 0;
477	477	}
478	478
479		−static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp)
	479	+static u32 get_max_inline_data(struct hns_roce_dev *hr_dev,
	480	+ struct ib_qp_cap *cap)
480	481	{
481		− /* GSI/UD QP only has extended sge */
482		− if (qp->ibqp.qp_type == IB_QPT_GSI \|\| qp->ibqp.qp_type == IB_QPT_UD)
483		− return qp->sq.max_gs;
484		−
485		− if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE)
486		− return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE;
	482	+ if (cap->max_inline_data) {
	483	+ cap->max_inline_data = roundup_pow_of_two(cap->max_inline_data);
	484	+ return min(cap->max_inline_data,
	485	+ hr_dev->caps.max_sq_inline);
	486	+ }
487	487
488	488	return 0;
489	489	}
490	490
	491	+static void update_inline_data(struct hns_roce_qp *hr_qp,
	492	+ struct ib_qp_cap *cap)
	493	+{
	494	+ u32 sge_num = hr_qp->sq.ext_sge_cnt;
	495	+
	496	+ if (hr_qp->config & HNS_ROCE_EXSGE_FLAGS) {
	497	+ if (!(hr_qp->ibqp.qp_type == IB_QPT_GSI \|\|
	498	+ hr_qp->ibqp.qp_type == IB_QPT_UD))
	499	+ sge_num = max((u32)HNS_ROCE_SGE_IN_WQE, sge_num);
	500	+
	501	+ cap->max_inline_data = max(cap->max_inline_data,
	502	+ sge_num * HNS_ROCE_SGE_SIZE);
	503	+ }
	504	+
	505	+ hr_qp->max_inline_data = cap->max_inline_data;
	506	+}
	507	+
	508	+static u32 get_sge_num_from_max_send_sge(bool is_ud_or_gsi,
	509	+ u32 max_send_sge)
	510	+{
	511	+ unsigned int std_sge_num;
	512	+ unsigned int min_sge;
	513	+
	514	+ std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;
	515	+ min_sge = is_ud_or_gsi ? 1 : 0;
	516	+ return max_send_sge > std_sge_num ? (max_send_sge - std_sge_num) :
	517	+ min_sge;
	518	+}
	519	+
	520	+static unsigned int get_sge_num_from_max_inl_data(bool is_ud_or_gsi,
	521	+ u32 max_inline_data)
	522	+{
	523	+ unsigned int inline_sge;
	524	+
	525	+ inline_sge = roundup_pow_of_two(max_inline_data) / HNS_ROCE_SGE_SIZE;
	526	+
	527	+ /*
	528	+ * if max_inline_data less than
	529	+ * HNS_ROCE_SGE_IN_WQE * HNS_ROCE_SGE_SIZE,
	530	+ * In addition to ud's mode, no need to extend sge.
	531	+ */
	532	+ if (!is_ud_or_gsi && inline_sge <= HNS_ROCE_SGE_IN_WQE)
	533	+ inline_sge = 0;
	534	+
	535	+ return inline_sge;
	536	+}
	537	+
491	538	static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
492	539	struct hns_roce_qp hr_qp, struct ib_qp_cap cap)
493	540	{
	541	+ bool is_ud_or_gsi = (hr_qp->ibqp.qp_type == IB_QPT_GSI \|\|
	542	+ hr_qp->ibqp.qp_type == IB_QPT_UD);
	543	+ unsigned int std_sge_num;
	544	+ u32 inline_ext_sge = 0;
	545	+ u32 ext_wqe_sge_cnt;
494	546	u32 total_sge_cnt;
495		− u32 wqe_sge_cnt;
	547	+
	548	+ cap->max_inline_data = get_max_inline_data(hr_dev, cap);
496	549
497	550	hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
	551	+ std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;
	552	+ ext_wqe_sge_cnt = get_sge_num_from_max_send_sge(is_ud_or_gsi,
	553	+ cap->max_send_sge);
498	554
499		− hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
	555	+ if (hr_qp->config & HNS_ROCE_EXSGE_FLAGS) {
	556	+ inline_ext_sge = max(ext_wqe_sge_cnt,
	557	+ get_sge_num_from_max_inl_data(is_ud_or_gsi,
	558	+ cap->max_inline_data));
	559	+ hr_qp->sq.ext_sge_cnt = inline_ext_sge ?
	560	+ roundup_pow_of_two(inline_ext_sge) : 0;
500	561
501		− wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp);
	562	+ hr_qp->sq.max_gs = max(1U, (hr_qp->sq.ext_sge_cnt + std_sge_num));
	563	+ hr_qp->sq.max_gs = min(hr_qp->sq.max_gs, hr_dev->caps.max_sq_sg);
	564	+
	565	+ ext_wqe_sge_cnt = hr_qp->sq.ext_sge_cnt;
	566	+ } else {
	567	+ hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
	568	+ hr_qp->sq.max_gs = min(hr_qp->sq.max_gs, hr_dev->caps.max_sq_sg);
	569	+ hr_qp->sq.ext_sge_cnt = hr_qp->sq.max_gs;
	570	+ }
502	571
503	572	/* If the number of extended sge is not zero, they MUST use the
504	573	* space of HNS_HW_PAGE_SIZE at least.
505	574	*/
506		− if (wqe_sge_cnt) {
507		− total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt);
	575	+ if (ext_wqe_sge_cnt) {
	576	+ total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * ext_wqe_sge_cnt);
508	577	hr_qp->sge.sge_cnt = max(total_sge_cnt,
509	578	(u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);
510	579	}
	580	+
	581	+ update_inline_data(hr_qp, cap);
511	582	}
512	583
513	584	static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
@@ -556,6 +627,7 @@ static int set_user_sq_size(struct hns_roce_dev *hr_dev, · 42 unmodified lines
556	627
557	628	hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
558	629	hr_qp->sq.wqe_cnt = cnt;
	630	+ cap->max_send_sge = hr_qp->sq.max_gs;
559	631
560	632	return 0;
561	633	}
@@ -986,13 +1058,9 @@ static int set_qp_param(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp, · 424 unmodified lines
986	1058	struct hns_roce_ib_create_qp *ucmd)
987	1059	{
988	1060	struct ib_device *ibdev = &hr_dev->ib_dev;
	1061	+ struct hns_roce_ucontext *uctx;
989	1062	int ret;
990	1063
991		− if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline)
992		− init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline;
993		−
994		− hr_qp->max_inline_data = init_attr->cap.max_inline_data;
995		−
996	1064	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
997	1065	hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
998	1066	else
@@ -1015,12 +1083,17 @@ static int set_qp_param(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp, · 16 unmodified lines
1015	1083	return ret;
1016	1084	}
1017	1085
	1086	+ uctx = rdma_udata_to_drv_context(udata, struct hns_roce_ucontext,
	1087	+ ibucontext);
	1088	+ hr_qp->config = uctx->config;
1018	1089	ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);
1019	1090	if (ret)
1020	1091	ibdev_err(ibdev,
1021	1092	"failed to set user SQ size, ret = %d.\n",
1022	1093	ret);
1023	1094	} else {
	1095	+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
	1096	+ hr_qp->config = HNS_ROCE_EXSGE_FLAGS;
1024	1097	ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);
1025	1098	if (ret)
1026	1099	ibdev_err(ibdev,

drivers/infiniband/hw/irdma/uk.c+107−63

@@ -566,21 +566,37 @@ static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe, · 565 unmodified lines
566	566
567	567	/**
568	568	* irdma_copy_inline_data_gen_1 - Copy inline data to wqe
569		− * @dest: pointer to wqe
570		− * @src: pointer to inline data
571		− * @len: length of inline data to copy
	569	+ * @wqe: pointer to wqe
	570	+ * @sge_list: table of pointers to inline data
	571	+ * @num_sges: Total inline data length
572	572	* @polarity: compatibility parameter
573	573	*/
574		−static void irdma_copy_inline_data_gen_1(u8 dest, u8 src, u32 len,
575		− u8 polarity)
	574	+static void irdma_copy_inline_data_gen_1(u8 wqe, struct ib_sge sge_list,
	575	+ u32 num_sges, u8 polarity)
576	576	{
577		− if (len <= 16) {
578		− memcpy(dest, src, len);
579		− } else {
580		− memcpy(dest, src, 16);
581		− src += 16;
582		− dest = dest + 32;
583		− memcpy(dest, src, len - 16);
	577	+ u32 quanta_bytes_remaining = 16;
	578	+ int i;
	579	+
	580	+ for (i = 0; i < num_sges; i++) {
	581	+ u8 cur_sge = (u8 )(uintptr_t)sge_list[i].addr;
	582	+ u32 sge_len = sge_list[i].length;
	583	+
	584	+ while (sge_len) {
	585	+ u32 bytes_copied;
	586	+
	587	+ bytes_copied = min(sge_len, quanta_bytes_remaining);
	588	+ memcpy(wqe, cur_sge, bytes_copied);
	589	+ wqe += bytes_copied;
	590	+ cur_sge += bytes_copied;
	591	+ quanta_bytes_remaining -= bytes_copied;
	592	+ sge_len -= bytes_copied;
	593	+
	594	+ if (!quanta_bytes_remaining) {
	595	+ /* Remaining inline bytes reside after hdr */
	596	+ wqe += 16;
	597	+ quanta_bytes_remaining = 32;
	598	+ }
	599	+ }
584	600	}
585	601	}
586	602
@@ -612,35 +628,51 @@ static void irdma_set_mw_bind_wqe(__le64 *wqe, · 25 unmodified lines
612	628
613	629	/**
614	630	* irdma_copy_inline_data - Copy inline data to wqe
615		− * @dest: pointer to wqe
616		− * @src: pointer to inline data
617		− * @len: length of inline data to copy
	631	+ * @wqe: pointer to wqe
	632	+ * @sge_list: table of pointers to inline data
	633	+ * @num_sges: number of SGE's
618	634	* @polarity: polarity of wqe valid bit
619	635	*/
620		−static void irdma_copy_inline_data(u8 dest, u8 src, u32 len, u8 polarity)
	636	+static void irdma_copy_inline_data(u8 wqe, struct ib_sge sge_list,
	637	+ u32 num_sges, u8 polarity)
621	638	{
622	639	u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
623		− u32 copy_size;
624		−
625		− dest += 8;
626		− if (len <= 8) {
627		− memcpy(dest, src, len);
628		− return;
629		− }
630		−
631		− ((u64 )dest) = ((u64 )src);
632		− len -= 8;
633		− src += 8;
634		− dest += 24; /* point to additional 32 byte quanta */
635		−
636		− while (len) {
637		− copy_size = len < 31 ? len : 31;
638		− memcpy(dest, src, copy_size);
639		− *(dest + 31) = inline_valid;
640		− len -= copy_size;
641		− dest += 32;
642		− src += copy_size;
	640	+ u32 quanta_bytes_remaining = 8;
	641	+ bool first_quanta = true;
	642	+ int i;
	643	+
	644	+ wqe += 8;
	645	+
	646	+ for (i = 0; i < num_sges; i++) {
	647	+ u8 cur_sge = (u8 )(uintptr_t)sge_list[i].addr;
	648	+ u32 sge_len = sge_list[i].length;
	649	+
	650	+ while (sge_len) {
	651	+ u32 bytes_copied;
	652	+
	653	+ bytes_copied = min(sge_len, quanta_bytes_remaining);
	654	+ memcpy(wqe, cur_sge, bytes_copied);
	655	+ wqe += bytes_copied;
	656	+ cur_sge += bytes_copied;
	657	+ quanta_bytes_remaining -= bytes_copied;
	658	+ sge_len -= bytes_copied;
	659	+
	660	+ if (!quanta_bytes_remaining) {
	661	+ quanta_bytes_remaining = 31;
	662	+
	663	+ /* Remaining inline bytes reside after hdr */
	664	+ if (first_quanta) {
	665	+ first_quanta = false;
	666	+ wqe += 16;
	667	+ } else {
	668	+ *wqe = inline_valid;
	669	+ wqe++;
	670	+ }
	671	+ }
	672	+ }
643	673	}
	674	+ if (!first_quanta && quanta_bytes_remaining < 31)
	675	+ *(wqe + quanta_bytes_remaining) = inline_valid;
644	676	}
645	677
646	678	/**
@@ -679,20 +711,27 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, · 32 unmodified lines
679	711	struct irdma_post_sq_info *info, bool post_sq)
680	712	{
681	713	__le64 *wqe;
682		− struct irdma_inline_rdma_write *op_info;
	714	+ struct irdma_rdma_write *op_info;
683	715	u64 hdr = 0;
684	716	u32 wqe_idx;
685	717	bool read_fence = false;
	718	+ u32 i, total_size = 0;
686	719	u16 quanta;
687	720
688	721	info->push_wqe = qp->push_db ? true : false;
689		− op_info = &info->op.inline_rdma_write;
	722	+ op_info = &info->op.rdma_write;
	723	+
	724	+ if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
	725	+ return -EINVAL;
	726	+
	727	+ for (i = 0; i < op_info->num_lo_sges; i++)
	728	+ total_size += op_info->lo_sg_list[i].length;
690	729
691		− if (op_info->len > qp->max_inline_data)
	730	+ if (unlikely(total_size > qp->max_inline_data))
692	731	return -EINVAL;
693	732
694		− quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
695		− wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
	733	+ quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
	734	+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
696	735	info);
697	736	if (!wqe)
698	737	return -ENOMEM;
@@ -705,7 +744,7 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, · 6 unmodified lines
705	744
706	745	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) \|
707	746	FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) \|
708		− FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, op_info->len) \|
	747	+ FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) \|
709	748	FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) \|
710	749	FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) \|
711	750	FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) \|
@@ -719,7 +758,8 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, · 7 unmodified lines
719	758	set_64bit_val(wqe, 0,
720	759	FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
721	760
722		− qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,
	761	+ qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
	762	+ op_info->num_lo_sges,
723	763	qp->swqe_polarity);
724	764	dma_wmb(); /* make sure WQE is populated before valid bit is set */
725	765
@@ -745,20 +785,27 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp, · 19 unmodified lines
745	785	struct irdma_post_sq_info *info, bool post_sq)
746	786	{
747	787	__le64 *wqe;
748		− struct irdma_post_inline_send *op_info;
	788	+ struct irdma_post_send *op_info;
749	789	u64 hdr;
750	790	u32 wqe_idx;
751	791	bool read_fence = false;
	792	+ u32 i, total_size = 0;
752	793	u16 quanta;
753	794
754	795	info->push_wqe = qp->push_db ? true : false;
755		− op_info = &info->op.inline_send;
	796	+ op_info = &info->op.send;
	797	+
	798	+ if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
	799	+ return -EINVAL;
756	800
757		− if (op_info->len > qp->max_inline_data)
	801	+ for (i = 0; i < op_info->num_sges; i++)
	802	+ total_size += op_info->sg_list[i].length;
	803	+
	804	+ if (unlikely(total_size > qp->max_inline_data))
758	805	return -EINVAL;
759	806
760		− quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
761		− wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
	807	+ quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
	808	+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
762	809	info);
763	810	if (!wqe)
764	811	return -ENOMEM;
@@ -773,7 +820,7 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp, · 8 unmodified lines
773	820	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) \|
774	821	FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) \|
775	822	FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) \|
776		− FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, op_info->len) \|
	823	+ FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) \|
777	824	FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
778	825	(info->imm_data_valid ? 1 : 0)) \|
779	826	FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) \|
@@ -789,8 +836,8 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp, · 9 unmodified lines
789	836	if (info->imm_data_valid)
790	837	set_64bit_val(wqe, 0,
791	838	FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
792		− qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,
793		− qp->swqe_polarity);
	839	+ qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
	840	+ op_info->num_sges, qp->swqe_polarity);
794	841
795	842	dma_wmb(); /* make sure WQE is populated before valid bit is set */
796	843
@@ -1002,11 +1049,10 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 205 unmodified lines
1002	1049	__le64 *cqe;
1003	1050	struct irdma_qp_uk *qp;
1004	1051	struct irdma_ring *pring = NULL;
1005		− u32 wqe_idx, q_type;
	1052	+ u32 wqe_idx;
1006	1053	int ret_code;
1007	1054	bool move_cq_head = true;
1008	1055	u8 polarity;
1009		− u8 op_type;
1010	1056	bool ext_valid;
1011	1057	__le64 *ext_cqe;
1012	1058
@@ -1074,7 +1120,7 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 61 unmodified lines
1074	1120	info->ud_vlan_valid = false;
1075	1121	}
1076	1122
1077		− q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
	1123	+ info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
1078	1124	info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
1079	1125	info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3);
1080	1126	info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
@@ -1113,8 +1159,9 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 32 unmodified lines
1113	1159	}
1114	1160	wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
1115	1161	info->qp_handle = (irdma_qp_handle)(unsigned long)qp;
	1162	+ info->op_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
1116	1163
1117		− if (q_type == IRDMA_CQE_QTYPE_RQ) {
	1164	+ if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
1118	1165	u32 array_idx;
1119	1166
1120	1167	array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
@@ -1134,10 +1181,6 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 13 unmodified lines
1134	1181
1135	1182	info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
1136	1183
1137		− if (info->imm_valid)
1138		− info->op_type = IRDMA_OP_TYPE_REC_IMM;
1139		− else
1140		− info->op_type = IRDMA_OP_TYPE_REC;
1141	1184	if (qword3 & IRDMACQ_STAG) {
1142	1185	info->stag_invalid_set = true;
1143	1186	info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);
@@ -1195,17 +1238,18 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 51 unmodified lines
1195	1238	sw_wqe = qp->sq_base[tail].elem;
1196	1239	get_64bit_val(sw_wqe, 24,
1197	1240	&wqe_qword);
1198		− op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
1199		− info->op_type = op_type;
	1241	+ info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE,
	1242	+ wqe_qword);
1200	1243	IRDMA_RING_SET_TAIL(qp->sq_ring,
1201	1244	tail + qp->sq_wrtrk_array[tail].quanta);
1202		− if (op_type != IRDMAQP_OP_NOP) {
	1245	+ if (info->op_type != IRDMAQP_OP_NOP) {
1203	1246	info->wr_id = qp->sq_wrtrk_array[tail].wrid;
1204	1247	info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
1205	1248	break;
1206	1249	}
1207	1250	} while (1);
1208		− if (op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR)
	1251	+ if (info->op_type == IRDMA_OP_TYPE_BIND_MW &&
	1252	+ info->minor_err == FLUSH_PROT_ERR)
1209	1253	info->minor_err = FLUSH_MW_BIND_ERR;
1210	1254	qp->sq_flush_seen = true;
1211	1255	if (!IRDMA_RING_MORE_WORK(qp->sq_ring))

drivers/infiniband/hw/irdma/user.h+3−17

@@ -173,14 +173,6 @@ struct irdma_post_send { · 172 unmodified lines
173	173	u32 ah_id;
174	174	};
175	175
176		−struct irdma_post_inline_send {
177		− void *data;
178		− u32 len;
179		− u32 qkey;
180		− u32 dest_qp;
181		− u32 ah_id;
182		−};
183		−
184	176	struct irdma_post_rq_info {
185	177	u64 wr_id;
186	178	struct ib_sge *sg_list;
@@ -193,12 +185,6 @@ struct irdma_rdma_write { · 6 unmodified lines
193	185	struct ib_sge rem_addr;
194	186	};
195	187
196		−struct irdma_inline_rdma_write {
197		− void *data;
198		− u32 len;
199		− struct ib_sge rem_addr;
200		−};
201		−
202	188	struct irdma_rdma_read {
203	189	struct ib_sge *lo_sg_list;
204	190	u32 num_lo_sges;
@@ -241,8 +227,6 @@ struct irdma_post_sq_info { · 36 unmodified lines
241	227	struct irdma_rdma_read rdma_read;
242	228	struct irdma_bind_window bind_window;
243	229	struct irdma_inv_local_stag inv_local_stag;
244		− struct irdma_inline_rdma_write inline_rdma_write;
245		− struct irdma_post_inline_send inline_send;
246	230	} op;
247	231	};
248	232
@@ -261,6 +245,7 @@ struct irdma_cq_poll_info { · 12 unmodified lines
261	245	u16 ud_vlan;
262	246	u8 ud_smac[6];
263	247	u8 op_type;
	248	+ u8 q_type;
264	249	bool stag_invalid_set:1; /* or L_R_Key set */
265	250	bool push_dropped:1;
266	251	bool error:1;
@@ -291,7 +276,8 @@ int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, · 24 unmodified lines
291	276	bool post_sq);
292	277
293	278	struct irdma_wqe_uk_ops {
294		− void (iw_copy_inline_data)(u8 dest, u8 *src, u32 len, u8 polarity);
	279	+ void (iw_copy_inline_data)(u8 dest, struct ib_sge *sge_list,
	280	+ u32 num_sges, u8 polarity);
295	281	u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
296	282	void (iw_set_fragment)(__le64 wqe, u32 offset, struct ib_sge *sge,
297	283	u8 valid);

drivers/infiniband/hw/irdma/utils.c+2−0

@@ -2591,6 +2591,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp) · 2590 unmodified lines
2591	2591	sw_wqe = qp->sq_base[wqe_idx].elem;
2592	2592	get_64bit_val(sw_wqe, 24, &wqe_qword);
2593	2593	cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
	2594	+ cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
2594	2595	/* remove the SQ WR by moving SQ tail*/
2595	2596	IRDMA_RING_SET_TAIL(*sq_ring,
2596	2597	sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
@@ -2629,6 +2630,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp) · 32 unmodified lines
2629	2630
2630	2631	cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2631	2632	cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
	2633	+ cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
2632	2634	/* remove the RQ WR by moving RQ tail */
2633	2635	IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2634	2636	ibdev_dbg(iwqp->iwrcq->ibcq.device,

drivers/infiniband/hw/irdma/verbs.c+34−111

@@ -63,36 +63,6 @@ static int irdma_query_device(struct ib_device *ibdev, · 62 unmodified lines
63	63	return 0;
64	64	}
65	65
66		−/**
67		− * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed
68		− * @link_speed: netdev phy link speed
69		− * @active_speed: IB port speed
70		− * @active_width: IB port width
71		− */
72		−static void irdma_get_eth_speed_and_width(u32 link_speed, u16 *active_speed,
73		− u8 *active_width)
74		−{
75		− if (link_speed <= SPEED_1000) {
76		− *active_width = IB_WIDTH_1X;
77		− *active_speed = IB_SPEED_SDR;
78		− } else if (link_speed <= SPEED_10000) {
79		− *active_width = IB_WIDTH_1X;
80		− *active_speed = IB_SPEED_FDR10;
81		− } else if (link_speed <= SPEED_20000) {
82		− *active_width = IB_WIDTH_4X;
83		− *active_speed = IB_SPEED_DDR;
84		− } else if (link_speed <= SPEED_25000) {
85		− *active_width = IB_WIDTH_1X;
86		− *active_speed = IB_SPEED_EDR;
87		− } else if (link_speed <= SPEED_40000) {
88		− *active_width = IB_WIDTH_4X;
89		− *active_speed = IB_SPEED_FDR10;
90		− } else {
91		− *active_width = IB_WIDTH_4X;
92		− *active_speed = IB_SPEED_EDR;
93		− }
94		−}
95		−
96	66	/**
97	67	* irdma_query_port - get port attributes
98	68	* @ibdev: device pointer from stack
@@ -120,8 +90,9 @@ static int irdma_query_port(struct ib_device *ibdev, u32 port, · 21 unmodified lines
120	90	props->state = IB_PORT_DOWN;
121	91	props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
122	92	}
123		− irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed,
124		− &props->active_width);
	93	+
	94	+ ib_get_eth_speed(ibdev, port, &props->active_speed,
	95	+ &props->active_width);
125	96
126	97	if (rdma_protocol_roce(ibdev, 1)) {
127	98	props->gid_tbl_len = 32;
@@ -1242,6 +1213,7 @@ int irdma_modify_qp_roce(struct ib_qp ibqp, struct ib_qp_attr attr, · 1114 unmodified lines
1242	1213	av->attrs = attr->ah_attr;
1243	1214	rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid);
1244	1215	rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid);
	1216	+ av->net_type = rdma_gid_attr_network_type(sgid_attr);
1245	1217	if (av->net_type == RDMA_NETWORK_IPV6) {
1246	1218	__be32 *daddr =
1247	1219	av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;
@@ -2358,9 +2330,10 @@ static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, · 1110 unmodified lines
2358	2330	* @rf: RDMA PCI function
2359	2331	* @iwmr: mr pointer for this memory registration
2360	2332	* @use_pbles: flag if to use pble's
	2333	+ * @lvl_1_only: request only level 1 pble if true
2361	2334	*/
2362	2335	static int irdma_setup_pbles(struct irdma_pci_f rf, struct irdma_mr iwmr,
2363		− bool use_pbles)
	2336	+ bool use_pbles, bool lvl_1_only)
2364	2337	{
2365	2338	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2366	2339	struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
@@ -2371,7 +2344,7 @@ static int irdma_setup_pbles(struct irdma_pci_f rf, struct irdma_mr iwmr, · 4 unmodified lines
2371	2344
2372	2345	if (use_pbles) {
2373	2346	status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,
2374		− false);
	2347	+ lvl_1_only);
2375	2348	if (status)
2376	2349	return status;
2377	2350
@@ -2414,16 +2387,10 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev, · 36 unmodified lines
2414	2387	bool ret = true;
2415	2388
2416	2389	pg_size = iwmr->page_size;
2417		− err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles);
	2390	+ err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, true);
2418	2391	if (err)
2419	2392	return err;
2420	2393
2421		− if (use_pbles && palloc->level != PBLE_LEVEL_1) {
2422		− irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
2423		− iwpbl->pbl_allocated = false;
2424		− return -ENOMEM;
2425		− }
2426		−
2427	2394	if (use_pbles)
2428	2395	arr = palloc->level1.addr;
2429	2396
@@ -2899,7 +2866,7 @@ static struct ib_mr irdma_reg_user_mr(struct ib_pd pd, u64 start, u64 len, · 469 unmodified lines
2899	2866	case IRDMA_MEMREG_TYPE_MEM:
2900	2867	use_pbles = (iwmr->page_cnt != 1);
2901	2868
2902		− err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles);
	2869	+ err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
2903	2870	if (err)
2904	2871	goto error;
2905	2872
@@ -3165,30 +3132,20 @@ static int irdma_post_send(struct ib_qp *ibqp, · 259 unmodified lines
3165	3132	info.stag_to_inv = ib_wr->ex.invalidate_rkey;
3166	3133	}
3167	3134
3168		− if (ib_wr->send_flags & IB_SEND_INLINE) {
3169		− info.op.inline_send.data = (void *)(unsigned long)
3170		− ib_wr->sg_list[0].addr;
3171		− info.op.inline_send.len = ib_wr->sg_list[0].length;
3172		− if (iwqp->ibqp.qp_type == IB_QPT_UD \|\|
3173		− iwqp->ibqp.qp_type == IB_QPT_GSI) {
3174		− ah = to_iwah(ud_wr(ib_wr)->ah);
3175		− info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx;
3176		− info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey;
3177		− info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn;
3178		− }
	3135	+ info.op.send.num_sges = ib_wr->num_sge;
	3136	+ info.op.send.sg_list = ib_wr->sg_list;
	3137	+ if (iwqp->ibqp.qp_type == IB_QPT_UD \|\|
	3138	+ iwqp->ibqp.qp_type == IB_QPT_GSI) {
	3139	+ ah = to_iwah(ud_wr(ib_wr)->ah);
	3140	+ info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
	3141	+ info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
	3142	+ info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
	3143	+ }
	3144	+
	3145	+ if (ib_wr->send_flags & IB_SEND_INLINE)
3179	3146	err = irdma_uk_inline_send(ukqp, &info, false);
3180		− } else {
3181		− info.op.send.num_sges = ib_wr->num_sge;
3182		− info.op.send.sg_list = ib_wr->sg_list;
3183		− if (iwqp->ibqp.qp_type == IB_QPT_UD \|\|
3184		− iwqp->ibqp.qp_type == IB_QPT_GSI) {
3185		− ah = to_iwah(ud_wr(ib_wr)->ah);
3186		− info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
3187		− info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
3188		− info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
3189		− }
	3147	+ else
3190	3148	err = irdma_uk_send(ukqp, &info, false);
3191		− }
3192	3149	break;
3193	3150	case IB_WR_RDMA_WRITE_WITH_IMM:
3194	3151	if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) {
@@ -3205,22 +3162,15 @@ static int irdma_post_send(struct ib_qp *ibqp, · 10 unmodified lines
3205	3162	else
3206	3163	info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
3207	3164
3208		− if (ib_wr->send_flags & IB_SEND_INLINE) {
3209		− info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
3210		− info.op.inline_rdma_write.len =
3211		− ib_wr->sg_list[0].length;
3212		− info.op.inline_rdma_write.rem_addr.addr =
3213		− rdma_wr(ib_wr)->remote_addr;
3214		− info.op.inline_rdma_write.rem_addr.lkey =
3215		− rdma_wr(ib_wr)->rkey;
	3165	+ info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
	3166	+ info.op.rdma_write.lo_sg_list = ib_wr->sg_list;
	3167	+ info.op.rdma_write.rem_addr.addr =
	3168	+ rdma_wr(ib_wr)->remote_addr;
	3169	+ info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
	3170	+ if (ib_wr->send_flags & IB_SEND_INLINE)
3216	3171	err = irdma_uk_inline_rdma_write(ukqp, &info, false);
3217		− } else {
3218		− info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
3219		− info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
3220		− info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
3221		− info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
	3172	+ else
3222	3173	err = irdma_uk_rdma_write(ukqp, &info, false);
3223		− }
3224	3174	break;
3225	3175	case IB_WR_RDMA_READ_WITH_INV:
3226	3176	inv_stag = true;
@@ -3380,7 +3330,6 @@ static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode · 153 unmodified lines
3380	3330	static void irdma_process_cqe(struct ib_wc *entry,
3381	3331	struct irdma_cq_poll_info *cq_poll_info)
3382	3332	{
3383		− struct irdma_qp *iwqp;
3384	3333	struct irdma_sc_qp *qp;
3385	3334
3386	3335	entry->wc_flags = 0;
@@ -3388,7 +3337,6 @@ static void irdma_process_cqe(struct ib_wc *entry, · 1 unmodified line
3388	3337	entry->wr_id = cq_poll_info->wr_id;
3389	3338
3390	3339	qp = cq_poll_info->qp_handle;
3391		− iwqp = qp->qp_uk.back_qp;
3392	3340	entry->qp = qp->qp_uk.back_qp;
3393	3341
3394	3342	if (cq_poll_info->error) {
@@ -3421,42 +3369,17 @@ static void irdma_process_cqe(struct ib_wc *entry, · 26 unmodified lines
3421	3369	}
3422	3370	}
3423	3371
3424		− switch (cq_poll_info->op_type) {
3425		− case IRDMA_OP_TYPE_RDMA_WRITE:
3426		− case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
3427		− entry->opcode = IB_WC_RDMA_WRITE;
3428		− break;
3429		− case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
3430		− case IRDMA_OP_TYPE_RDMA_READ:
3431		− entry->opcode = IB_WC_RDMA_READ;
3432		− break;
3433		− case IRDMA_OP_TYPE_SEND_INV:
3434		− case IRDMA_OP_TYPE_SEND_SOL:
3435		− case IRDMA_OP_TYPE_SEND_SOL_INV:
3436		− case IRDMA_OP_TYPE_SEND:
3437		− entry->opcode = IB_WC_SEND;
3438		− break;
3439		− case IRDMA_OP_TYPE_FAST_REG_NSMR:
3440		− entry->opcode = IB_WC_REG_MR;
3441		− break;
3442		− case IRDMA_OP_TYPE_INV_STAG:
3443		− entry->opcode = IB_WC_LOCAL_INV;
3444		− break;
3445		− case IRDMA_OP_TYPE_REC_IMM:
3446		− case IRDMA_OP_TYPE_REC:
3447		− entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ?
3448		− IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
	3372	+ if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) {
	3373	+ set_ib_wc_op_sq(cq_poll_info, entry);
	3374	+ } else {
	3375	+ set_ib_wc_op_rq(cq_poll_info, entry,
	3376	+ qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM ?
	3377	+ true : false);
3449	3378	if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD &&
3450	3379	cq_poll_info->stag_invalid_set) {
3451	3380	entry->ex.invalidate_rkey = cq_poll_info->inv_stag;
3452	3381	entry->wc_flags \|= IB_WC_WITH_INVALIDATE;
3453	3382	}
3454		− break;
3455		− default:
3456		− ibdev_err(&iwqp->iwdev->ibdev,
3457		− "Invalid opcode = %d in CQE\n", cq_poll_info->op_type);
3458		− entry->status = IB_WC_GENERAL_ERR;
3459		− return;
3460	3383	}
3461	3384
3462	3385	if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) {

drivers/infiniband/hw/irdma/verbs.h+53−0

@@ -232,6 +232,59 @@ static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev) · 231 unmodified lines
232	232	return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
233	233	}
234	234
	235	+static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
	236	+ struct ib_wc *entry)
	237	+{
	238	+ switch (cq_poll_info->op_type) {
	239	+ case IRDMA_OP_TYPE_RDMA_WRITE:
	240	+ case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
	241	+ entry->opcode = IB_WC_RDMA_WRITE;
	242	+ break;
	243	+ case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
	244	+ case IRDMA_OP_TYPE_RDMA_READ:
	245	+ entry->opcode = IB_WC_RDMA_READ;
	246	+ break;
	247	+ case IRDMA_OP_TYPE_SEND_SOL:
	248	+ case IRDMA_OP_TYPE_SEND_SOL_INV:
	249	+ case IRDMA_OP_TYPE_SEND_INV:
	250	+ case IRDMA_OP_TYPE_SEND:
	251	+ entry->opcode = IB_WC_SEND;
	252	+ break;
	253	+ case IRDMA_OP_TYPE_FAST_REG_NSMR:
	254	+ entry->opcode = IB_WC_REG_MR;
	255	+ break;
	256	+ case IRDMA_OP_TYPE_INV_STAG:
	257	+ entry->opcode = IB_WC_LOCAL_INV;
	258	+ break;
	259	+ default:
	260	+ entry->status = IB_WC_GENERAL_ERR;
	261	+ }
	262	+}
	263	+
	264	+static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
	265	+ struct ib_wc *entry, bool send_imm_support)
	266	+{
	267	+ /**
	268	+ * iWARP does not support sendImm, so the presence of Imm data
	269	+ * must be WriteImm.
	270	+ */
	271	+ if (!send_imm_support) {
	272	+ entry->opcode = cq_poll_info->imm_valid ?
	273	+ IB_WC_RECV_RDMA_WITH_IMM :
	274	+ IB_WC_RECV;
	275	+ return;
	276	+ }
	277	+
	278	+ switch (cq_poll_info->op_type) {
	279	+ case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
	280	+ case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
	281	+ entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
	282	+ break;
	283	+ default:
	284	+ entry->opcode = IB_WC_RECV;
	285	+ }
	286	+}
	287	+
235	288	void irdma_mcast_mac(u32 ip_addr, u8 mac, bool ipv4);
236	289	int irdma_ib_register_device(struct irdma_device *iwdev);
237	290	void irdma_ib_unregister_device(struct irdma_device *iwdev);

drivers/infiniband/hw/mana/Kconfig+10−0

@@ -0,0 +1,10 @@
	1	+# SPDX-License-Identifier: GPL-2.0-only
	2	+config MANA_INFINIBAND
	3	+ tristate "Microsoft Azure Network Adapter support"
	4	+ depends on NETDEVICES && ETHERNET && PCI && MICROSOFT_MANA
	5	+ help
	6	+ This driver provides low-level RDMA support for Microsoft Azure
	7	+ Network Adapter (MANA). MANA supports RDMA features that can be used
	8	+ for workloads (e.g. DPDK, MPI etc) that uses RDMA verbs to directly
	9	+ access hardware from user-mode processes in Microsoft Azure cloud
	10	+ environment.

drivers/infiniband/hw/mana/Makefile+4−0

@@ -0,0 +1,4 @@
	1	+# SPDX-License-Identifier: GPL-2.0-only
	2	+obj-$(CONFIG_MANA_INFINIBAND) += mana_ib.o
	3	+
	4	+mana_ib-y := device.o main.o wq.o qp.o cq.o mr.o

drivers/infiniband/hw/mana/cq.c+79−0

@@ -0,0 +1,79 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#include "mana_ib.h"
	7	+
	8	+int mana_ib_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
	9	+ struct ib_udata *udata)
	10	+{
	11	+ struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
	12	+ struct ib_device *ibdev = ibcq->device;
	13	+ struct mana_ib_create_cq ucmd = {};
	14	+ struct mana_ib_dev *mdev;
	15	+ int err;
	16	+
	17	+ mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	18	+
	19	+ if (udata->inlen < sizeof(ucmd))
	20	+ return -EINVAL;
	21	+
	22	+ err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
	23	+ if (err) {
	24	+ ibdev_dbg(ibdev,
	25	+ "Failed to copy from udata for create cq, %d\n", err);
	26	+ return err;
	27	+ }
	28	+
	29	+ if (attr->cqe > MAX_SEND_BUFFERS_PER_QUEUE) {
	30	+ ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
	31	+ return -EINVAL;
	32	+ }
	33	+
	34	+ cq->cqe = attr->cqe;
	35	+ cq->umem = ib_umem_get(ibdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
	36	+ IB_ACCESS_LOCAL_WRITE);
	37	+ if (IS_ERR(cq->umem)) {
	38	+ err = PTR_ERR(cq->umem);
	39	+ ibdev_dbg(ibdev, "Failed to get umem for create cq, err %d\n",
	40	+ err);
	41	+ return err;
	42	+ }
	43	+
	44	+ err = mana_ib_gd_create_dma_region(mdev, cq->umem, &cq->gdma_region);
	45	+ if (err) {
	46	+ ibdev_dbg(ibdev,
	47	+ "Failed to create dma region for create cq, %d\n",
	48	+ err);
	49	+ goto err_release_umem;
	50	+ }
	51	+
	52	+ ibdev_dbg(ibdev,
	53	+ "mana_ib_gd_create_dma_region ret %d gdma_region 0x%llx\n",
	54	+ err, cq->gdma_region);
	55	+
	56	+ /*
	57	+ * The CQ ID is not known at this time. The ID is generated at create_qp
	58	+ */
	59	+
	60	+ return 0;
	61	+
	62	+err_release_umem:
	63	+ ib_umem_release(cq->umem);
	64	+ return err;
	65	+}
	66	+
	67	+int mana_ib_destroy_cq(struct ib_cq ibcq, struct ib_udata udata)
	68	+{
	69	+ struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
	70	+ struct ib_device *ibdev = ibcq->device;
	71	+ struct mana_ib_dev *mdev;
	72	+
	73	+ mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	74	+
	75	+ mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
	76	+ ib_umem_release(cq->umem);
	77	+
	78	+ return 0;
	79	+}

drivers/infiniband/hw/mana/device.c+117−0

@@ -0,0 +1,117 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#include "mana_ib.h"
	7	+#include <net/mana/mana_auxiliary.h>
	8	+
	9	+MODULE_DESCRIPTION("Microsoft Azure Network Adapter IB driver");
	10	+MODULE_LICENSE("GPL");
	11	+MODULE_IMPORT_NS(NET_MANA);
	12	+
	13	+static const struct ib_device_ops mana_ib_dev_ops = {
	14	+ .owner = THIS_MODULE,
	15	+ .driver_id = RDMA_DRIVER_MANA,
	16	+ .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
	17	+
	18	+ .alloc_pd = mana_ib_alloc_pd,
	19	+ .alloc_ucontext = mana_ib_alloc_ucontext,
	20	+ .create_cq = mana_ib_create_cq,
	21	+ .create_qp = mana_ib_create_qp,
	22	+ .create_rwq_ind_table = mana_ib_create_rwq_ind_table,
	23	+ .create_wq = mana_ib_create_wq,
	24	+ .dealloc_pd = mana_ib_dealloc_pd,
	25	+ .dealloc_ucontext = mana_ib_dealloc_ucontext,
	26	+ .dereg_mr = mana_ib_dereg_mr,
	27	+ .destroy_cq = mana_ib_destroy_cq,
	28	+ .destroy_qp = mana_ib_destroy_qp,
	29	+ .destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table,
	30	+ .destroy_wq = mana_ib_destroy_wq,
	31	+ .disassociate_ucontext = mana_ib_disassociate_ucontext,
	32	+ .get_port_immutable = mana_ib_get_port_immutable,
	33	+ .mmap = mana_ib_mmap,
	34	+ .modify_qp = mana_ib_modify_qp,
	35	+ .modify_wq = mana_ib_modify_wq,
	36	+ .query_device = mana_ib_query_device,
	37	+ .query_gid = mana_ib_query_gid,
	38	+ .query_port = mana_ib_query_port,
	39	+ .reg_user_mr = mana_ib_reg_user_mr,
	40	+
	41	+ INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
	42	+ INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
	43	+ INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
	44	+ INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
	45	+ INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mana_ib_rwq_ind_table,
	46	+ ib_ind_table),
	47	+};
	48	+
	49	+static int mana_ib_probe(struct auxiliary_device *adev,
	50	+ const struct auxiliary_device_id *id)
	51	+{
	52	+ struct mana_adev *madev = container_of(adev, struct mana_adev, adev);
	53	+ struct gdma_dev *mdev = madev->mdev;
	54	+ struct mana_context *mc;
	55	+ struct mana_ib_dev *dev;
	56	+ int ret;
	57	+
	58	+ mc = mdev->driver_data;
	59	+
	60	+ dev = ib_alloc_device(mana_ib_dev, ib_dev);
	61	+ if (!dev)
	62	+ return -ENOMEM;
	63	+
	64	+ ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_ops);
	65	+
	66	+ dev->ib_dev.phys_port_cnt = mc->num_ports;
	67	+
	68	+ ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
	69	+ mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt);
	70	+
	71	+ dev->gdma_dev = mdev;
	72	+ dev->ib_dev.node_type = RDMA_NODE_IB_CA;
	73	+
	74	+ /*
	75	+ * num_comp_vectors needs to set to the max MSIX index
	76	+ * when interrupts and event queues are implemented
	77	+ */
	78	+ dev->ib_dev.num_comp_vectors = 1;
	79	+ dev->ib_dev.dev.parent = mdev->gdma_context->dev;
	80	+
	81	+ ret = ib_register_device(&dev->ib_dev, "mana_%d",
	82	+ mdev->gdma_context->dev);
	83	+ if (ret) {
	84	+ ib_dealloc_device(&dev->ib_dev);
	85	+ return ret;
	86	+ }
	87	+
	88	+ dev_set_drvdata(&adev->dev, dev);
	89	+
	90	+ return 0;
	91	+}
	92	+
	93	+static void mana_ib_remove(struct auxiliary_device *adev)
	94	+{
	95	+ struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev);
	96	+
	97	+ ib_unregister_device(&dev->ib_dev);
	98	+ ib_dealloc_device(&dev->ib_dev);
	99	+}
	100	+
	101	+static const struct auxiliary_device_id mana_id_table[] = {
	102	+ {
	103	+ .name = "mana.rdma",
	104	+ },
	105	+ {},
	106	+};
	107	+
	108	+MODULE_DEVICE_TABLE(auxiliary, mana_id_table);
	109	+
	110	+static struct auxiliary_driver mana_driver = {
	111	+ .name = "rdma",
	112	+ .probe = mana_ib_probe,
	113	+ .remove = mana_ib_remove,
	114	+ .id_table = mana_id_table,
	115	+};
	116	+
	117	+module_auxiliary_driver(mana_driver);

drivers/infiniband/hw/mana/main.c+521−0

@@ -0,0 +1,521 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#include "mana_ib.h"
	7	+
	8	+void mana_ib_uncfg_vport(struct mana_ib_dev dev, struct mana_ib_pd pd,
	9	+ u32 port)
	10	+{
	11	+ struct gdma_dev *gd = dev->gdma_dev;
	12	+ struct mana_port_context *mpc;
	13	+ struct net_device *ndev;
	14	+ struct mana_context *mc;
	15	+
	16	+ mc = gd->driver_data;
	17	+ ndev = mc->ports[port];
	18	+ mpc = netdev_priv(ndev);
	19	+
	20	+ mutex_lock(&pd->vport_mutex);
	21	+
	22	+ pd->vport_use_count--;
	23	+ WARN_ON(pd->vport_use_count < 0);
	24	+
	25	+ if (!pd->vport_use_count)
	26	+ mana_uncfg_vport(mpc);
	27	+
	28	+ mutex_unlock(&pd->vport_mutex);
	29	+}
	30	+
	31	+int mana_ib_cfg_vport(struct mana_ib_dev dev, u32 port, struct mana_ib_pd pd,
	32	+ u32 doorbell_id)
	33	+{
	34	+ struct gdma_dev *mdev = dev->gdma_dev;
	35	+ struct mana_port_context *mpc;
	36	+ struct mana_context *mc;
	37	+ struct net_device *ndev;
	38	+ int err;
	39	+
	40	+ mc = mdev->driver_data;
	41	+ ndev = mc->ports[port];
	42	+ mpc = netdev_priv(ndev);
	43	+
	44	+ mutex_lock(&pd->vport_mutex);
	45	+
	46	+ pd->vport_use_count++;
	47	+ if (pd->vport_use_count > 1) {
	48	+ ibdev_dbg(&dev->ib_dev,
	49	+ "Skip as this PD is already configured vport\n");
	50	+ mutex_unlock(&pd->vport_mutex);
	51	+ return 0;
	52	+ }
	53	+
	54	+ err = mana_cfg_vport(mpc, pd->pdn, doorbell_id);
	55	+ if (err) {
	56	+ pd->vport_use_count--;
	57	+ mutex_unlock(&pd->vport_mutex);
	58	+
	59	+ ibdev_dbg(&dev->ib_dev, "Failed to configure vPort %d\n", err);
	60	+ return err;
	61	+ }
	62	+
	63	+ mutex_unlock(&pd->vport_mutex);
	64	+
	65	+ pd->tx_shortform_allowed = mpc->tx_shortform_allowed;
	66	+ pd->tx_vp_offset = mpc->tx_vp_offset;
	67	+
	68	+ ibdev_dbg(&dev->ib_dev, "vport handle %llx pdid %x doorbell_id %x\n",
	69	+ mpc->port_handle, pd->pdn, doorbell_id);
	70	+
	71	+ return 0;
	72	+}
	73	+
	74	+int mana_ib_alloc_pd(struct ib_pd ibpd, struct ib_udata udata)
	75	+{
	76	+ struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	77	+ struct ib_device *ibdev = ibpd->device;
	78	+ struct gdma_create_pd_resp resp = {};
	79	+ struct gdma_create_pd_req req = {};
	80	+ enum gdma_pd_flags flags = 0;
	81	+ struct mana_ib_dev *dev;
	82	+ struct gdma_dev *mdev;
	83	+ int err;
	84	+
	85	+ dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	86	+ mdev = dev->gdma_dev;
	87	+
	88	+ mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
	89	+ sizeof(resp));
	90	+
	91	+ req.flags = flags;
	92	+ err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
	93	+ sizeof(resp), &resp);
	94	+
	95	+ if (err \|\| resp.hdr.status) {
	96	+ ibdev_dbg(&dev->ib_dev,
	97	+ "Failed to get pd_id err %d status %u\n", err,
	98	+ resp.hdr.status);
	99	+ if (!err)
	100	+ err = -EPROTO;
	101	+
	102	+ return err;
	103	+ }
	104	+
	105	+ pd->pd_handle = resp.pd_handle;
	106	+ pd->pdn = resp.pd_id;
	107	+ ibdev_dbg(&dev->ib_dev, "pd_handle 0x%llx pd_id %d\n",
	108	+ pd->pd_handle, pd->pdn);
	109	+
	110	+ mutex_init(&pd->vport_mutex);
	111	+ pd->vport_use_count = 0;
	112	+ return 0;
	113	+}
	114	+
	115	+int mana_ib_dealloc_pd(struct ib_pd ibpd, struct ib_udata udata)
	116	+{
	117	+ struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	118	+ struct ib_device *ibdev = ibpd->device;
	119	+ struct gdma_destory_pd_resp resp = {};
	120	+ struct gdma_destroy_pd_req req = {};
	121	+ struct mana_ib_dev *dev;
	122	+ struct gdma_dev *mdev;
	123	+ int err;
	124	+
	125	+ dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	126	+ mdev = dev->gdma_dev;
	127	+
	128	+ mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
	129	+ sizeof(resp));
	130	+
	131	+ req.pd_handle = pd->pd_handle;
	132	+ err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
	133	+ sizeof(resp), &resp);
	134	+
	135	+ if (err \|\| resp.hdr.status) {
	136	+ ibdev_dbg(&dev->ib_dev,
	137	+ "Failed to destroy pd_handle 0x%llx err %d status %u",
	138	+ pd->pd_handle, err, resp.hdr.status);
	139	+ if (!err)
	140	+ err = -EPROTO;
	141	+ }
	142	+
	143	+ return err;
	144	+}
	145	+
	146	+static int mana_gd_destroy_doorbell_page(struct gdma_context *gc,
	147	+ int doorbell_page)
	148	+{
	149	+ struct gdma_destroy_resource_range_req req = {};
	150	+ struct gdma_resp_hdr resp = {};
	151	+ int err;
	152	+
	153	+ mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_RESOURCE_RANGE,
	154	+ sizeof(req), sizeof(resp));
	155	+
	156	+ req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;
	157	+ req.num_resources = 1;
	158	+ req.allocated_resources = doorbell_page;
	159	+
	160	+ err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
	161	+ if (err \|\| resp.status) {
	162	+ dev_err(gc->dev,
	163	+ "Failed to destroy doorbell page: ret %d, 0x%x\n",
	164	+ err, resp.status);
	165	+ return err ?: -EPROTO;
	166	+ }
	167	+
	168	+ return 0;
	169	+}
	170	+
	171	+static int mana_gd_allocate_doorbell_page(struct gdma_context *gc,
	172	+ int *doorbell_page)
	173	+{
	174	+ struct gdma_allocate_resource_range_req req = {};
	175	+ struct gdma_allocate_resource_range_resp resp = {};
	176	+ int err;
	177	+
	178	+ mana_gd_init_req_hdr(&req.hdr, GDMA_ALLOCATE_RESOURCE_RANGE,
	179	+ sizeof(req), sizeof(resp));
	180	+
	181	+ req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;
	182	+ req.num_resources = 1;
	183	+ req.alignment = 1;
	184	+
	185	+ /* Have GDMA start searching from 0 */
	186	+ req.allocated_resources = 0;
	187	+
	188	+ err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
	189	+ if (err \|\| resp.hdr.status) {
	190	+ dev_err(gc->dev,
	191	+ "Failed to allocate doorbell page: ret %d, 0x%x\n",
	192	+ err, resp.hdr.status);
	193	+ return err ?: -EPROTO;
	194	+ }
	195	+
	196	+ *doorbell_page = resp.allocated_resources;
	197	+
	198	+ return 0;
	199	+}
	200	+
	201	+int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
	202	+ struct ib_udata *udata)
	203	+{
	204	+ struct mana_ib_ucontext *ucontext =
	205	+ container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
	206	+ struct ib_device *ibdev = ibcontext->device;
	207	+ struct mana_ib_dev *mdev;
	208	+ struct gdma_context *gc;
	209	+ struct gdma_dev *dev;
	210	+ int doorbell_page;
	211	+ int ret;
	212	+
	213	+ mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	214	+ dev = mdev->gdma_dev;
	215	+ gc = dev->gdma_context;
	216	+
	217	+ /* Allocate a doorbell page index */
	218	+ ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
	219	+ if (ret) {
	220	+ ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret);
	221	+ return ret;
	222	+ }
	223	+
	224	+ ibdev_dbg(ibdev, "Doorbell page allocated %d\n", doorbell_page);
	225	+
	226	+ ucontext->doorbell = doorbell_page;
	227	+
	228	+ return 0;
	229	+}
	230	+
	231	+void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
	232	+{
	233	+ struct mana_ib_ucontext *mana_ucontext =
	234	+ container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
	235	+ struct ib_device *ibdev = ibcontext->device;
	236	+ struct mana_ib_dev *mdev;
	237	+ struct gdma_context *gc;
	238	+ int ret;
	239	+
	240	+ mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	241	+ gc = mdev->gdma_dev->gdma_context;
	242	+
	243	+ ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
	244	+ if (ret)
	245	+ ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);
	246	+}
	247	+
	248	+static int
	249	+mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
	250	+ struct gdma_context *gc,
	251	+ struct gdma_create_dma_region_req *create_req,
	252	+ size_t num_pages, mana_handle_t *gdma_region)
	253	+{
	254	+ struct gdma_create_dma_region_resp create_resp = {};
	255	+ unsigned int create_req_msg_size;
	256	+ int err;
	257	+
	258	+ create_req_msg_size =
	259	+ struct_size(create_req, page_addr_list, num_pages);
	260	+ create_req->page_addr_list_len = num_pages;
	261	+
	262	+ err = mana_gd_send_request(gc, create_req_msg_size, create_req,
	263	+ sizeof(create_resp), &create_resp);
	264	+ if (err \|\| create_resp.hdr.status) {
	265	+ ibdev_dbg(&dev->ib_dev,
	266	+ "Failed to create DMA region: %d, 0x%x\n",
	267	+ err, create_resp.hdr.status);
	268	+ if (!err)
	269	+ err = -EPROTO;
	270	+
	271	+ return err;
	272	+ }
	273	+
	274	+ *gdma_region = create_resp.dma_region_handle;
	275	+ ibdev_dbg(&dev->ib_dev, "Created DMA region handle 0x%llx\n",
	276	+ *gdma_region);
	277	+
	278	+ return 0;
	279	+}
	280	+
	281	+static int
	282	+mana_ib_gd_add_dma_region(struct mana_ib_dev dev, struct gdma_context gc,
	283	+ struct gdma_dma_region_add_pages_req *add_req,
	284	+ unsigned int num_pages, u32 expected_status)
	285	+{
	286	+ unsigned int add_req_msg_size =
	287	+ struct_size(add_req, page_addr_list, num_pages);
	288	+ struct gdma_general_resp add_resp = {};
	289	+ int err;
	290	+
	291	+ mana_gd_init_req_hdr(&add_req->hdr, GDMA_DMA_REGION_ADD_PAGES,
	292	+ add_req_msg_size, sizeof(add_resp));
	293	+ add_req->page_addr_list_len = num_pages;
	294	+
	295	+ err = mana_gd_send_request(gc, add_req_msg_size, add_req,
	296	+ sizeof(add_resp), &add_resp);
	297	+ if (err \|\| add_resp.hdr.status != expected_status) {
	298	+ ibdev_dbg(&dev->ib_dev,
	299	+ "Failed to create DMA region: %d, 0x%x\n",
	300	+ err, add_resp.hdr.status);
	301	+
	302	+ if (!err)
	303	+ err = -EPROTO;
	304	+
	305	+ return err;
	306	+ }
	307	+
	308	+ return 0;
	309	+}
	310	+
	311	+int mana_ib_gd_create_dma_region(struct mana_ib_dev dev, struct ib_umem umem,
	312	+ mana_handle_t *gdma_region)
	313	+{
	314	+ struct gdma_dma_region_add_pages_req *add_req = NULL;
	315	+ size_t num_pages_processed = 0, num_pages_to_handle;
	316	+ struct gdma_create_dma_region_req *create_req;
	317	+ unsigned int create_req_msg_size;
	318	+ struct hw_channel_context *hwc;
	319	+ struct ib_block_iter biter;
	320	+ size_t max_pgs_add_cmd = 0;
	321	+ size_t max_pgs_create_cmd;
	322	+ struct gdma_context *gc;
	323	+ size_t num_pages_total;
	324	+ struct gdma_dev *mdev;
	325	+ unsigned long page_sz;
	326	+ unsigned int tail = 0;
	327	+ u64 *page_addr_list;
	328	+ void *request_buf;
	329	+ int err;
	330	+
	331	+ mdev = dev->gdma_dev;
	332	+ gc = mdev->gdma_context;
	333	+ hwc = gc->hwc.driver_data;
	334	+
	335	+ /* Hardware requires dma region to align to chosen page size */
	336	+ page_sz = ib_umem_find_best_pgsz(umem, PAGE_SZ_BM, 0);
	337	+ if (!page_sz) {
	338	+ ibdev_dbg(&dev->ib_dev, "failed to find page size.\n");
	339	+ return -ENOMEM;
	340	+ }
	341	+ num_pages_total = ib_umem_num_dma_blocks(umem, page_sz);
	342	+
	343	+ max_pgs_create_cmd =
	344	+ (hwc->max_req_msg_size - sizeof(*create_req)) / sizeof(u64);
	345	+ num_pages_to_handle =
	346	+ min_t(size_t, num_pages_total, max_pgs_create_cmd);
	347	+ create_req_msg_size =
	348	+ struct_size(create_req, page_addr_list, num_pages_to_handle);
	349	+
	350	+ request_buf = kzalloc(hwc->max_req_msg_size, GFP_KERNEL);
	351	+ if (!request_buf)
	352	+ return -ENOMEM;
	353	+
	354	+ create_req = request_buf;
	355	+ mana_gd_init_req_hdr(&create_req->hdr, GDMA_CREATE_DMA_REGION,
	356	+ create_req_msg_size,
	357	+ sizeof(struct gdma_create_dma_region_resp));
	358	+
	359	+ create_req->length = umem->length;
	360	+ create_req->offset_in_page = umem->address & (page_sz - 1);
	361	+ create_req->gdma_page_type = order_base_2(page_sz) - PAGE_SHIFT;
	362	+ create_req->page_count = num_pages_total;
	363	+
	364	+ ibdev_dbg(&dev->ib_dev, "size_dma_region %lu num_pages_total %lu\n",
	365	+ umem->length, num_pages_total);
	366	+
	367	+ ibdev_dbg(&dev->ib_dev, "page_sz %lu offset_in_page %u\n",
	368	+ page_sz, create_req->offset_in_page);
	369	+
	370	+ ibdev_dbg(&dev->ib_dev, "num_pages_to_handle %lu, gdma_page_type %u",
	371	+ num_pages_to_handle, create_req->gdma_page_type);
	372	+
	373	+ page_addr_list = create_req->page_addr_list;
	374	+ rdma_umem_for_each_dma_block(umem, &biter, page_sz) {
	375	+ page_addr_list[tail++] = rdma_block_iter_dma_address(&biter);
	376	+ if (tail < num_pages_to_handle)
	377	+ continue;
	378	+
	379	+ if (!num_pages_processed) {
	380	+ /* First create message */
	381	+ err = mana_ib_gd_first_dma_region(dev, gc, create_req,
	382	+ tail, gdma_region);
	383	+ if (err)
	384	+ goto out;
	385	+
	386	+ max_pgs_add_cmd = (hwc->max_req_msg_size -
	387	+ sizeof(*add_req)) / sizeof(u64);
	388	+
	389	+ add_req = request_buf;
	390	+ add_req->dma_region_handle = *gdma_region;
	391	+ add_req->reserved3 = 0;
	392	+ page_addr_list = add_req->page_addr_list;
	393	+ } else {
	394	+ /* Subsequent create messages */
	395	+ u32 expected_s = 0;
	396	+
	397	+ if (num_pages_processed + num_pages_to_handle <
	398	+ num_pages_total)
	399	+ expected_s = GDMA_STATUS_MORE_ENTRIES;
	400	+
	401	+ err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail,
	402	+ expected_s);
	403	+ if (err)
	404	+ break;
	405	+ }
	406	+
	407	+ num_pages_processed += tail;
	408	+ tail = 0;
	409	+
	410	+ /* The remaining pages to create */
	411	+ num_pages_to_handle =
	412	+ min_t(size_t,
	413	+ num_pages_total - num_pages_processed,
	414	+ max_pgs_add_cmd);
	415	+ }
	416	+
	417	+ if (err)
	418	+ mana_ib_gd_destroy_dma_region(dev, *gdma_region);
	419	+
	420	+out:
	421	+ kfree(request_buf);
	422	+ return err;
	423	+}
	424	+
	425	+int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
	426	+{
	427	+ struct gdma_dev *mdev = dev->gdma_dev;
	428	+ struct gdma_context *gc;
	429	+
	430	+ gc = mdev->gdma_context;
	431	+ ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
	432	+
	433	+ return mana_gd_destroy_dma_region(gc, gdma_region);
	434	+}
	435	+
	436	+int mana_ib_mmap(struct ib_ucontext ibcontext, struct vm_area_struct vma)
	437	+{
	438	+ struct mana_ib_ucontext *mana_ucontext =
	439	+ container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
	440	+ struct ib_device *ibdev = ibcontext->device;
	441	+ struct mana_ib_dev *mdev;
	442	+ struct gdma_context *gc;
	443	+ phys_addr_t pfn;
	444	+ pgprot_t prot;
	445	+ int ret;
	446	+
	447	+ mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	448	+ gc = mdev->gdma_dev->gdma_context;
	449	+
	450	+ if (vma->vm_pgoff != 0) {
	451	+ ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
	452	+ return -EINVAL;
	453	+ }
	454	+
	455	+ /* Map to the page indexed by ucontext->doorbell */
	456	+ pfn = (gc->phys_db_page_base +
	457	+ gc->db_page_size * mana_ucontext->doorbell) >>
	458	+ PAGE_SHIFT;
	459	+ prot = pgprot_writecombine(vma->vm_page_prot);
	460	+
	461	+ ret = rdma_user_mmap_io(ibcontext, vma, pfn, gc->db_page_size, prot,
	462	+ NULL);
	463	+ if (ret)
	464	+ ibdev_dbg(ibdev, "can't rdma_user_mmap_io ret %d\n", ret);
	465	+ else
	466	+ ibdev_dbg(ibdev, "mapped I/O pfn 0x%llx page_size %u, ret %d\n",
	467	+ pfn, gc->db_page_size, ret);
	468	+
	469	+ return ret;
	470	+}
	471	+
	472	+int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
	473	+ struct ib_port_immutable *immutable)
	474	+{
	475	+ /*
	476	+ * This version only support RAW_PACKET
	477	+ * other values need to be filled for other types
	478	+ */
	479	+ immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
	480	+
	481	+ return 0;
	482	+}
	483	+
	484	+int mana_ib_query_device(struct ib_device ibdev, struct ib_device_attr props,
	485	+ struct ib_udata *uhw)
	486	+{
	487	+ props->max_qp = MANA_MAX_NUM_QUEUES;
	488	+ props->max_qp_wr = MAX_SEND_BUFFERS_PER_QUEUE;
	489	+
	490	+ /*
	491	+ * max_cqe could be potentially much bigger.
	492	+ * As this version of driver only support RAW QP, set it to the same
	493	+ * value as max_qp_wr
	494	+ */
	495	+ props->max_cqe = MAX_SEND_BUFFERS_PER_QUEUE;
	496	+
	497	+ props->max_mr_size = MANA_IB_MAX_MR_SIZE;
	498	+ props->max_mr = MANA_IB_MAX_MR;
	499	+ props->max_send_sge = MAX_TX_WQE_SGL_ENTRIES;
	500	+ props->max_recv_sge = MAX_RX_WQE_SGL_ENTRIES;
	501	+
	502	+ return 0;
	503	+}
	504	+
	505	+int mana_ib_query_port(struct ib_device *ibdev, u32 port,
	506	+ struct ib_port_attr *props)
	507	+{
	508	+ /* This version doesn't return port properties */
	509	+ return 0;
	510	+}
	511	+
	512	+int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
	513	+ union ib_gid *gid)
	514	+{
	515	+ /* This version doesn't return GID properties */
	516	+ return 0;
	517	+}
	518	+
	519	+void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
	520	+{
	521	+}

drivers/infiniband/hw/mana/mana_ib.h+162−0

@@ -0,0 +1,162 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
	2	+/*
	3	+ * Copyright (c) 2022 Microsoft Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#ifndef _MANA_IB_H_
	7	+#define _MANA_IB_H_
	8	+
	9	+#include <rdma/ib_verbs.h>
	10	+#include <rdma/ib_mad.h>
	11	+#include <rdma/ib_umem.h>
	12	+#include <rdma/mana-abi.h>
	13	+#include <rdma/uverbs_ioctl.h>
	14	+
	15	+#include <net/mana/mana.h>
	16	+
	17	+#define PAGE_SZ_BM \
	18	+ (SZ_4K \| SZ_8K \| SZ_16K \| SZ_32K \| SZ_64K \| SZ_128K \| SZ_256K \| \
	19	+ SZ_512K \| SZ_1M \| SZ_2M)
	20	+
	21	+/* MANA doesn't have any limit for MR size */
	22	+#define MANA_IB_MAX_MR_SIZE U64_MAX
	23	+
	24	+/*
	25	+ * The hardware limit of number of MRs is greater than maximum number of MRs
	26	+ * that can possibly represent in 24 bits
	27	+ */
	28	+#define MANA_IB_MAX_MR 0xFFFFFFu
	29	+
	30	+struct mana_ib_dev {
	31	+ struct ib_device ib_dev;
	32	+ struct gdma_dev *gdma_dev;
	33	+};
	34	+
	35	+struct mana_ib_wq {
	36	+ struct ib_wq ibwq;
	37	+ struct ib_umem *umem;
	38	+ int wqe;
	39	+ u32 wq_buf_size;
	40	+ u64 gdma_region;
	41	+ u64 id;
	42	+ mana_handle_t rx_object;
	43	+};
	44	+
	45	+struct mana_ib_pd {
	46	+ struct ib_pd ibpd;
	47	+ u32 pdn;
	48	+ mana_handle_t pd_handle;
	49	+
	50	+ /* Mutex for sharing access to vport_use_count */
	51	+ struct mutex vport_mutex;
	52	+ int vport_use_count;
	53	+
	54	+ bool tx_shortform_allowed;
	55	+ u32 tx_vp_offset;
	56	+};
	57	+
	58	+struct mana_ib_mr {
	59	+ struct ib_mr ibmr;
	60	+ struct ib_umem *umem;
	61	+ mana_handle_t mr_handle;
	62	+};
	63	+
	64	+struct mana_ib_cq {
	65	+ struct ib_cq ibcq;
	66	+ struct ib_umem *umem;
	67	+ int cqe;
	68	+ u64 gdma_region;
	69	+ u64 id;
	70	+};
	71	+
	72	+struct mana_ib_qp {
	73	+ struct ib_qp ibqp;
	74	+
	75	+ /* Work queue info */
	76	+ struct ib_umem *sq_umem;
	77	+ int sqe;
	78	+ u64 sq_gdma_region;
	79	+ u64 sq_id;
	80	+ mana_handle_t tx_object;
	81	+
	82	+ /* The port on the IB device, starting with 1 */
	83	+ u32 port;
	84	+};
	85	+
	86	+struct mana_ib_ucontext {
	87	+ struct ib_ucontext ibucontext;
	88	+ u32 doorbell;
	89	+};
	90	+
	91	+struct mana_ib_rwq_ind_table {
	92	+ struct ib_rwq_ind_table ib_ind_table;
	93	+};
	94	+
	95	+int mana_ib_gd_create_dma_region(struct mana_ib_dev dev, struct ib_umem umem,
	96	+ mana_handle_t *gdma_region);
	97	+
	98	+int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
	99	+ mana_handle_t gdma_region);
	100	+
	101	+struct ib_wq mana_ib_create_wq(struct ib_pd pd,
	102	+ struct ib_wq_init_attr *init_attr,
	103	+ struct ib_udata *udata);
	104	+
	105	+int mana_ib_modify_wq(struct ib_wq wq, struct ib_wq_attr wq_attr,
	106	+ u32 wq_attr_mask, struct ib_udata *udata);
	107	+
	108	+int mana_ib_destroy_wq(struct ib_wq ibwq, struct ib_udata udata);
	109	+
	110	+int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
	111	+ struct ib_rwq_ind_table_init_attr *init_attr,
	112	+ struct ib_udata *udata);
	113	+
	114	+int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl);
	115	+
	116	+struct ib_mr mana_ib_get_dma_mr(struct ib_pd ibpd, int access_flags);
	117	+
	118	+struct ib_mr mana_ib_reg_user_mr(struct ib_pd pd, u64 start, u64 length,
	119	+ u64 iova, int access_flags,
	120	+ struct ib_udata *udata);
	121	+
	122	+int mana_ib_dereg_mr(struct ib_mr ibmr, struct ib_udata udata);
	123	+
	124	+int mana_ib_create_qp(struct ib_qp qp, struct ib_qp_init_attr qp_init_attr,
	125	+ struct ib_udata *udata);
	126	+
	127	+int mana_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
	128	+ int attr_mask, struct ib_udata *udata);
	129	+
	130	+int mana_ib_destroy_qp(struct ib_qp ibqp, struct ib_udata udata);
	131	+
	132	+int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id,
	133	+ struct mana_ib_pd *pd, u32 doorbell_id);
	134	+void mana_ib_uncfg_vport(struct mana_ib_dev dev, struct mana_ib_pd pd,
	135	+ u32 port);
	136	+
	137	+int mana_ib_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
	138	+ struct ib_udata *udata);
	139	+
	140	+int mana_ib_destroy_cq(struct ib_cq ibcq, struct ib_udata udata);
	141	+
	142	+int mana_ib_alloc_pd(struct ib_pd ibpd, struct ib_udata udata);
	143	+int mana_ib_dealloc_pd(struct ib_pd ibpd, struct ib_udata udata);
	144	+
	145	+int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
	146	+ struct ib_udata *udata);
	147	+void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);
	148	+
	149	+int mana_ib_mmap(struct ib_ucontext ibcontext, struct vm_area_struct vma);
	150	+
	151	+int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
	152	+ struct ib_port_immutable *immutable);
	153	+int mana_ib_query_device(struct ib_device ibdev, struct ib_device_attr props,
	154	+ struct ib_udata *uhw);
	155	+int mana_ib_query_port(struct ib_device *ibdev, u32 port,
	156	+ struct ib_port_attr *props);
	157	+int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
	158	+ union ib_gid *gid);
	159	+
	160	+void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);
	161	+
	162	+#endif

drivers/infiniband/hw/mana/mr.c+197−0

@@ -0,0 +1,197 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#include "mana_ib.h"
	7	+
	8	+#define VALID_MR_FLAGS \
	9	+ (IB_ACCESS_LOCAL_WRITE \| IB_ACCESS_REMOTE_WRITE \| IB_ACCESS_REMOTE_READ)
	10	+
	11	+static enum gdma_mr_access_flags
	12	+mana_ib_verbs_to_gdma_access_flags(int access_flags)
	13	+{
	14	+ enum gdma_mr_access_flags flags = GDMA_ACCESS_FLAG_LOCAL_READ;
	15	+
	16	+ if (access_flags & IB_ACCESS_LOCAL_WRITE)
	17	+ flags \|= GDMA_ACCESS_FLAG_LOCAL_WRITE;
	18	+
	19	+ if (access_flags & IB_ACCESS_REMOTE_WRITE)
	20	+ flags \|= GDMA_ACCESS_FLAG_REMOTE_WRITE;
	21	+
	22	+ if (access_flags & IB_ACCESS_REMOTE_READ)
	23	+ flags \|= GDMA_ACCESS_FLAG_REMOTE_READ;
	24	+
	25	+ return flags;
	26	+}
	27	+
	28	+static int mana_ib_gd_create_mr(struct mana_ib_dev dev, struct mana_ib_mr mr,
	29	+ struct gdma_create_mr_params *mr_params)
	30	+{
	31	+ struct gdma_create_mr_response resp = {};
	32	+ struct gdma_create_mr_request req = {};
	33	+ struct gdma_dev *mdev = dev->gdma_dev;
	34	+ struct gdma_context *gc;
	35	+ int err;
	36	+
	37	+ gc = mdev->gdma_context;
	38	+
	39	+ mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
	40	+ sizeof(resp));
	41	+ req.pd_handle = mr_params->pd_handle;
	42	+ req.mr_type = mr_params->mr_type;
	43	+
	44	+ switch (mr_params->mr_type) {
	45	+ case GDMA_MR_TYPE_GVA:
	46	+ req.gva.dma_region_handle = mr_params->gva.dma_region_handle;
	47	+ req.gva.virtual_address = mr_params->gva.virtual_address;
	48	+ req.gva.access_flags = mr_params->gva.access_flags;
	49	+ break;
	50	+
	51	+ default:
	52	+ ibdev_dbg(&dev->ib_dev,
	53	+ "invalid param (GDMA_MR_TYPE) passed, type %d\n",
	54	+ req.mr_type);
	55	+ return -EINVAL;
	56	+ }
	57	+
	58	+ err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
	59	+
	60	+ if (err \|\| resp.hdr.status) {
	61	+ ibdev_dbg(&dev->ib_dev, "Failed to create mr %d, %u", err,
	62	+ resp.hdr.status);
	63	+ if (!err)
	64	+ err = -EPROTO;
	65	+
	66	+ return err;
	67	+ }
	68	+
	69	+ mr->ibmr.lkey = resp.lkey;
	70	+ mr->ibmr.rkey = resp.rkey;
	71	+ mr->mr_handle = resp.mr_handle;
	72	+
	73	+ return 0;
	74	+}
	75	+
	76	+static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
	77	+{
	78	+ struct gdma_destroy_mr_response resp = {};
	79	+ struct gdma_destroy_mr_request req = {};
	80	+ struct gdma_dev *mdev = dev->gdma_dev;
	81	+ struct gdma_context *gc;
	82	+ int err;
	83	+
	84	+ gc = mdev->gdma_context;
	85	+
	86	+ mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
	87	+ sizeof(resp));
	88	+
	89	+ req.mr_handle = mr_handle;
	90	+
	91	+ err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
	92	+ if (err \|\| resp.hdr.status) {
	93	+ dev_err(gc->dev, "Failed to destroy MR: %d, 0x%x\n", err,
	94	+ resp.hdr.status);
	95	+ if (!err)
	96	+ err = -EPROTO;
	97	+ return err;
	98	+ }
	99	+
	100	+ return 0;
	101	+}
	102	+
	103	+struct ib_mr mana_ib_reg_user_mr(struct ib_pd ibpd, u64 start, u64 length,
	104	+ u64 iova, int access_flags,
	105	+ struct ib_udata *udata)
	106	+{
	107	+ struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	108	+ struct gdma_create_mr_params mr_params = {};
	109	+ struct ib_device *ibdev = ibpd->device;
	110	+ struct mana_ib_dev *dev;
	111	+ struct mana_ib_mr *mr;
	112	+ u64 dma_region_handle;
	113	+ int err;
	114	+
	115	+ dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	116	+
	117	+ ibdev_dbg(ibdev,
	118	+ "start 0x%llx, iova 0x%llx length 0x%llx access_flags 0x%x",
	119	+ start, iova, length, access_flags);
	120	+
	121	+ if (access_flags & ~VALID_MR_FLAGS)
	122	+ return ERR_PTR(-EINVAL);
	123	+
	124	+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
	125	+ if (!mr)
	126	+ return ERR_PTR(-ENOMEM);
	127	+
	128	+ mr->umem = ib_umem_get(ibdev, start, length, access_flags);
	129	+ if (IS_ERR(mr->umem)) {
	130	+ err = PTR_ERR(mr->umem);
	131	+ ibdev_dbg(ibdev,
	132	+ "Failed to get umem for register user-mr, %d\n", err);
	133	+ goto err_free;
	134	+ }
	135	+
	136	+ err = mana_ib_gd_create_dma_region(dev, mr->umem, &dma_region_handle);
	137	+ if (err) {
	138	+ ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n",
	139	+ err);
	140	+ goto err_umem;
	141	+ }
	142	+
	143	+ ibdev_dbg(ibdev,
	144	+ "mana_ib_gd_create_dma_region ret %d gdma_region %llx\n", err,
	145	+ dma_region_handle);
	146	+
	147	+ mr_params.pd_handle = pd->pd_handle;
	148	+ mr_params.mr_type = GDMA_MR_TYPE_GVA;
	149	+ mr_params.gva.dma_region_handle = dma_region_handle;
	150	+ mr_params.gva.virtual_address = iova;
	151	+ mr_params.gva.access_flags =
	152	+ mana_ib_verbs_to_gdma_access_flags(access_flags);
	153	+
	154	+ err = mana_ib_gd_create_mr(dev, mr, &mr_params);
	155	+ if (err)
	156	+ goto err_dma_region;
	157	+
	158	+ /*
	159	+ * There is no need to keep track of dma_region_handle after MR is
	160	+ * successfully created. The dma_region_handle is tracked in the PF
	161	+ * as part of the lifecycle of this MR.
	162	+ */
	163	+
	164	+ return &mr->ibmr;
	165	+
	166	+err_dma_region:
	167	+ mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
	168	+ dma_region_handle);
	169	+
	170	+err_umem:
	171	+ ib_umem_release(mr->umem);
	172	+
	173	+err_free:
	174	+ kfree(mr);
	175	+ return ERR_PTR(err);
	176	+}
	177	+
	178	+int mana_ib_dereg_mr(struct ib_mr ibmr, struct ib_udata udata)
	179	+{
	180	+ struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);
	181	+ struct ib_device *ibdev = ibmr->device;
	182	+ struct mana_ib_dev *dev;
	183	+ int err;
	184	+
	185	+ dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	186	+
	187	+ err = mana_ib_gd_destroy_mr(dev, mr->mr_handle);
	188	+ if (err)
	189	+ return err;
	190	+
	191	+ if (mr->umem)
	192	+ ib_umem_release(mr->umem);
	193	+
	194	+ kfree(mr);
	195	+
	196	+ return 0;
	197	+}

drivers/infiniband/hw/mana/qp.c+506−0

@@ -0,0 +1,506 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#include "mana_ib.h"
	7	+
	8	+static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
	9	+ struct net_device *ndev,
	10	+ mana_handle_t default_rxobj,
	11	+ mana_handle_t ind_table[],
	12	+ u32 log_ind_tbl_size, u32 rx_hash_key_len,
	13	+ u8 *rx_hash_key)
	14	+{
	15	+ struct mana_port_context *mpc = netdev_priv(ndev);
	16	+ struct mana_cfg_rx_steer_req *req = NULL;
	17	+ struct mana_cfg_rx_steer_resp resp = {};
	18	+ mana_handle_t *req_indir_tab;
	19	+ struct gdma_context *gc;
	20	+ struct gdma_dev *mdev;
	21	+ u32 req_buf_size;
	22	+ int i, err;
	23	+
	24	+ mdev = dev->gdma_dev;
	25	+ gc = mdev->gdma_context;
	26	+
	27	+ req_buf_size =
	28	+ sizeof(req) + sizeof(mana_handle_t) MANA_INDIRECT_TABLE_SIZE;
	29	+ req = kzalloc(req_buf_size, GFP_KERNEL);
	30	+ if (!req)
	31	+ return -ENOMEM;
	32	+
	33	+ mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
	34	+ sizeof(resp));
	35	+
	36	+ req->vport = mpc->port_handle;
	37	+ req->rx_enable = 1;
	38	+ req->update_default_rxobj = 1;
	39	+ req->default_rxobj = default_rxobj;
	40	+ req->hdr.dev_id = mdev->dev_id;
	41	+
	42	+ /* If there are more than 1 entries in indirection table, enable RSS */
	43	+ if (log_ind_tbl_size)
	44	+ req->rss_enable = true;
	45	+
	46	+ req->num_indir_entries = MANA_INDIRECT_TABLE_SIZE;
	47	+ req->indir_tab_offset = sizeof(*req);
	48	+ req->update_indir_tab = true;
	49	+
	50	+ req_indir_tab = (mana_handle_t *)(req + 1);
	51	+ /* The ind table passed to the hardware must have
	52	+ * MANA_INDIRECT_TABLE_SIZE entries. Adjust the verb
	53	+ * ind_table to MANA_INDIRECT_TABLE_SIZE if required
	54	+ */
	55	+ ibdev_dbg(&dev->ib_dev, "ind table size %u\n", 1 << log_ind_tbl_size);
	56	+ for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) {
	57	+ req_indir_tab[i] = ind_table[i % (1 << log_ind_tbl_size)];
	58	+ ibdev_dbg(&dev->ib_dev, "index %u handle 0x%llx\n", i,
	59	+ req_indir_tab[i]);
	60	+ }
	61	+
	62	+ req->update_hashkey = true;
	63	+ if (rx_hash_key_len)
	64	+ memcpy(req->hashkey, rx_hash_key, rx_hash_key_len);
	65	+ else
	66	+ netdev_rss_key_fill(req->hashkey, MANA_HASH_KEY_SIZE);
	67	+
	68	+ ibdev_dbg(&dev->ib_dev, "vport handle %llu default_rxobj 0x%llx\n",
	69	+ req->vport, default_rxobj);
	70	+
	71	+ err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &resp);
	72	+ if (err) {
	73	+ netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);
	74	+ goto out;
	75	+ }
	76	+
	77	+ if (resp.hdr.status) {
	78	+ netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
	79	+ resp.hdr.status);
	80	+ err = -EPROTO;
	81	+ goto out;
	82	+ }
	83	+
	84	+ netdev_info(ndev, "Configured steering vPort %llu log_entries %u\n",
	85	+ mpc->port_handle, log_ind_tbl_size);
	86	+
	87	+out:
	88	+ kfree(req);
	89	+ return err;
	90	+}
	91	+
	92	+static int mana_ib_create_qp_rss(struct ib_qp ibqp, struct ib_pd pd,
	93	+ struct ib_qp_init_attr *attr,
	94	+ struct ib_udata *udata)
	95	+{
	96	+ struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
	97	+ struct mana_ib_dev *mdev =
	98	+ container_of(pd->device, struct mana_ib_dev, ib_dev);
	99	+ struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl;
	100	+ struct mana_ib_create_qp_rss_resp resp = {};
	101	+ struct mana_ib_create_qp_rss ucmd = {};
	102	+ struct gdma_dev *gd = mdev->gdma_dev;
	103	+ mana_handle_t *mana_ind_table;
	104	+ struct mana_port_context *mpc;
	105	+ struct mana_context *mc;
	106	+ struct net_device *ndev;
	107	+ struct mana_ib_cq *cq;
	108	+ struct mana_ib_wq *wq;
	109	+ unsigned int ind_tbl_size;
	110	+ struct ib_cq *ibcq;
	111	+ struct ib_wq *ibwq;
	112	+ int i = 0;
	113	+ u32 port;
	114	+ int ret;
	115	+
	116	+ mc = gd->driver_data;
	117	+
	118	+ if (!udata \|\| udata->inlen < sizeof(ucmd))
	119	+ return -EINVAL;
	120	+
	121	+ ret = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
	122	+ if (ret) {
	123	+ ibdev_dbg(&mdev->ib_dev,
	124	+ "Failed copy from udata for create rss-qp, err %d\n",
	125	+ ret);
	126	+ return ret;
	127	+ }
	128	+
	129	+ if (attr->cap.max_recv_wr > MAX_SEND_BUFFERS_PER_QUEUE) {
	130	+ ibdev_dbg(&mdev->ib_dev,
	131	+ "Requested max_recv_wr %d exceeding limit\n",
	132	+ attr->cap.max_recv_wr);
	133	+ return -EINVAL;
	134	+ }
	135	+
	136	+ if (attr->cap.max_recv_sge > MAX_RX_WQE_SGL_ENTRIES) {
	137	+ ibdev_dbg(&mdev->ib_dev,
	138	+ "Requested max_recv_sge %d exceeding limit\n",
	139	+ attr->cap.max_recv_sge);
	140	+ return -EINVAL;
	141	+ }
	142	+
	143	+ ind_tbl_size = 1 << ind_tbl->log_ind_tbl_size;
	144	+ if (ind_tbl_size > MANA_INDIRECT_TABLE_SIZE) {
	145	+ ibdev_dbg(&mdev->ib_dev,
	146	+ "Indirect table size %d exceeding limit\n",
	147	+ ind_tbl_size);
	148	+ return -EINVAL;
	149	+ }
	150	+
	151	+ if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) {
	152	+ ibdev_dbg(&mdev->ib_dev,
	153	+ "RX Hash function is not supported, %d\n",
	154	+ ucmd.rx_hash_function);
	155	+ return -EINVAL;
	156	+ }
	157	+
	158	+ /* IB ports start with 1, MANA start with 0 */
	159	+ port = ucmd.port;
	160	+ if (port < 1 \|\| port > mc->num_ports) {
	161	+ ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n",
	162	+ port);
	163	+ return -EINVAL;
	164	+ }
	165	+ ndev = mc->ports[port - 1];
	166	+ mpc = netdev_priv(ndev);
	167	+
	168	+ ibdev_dbg(&mdev->ib_dev, "rx_hash_function %d port %d\n",
	169	+ ucmd.rx_hash_function, port);
	170	+
	171	+ mana_ind_table = kcalloc(ind_tbl_size, sizeof(mana_handle_t),
	172	+ GFP_KERNEL);
	173	+ if (!mana_ind_table) {
	174	+ ret = -ENOMEM;
	175	+ goto fail;
	176	+ }
	177	+
	178	+ qp->port = port;
	179	+
	180	+ for (i = 0; i < ind_tbl_size; i++) {
	181	+ struct mana_obj_spec wq_spec = {};
	182	+ struct mana_obj_spec cq_spec = {};
	183	+
	184	+ ibwq = ind_tbl->ind_tbl[i];
	185	+ wq = container_of(ibwq, struct mana_ib_wq, ibwq);
	186	+
	187	+ ibcq = ibwq->cq;
	188	+ cq = container_of(ibcq, struct mana_ib_cq, ibcq);
	189	+
	190	+ wq_spec.gdma_region = wq->gdma_region;
	191	+ wq_spec.queue_size = wq->wq_buf_size;
	192	+
	193	+ cq_spec.gdma_region = cq->gdma_region;
	194	+ cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE;
	195	+ cq_spec.modr_ctx_id = 0;
	196	+ cq_spec.attached_eq = GDMA_CQ_NO_EQ;
	197	+
	198	+ ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ,
	199	+ &wq_spec, &cq_spec, &wq->rx_object);
	200	+ if (ret)
	201	+ goto fail;
	202	+
	203	+ /* The GDMA regions are now owned by the WQ object */
	204	+ wq->gdma_region = GDMA_INVALID_DMA_REGION;
	205	+ cq->gdma_region = GDMA_INVALID_DMA_REGION;
	206	+
	207	+ wq->id = wq_spec.queue_index;
	208	+ cq->id = cq_spec.queue_index;
	209	+
	210	+ ibdev_dbg(&mdev->ib_dev,
	211	+ "ret %d rx_object 0x%llx wq id %llu cq id %llu\n",
	212	+ ret, wq->rx_object, wq->id, cq->id);
	213	+
	214	+ resp.entries[i].cqid = cq->id;
	215	+ resp.entries[i].wqid = wq->id;
	216	+
	217	+ mana_ind_table[i] = wq->rx_object;
	218	+ }
	219	+ resp.num_entries = i;
	220	+
	221	+ ret = mana_ib_cfg_vport_steering(mdev, ndev, wq->rx_object,
	222	+ mana_ind_table,
	223	+ ind_tbl->log_ind_tbl_size,
	224	+ ucmd.rx_hash_key_len,
	225	+ ucmd.rx_hash_key);
	226	+ if (ret)
	227	+ goto fail;
	228	+
	229	+ ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
	230	+ if (ret) {
	231	+ ibdev_dbg(&mdev->ib_dev,
	232	+ "Failed to copy to udata create rss-qp, %d\n",
	233	+ ret);
	234	+ goto fail;
	235	+ }
	236	+
	237	+ kfree(mana_ind_table);
	238	+
	239	+ return 0;
	240	+
	241	+fail:
	242	+ while (i-- > 0) {
	243	+ ibwq = ind_tbl->ind_tbl[i];
	244	+ wq = container_of(ibwq, struct mana_ib_wq, ibwq);
	245	+ mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
	246	+ }
	247	+
	248	+ kfree(mana_ind_table);
	249	+
	250	+ return ret;
	251	+}
	252	+
	253	+static int mana_ib_create_qp_raw(struct ib_qp ibqp, struct ib_pd ibpd,
	254	+ struct ib_qp_init_attr *attr,
	255	+ struct ib_udata *udata)
	256	+{
	257	+ struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	258	+ struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
	259	+ struct mana_ib_dev *mdev =
	260	+ container_of(ibpd->device, struct mana_ib_dev, ib_dev);
	261	+ struct mana_ib_cq *send_cq =
	262	+ container_of(attr->send_cq, struct mana_ib_cq, ibcq);
	263	+ struct mana_ib_ucontext *mana_ucontext =
	264	+ rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
	265	+ ibucontext);
	266	+ struct mana_ib_create_qp_resp resp = {};
	267	+ struct gdma_dev *gd = mdev->gdma_dev;
	268	+ struct mana_ib_create_qp ucmd = {};
	269	+ struct mana_obj_spec wq_spec = {};
	270	+ struct mana_obj_spec cq_spec = {};
	271	+ struct mana_port_context *mpc;
	272	+ struct mana_context *mc;
	273	+ struct net_device *ndev;
	274	+ struct ib_umem *umem;
	275	+ int err;
	276	+ u32 port;
	277	+
	278	+ mc = gd->driver_data;
	279	+
	280	+ if (!mana_ucontext \|\| udata->inlen < sizeof(ucmd))
	281	+ return -EINVAL;
	282	+
	283	+ err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
	284	+ if (err) {
	285	+ ibdev_dbg(&mdev->ib_dev,
	286	+ "Failed to copy from udata create qp-raw, %d\n", err);
	287	+ return err;
	288	+ }
	289	+
	290	+ /* IB ports start with 1, MANA Ethernet ports start with 0 */
	291	+ port = ucmd.port;
	292	+ if (ucmd.port > mc->num_ports)
	293	+ return -EINVAL;
	294	+
	295	+ if (attr->cap.max_send_wr > MAX_SEND_BUFFERS_PER_QUEUE) {
	296	+ ibdev_dbg(&mdev->ib_dev,
	297	+ "Requested max_send_wr %d exceeding limit\n",
	298	+ attr->cap.max_send_wr);
	299	+ return -EINVAL;
	300	+ }
	301	+
	302	+ if (attr->cap.max_send_sge > MAX_TX_WQE_SGL_ENTRIES) {
	303	+ ibdev_dbg(&mdev->ib_dev,
	304	+ "Requested max_send_sge %d exceeding limit\n",
	305	+ attr->cap.max_send_sge);
	306	+ return -EINVAL;
	307	+ }
	308	+
	309	+ ndev = mc->ports[port - 1];
	310	+ mpc = netdev_priv(ndev);
	311	+ ibdev_dbg(&mdev->ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc);
	312	+
	313	+ err = mana_ib_cfg_vport(mdev, port - 1, pd, mana_ucontext->doorbell);
	314	+ if (err)
	315	+ return -ENODEV;
	316	+
	317	+ qp->port = port;
	318	+
	319	+ ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",
	320	+ ucmd.sq_buf_addr, ucmd.port);
	321	+
	322	+ umem = ib_umem_get(ibpd->device, ucmd.sq_buf_addr, ucmd.sq_buf_size,
	323	+ IB_ACCESS_LOCAL_WRITE);
	324	+ if (IS_ERR(umem)) {
	325	+ err = PTR_ERR(umem);
	326	+ ibdev_dbg(&mdev->ib_dev,
	327	+ "Failed to get umem for create qp-raw, err %d\n",
	328	+ err);
	329	+ goto err_free_vport;
	330	+ }
	331	+ qp->sq_umem = umem;
	332	+
	333	+ err = mana_ib_gd_create_dma_region(mdev, qp->sq_umem,
	334	+ &qp->sq_gdma_region);
	335	+ if (err) {
	336	+ ibdev_dbg(&mdev->ib_dev,
	337	+ "Failed to create dma region for create qp-raw, %d\n",
	338	+ err);
	339	+ goto err_release_umem;
	340	+ }
	341	+
	342	+ ibdev_dbg(&mdev->ib_dev,
	343	+ "mana_ib_gd_create_dma_region ret %d gdma_region 0x%llx\n",
	344	+ err, qp->sq_gdma_region);
	345	+
	346	+ /* Create a WQ on the same port handle used by the Ethernet */
	347	+ wq_spec.gdma_region = qp->sq_gdma_region;
	348	+ wq_spec.queue_size = ucmd.sq_buf_size;
	349	+
	350	+ cq_spec.gdma_region = send_cq->gdma_region;
	351	+ cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE;
	352	+ cq_spec.modr_ctx_id = 0;
	353	+ cq_spec.attached_eq = GDMA_CQ_NO_EQ;
	354	+
	355	+ err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec,
	356	+ &cq_spec, &qp->tx_object);
	357	+ if (err) {
	358	+ ibdev_dbg(&mdev->ib_dev,
	359	+ "Failed to create wq for create raw-qp, err %d\n",
	360	+ err);
	361	+ goto err_destroy_dma_region;
	362	+ }
	363	+
	364	+ /* The GDMA regions are now owned by the WQ object */
	365	+ qp->sq_gdma_region = GDMA_INVALID_DMA_REGION;
	366	+ send_cq->gdma_region = GDMA_INVALID_DMA_REGION;
	367	+
	368	+ qp->sq_id = wq_spec.queue_index;
	369	+ send_cq->id = cq_spec.queue_index;
	370	+
	371	+ ibdev_dbg(&mdev->ib_dev,
	372	+ "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
	373	+ qp->tx_object, qp->sq_id, send_cq->id);
	374	+
	375	+ resp.sqid = qp->sq_id;
	376	+ resp.cqid = send_cq->id;
	377	+ resp.tx_vp_offset = pd->tx_vp_offset;
	378	+
	379	+ err = ib_copy_to_udata(udata, &resp, sizeof(resp));
	380	+ if (err) {
	381	+ ibdev_dbg(&mdev->ib_dev,
	382	+ "Failed copy udata for create qp-raw, %d\n",
	383	+ err);
	384	+ goto err_destroy_wq_obj;
	385	+ }
	386	+
	387	+ return 0;
	388	+
	389	+err_destroy_wq_obj:
	390	+ mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
	391	+
	392	+err_destroy_dma_region:
	393	+ mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region);
	394	+
	395	+err_release_umem:
	396	+ ib_umem_release(umem);
	397	+
	398	+err_free_vport:
	399	+ mana_ib_uncfg_vport(mdev, pd, port - 1);
	400	+
	401	+ return err;
	402	+}
	403	+
	404	+int mana_ib_create_qp(struct ib_qp ibqp, struct ib_qp_init_attr attr,
	405	+ struct ib_udata *udata)
	406	+{
	407	+ switch (attr->qp_type) {
	408	+ case IB_QPT_RAW_PACKET:
	409	+ /* When rwq_ind_tbl is used, it's for creating WQs for RSS */
	410	+ if (attr->rwq_ind_tbl)
	411	+ return mana_ib_create_qp_rss(ibqp, ibqp->pd, attr,
	412	+ udata);
	413	+
	414	+ return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
	415	+ default:
	416	+ /* Creating QP other than IB_QPT_RAW_PACKET is not supported */
	417	+ ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n",
	418	+ attr->qp_type);
	419	+ }
	420	+
	421	+ return -EINVAL;
	422	+}
	423	+
	424	+int mana_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
	425	+ int attr_mask, struct ib_udata *udata)
	426	+{
	427	+ /* modify_qp is not supported by this version of the driver */
	428	+ return -EOPNOTSUPP;
	429	+}
	430	+
	431	+static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
	432	+ struct ib_rwq_ind_table *ind_tbl,
	433	+ struct ib_udata *udata)
	434	+{
	435	+ struct mana_ib_dev *mdev =
	436	+ container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
	437	+ struct gdma_dev *gd = mdev->gdma_dev;
	438	+ struct mana_port_context *mpc;
	439	+ struct mana_context *mc;
	440	+ struct net_device *ndev;
	441	+ struct mana_ib_wq *wq;
	442	+ struct ib_wq *ibwq;
	443	+ int i;
	444	+
	445	+ mc = gd->driver_data;
	446	+ ndev = mc->ports[qp->port - 1];
	447	+ mpc = netdev_priv(ndev);
	448	+
	449	+ for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
	450	+ ibwq = ind_tbl->ind_tbl[i];
	451	+ wq = container_of(ibwq, struct mana_ib_wq, ibwq);
	452	+ ibdev_dbg(&mdev->ib_dev, "destroying wq->rx_object %llu\n",
	453	+ wq->rx_object);
	454	+ mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
	455	+ }
	456	+
	457	+ return 0;
	458	+}
	459	+
	460	+static int mana_ib_destroy_qp_raw(struct mana_ib_qp qp, struct ib_udata udata)
	461	+{
	462	+ struct mana_ib_dev *mdev =
	463	+ container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
	464	+ struct gdma_dev *gd = mdev->gdma_dev;
	465	+ struct ib_pd *ibpd = qp->ibqp.pd;
	466	+ struct mana_port_context *mpc;
	467	+ struct mana_context *mc;
	468	+ struct net_device *ndev;
	469	+ struct mana_ib_pd *pd;
	470	+
	471	+ mc = gd->driver_data;
	472	+ ndev = mc->ports[qp->port - 1];
	473	+ mpc = netdev_priv(ndev);
	474	+ pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	475	+
	476	+ mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
	477	+
	478	+ if (qp->sq_umem) {
	479	+ mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region);
	480	+ ib_umem_release(qp->sq_umem);
	481	+ }
	482	+
	483	+ mana_ib_uncfg_vport(mdev, pd, qp->port - 1);
	484	+
	485	+ return 0;
	486	+}
	487	+
	488	+int mana_ib_destroy_qp(struct ib_qp ibqp, struct ib_udata udata)
	489	+{
	490	+ struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
	491	+
	492	+ switch (ibqp->qp_type) {
	493	+ case IB_QPT_RAW_PACKET:
	494	+ if (ibqp->rwq_ind_tbl)
	495	+ return mana_ib_destroy_qp_rss(qp, ibqp->rwq_ind_tbl,
	496	+ udata);
	497	+
	498	+ return mana_ib_destroy_qp_raw(qp, udata);
	499	+
	500	+ default:
	501	+ ibdev_dbg(ibqp->device, "Unexpected QP type %u\n",
	502	+ ibqp->qp_type);
	503	+ }
	504	+
	505	+ return -ENOENT;
	506	+}

drivers/infiniband/hw/mana/wq.c+115−0

@@ -0,0 +1,115 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#include "mana_ib.h"
	7	+
	8	+struct ib_wq mana_ib_create_wq(struct ib_pd pd,
	9	+ struct ib_wq_init_attr *init_attr,
	10	+ struct ib_udata *udata)
	11	+{
	12	+ struct mana_ib_dev *mdev =
	13	+ container_of(pd->device, struct mana_ib_dev, ib_dev);
	14	+ struct mana_ib_create_wq ucmd = {};
	15	+ struct mana_ib_wq *wq;
	16	+ struct ib_umem *umem;
	17	+ int err;
	18	+
	19	+ if (udata->inlen < sizeof(ucmd))
	20	+ return ERR_PTR(-EINVAL);
	21	+
	22	+ err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
	23	+ if (err) {
	24	+ ibdev_dbg(&mdev->ib_dev,
	25	+ "Failed to copy from udata for create wq, %d\n", err);
	26	+ return ERR_PTR(err);
	27	+ }
	28	+
	29	+ wq = kzalloc(sizeof(*wq), GFP_KERNEL);
	30	+ if (!wq)
	31	+ return ERR_PTR(-ENOMEM);
	32	+
	33	+ ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr);
	34	+
	35	+ umem = ib_umem_get(pd->device, ucmd.wq_buf_addr, ucmd.wq_buf_size,
	36	+ IB_ACCESS_LOCAL_WRITE);
	37	+ if (IS_ERR(umem)) {
	38	+ err = PTR_ERR(umem);
	39	+ ibdev_dbg(&mdev->ib_dev,
	40	+ "Failed to get umem for create wq, err %d\n", err);
	41	+ goto err_free_wq;
	42	+ }
	43	+
	44	+ wq->umem = umem;
	45	+ wq->wqe = init_attr->max_wr;
	46	+ wq->wq_buf_size = ucmd.wq_buf_size;
	47	+ wq->rx_object = INVALID_MANA_HANDLE;
	48	+
	49	+ err = mana_ib_gd_create_dma_region(mdev, wq->umem, &wq->gdma_region);
	50	+ if (err) {
	51	+ ibdev_dbg(&mdev->ib_dev,
	52	+ "Failed to create dma region for create wq, %d\n",
	53	+ err);
	54	+ goto err_release_umem;
	55	+ }
	56	+
	57	+ ibdev_dbg(&mdev->ib_dev,
	58	+ "mana_ib_gd_create_dma_region ret %d gdma_region 0x%llx\n",
	59	+ err, wq->gdma_region);
	60	+
	61	+ /* WQ ID is returned at wq_create time, doesn't know the value yet */
	62	+
	63	+ return &wq->ibwq;
	64	+
	65	+err_release_umem:
	66	+ ib_umem_release(umem);
	67	+
	68	+err_free_wq:
	69	+ kfree(wq);
	70	+
	71	+ return ERR_PTR(err);
	72	+}
	73	+
	74	+int mana_ib_modify_wq(struct ib_wq wq, struct ib_wq_attr wq_attr,
	75	+ u32 wq_attr_mask, struct ib_udata *udata)
	76	+{
	77	+ /* modify_wq is not supported by this version of the driver */
	78	+ return -EOPNOTSUPP;
	79	+}
	80	+
	81	+int mana_ib_destroy_wq(struct ib_wq ibwq, struct ib_udata udata)
	82	+{
	83	+ struct mana_ib_wq *wq = container_of(ibwq, struct mana_ib_wq, ibwq);
	84	+ struct ib_device *ib_dev = ibwq->device;
	85	+ struct mana_ib_dev *mdev;
	86	+
	87	+ mdev = container_of(ib_dev, struct mana_ib_dev, ib_dev);
	88	+
	89	+ mana_ib_gd_destroy_dma_region(mdev, wq->gdma_region);
	90	+ ib_umem_release(wq->umem);
	91	+
	92	+ kfree(wq);
	93	+
	94	+ return 0;
	95	+}
	96	+
	97	+int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
	98	+ struct ib_rwq_ind_table_init_attr *init_attr,
	99	+ struct ib_udata *udata)
	100	+{
	101	+ /*
	102	+ * There is no additional data in ind_table to be maintained by this
	103	+ * driver, do nothing
	104	+ */
	105	+ return 0;
	106	+}
	107	+
	108	+int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
	109	+{
	110	+ /*
	111	+ * There is no additional data in ind_table to be maintained by this
	112	+ * driver, do nothing
	113	+ */
	114	+ return 0;
	115	+}

drivers/infiniband/hw/mlx4/main.c+4−8

@@ -144,8 +144,7 @@ static struct net_device mlx4_ib_get_netdev(struct ib_device device, · 143 unmodified lines
144	144	}
145	145	}
146	146	}
147		− if (dev)
148		− dev_hold(dev);
	147	+ dev_hold(dev);
149	148
150	149	rcu_read_unlock();
151	150	return dev;
@@ -1307,8 +1306,7 @@ int mlx4_ib_add_mc(struct mlx4_ib_dev mdev, struct mlx4_ib_qp mqp, · 1155 unmodified lines
1307	1306
1308	1307	spin_lock_bh(&mdev->iboe.lock);
1309	1308	ndev = mdev->iboe.netdevs[mqp->port - 1];
1310		− if (ndev)
1311		− dev_hold(ndev);
	1309	+ dev_hold(ndev);
1312	1310	spin_unlock_bh(&mdev->iboe.lock);
1313	1311
1314	1312	if (ndev) {
@@ -1955,11 +1953,9 @@ static int mlx4_ib_mcg_detach(struct ib_qp ibqp, union ib_gid gid, u16 lid) · 640 unmodified lines
1955	1953	if (ge) {
1956	1954	spin_lock_bh(&mdev->iboe.lock);
1957	1955	ndev = ge->added ? mdev->iboe.netdevs[ge->port - 1] : NULL;
1958		− if (ndev)
1959		− dev_hold(ndev);
	1956	+ dev_hold(ndev);
1960	1957	spin_unlock_bh(&mdev->iboe.lock);
1961		− if (ndev)
1962		− dev_put(ndev);
	1958	+ dev_put(ndev);
1963	1959	list_del(&ge->list);
1964	1960	kfree(ge);
1965	1961	} else

drivers/infiniband/hw/mlx5/cq.c+15−12

@@ -267,17 +267,20 @@ static void handle_responder(struct ib_wc wc, struct mlx5_cqe64 cqe, · 266 unmodified lines
267	267	wc->wc_flags \|= IB_WC_WITH_NETWORK_HDR_TYPE;
268	268	}
269	269
270		−static void dump_cqe(struct mlx5_ib_dev dev, struct mlx5_err_cqe cqe)
	270	+static void dump_cqe(struct mlx5_ib_dev dev, struct mlx5_err_cqe cqe,
	271	+ struct ib_wc wc, const char level)
271	272	{
272		− mlx5_ib_warn(dev, "dump error cqe\n");
273		− mlx5_dump_err_cqe(dev->mdev, cqe);
	273	+ mlx5_ib_log(level, dev, "WC error: %d, Message: %s\n", wc->status,
	274	+ ib_wc_status_msg(wc->status));
	275	+ print_hex_dump(level, "cqe_dump: ", DUMP_PREFIX_OFFSET, 16, 1,
	276	+ cqe, sizeof(*cqe), false);
274	277	}
275	278
276	279	static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,
277	280	struct mlx5_err_cqe *cqe,
278	281	struct ib_wc *wc)
279	282	{
280		− int dump = 1;
	283	+ const char *dump = KERN_WARNING;
281	284
282	285	switch (cqe->syndrome) {
283	286	case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR:
@@ -287,10 +290,11 @@ static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev, · 3 unmodified lines
287	290	wc->status = IB_WC_LOC_QP_OP_ERR;
288	291	break;
289	292	case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR:
	293	+ dump = KERN_DEBUG;
290	294	wc->status = IB_WC_LOC_PROT_ERR;
291	295	break;
292	296	case MLX5_CQE_SYNDROME_WR_FLUSH_ERR:
293		− dump = 0;
	297	+ dump = NULL;
294	298	wc->status = IB_WC_WR_FLUSH_ERR;
295	299	break;
296	300	case MLX5_CQE_SYNDROME_MW_BIND_ERR:
@@ -306,18 +310,20 @@ static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev, · 9 unmodified lines
306	310	wc->status = IB_WC_REM_INV_REQ_ERR;
307	311	break;
308	312	case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR:
	313	+ dump = KERN_DEBUG;
309	314	wc->status = IB_WC_REM_ACCESS_ERR;
310	315	break;
311	316	case MLX5_CQE_SYNDROME_REMOTE_OP_ERR:
	317	+ dump = KERN_DEBUG;
312	318	wc->status = IB_WC_REM_OP_ERR;
313	319	break;
314	320	case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
	321	+ dump = NULL;
315	322	wc->status = IB_WC_RETRY_EXC_ERR;
316		− dump = 0;
317	323	break;
318	324	case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
	325	+ dump = NULL;
319	326	wc->status = IB_WC_RNR_RETRY_EXC_ERR;
320		− dump = 0;
321	327	break;
322	328	case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR:
323	329	wc->status = IB_WC_REM_ABORT_ERR;
@@ -328,11 +334,8 @@ static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev, · 4 unmodified lines
328	334	}
329	335
330	336	wc->vendor_err = cqe->vendor_err_synd;
331		− if (dump) {
332		− mlx5_ib_warn(dev, "WC error: %d, Message: %s\n", wc->status,
333		− ib_wc_status_msg(wc->status));
334		− dump_cqe(dev, cqe);
335		− }
	337	+ if (dump)
	338	+ dump_cqe(dev, cqe, wc, dump);
336	339	}
337	340
338	341	static void handle_atomics(struct mlx5_ib_qp qp, struct mlx5_cqe64 cqe64,

drivers/infiniband/hw/mlx5/fs.c+0−1

@@ -127,7 +127,6 @@ static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) · 126 unmodified lines
127	127	}
128	128
129	129	#define LAST_ETH_FIELD vlan_tag
130		−#define LAST_IB_FIELD sl
131	130	#define LAST_IPV4_FIELD tos
132	131	#define LAST_IPV6_FIELD traffic_class
133	132	#define LAST_TCP_UDP_FIELD src_port

drivers/infiniband/hw/mlx5/mlx5_ib.h+4−0

@@ -38,6 +38,10 @@ · 37 unmodified lines
38	38	dev_warn(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__, \
39	39	__LINE__, current->pid, ##arg)
40	40
	41	+#define mlx5_ib_log(lvl, _dev, format, arg...) \
	42	+ dev_printk(lvl, &(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, \
	43	+ __func__, __LINE__, current->pid, ##arg)
	44	+
41	45	#define MLX5_IB_DEFAULT_UIDX 0xffffff
42	46	#define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
43	47

drivers/infiniband/hw/mlx5/mr.c+2−4

@@ -1929,10 +1929,8 @@ int mlx5_ib_alloc_mw(struct ib_mw ibmw, struct ib_udata udata) · 1928 unmodified lines
1929	1929	ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
1930	1930
1931	1931	in = kzalloc(inlen, GFP_KERNEL);
1932		− if (!in) {
1933		− err = -ENOMEM;
1934		− goto free;
1935		− }
	1932	+ if (!in)
	1933	+ return -ENOMEM;
1936	1934
1937	1935	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1938	1936

drivers/infiniband/hw/qedr/main.c+1−1

@@ -479,7 +479,7 @@ static irqreturn_t qedr_irq_handler(int irq, void *handle) · 478 unmodified lines
479	479	/* The CQ's CNQ notification counter is checked before
480	480	* destroying the CQ in a busy-wait loop that waits for all of
481	481	* the CQ's CNQ interrupts to be processed. It is increased
482		− * here, only after the completion handler, to ensure that the
	482	+ * here, only after the completion handler, to ensure that
483	483	* the handler is not running when the CQ is destroyed.
484	484	*/
485	485	cq->cnq_notif++;

drivers/infiniband/hw/qib/qib_iba6120.c+2−5

@@ -799,12 +799,9 @@ static void qib_handle_6120_hwerrors(struct qib_devdata dd, char msg, · 798 unmodified lines
799	799	hwerrs &= ~TXE_PIO_PARITY;
800	800	}
801	801
802		− if (!hwerrs) {
803		− static u32 freeze_cnt;
804		−
805		− freeze_cnt++;
	802	+ if (!hwerrs)
806	803	qib_6120_clear_freeze(dd);
807		− } else
	804	+ else
808	805	isfatal = 1;
809	806	}
810	807

drivers/infiniband/hw/qib/qib_tx.c+1−4

@@ -82,7 +82,6 @@ int qib_disarm_piobufs_ifneeded(struct qib_ctxtdata *rcd) · 81 unmodified lines
82	82	struct qib_devdata *dd = rcd->dd;
83	83	unsigned i;
84	84	unsigned last;
85		− unsigned n = 0;
86	85
87	86	last = rcd->pio_base + rcd->piocnt;
88	87	/*
@@ -102,10 +101,8 @@ int qib_disarm_piobufs_ifneeded(struct qib_ctxtdata *rcd) · 13 unmodified lines
102	101	}
103	102	spin_lock_irq(&dd->pioavail_lock);
104	103	for (i = rcd->pio_base; i < last; i++) {
105		− if (__test_and_clear_bit(i, dd->pio_need_disarm)) {
106		− n++;
	104	+ if (__test_and_clear_bit(i, dd->pio_need_disarm))
107	105	dd->f_sendctrl(rcd->ppd, QIB_SENDCTRL_DISARM_BUF(i));
108		− }
109	106	}
110	107	spin_unlock_irq(&dd->pioavail_lock);
111	108	return 0;

drivers/infiniband/hw/qib/qib_user_sdma.c+1−1

@@ -851,7 +851,7 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd, · 850 unmodified lines
851	851	}
852	852
853	853	/*
854		− * This assignment is a bit strange. it's because the
	854	+ * This assignment is a bit strange. it's because
855	855	* the pbc counts the number of 32 bit words in the full
856	856	* packet _except_ the first word of the pbc itself...
857	857	*/

drivers/infiniband/sw/rxe/rxe.c+2−2

@@ -187,14 +187,14 @@ static int rxe_newlink(const char ibdev_name, struct net_device ndev) · 186 unmodified lines
187	187	exists = rxe_get_dev_from_net(ndev);
188	188	if (exists) {
189	189	ib_device_put(&exists->ib_dev);
190		− pr_err("already configured on %s\n", ndev->name);
	190	+ rxe_dbg(exists, "already configured on %s\n", ndev->name);
191	191	err = -EEXIST;
192	192	goto err;
193	193	}
194	194
195	195	err = rxe_net_add(ibdev_name, ndev);
196	196	if (err) {
197		− pr_err("failed to add %s\n", ndev->name);
	197	+ rxe_dbg(exists, "failed to add %s\n", ndev->name);
198	198	goto err;
199	199	}
200	200	err:

drivers/infiniband/sw/rxe/rxe.h+19−0

@@ -38,6 +38,25 @@ · 37 unmodified lines
38	38
39	39	#define RXE_ROCE_V2_SPORT (0xc000)
40	40
	41	+#define rxe_dbg(rxe, fmt, ...) ibdev_dbg(&(rxe)->ib_dev, \
	42	+ "%s: " fmt, __func__, ##__VA_ARGS__)
	43	+#define rxe_dbg_uc(uc, fmt, ...) ibdev_dbg((uc)->ibuc.device, \
	44	+ "uc#%d %s: " fmt, (uc)->elem.index, __func__, ##__VA_ARGS__)
	45	+#define rxe_dbg_pd(pd, fmt, ...) ibdev_dbg((pd)->ibpd.device, \
	46	+ "pd#%d %s: " fmt, (pd)->elem.index, __func__, ##__VA_ARGS__)
	47	+#define rxe_dbg_ah(ah, fmt, ...) ibdev_dbg((ah)->ibah.device, \
	48	+ "ah#%d %s: " fmt, (ah)->elem.index, __func__, ##__VA_ARGS__)
	49	+#define rxe_dbg_srq(srq, fmt, ...) ibdev_dbg((srq)->ibsrq.device, \
	50	+ "srq#%d %s: " fmt, (srq)->elem.index, __func__, ##__VA_ARGS__)
	51	+#define rxe_dbg_qp(qp, fmt, ...) ibdev_dbg((qp)->ibqp.device, \
	52	+ "qp#%d %s: " fmt, (qp)->elem.index, __func__, ##__VA_ARGS__)
	53	+#define rxe_dbg_cq(cq, fmt, ...) ibdev_dbg((cq)->ibcq.device, \
	54	+ "cq#%d %s: " fmt, (cq)->elem.index, __func__, ##__VA_ARGS__)
	55	+#define rxe_dbg_mr(mr, fmt, ...) ibdev_dbg((mr)->ibmr.device, \
	56	+ "mr#%d %s: " fmt, (mr)->elem.index, __func__, ##__VA_ARGS__)
	57	+#define rxe_dbg_mw(mw, fmt, ...) ibdev_dbg((mw)->ibmw.device, \
	58	+ "mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
	59	+
41	60	void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
42	61
43	62	int rxe_add(struct rxe_dev rxe, unsigned int mtu, const char ibdev_name);

drivers/infiniband/sw/rxe/rxe_av.c+36−7

@@ -14,26 +14,45 @@ void rxe_init_av(struct rdma_ah_attr attr, struct rxe_av av) · 13 unmodified lines
14	14	memcpy(av->dmac, attr->roce.dmac, ETH_ALEN);
15	15	}
16	16
17		−int rxe_av_chk_attr(struct rxe_dev rxe, struct rdma_ah_attr attr)
	17	+static int chk_attr(void obj, struct rdma_ah_attr attr, bool obj_is_ah)
18	18	{
19	19	const struct ib_global_route *grh = rdma_ah_read_grh(attr);
20	20	struct rxe_port *port;
	21	+ struct rxe_dev *rxe;
	22	+ struct rxe_qp *qp;
	23	+ struct rxe_ah *ah;
21	24	int type;
22	25
	26	+ if (obj_is_ah) {
	27	+ ah = obj;
	28	+ rxe = to_rdev(ah->ibah.device);
	29	+ } else {
	30	+ qp = obj;
	31	+ rxe = to_rdev(qp->ibqp.device);
	32	+ }
	33	+
23	34	port = &rxe->port;
24	35
25	36	if (rdma_ah_get_ah_flags(attr) & IB_AH_GRH) {
26	37	if (grh->sgid_index > port->attr.gid_tbl_len) {
27		− pr_warn("invalid sgid index = %d\n",
28		− grh->sgid_index);
	38	+ if (obj_is_ah)
	39	+ rxe_dbg_ah(ah, "invalid sgid index = %d\n",
	40	+ grh->sgid_index);
	41	+ else
	42	+ rxe_dbg_qp(qp, "invalid sgid index = %d\n",
	43	+ grh->sgid_index);
29	44	return -EINVAL;
30	45	}
31	46
32	47	type = rdma_gid_attr_network_type(grh->sgid_attr);
33	48	if (type < RDMA_NETWORK_IPV4 \|\|
34	49	type > RDMA_NETWORK_IPV6) {
35		− pr_warn("invalid network type for rdma_rxe = %d\n",
36		− type);
	50	+ if (obj_is_ah)
	51	+ rxe_dbg_ah(ah, "invalid network type for rdma_rxe = %d\n",
	52	+ type);
	53	+ else
	54	+ rxe_dbg_qp(qp, "invalid network type for rdma_rxe = %d\n",
	55	+ type);
37	56	return -EINVAL;
38	57	}
39	58	}
@@ -41,6 +60,16 @@ int rxe_av_chk_attr(struct rxe_dev rxe, struct rdma_ah_attr attr) · 1 unmodified line
41	60	return 0;
42	61	}
43	62
	63	+int rxe_av_chk_attr(struct rxe_qp qp, struct rdma_ah_attr attr)
	64	+{
	65	+ return chk_attr(qp, attr, false);
	66	+}
	67	+
	68	+int rxe_ah_chk_attr(struct rxe_ah ah, struct rdma_ah_attr attr)
	69	+{
	70	+ return chk_attr(ah, attr, true);
	71	+}
	72	+
44	73	void rxe_av_from_attr(u8 port_num, struct rxe_av *av,
45	74	struct rdma_ah_attr *attr)
46	75	{
@@ -121,12 +150,12 @@ struct rxe_av rxe_get_av(struct rxe_pkt_info pkt, struct rxe_ah **ahp) · 74 unmodified lines
121	150	/* only new user provider or kernel client */
122	151	ah = rxe_pool_get_index(&pkt->rxe->ah_pool, ah_num);
123	152	if (!ah) {
124		− pr_warn("Unable to find AH matching ah_num\n");
	153	+ rxe_dbg_qp(pkt->qp, "Unable to find AH matching ah_num\n");
125	154	return NULL;
126	155	}
127	156
128	157	if (rxe_ah_pd(ah) != pkt->qp->pd) {
129		− pr_warn("PDs don't match for AH and QP\n");
	158	+ rxe_dbg_qp(pkt->qp, "PDs don't match for AH and QP\n");
130	159	rxe_put(ah);
131	160	return NULL;
132	161	}

drivers/infiniband/sw/rxe/rxe_comp.c+31−16

@@ -104,6 +104,8 @@ static enum ib_wc_opcode wr_to_wc_opcode(enum ib_wr_opcode opcode) · 103 unmodified lines
104	104	case IB_WR_LOCAL_INV: return IB_WC_LOCAL_INV;
105	105	case IB_WR_REG_MR: return IB_WC_REG_MR;
106	106	case IB_WR_BIND_MW: return IB_WC_BIND_MW;
	107	+ case IB_WR_ATOMIC_WRITE: return IB_WC_ATOMIC_WRITE;
	108	+ case IB_WR_FLUSH: return IB_WC_FLUSH;
107	109
108	110	default:
109	111	return 0xff;
@@ -114,11 +116,11 @@ void retransmit_timer(struct timer_list *t) · 4 unmodified lines
114	116	{
115	117	struct rxe_qp *qp = from_timer(qp, t, retrans_timer);
116	118
117		− pr_debug("%s: fired for qp#%d\n", __func__, qp->elem.index);
	119	+ rxe_dbg_qp(qp, "retransmit timer fired\n");
118	120
119	121	if (qp->valid) {
120	122	qp->comp.timeout = 1;
121		− rxe_run_task(&qp->comp.task, 1);
	123	+ rxe_sched_task(&qp->comp.task);
122	124	}
123	125	}
124	126
@@ -132,7 +134,10 @@ void rxe_comp_queue_pkt(struct rxe_qp qp, struct sk_buff skb) · 7 unmodified lines
132	134	if (must_sched != 0)
133	135	rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED);
134	136
135		− rxe_run_task(&qp->comp.task, must_sched);
	137	+ if (must_sched)
	138	+ rxe_sched_task(&qp->comp.task);
	139	+ else
	140	+ rxe_run_task(&qp->comp.task);
136	141	}
137	142
138	143	static inline enum comp_state get_wqe(struct rxe_qp *qp,
@@ -200,6 +205,10 @@ static inline enum comp_state check_psn(struct rxe_qp *qp, · 61 unmodified lines
200	205	*/
201	206	if (pkt->psn == wqe->last_psn)
202	207	return COMPST_COMP_ACK;
	208	+ else if (pkt->opcode == IB_OPCODE_RC_ACKNOWLEDGE &&
	209	+ (qp->comp.opcode == IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST \|\|
	210	+ qp->comp.opcode == IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE))
	211	+ return COMPST_CHECK_ACK;
203	212	else
204	213	return COMPST_DONE;
205	214	} else if ((diff > 0) && (wqe->mask & WR_ATOMIC_OR_READ_MASK)) {
@@ -228,6 +237,10 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, · 22 unmodified lines
228	237
229	238	case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
230	239	case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
	240	+ /* Check NAK code to handle a remote error */
	241	+ if (pkt->opcode == IB_OPCODE_RC_ACKNOWLEDGE)
	242	+ break;
	243	+
231	244	if (pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE &&
232	245	pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST) {
233	246	/* read retries of partial data may restart from
@@ -258,12 +271,16 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, · 24 unmodified lines
258	271	if ((syn & AETH_TYPE_MASK) != AETH_ACK)
259	272	return COMPST_ERROR;
260	273
	274	+ if (wqe->wr.opcode == IB_WR_ATOMIC_WRITE)
	275	+ return COMPST_WRITE_SEND;
	276	+
261	277	fallthrough;
262	278	/* (IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE doesn't have an AETH)
263	279	*/
264	280	case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
265	281	if (wqe->wr.opcode != IB_WR_RDMA_READ &&
266		− wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV) {
	282	+ wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV &&
	283	+ wqe->wr.opcode != IB_WR_FLUSH) {
267	284	wqe->status = IB_WC_FATAL_ERR;
268	285	return COMPST_ERROR;
269	286	}
@@ -305,7 +322,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, · 35 unmodified lines
305	322	qp->comp.psn = pkt->psn;
306	323	if (qp->req.wait_psn) {
307	324	qp->req.wait_psn = 0;
308		− rxe_run_task(&qp->req.task, 0);
	325	+ rxe_run_task(&qp->req.task);
309	326	}
310	327	}
311	328	return COMPST_ERROR_RETRY;
@@ -323,7 +340,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, · 11 unmodified lines
323	340	return COMPST_ERROR;
324	341
325	342	default:
326		− pr_warn("unexpected nak %x\n", syn);
	343	+ rxe_dbg_qp(qp, "unexpected nak %x\n", syn);
327	344	wqe->status = IB_WC_REM_OP_ERR;
328	345	return COMPST_ERROR;
329	346	}
@@ -334,7 +351,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, · 4 unmodified lines
334	351	break;
335	352
336	353	default:
337		− pr_warn("unexpected opcode\n");
	354	+ rxe_dbg_qp(qp, "unexpected opcode\n");
338	355	}
339	356
340	357	return COMPST_ERROR;
@@ -452,7 +469,7 @@ static void do_complete(struct rxe_qp qp, struct rxe_send_wqe wqe) · 111 unmodified lines
452	469	*/
453	470	if (qp->req.wait_fence) {
454	471	qp->req.wait_fence = 0;
455		− rxe_run_task(&qp->req.task, 0);
	472	+ rxe_run_task(&qp->req.task);
456	473	}
457	474	}
458	475
@@ -466,7 +483,7 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, · 7 unmodified lines
466	483	if (qp->req.need_rd_atomic) {
467	484	qp->comp.timeout_retry = 0;
468	485	qp->req.need_rd_atomic = 0;
469		− rxe_run_task(&qp->req.task, 0);
	486	+ rxe_run_task(&qp->req.task);
470	487	}
471	488	}
472	489
@@ -512,7 +529,7 @@ static inline enum comp_state complete_wqe(struct rxe_qp *qp, · 39 unmodified lines
512	529
513	530	if (qp->req.wait_psn) {
514	531	qp->req.wait_psn = 0;
515		− rxe_run_task(&qp->req.task, 1);
	532	+ rxe_sched_task(&qp->req.task);
516	533	}
517	534	}
518	535
@@ -587,8 +604,7 @@ int rxe_completer(void *arg) · 68 unmodified lines
587	604	state = COMPST_GET_ACK;
588	605
589	606	while (1) {
590		− pr_debug("qp#%d state = %s\n", qp_num(qp),
591		− comp_state_name[state]);
	607	+ rxe_dbg_qp(qp, "state = %s\n", comp_state_name[state]);
592	608	switch (state) {
593	609	case COMPST_GET_ACK:
594	610	skb = skb_dequeue(&qp->resp_pkts);
@@ -646,7 +662,7 @@ int rxe_completer(void *arg) · 51 unmodified lines
646	662
647	663	if (qp->req.wait_psn) {
648	664	qp->req.wait_psn = 0;
649		− rxe_run_task(&qp->req.task, 1);
	665	+ rxe_sched_task(&qp->req.task);
650	666	}
651	667
652	668	state = COMPST_DONE;
@@ -714,7 +730,7 @@ int rxe_completer(void *arg) · 61 unmodified lines
714	730	RXE_CNT_COMP_RETRY);
715	731	qp->req.need_retry = 1;
716	732	qp->comp.started_retry = 1;
717		− rxe_run_task(&qp->req.task, 0);
	733	+ rxe_run_task(&qp->req.task);
718	734	}
719	735	goto done;
720	736
@@ -735,8 +751,7 @@ int rxe_completer(void *arg) · 14 unmodified lines
735	751	* rnr timer has fired
736	752	*/
737	753	qp->req.wait_for_rnr_timer = 1;
738		− pr_debug("qp#%d set rnr nak timer\n",
739		− qp_num(qp));
	754	+ rxe_dbg_qp(qp, "set rnr nak timer\n");
740	755	mod_timer(&qp->rnr_nak_timer,
741	756	jiffies + rnrnak_jiffies(aeth_syn(pkt)
742	757	& ~AETH_TYPE_MASK));

drivers/infiniband/sw/rxe/rxe_cq.c+4−4

@@ -14,12 +14,12 @@ int rxe_cq_chk_attr(struct rxe_dev rxe, struct rxe_cq cq, · 13 unmodified lines
14	14	int count;
15	15
16	16	if (cqe <= 0) {
17		− pr_warn("cqe(%d) <= 0\n", cqe);
	17	+ rxe_dbg(rxe, "cqe(%d) <= 0\n", cqe);
18	18	goto err1;
19	19	}
20	20
21	21	if (cqe > rxe->attr.max_cqe) {
22		− pr_debug("cqe(%d) > max_cqe(%d)\n",
	22	+ rxe_dbg(rxe, "cqe(%d) > max_cqe(%d)\n",
23	23	cqe, rxe->attr.max_cqe);
24	24	goto err1;
25	25	}
@@ -27,7 +27,7 @@ int rxe_cq_chk_attr(struct rxe_dev rxe, struct rxe_cq cq, · 1 unmodified line
27	27	if (cq) {
28	28	count = queue_count(cq->queue, QUEUE_TYPE_TO_CLIENT);
29	29	if (cqe < count) {
30		− pr_debug("cqe(%d) < current # elements in queue (%d)",
	30	+ rxe_dbg_cq(cq, "cqe(%d) < current # elements in queue (%d)",
31	31	cqe, count);
32	32	goto err1;
33	33	}
@@ -65,7 +65,7 @@ int rxe_cq_from_init(struct rxe_dev rxe, struct rxe_cq cq, int cqe, · 31 unmodified lines
65	65	cq->queue = rxe_queue_init(rxe, &cqe,
66	66	sizeof(struct rxe_cqe), type);
67	67	if (!cq->queue) {
68		− pr_warn("unable to create cq\n");
	68	+ rxe_dbg(rxe, "unable to create cq\n");
69	69	return -ENOMEM;
70	70	}
71	71

drivers/infiniband/sw/rxe/rxe_hdr.h+47−1

@@ -607,6 +607,52 @@ static inline void reth_set_len(struct rxe_pkt_info *pkt, u32 len) · 606 unmodified lines
607	607	rxe_opcode[pkt->opcode].offset[RXE_RETH], len);
608	608	}
609	609
	610	+/******************************************************************************
	611	+ * FLUSH Extended Transport Header
	612	+ ******************************************************************************/
	613	+
	614	+struct rxe_feth {
	615	+ __be32 bits;
	616	+};
	617	+
	618	+#define FETH_PLT_MASK (0x0000000f) /* bits 3-0 */
	619	+#define FETH_SEL_MASK (0x00000030) /* bits 5-4 */
	620	+#define FETH_SEL_SHIFT (4U)
	621	+
	622	+static inline u32 __feth_plt(void *arg)
	623	+{
	624	+ struct rxe_feth *feth = arg;
	625	+
	626	+ return be32_to_cpu(feth->bits) & FETH_PLT_MASK;
	627	+}
	628	+
	629	+static inline u32 __feth_sel(void *arg)
	630	+{
	631	+ struct rxe_feth *feth = arg;
	632	+
	633	+ return (be32_to_cpu(feth->bits) & FETH_SEL_MASK) >> FETH_SEL_SHIFT;
	634	+}
	635	+
	636	+static inline u32 feth_plt(struct rxe_pkt_info *pkt)
	637	+{
	638	+ return __feth_plt(pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
	639	+}
	640	+
	641	+static inline u32 feth_sel(struct rxe_pkt_info *pkt)
	642	+{
	643	+ return __feth_sel(pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
	644	+}
	645	+
	646	+static inline void feth_init(struct rxe_pkt_info *pkt, u8 type, u8 level)
	647	+{
	648	+ struct rxe_feth feth = (struct rxe_feth )
	649	+ (pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
	650	+ u32 bits = ((level << FETH_SEL_SHIFT) & FETH_SEL_MASK) \|
	651	+ (type & FETH_PLT_MASK);
	652	+
	653	+ feth->bits = cpu_to_be32(bits);
	654	+}
	655	+
610	656	/******************************************************************************
611	657	* Atomic Extended Transport Header
612	658	******************************************************************************/
@@ -742,7 +788,6 @@ enum aeth_syndrome { · 129 unmodified lines
742	788	AETH_NAK_INVALID_REQ = 0x61,
743	789	AETH_NAK_REM_ACC_ERR = 0x62,
744	790	AETH_NAK_REM_OP_ERR = 0x63,
745		− AETH_NAK_INV_RD_REQ = 0x64,
746	791	};
747	792
748	793	static inline u8 __aeth_syn(void *arg)
@@ -910,6 +955,7 @@ enum rxe_hdr_length { · 161 unmodified lines
910	955	RXE_ATMETH_BYTES = sizeof(struct rxe_atmeth),
911	956	RXE_IETH_BYTES = sizeof(struct rxe_ieth),
912	957	RXE_RDETH_BYTES = sizeof(struct rxe_rdeth),
	958	+ RXE_FETH_BYTES = sizeof(struct rxe_feth),
913	959	};
914	960
915	961	static inline size_t header_size(struct rxe_pkt_info *pkt)

drivers/infiniband/sw/rxe/rxe_icrc.c+2−2

@@ -21,7 +21,7 @@ int rxe_icrc_init(struct rxe_dev *rxe) · 20 unmodified lines
21	21
22	22	tfm = crypto_alloc_shash("crc32", 0, 0);
23	23	if (IS_ERR(tfm)) {
24		− pr_warn("failed to init crc32 algorithm err:%ld\n",
	24	+ rxe_dbg(rxe, "failed to init crc32 algorithm err: %ld\n",
25	25	PTR_ERR(tfm));
26	26	return PTR_ERR(tfm);
27	27	}
@@ -51,7 +51,7 @@ static __be32 rxe_crc32(struct rxe_dev rxe, __be32 crc, void next, size_t len) · 23 unmodified lines
51	51	(__be32 )shash_desc_ctx(shash) = crc;
52	52	err = crypto_shash_update(shash, next, len);
53	53	if (unlikely(err)) {
54		− pr_warn_ratelimited("failed crc calculation, err: %d\n", err);
	54	+ rxe_dbg(rxe, "failed crc calculation, err: %d\n", err);
55	55	return (__force __be32)crc32_le((__force u32)crc, next, len);
56	56	}
57	57

drivers/infiniband/sw/rxe/rxe_loc.h+3−6

@@ -9,16 +9,12 @@ · 8 unmodified lines
9	9
10	10	/* rxe_av.c */
11	11	void rxe_init_av(struct rdma_ah_attr attr, struct rxe_av av);
12		−
13		−int rxe_av_chk_attr(struct rxe_dev rxe, struct rdma_ah_attr attr);
14		−
	12	+int rxe_av_chk_attr(struct rxe_qp qp, struct rdma_ah_attr attr);
	13	+int rxe_ah_chk_attr(struct rxe_ah ah, struct rdma_ah_attr attr);
15	14	void rxe_av_from_attr(u8 port_num, struct rxe_av *av,
16	15	struct rdma_ah_attr *attr);
17		−
18	16	void rxe_av_to_attr(struct rxe_av av, struct rdma_ah_attr attr);
19		−
20	17	void rxe_av_fill_ip_info(struct rxe_av av, struct rdma_ah_attr attr);
21		−
22	18	struct rxe_av rxe_get_av(struct rxe_pkt_info pkt, struct rxe_ah **ahp);
23	19
24	20	/* rxe_cq.c */
@@ -68,6 +64,7 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr); · 43 unmodified lines
68	64	int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
69	65	int access, struct rxe_mr *mr);
70	66	int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
	67	+int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length);
71	68	int rxe_mr_copy(struct rxe_mr mr, u64 iova, void addr, int length,
72	69	enum rxe_mr_copy_dir dir);
73	70	int copy_data(struct rxe_pd pd, int access, struct rxe_dma_info dma,

drivers/infiniband/sw/rxe/rxe_mmap.c+3−3

@@ -79,7 +79,7 @@ int rxe_mmap(struct ib_ucontext context, struct vm_area_struct vma) · 78 unmodified lines
79	79
80	80	/* Don't allow a mmap larger than the object. */
81	81	if (size > ip->info.size) {
82		− pr_err("mmap region is larger than the object!\n");
	82	+ rxe_dbg(rxe, "mmap region is larger than the object!\n");
83	83	spin_unlock_bh(&rxe->pending_lock);
84	84	ret = -EINVAL;
85	85	goto done;
@@ -87,7 +87,7 @@ int rxe_mmap(struct ib_ucontext context, struct vm_area_struct vma) · 1 unmodified line
87	87
88	88	goto found_it;
89	89	}
90		− pr_warn("unable to find pending mmap info\n");
	90	+ rxe_dbg(rxe, "unable to find pending mmap info\n");
91	91	spin_unlock_bh(&rxe->pending_lock);
92	92	ret = -EINVAL;
93	93	goto done;
@@ -98,7 +98,7 @@ found_it: · 4 unmodified lines
98	98
99	99	ret = remap_vmalloc_range(vma, ip->obj, 0);
100	100	if (ret) {
101		− pr_err("err %d from remap_vmalloc_range\n", ret);
	101	+ rxe_dbg(rxe, "err %d from remap_vmalloc_range\n", ret);
102	102	goto done;
103	103	}
104	104

drivers/infiniband/sw/rxe/rxe_mr.c+82−40

@@ -4,6 +4,8 @@ · 3 unmodified lines
4	4	* Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5	5	*/
6	6
	7	+#include <linux/libnvdimm.h>
	8	+
7	9	#include "rxe.h"
8	10	#include "rxe_loc.h"
9	11
@@ -26,7 +28,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) · 16 unmodified lines
26	28	{
27	29
28	30
29		− switch (mr->type) {
	31	+ switch (mr->ibmr.type) {
30	32	case IB_MR_TYPE_DMA:
31	33	return 0;
32	34
@@ -38,8 +40,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) · 5 unmodified lines
38	40	return 0;
39	41
40	42	default:
41		− pr_warn("%s: mr type (%d) not supported\n",
42		− __func__, mr->type);
	43	+ rxe_dbg_mr(mr, "type (%d) not supported\n", mr->ibmr.type);
43	44	return -EFAULT;
44	45	}
45	46	}
@@ -62,7 +63,6 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) · 16 unmodified lines
62	63	mr->rkey = mr->ibmr.rkey = rkey;
63	64
64	65	mr->state = RXE_MR_STATE_INVALID;
65		− mr->map_shift = ilog2(RXE_BUF_PER_MAP);
66	66	}
67	67
68	68	static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf)
@@ -99,6 +99,7 @@ err2: · 30 unmodified lines
99	99	kfree(mr->map[i]);
100	100
101	101	kfree(mr->map);
	102	+ mr->map = NULL;
102	103	err1:
103	104	return -ENOMEM;
104	105	}
@@ -109,7 +110,16 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr) · 4 unmodified lines
109	110
110	111	mr->access = access;
111	112	mr->state = RXE_MR_STATE_VALID;
112		− mr->type = IB_MR_TYPE_DMA;
	113	+ mr->ibmr.type = IB_MR_TYPE_DMA;
	114	+}
	115	+
	116	+static bool is_pmem_page(struct page *pg)
	117	+{
	118	+ unsigned long paddr = page_to_phys(pg);
	119	+
	120	+ return REGION_INTERSECTS ==
	121	+ region_intersects(paddr, PAGE_SIZE, IORESOURCE_MEM,
	122	+ IORES_DESC_PERSISTENT_MEMORY);
113	123	}
114	124
115	125	int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
@@ -122,12 +132,11 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, · 6 unmodified lines
122	132	int num_buf;
123	133	void *vaddr;
124	134	int err;
125		− int i;
126	135
127	136	umem = ib_umem_get(&rxe->ib_dev, start, length, access);
128	137	if (IS_ERR(umem)) {
129		− pr_warn("%s: Unable to pin memory region err = %d\n",
130		− __func__, (int)PTR_ERR(umem));
	138	+ rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
	139	+ (int)PTR_ERR(umem));
131	140	err = PTR_ERR(umem);
132	141	goto err_out;
133	142	}
@@ -138,8 +147,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, · 4 unmodified lines
138	147
139	148	err = rxe_mr_alloc(mr, num_buf);
140	149	if (err) {
141		− pr_warn("%s: Unable to allocate memory for map\n",
142		− __func__);
	150	+ rxe_dbg_mr(mr, "Unable to allocate memory for map\n");
143	151	goto err_release_umem;
144	152	}
145	153
@@ -149,23 +157,30 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, · 3 unmodified lines
149	157	num_buf = 0;
150	158	map = mr->map;
151	159	if (length > 0) {
152		− buf = map[0]->buf;
	160	+ bool persistent_access = access & IB_ACCESS_FLUSH_PERSISTENT;
153	161
	162	+ buf = map[0]->buf;
154	163	for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {
	164	+ struct page *pg = sg_page_iter_page(&sg_iter);
	165	+
	166	+ if (persistent_access && !is_pmem_page(pg)) {
	167	+ rxe_dbg_mr(mr, "Unable to register persistent access to non-pmem device\n");
	168	+ err = -EINVAL;
	169	+ goto err_release_umem;
	170	+ }
	171	+
155	172	if (num_buf >= RXE_BUF_PER_MAP) {
156	173	map++;
157	174	buf = map[0]->buf;
158	175	num_buf = 0;
159	176	}
160	177
161		− vaddr = page_address(sg_page_iter_page(&sg_iter));
	178	+ vaddr = page_address(pg);
162	179	if (!vaddr) {
163		− pr_warn("%s: Unable to get virtual address\n",
164		− __func__);
	180	+ rxe_dbg_mr(mr, "Unable to get virtual address\n");
165	181	err = -ENOMEM;
166		− goto err_cleanup_map;
	182	+ goto err_release_umem;
167	183	}
168		−
169	184	buf->addr = (uintptr_t)vaddr;
170	185	buf->size = PAGE_SIZE;
171	186	num_buf++;
@@ -178,14 +193,11 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, · 6 unmodified lines
178	193	mr->access = access;
179	194	mr->offset = ib_umem_offset(umem);
180	195	mr->state = RXE_MR_STATE_VALID;
181		− mr->type = IB_MR_TYPE_USER;
	196	+ mr->ibmr.type = IB_MR_TYPE_USER;
	197	+ mr->ibmr.page_size = PAGE_SIZE;
182	198
183	199	return 0;
184	200
185		−err_cleanup_map:
186		− for (i = 0; i < mr->num_map; i++)
187		− kfree(mr->map[i]);
188		− kfree(mr->map);
189	201	err_release_umem:
190	202	ib_umem_release(umem);
191	203	err_out:
@@ -205,7 +217,7 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) · 13 unmodified lines
205	217
206	218	mr->max_buf = max_pages;
207	219	mr->state = RXE_MR_STATE_FREE;
208		− mr->type = IB_MR_TYPE_MEM_REG;
	220	+ mr->ibmr.type = IB_MR_TYPE_MEM_REG;
209	221
210	222	return 0;
211	223
@@ -256,7 +268,7 @@ void iova_to_vaddr(struct rxe_mr mr, u64 iova, int length) · 44 unmodified lines
256	268	void *addr;
257	269
258	270	if (mr->state != RXE_MR_STATE_VALID) {
259		− pr_warn("mr not in valid state\n");
	271	+ rxe_dbg_mr(mr, "Not in valid state\n");
260	272	addr = NULL;
261	273	goto out;
262	274	}
@@ -267,7 +279,7 @@ void iova_to_vaddr(struct rxe_mr mr, u64 iova, int length) · 4 unmodified lines
267	279	}
268	280
269	281	if (mr_check_range(mr, iova, length)) {
270		− pr_warn("range violation\n");
	282	+ rxe_dbg_mr(mr, "Range violation\n");
271	283	addr = NULL;
272	284	goto out;
273	285	}
@@ -275,7 +287,7 @@ void iova_to_vaddr(struct rxe_mr mr, u64 iova, int length) · 1 unmodified line
275	287	lookup_iova(mr, iova, &m, &n, &offset);
276	288
277	289	if (offset + length > mr->map[m]->buf[n].size) {
278		− pr_warn("crosses page boundary\n");
	290	+ rxe_dbg_mr(mr, "Crosses page boundary\n");
279	291	addr = NULL;
280	292	goto out;
281	293	}
@@ -286,6 +298,39 @@ out: · 4 unmodified lines
286	298	return addr;
287	299	}
288	300
	301	+int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)
	302	+{
	303	+ size_t offset;
	304	+
	305	+ if (length == 0)
	306	+ return 0;
	307	+
	308	+ if (mr->ibmr.type == IB_MR_TYPE_DMA)
	309	+ return -EFAULT;
	310	+
	311	+ offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask;
	312	+ while (length > 0) {
	313	+ u8 *va;
	314	+ int bytes;
	315	+
	316	+ bytes = mr->ibmr.page_size - offset;
	317	+ if (bytes > length)
	318	+ bytes = length;
	319	+
	320	+ va = iova_to_vaddr(mr, iova, length);
	321	+ if (!va)
	322	+ return -EFAULT;
	323	+
	324	+ arch_wb_cache_pmem(va, bytes);
	325	+
	326	+ length -= bytes;
	327	+ iova += bytes;
	328	+ offset = 0;
	329	+ }
	330	+
	331	+ return 0;
	332	+}
	333	+
289	334	/* copy data from a range (vaddr, vaddr+length-1) to or from
290	335	* a mr object starting at iova.
291	336	*/
@@ -304,7 +349,7 @@ int rxe_mr_copy(struct rxe_mr mr, u64 iova, void addr, int length, · 12 unmodified lines
304	349	if (length == 0)
305	350	return 0;
306	351
307		− if (mr->type == IB_MR_TYPE_DMA) {
	352	+ if (mr->ibmr.type == IB_MR_TYPE_DMA) {
308	353	u8 src, dest;
309	354
310	355	src = (dir == RXE_TO_MR_OBJ) ? addr : ((void *)(uintptr_t)iova);
@@ -511,7 +556,7 @@ struct rxe_mr lookup_mr(struct rxe_pd pd, int access, u32 key, · 200 unmodified lines
511	556
512	557	if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) \|\|
513	558	(type == RXE_LOOKUP_REMOTE && mr->rkey != key) \|\|
514		− mr_pd(mr) != pd \|\| (access && !(access & mr->access)) \|\|
	559	+ mr_pd(mr) != pd \|\| ((access & mr->access) != access) \|\|
515	560	mr->state != RXE_MR_STATE_VALID)) {
516	561	rxe_put(mr);
517	562	mr = NULL;
@@ -528,27 +573,26 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 key) · 10 unmodified lines
528	573
529	574	mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
530	575	if (!mr) {
531		− pr_err("%s: No MR for key %#x\n", __func__, key);
	576	+ rxe_dbg_qp(qp, "No MR for key %#x\n", key);
532	577	ret = -EINVAL;
533	578	goto err;
534	579	}
535	580
536	581	if (mr->rkey ? (key != mr->rkey) : (key != mr->lkey)) {
537		− pr_err("%s: wr key (%#x) doesn't match mr key (%#x)\n",
538		− __func__, key, (mr->rkey ? mr->rkey : mr->lkey));
	582	+ rxe_dbg_mr(mr, "wr key (%#x) doesn't match mr key (%#x)\n",
	583	+ key, (mr->rkey ? mr->rkey : mr->lkey));
539	584	ret = -EINVAL;
540	585	goto err_drop_ref;
541	586	}
542	587
543	588	if (atomic_read(&mr->num_mw) > 0) {
544		− pr_warn("%s: Attempt to invalidate an MR while bound to MWs\n",
545		− __func__);
	589	+ rxe_dbg_mr(mr, "Attempt to invalidate an MR while bound to MWs\n");
546	590	ret = -EINVAL;
547	591	goto err_drop_ref;
548	592	}
549	593
550		− if (unlikely(mr->type != IB_MR_TYPE_MEM_REG)) {
551		− pr_warn("%s: mr->type (%d) is wrong type\n", __func__, mr->type);
	594	+ if (unlikely(mr->ibmr.type != IB_MR_TYPE_MEM_REG)) {
	595	+ rxe_dbg_mr(mr, "Type (%d) is wrong\n", mr->ibmr.type);
552	596	ret = -EINVAL;
553	597	goto err_drop_ref;
554	598	}
@@ -577,22 +621,20 @@ int rxe_reg_fast_mr(struct rxe_qp qp, struct rxe_send_wqe wqe) · 22 unmodified lines
577	621
578	622	/* user can only register MR in free state */
579	623	if (unlikely(mr->state != RXE_MR_STATE_FREE)) {
580		− pr_warn("%s: mr->lkey = 0x%x not free\n",
581		− __func__, mr->lkey);
	624	+ rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n", mr->lkey);
582	625	return -EINVAL;
583	626	}
584	627
585	628	/* user can only register mr with qp in same protection domain */
586	629	if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {
587		− pr_warn("%s: qp->pd and mr->pd don't match\n",
588		− __func__);
	630	+ rxe_dbg_mr(mr, "qp->pd and mr->pd don't match\n");
589	631	return -EINVAL;
590	632	}
591	633
592	634	/* user is only allowed to change key portion of l/rkey */
593	635	if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {
594		− pr_warn("%s: key = 0x%x has wrong index mr->lkey = 0x%x\n",
595		− __func__, key, mr->lkey);
	636	+ rxe_dbg_mr(mr, "key = 0x%x has wrong index mr->lkey = 0x%x\n",
	637	+ key, mr->lkey);
596	638	return -EINVAL;
597	639	}
598	640

drivers/infiniband/sw/rxe/rxe_mw.c+11−12

@@ -52,14 +52,14 @@ static int rxe_check_bind_mw(struct rxe_qp qp, struct rxe_send_wqe wqe, · 51 unmodified lines
52	52	{
53	53	if (mw->ibmw.type == IB_MW_TYPE_1) {
54	54	if (unlikely(mw->state != RXE_MW_STATE_VALID)) {
55		− pr_err_once(
	55	+ rxe_dbg_mw(mw,
56	56	"attempt to bind a type 1 MW not in the valid state\n");
57	57	return -EINVAL;
58	58	}
59	59
60	60	/* o10-36.2.2 */
61	61	if (unlikely((mw->access & IB_ZERO_BASED))) {
62		− pr_err_once("attempt to bind a zero based type 1 MW\n");
	62	+ rxe_dbg_mw(mw, "attempt to bind a zero based type 1 MW\n");
63	63	return -EINVAL;
64	64	}
65	65	}
@@ -67,21 +67,21 @@ static int rxe_check_bind_mw(struct rxe_qp qp, struct rxe_send_wqe wqe, · 1 unmodified line
67	67	if (mw->ibmw.type == IB_MW_TYPE_2) {
68	68	/* o10-37.2.30 */
69	69	if (unlikely(mw->state != RXE_MW_STATE_FREE)) {
70		− pr_err_once(
	70	+ rxe_dbg_mw(mw,
71	71	"attempt to bind a type 2 MW not in the free state\n");
72	72	return -EINVAL;
73	73	}
74	74
75	75	/* C10-72 */
76	76	if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) {
77		− pr_err_once(
	77	+ rxe_dbg_mw(mw,
78	78	"attempt to bind type 2 MW with qp with different PD\n");
79	79	return -EINVAL;
80	80	}
81	81
82	82	/* o10-37.2.40 */
83	83	if (unlikely(!mr \|\| wqe->wr.wr.mw.length == 0)) {
84		− pr_err_once(
	84	+ rxe_dbg_mw(mw,
85	85	"attempt to invalidate type 2 MW by binding with NULL or zero length MR\n");
86	86	return -EINVAL;
87	87	}
@@ -92,13 +92,13 @@ static int rxe_check_bind_mw(struct rxe_qp qp, struct rxe_send_wqe wqe, · 4 unmodified lines
92	92	return 0;
93	93
94	94	if (unlikely(mr->access & IB_ZERO_BASED)) {
95		− pr_err_once("attempt to bind MW to zero based MR\n");
	95	+ rxe_dbg_mw(mw, "attempt to bind MW to zero based MR\n");
96	96	return -EINVAL;
97	97	}
98	98
99	99	/* C10-73 */
100	100	if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) {
101		− pr_err_once(
	101	+ rxe_dbg_mw(mw,
102	102	"attempt to bind an MW to an MR without bind access\n");
103	103	return -EINVAL;
104	104	}
@@ -107,7 +107,7 @@ static int rxe_check_bind_mw(struct rxe_qp qp, struct rxe_send_wqe wqe, · 2 unmodified lines
107	107	if (unlikely((mw->access &
108	108	(IB_ACCESS_REMOTE_WRITE \| IB_ACCESS_REMOTE_ATOMIC)) &&
109	109	!(mr->access & IB_ACCESS_LOCAL_WRITE))) {
110		− pr_err_once(
	110	+ rxe_dbg_mw(mw,
111	111	"attempt to bind an Writable MW to an MR without local write access\n");
112	112	return -EINVAL;
113	113	}
@@ -115,7 +115,7 @@ static int rxe_check_bind_mw(struct rxe_qp qp, struct rxe_send_wqe wqe, · 1 unmodified line
115	115	/* C10-75 */
116	116	if (mw->access & IB_ZERO_BASED) {
117	117	if (unlikely(wqe->wr.wr.mw.length > mr->ibmr.length)) {
118		− pr_err_once(
	118	+ rxe_dbg_mw(mw,
119	119	"attempt to bind a ZB MW outside of the MR\n");
120	120	return -EINVAL;
121	121	}
@@ -123,7 +123,7 @@ static int rxe_check_bind_mw(struct rxe_qp qp, struct rxe_send_wqe wqe, · 1 unmodified line
123	123	if (unlikely((wqe->wr.wr.mw.addr < mr->ibmr.iova) \|\|
124	124	((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) >
125	125	(mr->ibmr.iova + mr->ibmr.length)))) {
126		− pr_err_once(
	126	+ rxe_dbg_mw(mw,
127	127	"attempt to bind a VA MW outside of the MR\n");
128	128	return -EINVAL;
129	129	}
@@ -293,8 +293,7 @@ struct rxe_mw rxe_lookup_mw(struct rxe_qp qp, int access, u32 rkey) · 163 unmodified lines
293	293
294	294	if (unlikely((mw->rkey != rkey) \|\| rxe_mw_pd(mw) != pd \|\|
295	295	(mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) \|\|
296		− (mw->length == 0) \|\|
297		− (access && !(access & mw->access)) \|\|
	296	+ (mw->length == 0) \|\| ((access & mw->access) != access) \|\|
298	297	mw->state != RXE_MW_STATE_VALID)) {
299	298	rxe_put(mw);
300	299	return NULL;

drivers/infiniband/sw/rxe/rxe_net.c+23−19

@@ -20,9 +20,10 @@ · 19 unmodified lines
20	20
21	21	static struct rxe_recv_sockets recv_sockets;
22	22
23		−static struct dst_entry rxe_find_route4(struct net_device ndev,
24		− struct in_addr *saddr,
25		− struct in_addr *daddr)
	23	+static struct dst_entry rxe_find_route4(struct rxe_qp qp,
	24	+ struct net_device *ndev,
	25	+ struct in_addr *saddr,
	26	+ struct in_addr *daddr)
26	27	{
27	28	struct rtable *rt;
28	29	struct flowi4 fl = { { 0 } };
@@ -35,7 +36,7 @@ static struct dst_entry rxe_find_route4(struct net_device ndev, · 6 unmodified lines
35	36
36	37	rt = ip_route_output_key(&init_net, &fl);
37	38	if (IS_ERR(rt)) {
38		− pr_err_ratelimited("no route to %pI4\n", &daddr->s_addr);
	39	+ rxe_dbg_qp(qp, "no route to %pI4\n", &daddr->s_addr);
39	40	return NULL;
40	41	}
41	42
@@ -43,7 +44,8 @@ static struct dst_entry rxe_find_route4(struct net_device ndev, · 1 unmodified line
43	44	}
44	45
45	46	#if IS_ENABLED(CONFIG_IPV6)
46		−static struct dst_entry rxe_find_route6(struct net_device ndev,
	47	+static struct dst_entry rxe_find_route6(struct rxe_qp qp,
	48	+ struct net_device *ndev,
47	49	struct in6_addr *saddr,
48	50	struct in6_addr *daddr)
49	51	{
@@ -60,12 +62,12 @@ static struct dst_entry rxe_find_route6(struct net_device ndev, · 10 unmodified lines
60	62	recv_sockets.sk6->sk, &fl6,
61	63	NULL);
62	64	if (IS_ERR(ndst)) {
63		− pr_err_ratelimited("no route to %pI6\n", daddr);
	65	+ rxe_dbg_qp(qp, "no route to %pI6\n", daddr);
64	66	return NULL;
65	67	}
66	68
67	69	if (unlikely(ndst->error)) {
68		− pr_err("no route to %pI6\n", daddr);
	70	+ rxe_dbg_qp(qp, "no route to %pI6\n", daddr);
69	71	goto put;
70	72	}
71	73
@@ -77,7 +79,8 @@ put: · 5 unmodified lines
77	79
78	80	#else
79	81
80		−static struct dst_entry rxe_find_route6(struct net_device ndev,
	82	+static struct dst_entry rxe_find_route6(struct rxe_qp qp,
	83	+ struct net_device *ndev,
81	84	struct in6_addr *saddr,
82	85	struct in6_addr *daddr)
83	86	{
@@ -105,14 +108,14 @@ static struct dst_entry rxe_find_route(struct net_device ndev, · 21 unmodified lines
105	108
106	109	saddr = &av->sgid_addr._sockaddr_in.sin_addr;
107	110	daddr = &av->dgid_addr._sockaddr_in.sin_addr;
108		− dst = rxe_find_route4(ndev, saddr, daddr);
	111	+ dst = rxe_find_route4(qp, ndev, saddr, daddr);
109	112	} else if (av->network_type == RXE_NETWORK_TYPE_IPV6) {
110	113	struct in6_addr *saddr6;
111	114	struct in6_addr *daddr6;
112	115
113	116	saddr6 = &av->sgid_addr._sockaddr_in6.sin6_addr;
114	117	daddr6 = &av->dgid_addr._sockaddr_in6.sin6_addr;
115		− dst = rxe_find_route6(ndev, saddr6, daddr6);
	118	+ dst = rxe_find_route6(qp, ndev, saddr6, daddr6);
116	119	#if IS_ENABLED(CONFIG_IPV6)
117	120	if (dst)
118	121	qp->dst_cookie =
@@ -282,7 +285,7 @@ static int prepare4(struct rxe_av av, struct rxe_pkt_info pkt, · 163 unmodified lines
282	285
283	286	dst = rxe_find_route(skb->dev, qp, av);
284	287	if (!dst) {
285		− pr_err("Host not reachable\n");
	288	+ rxe_dbg_qp(qp, "Host not reachable\n");
286	289	return -EHOSTUNREACH;
287	290	}
288	291
@@ -306,7 +309,7 @@ static int prepare6(struct rxe_av av, struct rxe_pkt_info pkt, · 17 unmodified lines
306	309
307	310	dst = rxe_find_route(skb->dev, qp, av);
308	311	if (!dst) {
309		− pr_err("Host not reachable\n");
	312	+ rxe_dbg_qp(qp, "Host not reachable\n");
310	313	return -EHOSTUNREACH;
311	314	}
312	315
@@ -345,7 +348,7 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb) · 32 unmodified lines
345	348
346	349	if (unlikely(qp->need_req_skb &&
347	350	skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW))
348		− rxe_run_task(&qp->req.task, 1);
	351	+ rxe_sched_task(&qp->req.task);
349	352
350	353	rxe_put(qp);
351	354	}
@@ -365,7 +368,8 @@ static int rxe_send(struct sk_buff skb, struct rxe_pkt_info pkt) · 13 unmodified lines
365	368	} else if (skb->protocol == htons(ETH_P_IPV6)) {
366	369	err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
367	370	} else {
368		− pr_err("Unknown layer 3 protocol: %d\n", skb->protocol);
	371	+ rxe_dbg_qp(pkt->qp, "Unknown layer 3 protocol: %d\n",
	372	+ skb->protocol);
369	373	atomic_dec(&pkt->qp->skb_out);
370	374	rxe_put(pkt->qp);
371	375	kfree_skb(skb);
@@ -373,7 +377,7 @@ static int rxe_send(struct sk_buff skb, struct rxe_pkt_info pkt) · 1 unmodified line
373	377	}
374	378
375	379	if (unlikely(net_xmit_eval(err))) {
376		− pr_debug("error sending packet: %d\n", err);
	380	+ rxe_dbg_qp(pkt->qp, "error sending packet: %d\n", err);
377	381	return -EAGAIN;
378	382	}
379	383
@@ -411,7 +415,7 @@ int rxe_xmit_packet(struct rxe_qp qp, struct rxe_pkt_info pkt, · 31 unmodified lines
411	415
412	416	if ((is_request && (qp->req.state != QP_STATE_READY)) \|\|
413	417	(!is_request && (qp->resp.state != QP_STATE_READY))) {
414		− pr_info("Packet dropped. QP is not in ready state\n");
	418	+ rxe_dbg_qp(qp, "Packet dropped. QP is not in ready state\n");
415	419	goto drop;
416	420	}
417	421
@@ -429,7 +433,7 @@ int rxe_xmit_packet(struct rxe_qp qp, struct rxe_pkt_info pkt, · 11 unmodified lines
429	433	if ((qp_type(qp) != IB_QPT_RC) &&
430	434	(pkt->mask & RXE_END_MASK)) {
431	435	pkt->wqe->state = wqe_state_done;
432		− rxe_run_task(&qp->comp.task, 1);
	436	+ rxe_sched_task(&qp->comp.task);
433	437	}
434	438
435	439	rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
@@ -592,7 +596,7 @@ static int rxe_notify(struct notifier_block *not_blk, · 156 unmodified lines
592	596	rxe_port_down(rxe);
593	597	break;
594	598	case NETDEV_CHANGEMTU:
595		− pr_info("%s changed mtu to %d\n", ndev->name, ndev->mtu);
	599	+ rxe_dbg(rxe, "%s changed mtu to %d\n", ndev->name, ndev->mtu);
596	600	rxe_set_mtu(rxe, ndev->mtu);
597	601	break;
598	602	case NETDEV_CHANGE:
@@ -604,7 +608,7 @@ static int rxe_notify(struct notifier_block *not_blk, · 5 unmodified lines
604	608	case NETDEV_CHANGENAME:
605	609	case NETDEV_FEAT_CHANGE:
606	610	default:
607		− pr_info("ignoring netdev event = %ld for %s\n",
	611	+ rxe_dbg(rxe, "ignoring netdev event = %ld for %s\n",
608	612	event, ndev->name);
609	613	break;
610	614	}

drivers/infiniband/sw/rxe/rxe_opcode.c+35−0

@@ -101,6 +101,18 @@ struct rxe_wr_opcode_info rxe_wr_opcode_info[] = { · 100 unmodified lines
101	101	[IB_QPT_UC] = WR_LOCAL_OP_MASK,
102	102	},
103	103	},
	104	+ [IB_WR_FLUSH] = {
	105	+ .name = "IB_WR_FLUSH",
	106	+ .mask = {
	107	+ [IB_QPT_RC] = WR_FLUSH_MASK,
	108	+ },
	109	+ },
	110	+ [IB_WR_ATOMIC_WRITE] = {
	111	+ .name = "IB_WR_ATOMIC_WRITE",
	112	+ .mask = {
	113	+ [IB_QPT_RC] = WR_ATOMIC_WRITE_MASK,
	114	+ },
	115	+ },
104	116	};
105	117
106	118	struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = {
@@ -378,6 +390,29 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 271 unmodified lines
378	390	RXE_IETH_BYTES,
379	391	}
380	392	},
	393	+ [IB_OPCODE_RC_FLUSH] = {
	394	+ .name = "IB_OPCODE_RC_FLUSH",
	395	+ .mask = RXE_FETH_MASK \| RXE_RETH_MASK \| RXE_FLUSH_MASK \|
	396	+ RXE_START_MASK \| RXE_END_MASK \| RXE_REQ_MASK,
	397	+ .length = RXE_BTH_BYTES + RXE_FETH_BYTES + RXE_RETH_BYTES,
	398	+ .offset = {
	399	+ [RXE_BTH] = 0,
	400	+ [RXE_FETH] = RXE_BTH_BYTES,
	401	+ [RXE_RETH] = RXE_BTH_BYTES + RXE_FETH_BYTES,
	402	+ }
	403	+ },
	404	+ [IB_OPCODE_RC_ATOMIC_WRITE] = {
	405	+ .name = "IB_OPCODE_RC_ATOMIC_WRITE",
	406	+ .mask = RXE_RETH_MASK \| RXE_PAYLOAD_MASK \| RXE_REQ_MASK \|
	407	+ RXE_ATOMIC_WRITE_MASK \| RXE_START_MASK \|
	408	+ RXE_END_MASK,
	409	+ .length = RXE_BTH_BYTES + RXE_RETH_BYTES,
	410	+ .offset = {
	411	+ [RXE_BTH] = 0,
	412	+ [RXE_RETH] = RXE_BTH_BYTES,
	413	+ [RXE_PAYLOAD] = RXE_BTH_BYTES + RXE_RETH_BYTES,
	414	+ }
	415	+ },
381	416
382	417	/* UC */
383	418	[IB_OPCODE_UC_SEND_FIRST] = {

drivers/infiniband/sw/rxe/rxe_opcode.h+12−5

@@ -20,6 +20,8 @@ enum rxe_wr_mask { · 19 unmodified lines
20	20	WR_READ_MASK = BIT(3),
21	21	WR_WRITE_MASK = BIT(4),
22	22	WR_LOCAL_OP_MASK = BIT(5),
	23	+ WR_FLUSH_MASK = BIT(6),
	24	+ WR_ATOMIC_WRITE_MASK = BIT(7),
23	25
24	26	WR_READ_OR_WRITE_MASK = WR_READ_MASK \| WR_WRITE_MASK,
25	27	WR_WRITE_OR_SEND_MASK = WR_WRITE_MASK \| WR_SEND_MASK,
@@ -47,6 +49,7 @@ enum rxe_hdr_type { · 21 unmodified lines
47	49	RXE_RDETH,
48	50	RXE_DETH,
49	51	RXE_IMMDT,
	52	+ RXE_FETH,
50	53	RXE_PAYLOAD,
51	54	NUM_HDR_TYPES
52	55	};
@@ -63,6 +66,7 @@ enum rxe_hdr_mask { · 10 unmodified lines
63	66	RXE_IETH_MASK = BIT(RXE_IETH),
64	67	RXE_RDETH_MASK = BIT(RXE_RDETH),
65	68	RXE_DETH_MASK = BIT(RXE_DETH),
	69	+ RXE_FETH_MASK = BIT(RXE_FETH),
66	70	RXE_PAYLOAD_MASK = BIT(RXE_PAYLOAD),
67	71
68	72	RXE_REQ_MASK = BIT(NUM_HDR_TYPES + 0),
@@ -71,16 +75,19 @@ enum rxe_hdr_mask { · 2 unmodified lines
71	75	RXE_WRITE_MASK = BIT(NUM_HDR_TYPES + 3),
72	76	RXE_READ_MASK = BIT(NUM_HDR_TYPES + 4),
73	77	RXE_ATOMIC_MASK = BIT(NUM_HDR_TYPES + 5),
	78	+ RXE_FLUSH_MASK = BIT(NUM_HDR_TYPES + 6),
74	79
75		− RXE_RWR_MASK = BIT(NUM_HDR_TYPES + 6),
76		− RXE_COMP_MASK = BIT(NUM_HDR_TYPES + 7),
	80	+ RXE_RWR_MASK = BIT(NUM_HDR_TYPES + 7),
	81	+ RXE_COMP_MASK = BIT(NUM_HDR_TYPES + 8),
77	82
78		− RXE_START_MASK = BIT(NUM_HDR_TYPES + 8),
79		− RXE_MIDDLE_MASK = BIT(NUM_HDR_TYPES + 9),
80		− RXE_END_MASK = BIT(NUM_HDR_TYPES + 10),
	83	+ RXE_START_MASK = BIT(NUM_HDR_TYPES + 9),
	84	+ RXE_MIDDLE_MASK = BIT(NUM_HDR_TYPES + 10),
	85	+ RXE_END_MASK = BIT(NUM_HDR_TYPES + 11),
81	86
82	87	RXE_LOOPBACK_MASK = BIT(NUM_HDR_TYPES + 12),
83	88
	89	+ RXE_ATOMIC_WRITE_MASK = BIT(NUM_HDR_TYPES + 14),
	90	+
84	91	RXE_READ_OR_ATOMIC_MASK = (RXE_READ_MASK \| RXE_ATOMIC_MASK),
85	92	RXE_WRITE_OR_SEND_MASK = (RXE_WRITE_MASK \| RXE_SEND_MASK),
86	93	RXE_READ_OR_WRITE_MASK = (RXE_READ_MASK \| RXE_WRITE_MASK),

drivers/infiniband/sw/rxe/rxe_param.h+7−0

@@ -51,7 +51,14 @@ enum rxe_device_param { · 50 unmodified lines
51	51	\| IB_DEVICE_SRQ_RESIZE
52	52	\| IB_DEVICE_MEM_MGT_EXTENSIONS
53	53	\| IB_DEVICE_MEM_WINDOW
	54	+ \| IB_DEVICE_FLUSH_GLOBAL
	55	+ \| IB_DEVICE_FLUSH_PERSISTENT
	56	+#ifdef CONFIG_64BIT
	57	+ \| IB_DEVICE_MEM_WINDOW_TYPE_2B
	58	+ \| IB_DEVICE_ATOMIC_WRITE,
	59	+#else
54	60	\| IB_DEVICE_MEM_WINDOW_TYPE_2B,
	61	+#endif /* CONFIG_64BIT */
55	62	RXE_MAX_SGE = 32,
56	63	RXE_MAX_WQE_SIZE = sizeof(struct rxe_send_wqe) +
57	64	sizeof(struct ib_sge) * RXE_MAX_SGE,

drivers/infiniband/sw/rxe/rxe_qp.c+43−55

@@ -19,33 +19,33 @@ static int rxe_qp_chk_cap(struct rxe_dev rxe, struct ib_qp_cap cap, · 18 unmodified lines
19	19	int has_srq)
20	20	{
21	21	if (cap->max_send_wr > rxe->attr.max_qp_wr) {
22		− pr_debug("invalid send wr = %u > %d\n",
	22	+ rxe_dbg(rxe, "invalid send wr = %u > %d\n",
23	23	cap->max_send_wr, rxe->attr.max_qp_wr);
24	24	goto err1;
25	25	}
26	26
27	27	if (cap->max_send_sge > rxe->attr.max_send_sge) {
28		− pr_debug("invalid send sge = %u > %d\n",
	28	+ rxe_dbg(rxe, "invalid send sge = %u > %d\n",
29	29	cap->max_send_sge, rxe->attr.max_send_sge);
30	30	goto err1;
31	31	}
32	32
33	33	if (!has_srq) {
34	34	if (cap->max_recv_wr > rxe->attr.max_qp_wr) {
35		− pr_debug("invalid recv wr = %u > %d\n",
	35	+ rxe_dbg(rxe, "invalid recv wr = %u > %d\n",
36	36	cap->max_recv_wr, rxe->attr.max_qp_wr);
37	37	goto err1;
38	38	}
39	39
40	40	if (cap->max_recv_sge > rxe->attr.max_recv_sge) {
41		− pr_debug("invalid recv sge = %u > %d\n",
	41	+ rxe_dbg(rxe, "invalid recv sge = %u > %d\n",
42	42	cap->max_recv_sge, rxe->attr.max_recv_sge);
43	43	goto err1;
44	44	}
45	45	}
46	46
47	47	if (cap->max_inline_data > rxe->max_inline_data) {
48		− pr_debug("invalid max inline data = %u > %d\n",
	48	+ rxe_dbg(rxe, "invalid max inline data = %u > %d\n",
49	49	cap->max_inline_data, rxe->max_inline_data);
50	50	goto err1;
51	51	}
@@ -73,7 +73,7 @@ int rxe_qp_chk_init(struct rxe_dev rxe, struct ib_qp_init_attr init) · 21 unmodified lines
73	73	}
74	74
75	75	if (!init->recv_cq \|\| !init->send_cq) {
76		− pr_debug("missing cq\n");
	76	+ rxe_dbg(rxe, "missing cq\n");
77	77	goto err1;
78	78	}
79	79
@@ -82,14 +82,14 @@ int rxe_qp_chk_init(struct rxe_dev rxe, struct ib_qp_init_attr init) · 2 unmodified lines
82	82
83	83	if (init->qp_type == IB_QPT_GSI) {
84	84	if (!rdma_is_port_valid(&rxe->ib_dev, port_num)) {
85		− pr_debug("invalid port = %d\n", port_num);
	85	+ rxe_dbg(rxe, "invalid port = %d\n", port_num);
86	86	goto err1;
87	87	}
88	88
89	89	port = &rxe->port;
90	90
91	91	if (init->qp_type == IB_QPT_GSI && port->qp_gsi_index) {
92		− pr_debug("GSI QP exists for port %d\n", port_num);
	92	+ rxe_dbg(rxe, "GSI QP exists for port %d\n", port_num);
93	93	goto err1;
94	94	}
95	95	}
@@ -172,10 +172,6 @@ static void rxe_qp_init_misc(struct rxe_dev rxe, struct rxe_qp qp, · 76 unmodified lines
172	172
173	173	spin_lock_init(&qp->state_lock);
174	174
175		− spin_lock_init(&qp->req.task.state_lock);
176		− spin_lock_init(&qp->resp.task.state_lock);
177		− spin_lock_init(&qp->comp.task.state_lock);
178		−
179	175	spin_lock_init(&qp->sq.sq_lock);
180	176	spin_lock_init(&qp->rq.producer_lock);
181	177	spin_lock_init(&qp->rq.consumer_lock);
@@ -242,10 +238,8 @@ static int rxe_qp_init_req(struct rxe_dev rxe, struct rxe_qp qp, · 60 unmodified lines
242	238
243	239	skb_queue_head_init(&qp->req_pkts);
244	240
245		− rxe_init_task(&qp->req.task, qp,
246		− rxe_requester, "req");
247		− rxe_init_task(&qp->comp.task, qp,
248		− rxe_completer, "comp");
	241	+ rxe_init_task(&qp->req.task, qp, rxe_requester);
	242	+ rxe_init_task(&qp->comp.task, qp, rxe_completer);
249	243
250	244	qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */
251	245	if (init->qp_type == IB_QPT_RC) {
@@ -270,9 +264,6 @@ static int rxe_qp_init_resp(struct rxe_dev rxe, struct rxe_qp qp, · 18 unmodified lines
270	264
271	265	wqe_size = rcv_wqe_size(qp->rq.max_sge);
272	266
273		− pr_debug("qp#%d max_wr = %d, max_sge = %d, wqe_size = %d\n",
274		− qp_num(qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size);
275		−
276	267	type = QUEUE_TYPE_FROM_CLIENT;
277	268	qp->rq.queue = rxe_queue_init(rxe, &qp->rq.max_wr,
278	269	wqe_size, type);
@@ -292,8 +283,7 @@ static int rxe_qp_init_resp(struct rxe_dev rxe, struct rxe_qp qp, · 13 unmodified lines
292	283
293	284	skb_queue_head_init(&qp->resp_pkts);
294	285
295		− rxe_init_task(&qp->resp.task, qp,
296		− rxe_responder, "resp");
	286	+ rxe_init_task(&qp->resp.task, qp, rxe_responder);
297	287
298	288	qp->resp.opcode = OPCODE_NONE;
299	289	qp->resp.msn = 0;
@@ -402,7 +392,7 @@ int rxe_qp_chk_attr(struct rxe_dev rxe, struct rxe_qp qp, · 102 unmodified lines
402	392	attr->qp_state : cur_state;
403	393
404	394	if (!ib_modify_qp_is_ok(cur_state, new_state, qp_type(qp), mask)) {
405		− pr_debug("invalid mask or state for qp\n");
	395	+ rxe_dbg_qp(qp, "invalid mask or state\n");
406	396	goto err1;
407	397	}
408	398
@@ -416,7 +406,7 @@ int rxe_qp_chk_attr(struct rxe_dev rxe, struct rxe_qp qp, · 7 unmodified lines
416	406
417	407	if (mask & IB_QP_PORT) {
418	408	if (!rdma_is_port_valid(&rxe->ib_dev, attr->port_num)) {
419		− pr_debug("invalid port %d\n", attr->port_num);
	409	+ rxe_dbg_qp(qp, "invalid port %d\n", attr->port_num);
420	410	goto err1;
421	411	}
422	412	}
@@ -424,18 +414,18 @@ int rxe_qp_chk_attr(struct rxe_dev rxe, struct rxe_qp qp, · 1 unmodified line
424	414	if (mask & IB_QP_CAP && rxe_qp_chk_cap(rxe, &attr->cap, !!qp->srq))
425	415	goto err1;
426	416
427		− if (mask & IB_QP_AV && rxe_av_chk_attr(rxe, &attr->ah_attr))
	417	+ if (mask & IB_QP_AV && rxe_av_chk_attr(qp, &attr->ah_attr))
428	418	goto err1;
429	419
430	420	if (mask & IB_QP_ALT_PATH) {
431		− if (rxe_av_chk_attr(rxe, &attr->alt_ah_attr))
	421	+ if (rxe_av_chk_attr(qp, &attr->alt_ah_attr))
432	422	goto err1;
433	423	if (!rdma_is_port_valid(&rxe->ib_dev, attr->alt_port_num)) {
434		− pr_debug("invalid alt port %d\n", attr->alt_port_num);
	424	+ rxe_dbg_qp(qp, "invalid alt port %d\n", attr->alt_port_num);
435	425	goto err1;
436	426	}
437	427	if (attr->alt_timeout > 31) {
438		− pr_debug("invalid QP alt timeout %d > 31\n",
	428	+ rxe_dbg_qp(qp, "invalid alt timeout %d > 31\n",
439	429	attr->alt_timeout);
440	430	goto err1;
441	431	}
@@ -448,7 +438,7 @@ int rxe_qp_chk_attr(struct rxe_dev rxe, struct rxe_qp qp, · 6 unmodified lines
448	438	enum ib_mtu mtu = attr->path_mtu;
449	439
450	440	if (mtu > max_mtu) {
451		− pr_debug("invalid mtu (%d) > (%d)\n",
	441	+ rxe_dbg_qp(qp, "invalid mtu (%d) > (%d)\n",
452	442	ib_mtu_enum_to_int(mtu),
453	443	ib_mtu_enum_to_int(max_mtu));
454	444	goto err1;
@@ -457,7 +447,7 @@ int rxe_qp_chk_attr(struct rxe_dev rxe, struct rxe_qp qp, · 2 unmodified lines
457	447
458	448	if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
459	449	if (attr->max_rd_atomic > rxe->attr.max_qp_rd_atom) {
460		− pr_debug("invalid max_rd_atomic %d > %d\n",
	450	+ rxe_dbg_qp(qp, "invalid max_rd_atomic %d > %d\n",
461	451	attr->max_rd_atomic,
462	452	rxe->attr.max_qp_rd_atom);
463	453	goto err1;
@@ -466,7 +456,8 @@ int rxe_qp_chk_attr(struct rxe_dev rxe, struct rxe_qp qp, · 2 unmodified lines
466	456
467	457	if (mask & IB_QP_TIMEOUT) {
468	458	if (attr->timeout > 31) {
469		− pr_debug("invalid QP timeout %d > 31\n", attr->timeout);
	459	+ rxe_dbg_qp(qp, "invalid timeout %d > 31\n",
	460	+ attr->timeout);
470	461	goto err1;
471	462	}
472	463	}
@@ -543,10 +534,10 @@ static void rxe_qp_drain(struct rxe_qp *qp) · 70 unmodified lines
543	534	if (qp->req.state != QP_STATE_DRAINED) {
544	535	qp->req.state = QP_STATE_DRAIN;
545	536	if (qp_type(qp) == IB_QPT_RC)
546		− rxe_run_task(&qp->comp.task, 1);
	537	+ rxe_sched_task(&qp->comp.task);
547	538	else
548	539	__rxe_do_task(&qp->comp.task);
549		− rxe_run_task(&qp->req.task, 1);
	540	+ rxe_sched_task(&qp->req.task);
550	541	}
551	542	}
552	543	}
@@ -560,13 +551,13 @@ void rxe_qp_error(struct rxe_qp *qp) · 7 unmodified lines
560	551	qp->attr.qp_state = IB_QPS_ERR;
561	552
562	553	/* drain work and packet queues */
563		− rxe_run_task(&qp->resp.task, 1);
	554	+ rxe_sched_task(&qp->resp.task);
564	555
565	556	if (qp_type(qp) == IB_QPT_RC)
566		− rxe_run_task(&qp->comp.task, 1);
	557	+ rxe_sched_task(&qp->comp.task);
567	558	else
568	559	__rxe_do_task(&qp->comp.task);
569		− rxe_run_task(&qp->req.task, 1);
	560	+ rxe_sched_task(&qp->req.task);
570	561	}
571	562
572	563	/* called by the modify qp verb */
@@ -644,27 +635,24 @@ int rxe_qp_from_attr(struct rxe_qp qp, struct ib_qp_attr attr, int mask, · 71 unmodified lines
644	635	if (mask & IB_QP_RETRY_CNT) {
645	636	qp->attr.retry_cnt = attr->retry_cnt;
646	637	qp->comp.retry_cnt = attr->retry_cnt;
647		− pr_debug("qp#%d set retry count = %d\n", qp_num(qp),
648		− attr->retry_cnt);
	638	+ rxe_dbg_qp(qp, "set retry count = %d\n", attr->retry_cnt);
649	639	}
650	640
651	641	if (mask & IB_QP_RNR_RETRY) {
652	642	qp->attr.rnr_retry = attr->rnr_retry;
653	643	qp->comp.rnr_retry = attr->rnr_retry;
654		− pr_debug("qp#%d set rnr retry count = %d\n", qp_num(qp),
655		− attr->rnr_retry);
	644	+ rxe_dbg_qp(qp, "set rnr retry count = %d\n", attr->rnr_retry);
656	645	}
657	646
658	647	if (mask & IB_QP_RQ_PSN) {
659	648	qp->attr.rq_psn = (attr->rq_psn & BTH_PSN_MASK);
660	649	qp->resp.psn = qp->attr.rq_psn;
661		− pr_debug("qp#%d set resp psn = 0x%x\n", qp_num(qp),
662		− qp->resp.psn);
	650	+ rxe_dbg_qp(qp, "set resp psn = 0x%x\n", qp->resp.psn);
663	651	}
664	652
665	653	if (mask & IB_QP_MIN_RNR_TIMER) {
666	654	qp->attr.min_rnr_timer = attr->min_rnr_timer;
667		− pr_debug("qp#%d set min rnr timer = 0x%x\n", qp_num(qp),
	655	+ rxe_dbg_qp(qp, "set min rnr timer = 0x%x\n",
668	656	attr->min_rnr_timer);
669	657	}
670	658
@@ -672,7 +660,7 @@ int rxe_qp_from_attr(struct rxe_qp qp, struct ib_qp_attr attr, int mask, · 1 unmodified line
672	660	qp->attr.sq_psn = (attr->sq_psn & BTH_PSN_MASK);
673	661	qp->req.psn = qp->attr.sq_psn;
674	662	qp->comp.psn = qp->attr.sq_psn;
675		− pr_debug("qp#%d set req psn = 0x%x\n", qp_num(qp), qp->req.psn);
	663	+ rxe_dbg_qp(qp, "set req psn = 0x%x\n", qp->req.psn);
676	664	}
677	665
678	666	if (mask & IB_QP_PATH_MIG_STATE)
@@ -686,40 +674,40 @@ int rxe_qp_from_attr(struct rxe_qp qp, struct ib_qp_attr attr, int mask, · 7 unmodified lines
686	674
687	675	switch (attr->qp_state) {
688	676	case IB_QPS_RESET:
689		− pr_debug("qp#%d state -> RESET\n", qp_num(qp));
	677	+ rxe_dbg_qp(qp, "state -> RESET\n");
690	678	rxe_qp_reset(qp);
691	679	break;
692	680
693	681	case IB_QPS_INIT:
694		− pr_debug("qp#%d state -> INIT\n", qp_num(qp));
	682	+ rxe_dbg_qp(qp, "state -> INIT\n");
695	683	qp->req.state = QP_STATE_INIT;
696	684	qp->resp.state = QP_STATE_INIT;
697	685	qp->comp.state = QP_STATE_INIT;
698	686	break;
699	687
700	688	case IB_QPS_RTR:
701		− pr_debug("qp#%d state -> RTR\n", qp_num(qp));
	689	+ rxe_dbg_qp(qp, "state -> RTR\n");
702	690	qp->resp.state = QP_STATE_READY;
703	691	break;
704	692
705	693	case IB_QPS_RTS:
706		− pr_debug("qp#%d state -> RTS\n", qp_num(qp));
	694	+ rxe_dbg_qp(qp, "state -> RTS\n");
707	695	qp->req.state = QP_STATE_READY;
708	696	qp->comp.state = QP_STATE_READY;
709	697	break;
710	698
711	699	case IB_QPS_SQD:
712		− pr_debug("qp#%d state -> SQD\n", qp_num(qp));
	700	+ rxe_dbg_qp(qp, "state -> SQD\n");
713	701	rxe_qp_drain(qp);
714	702	break;
715	703
716	704	case IB_QPS_SQE:
717		− pr_warn("qp#%d state -> SQE !!?\n", qp_num(qp));
	705	+ rxe_dbg_qp(qp, "state -> SQE !!?\n");
718	706	/* Not possible from modify_qp. */
719	707	break;
720	708
721	709	case IB_QPS_ERR:
722		− pr_debug("qp#%d state -> ERR\n", qp_num(qp));
	710	+ rxe_dbg_qp(qp, "state -> ERR\n");
723	711	rxe_qp_error(qp);
724	712	break;
725	713	}
@@ -759,7 +747,7 @@ int rxe_qp_to_attr(struct rxe_qp qp, struct ib_qp_attr attr, int mask) · 33 unmodified lines
759	747	attr->sq_draining = 0;
760	748	}
761	749
762		− pr_debug("attr->sq_draining = %d\n", attr->sq_draining);
	750	+ rxe_dbg_qp(qp, "attr->sq_draining = %d\n", attr->sq_draining);
763	751
764	752	return 0;
765	753	}
@@ -771,7 +759,7 @@ int rxe_qp_chk_destroy(struct rxe_qp *qp) · 5 unmodified lines
771	759	* will fail immediately.
772	760	*/
773	761	if (atomic_read(&qp->mcg_num)) {
774		− pr_debug("Attempt to destroy QP while attached to multicast group\n");
	762	+ rxe_dbg_qp(qp, "Attempt to destroy while attached to multicast group\n");
775	763	return -EBUSY;
776	764	}
777	765
@@ -829,12 +817,12 @@ static void rxe_qp_do_cleanup(struct work_struct *work) · 51 unmodified lines
829	817	if (qp->resp.mr)
830	818	rxe_put(qp->resp.mr);
831	819
832		− if (qp_type(qp) == IB_QPT_RC)
833		− sk_dst_reset(qp->sk->sk);
834		−
835	820	free_rd_atomic_resources(qp);
836	821
837	822	if (qp->sk) {
	823	+ if (qp_type(qp) == IB_QPT_RC)
	824	+ sk_dst_reset(qp->sk->sk);
	825	+
838	826	kernel_sock_shutdown(qp->sk, SHUT_RDWR);
839	827	sock_release(qp->sk);
840	828	}

drivers/infiniband/sw/rxe/rxe_req.c+37−13

@@ -100,12 +100,12 @@ void rnr_nak_timer(struct timer_list *t) · 99 unmodified lines
100	100	{
101	101	struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
102	102
103		− pr_debug("%s: fired for qp#%d\n", __func__, qp_num(qp));
	103	+ rxe_dbg_qp(qp, "nak timer fired\n");
104	104
105	105	/* request a send queue retry */
106	106	qp->req.need_retry = 1;
107	107	qp->req.wait_for_rnr_timer = 0;
108		− rxe_run_task(&qp->req.task, 1);
	108	+ rxe_sched_task(&qp->req.task);
109	109	}
110	110
111	111	static struct rxe_send_wqe req_next_wqe(struct rxe_qp qp)
@@ -241,6 +241,9 @@ static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits) · 129 unmodified lines
241	241	IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE :
242	242	IB_OPCODE_RC_SEND_FIRST;
243	243
	244	+ case IB_WR_FLUSH:
	245	+ return IB_OPCODE_RC_FLUSH;
	246	+
244	247	case IB_WR_RDMA_READ:
245	248	return IB_OPCODE_RC_RDMA_READ_REQUEST;
246	249
@@ -258,6 +261,10 @@ static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits) · 11 unmodified lines
258	261	else
259	262	return fits ? IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE :
260	263	IB_OPCODE_RC_SEND_FIRST;
	264	+
	265	+ case IB_WR_ATOMIC_WRITE:
	266	+ return IB_OPCODE_RC_ATOMIC_WRITE;
	267	+
261	268	case IB_WR_REG_MR:
262	269	case IB_WR_LOCAL_INV:
263	270	return opcode;
@@ -421,11 +428,18 @@ static struct sk_buff init_req_packet(struct rxe_qp qp, · 157 unmodified lines
421	428
422	429	/* init optional headers */
423	430	if (pkt->mask & RXE_RETH_MASK) {
424		− reth_set_rkey(pkt, ibwr->wr.rdma.rkey);
	431	+ if (pkt->mask & RXE_FETH_MASK)
	432	+ reth_set_rkey(pkt, ibwr->wr.flush.rkey);
	433	+ else
	434	+ reth_set_rkey(pkt, ibwr->wr.rdma.rkey);
425	435	reth_set_va(pkt, wqe->iova);
426	436	reth_set_len(pkt, wqe->dma.resid);
427	437	}
428	438
	439	+ /* Fill Flush Extension Transport Header */
	440	+ if (pkt->mask & RXE_FETH_MASK)
	441	+ feth_init(pkt, ibwr->wr.flush.type, ibwr->wr.flush.level);
	442	+
429	443	if (pkt->mask & RXE_IMMDT_MASK)
430	444	immdt_set_imm(pkt, ibwr->ex.imm_data);
431	445
@@ -484,6 +498,14 @@ static int finish_packet(struct rxe_qp qp, struct rxe_av av, · 52 unmodified lines
484	498
485	499	memset(pad, 0, bth_pad(pkt));
486	500	}
	501	+ } else if (pkt->mask & RXE_FLUSH_MASK) {
	502	+ /* oA19-2: shall have no payload. */
	503	+ wqe->dma.resid = 0;
	504	+ }
	505	+
	506	+ if (pkt->mask & RXE_ATOMIC_WRITE_MASK) {
	507	+ memcpy(payload_addr(pkt), wqe->dma.atomic_wr, payload);
	508	+ wqe->dma.resid -= payload;
487	509	}
488	510
489	511	return 0;
@@ -595,7 +617,7 @@ static int rxe_do_local_ops(struct rxe_qp qp, struct rxe_send_wqe wqe) · 105 unmodified lines
595	617	}
596	618	break;
597	619	default:
598		− pr_err("Unexpected send wqe opcode %d\n", opcode);
	620	+ rxe_dbg_qp(qp, "Unexpected send wqe opcode %d\n", opcode);
599	621	wqe->status = IB_WC_LOC_QP_OP_ERR;
600	622	return -EINVAL;
601	623	}
@@ -608,7 +630,7 @@ static int rxe_do_local_ops(struct rxe_qp qp, struct rxe_send_wqe wqe) · 6 unmodified lines
608	630	* which can lead to a deadlock. So go ahead and complete
609	631	* it now.
610	632	*/
611		− rxe_run_task(&qp->comp.task, 1);
	633	+ rxe_sched_task(&qp->comp.task);
612	634
613	635	return 0;
614	636	}
@@ -709,13 +731,15 @@ int rxe_requester(void *arg) · 94 unmodified lines
709	731	}
710	732
711	733	mask = rxe_opcode[opcode].mask;
712		− if (unlikely(mask & RXE_READ_OR_ATOMIC_MASK)) {
	734	+ if (unlikely(mask & (RXE_READ_OR_ATOMIC_MASK \|
	735	+ RXE_ATOMIC_WRITE_MASK))) {
713	736	if (check_init_depth(qp, wqe))
714	737	goto exit;
715	738	}
716	739
717	740	mtu = get_mtu(qp);
718		− payload = (mask & RXE_WRITE_OR_SEND_MASK) ? wqe->dma.resid : 0;
	741	+ payload = (mask & (RXE_WRITE_OR_SEND_MASK \| RXE_ATOMIC_WRITE_MASK)) ?
	742	+ wqe->dma.resid : 0;
719	743	if (payload > mtu) {
720	744	if (qp_type(qp) == IB_QPT_UD) {
721	745	/* C10-93.1.1: If the total sum of all the buffer lengths specified for a
@@ -733,7 +757,7 @@ int rxe_requester(void *arg) · 11 unmodified lines
733	757	qp->req.wqe_index);
734	758	wqe->state = wqe_state_done;
735	759	wqe->status = IB_WC_SUCCESS;
736		− rxe_run_task(&qp->comp.task, 0);
	760	+ rxe_run_task(&qp->comp.task);
737	761	goto done;
738	762	}
739	763	payload = mtu;
@@ -748,14 +772,14 @@ int rxe_requester(void *arg) · 8 unmodified lines
748	772
749	773	av = rxe_get_av(&pkt, &ah);
750	774	if (unlikely(!av)) {
751		− pr_err("qp#%d Failed no address vector\n", qp_num(qp));
	775	+ rxe_dbg_qp(qp, "Failed no address vector\n");
752	776	wqe->status = IB_WC_LOC_QP_OP_ERR;
753	777	goto err;
754	778	}
755	779
756	780	skb = init_req_packet(qp, av, wqe, opcode, payload, &pkt);
757	781	if (unlikely(!skb)) {
758		− pr_err("qp#%d Failed allocating skb\n", qp_num(qp));
	782	+ rxe_dbg_qp(qp, "Failed allocating skb\n");
759	783	wqe->status = IB_WC_LOC_QP_OP_ERR;
760	784	if (ah)
761	785	rxe_put(ah);
@@ -764,7 +788,7 @@ int rxe_requester(void *arg) · 2 unmodified lines
764	788
765	789	err = finish_packet(qp, av, wqe, &pkt, skb, payload);
766	790	if (unlikely(err)) {
767		− pr_debug("qp#%d Error during finish packet\n", qp_num(qp));
	791	+ rxe_dbg_qp(qp, "Error during finish packet\n");
768	792	if (err == -EFAULT)
769	793	wqe->status = IB_WC_LOC_PROT_ERR;
770	794	else
@@ -795,7 +819,7 @@ int rxe_requester(void *arg) · 24 unmodified lines
795	819	rollback_state(wqe, qp, &rollback_wqe, rollback_psn);
796	820
797	821	if (err == -EAGAIN) {
798		− rxe_run_task(&qp->req.task, 1);
	822	+ rxe_sched_task(&qp->req.task);
799	823	goto exit;
800	824	}
801	825
@@ -817,7 +841,7 @@ err: · 15 unmodified lines
817	841	qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index);
818	842	wqe->state = wqe_state_error;
819	843	qp->req.state = QP_STATE_ERROR;
820		− rxe_run_task(&qp->comp.task, 0);
	844	+ rxe_run_task(&qp->comp.task);
821	845	exit:
822	846	ret = -EAGAIN;
823	847	out:

drivers/infiniband/sw/rxe/rxe_resp.c+279−50

@@ -22,6 +22,8 @@ enum resp_states { · 21 unmodified lines
22	22	RESPST_EXECUTE,
23	23	RESPST_READ_REPLY,
24	24	RESPST_ATOMIC_REPLY,
	25	+ RESPST_ATOMIC_WRITE_REPLY,
	26	+ RESPST_PROCESS_FLUSH,
25	27	RESPST_COMPLETE,
26	28	RESPST_ACKNOWLEDGE,
27	29	RESPST_CLEANUP,
@@ -57,6 +59,8 @@ static char *resp_state_name[] = { · 29 unmodified lines
57	59	[RESPST_EXECUTE] = "EXECUTE",
58	60	[RESPST_READ_REPLY] = "READ_REPLY",
59	61	[RESPST_ATOMIC_REPLY] = "ATOMIC_REPLY",
	62	+ [RESPST_ATOMIC_WRITE_REPLY] = "ATOMIC_WRITE_REPLY",
	63	+ [RESPST_PROCESS_FLUSH] = "PROCESS_FLUSH",
60	64	[RESPST_COMPLETE] = "COMPLETE",
61	65	[RESPST_ACKNOWLEDGE] = "ACKNOWLEDGE",
62	66	[RESPST_CLEANUP] = "CLEANUP",
@@ -91,7 +95,10 @@ void rxe_resp_queue_pkt(struct rxe_qp qp, struct sk_buff skb) · 28 unmodified lines
91	95	must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) \|\|
92	96	(skb_queue_len(&qp->req_pkts) > 1);
93	97
94		− rxe_run_task(&qp->resp.task, must_sched);
	98	+ if (must_sched)
	99	+ rxe_sched_task(&qp->resp.task);
	100	+ else
	101	+ rxe_run_task(&qp->resp.task);
95	102	}
96	103
97	104	static inline enum resp_states get_req(struct rxe_qp *qp,
@@ -253,19 +260,37 @@ static enum resp_states check_op_seq(struct rxe_qp *qp, · 155 unmodified lines
253	260	}
254	261	}
255	262
	263	+static bool check_qp_attr_access(struct rxe_qp *qp,
	264	+ struct rxe_pkt_info *pkt)
	265	+{
	266	+ if (((pkt->mask & RXE_READ_MASK) &&
	267	+ !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) \|\|
	268	+ ((pkt->mask & (RXE_WRITE_MASK \| RXE_ATOMIC_WRITE_MASK)) &&
	269	+ !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) \|\|
	270	+ ((pkt->mask & RXE_ATOMIC_MASK) &&
	271	+ !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
	272	+ return false;
	273	+
	274	+ if (pkt->mask & RXE_FLUSH_MASK) {
	275	+ u32 flush_type = feth_plt(pkt);
	276	+
	277	+ if ((flush_type & IB_FLUSH_GLOBAL &&
	278	+ !(qp->attr.qp_access_flags & IB_ACCESS_FLUSH_GLOBAL)) \|\|
	279	+ (flush_type & IB_FLUSH_PERSISTENT &&
	280	+ !(qp->attr.qp_access_flags & IB_ACCESS_FLUSH_PERSISTENT)))
	281	+ return false;
	282	+ }
	283	+
	284	+ return true;
	285	+}
	286	+
256	287	static enum resp_states check_op_valid(struct rxe_qp *qp,
257	288	struct rxe_pkt_info *pkt)
258	289	{
259	290	switch (qp_type(qp)) {
260	291	case IB_QPT_RC:
261		− if (((pkt->mask & RXE_READ_MASK) &&
262		− !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) \|\|
263		− ((pkt->mask & RXE_WRITE_MASK) &&
264		− !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) \|\|
265		− ((pkt->mask & RXE_ATOMIC_MASK) &&
266		− !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) {
	292	+ if (!check_qp_attr_access(qp, pkt))
267	293	return RESPST_ERR_UNSUPPORTED_OPCODE;
268		− }
269	294
270	295	break;
271	296
@@ -314,7 +339,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) · 42 unmodified lines
314	339	/* don't trust user space data */
315	340	if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) {
316	341	spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
317		− pr_warn("%s: invalid num_sge in SRQ entry\n", __func__);
	342	+ rxe_dbg_qp(qp, "invalid num_sge in SRQ entry\n");
318	343	return RESPST_ERR_MALFORMED_WQE;
319	344	}
320	345	size = sizeof(wqe) + wqe->dma.num_sgesizeof(struct rxe_sge);
@@ -364,7 +389,7 @@ static enum resp_states check_resource(struct rxe_qp *qp, · 43 unmodified lines
364	389	}
365	390	}
366	391
367		− if (pkt->mask & RXE_READ_OR_ATOMIC_MASK) {
	392	+ if (pkt->mask & (RXE_READ_OR_ATOMIC_MASK \| RXE_ATOMIC_WRITE_MASK)) {
368	393	/* it is the requesters job to not send
369	394	* too many read/atomic ops, we just
370	395	* recycle the responder resource queue
@@ -387,19 +412,66 @@ static enum resp_states check_resource(struct rxe_qp *qp, · 16 unmodified lines
387	412	return RESPST_CHK_LENGTH;
388	413	}
389	414
390		−static enum resp_states check_length(struct rxe_qp *qp,
391		− struct rxe_pkt_info *pkt)
	415	+static enum resp_states rxe_resp_check_length(struct rxe_qp *qp,
	416	+ struct rxe_pkt_info *pkt)
392	417	{
393		− switch (qp_type(qp)) {
394		− case IB_QPT_RC:
395		− return RESPST_CHK_RKEY;
396		−
397		− case IB_QPT_UC:
398		− return RESPST_CHK_RKEY;
	418	+ /*
	419	+ * See IBA C9-92
	420	+ * For UD QPs we only check if the packet will fit in the
	421	+ * receive buffer later. For rmda operations additional
	422	+ * length checks are performed in check_rkey.
	423	+ */
	424	+ if (pkt->mask & RXE_PAYLOAD_MASK && ((qp_type(qp) == IB_QPT_RC) \|\|
	425	+ (qp_type(qp) == IB_QPT_UC))) {
	426	+ unsigned int mtu = qp->mtu;
	427	+ unsigned int payload = payload_size(pkt);
	428	+
	429	+ if ((pkt->mask & RXE_START_MASK) &&
	430	+ (pkt->mask & RXE_END_MASK)) {
	431	+ if (unlikely(payload > mtu)) {
	432	+ rxe_dbg_qp(qp, "only packet too long");
	433	+ return RESPST_ERR_LENGTH;
	434	+ }
	435	+ } else if ((pkt->mask & RXE_START_MASK) \|\|
	436	+ (pkt->mask & RXE_MIDDLE_MASK)) {
	437	+ if (unlikely(payload != mtu)) {
	438	+ rxe_dbg_qp(qp, "first or middle packet not mtu");
	439	+ return RESPST_ERR_LENGTH;
	440	+ }
	441	+ } else if (pkt->mask & RXE_END_MASK) {
	442	+ if (unlikely((payload == 0) \|\| (payload > mtu))) {
	443	+ rxe_dbg_qp(qp, "last packet zero or too long");
	444	+ return RESPST_ERR_LENGTH;
	445	+ }
	446	+ }
	447	+ }
399	448
400		− default:
401		− return RESPST_CHK_RKEY;
	449	+ /* See IBA C9-94 */
	450	+ if (pkt->mask & RXE_RETH_MASK) {
	451	+ if (reth_len(pkt) > (1U << 31)) {
	452	+ rxe_dbg_qp(qp, "dma length too long");
	453	+ return RESPST_ERR_LENGTH;
	454	+ }
402	455	}
	456	+
	457	+ return RESPST_CHK_RKEY;
	458	+}
	459	+
	460	+static void qp_resp_from_reth(struct rxe_qp qp, struct rxe_pkt_info pkt)
	461	+{
	462	+ qp->resp.va = reth_va(pkt);
	463	+ qp->resp.offset = 0;
	464	+ qp->resp.rkey = reth_rkey(pkt);
	465	+ qp->resp.resid = reth_len(pkt);
	466	+ qp->resp.length = reth_len(pkt);
	467	+}
	468	+
	469	+static void qp_resp_from_atmeth(struct rxe_qp qp, struct rxe_pkt_info pkt)
	470	+{
	471	+ qp->resp.va = atmeth_va(pkt);
	472	+ qp->resp.offset = 0;
	473	+ qp->resp.rkey = atmeth_rkey(pkt);
	474	+ qp->resp.resid = sizeof(u64);
403	475	}
404	476
405	477	static enum resp_states check_rkey(struct rxe_qp *qp,
@@ -413,29 +485,32 @@ static enum resp_states check_rkey(struct rxe_qp *qp, · 7 unmodified lines
413	485	u32 pktlen;
414	486	int mtu = qp->mtu;
415	487	enum resp_states state;
416		− int access;
417		−
418		− if (pkt->mask & RXE_READ_OR_WRITE_MASK) {
419		− if (pkt->mask & RXE_RETH_MASK) {
420		− qp->resp.va = reth_va(pkt);
421		− qp->resp.offset = 0;
422		− qp->resp.rkey = reth_rkey(pkt);
423		− qp->resp.resid = reth_len(pkt);
424		− qp->resp.length = reth_len(pkt);
425		− }
	488	+ int access = 0;
	489	+
	490	+ if (pkt->mask & (RXE_READ_OR_WRITE_MASK \| RXE_ATOMIC_WRITE_MASK)) {
	491	+ if (pkt->mask & RXE_RETH_MASK)
	492	+ qp_resp_from_reth(qp, pkt);
	493	+
426	494	access = (pkt->mask & RXE_READ_MASK) ? IB_ACCESS_REMOTE_READ
427	495	: IB_ACCESS_REMOTE_WRITE;
	496	+ } else if (pkt->mask & RXE_FLUSH_MASK) {
	497	+ u32 flush_type = feth_plt(pkt);
	498	+
	499	+ if (pkt->mask & RXE_RETH_MASK)
	500	+ qp_resp_from_reth(qp, pkt);
	501	+
	502	+ if (flush_type & IB_FLUSH_GLOBAL)
	503	+ access \|= IB_ACCESS_FLUSH_GLOBAL;
	504	+ if (flush_type & IB_FLUSH_PERSISTENT)
	505	+ access \|= IB_ACCESS_FLUSH_PERSISTENT;
428	506	} else if (pkt->mask & RXE_ATOMIC_MASK) {
429		− qp->resp.va = atmeth_va(pkt);
430		− qp->resp.offset = 0;
431		− qp->resp.rkey = atmeth_rkey(pkt);
432		− qp->resp.resid = sizeof(u64);
	507	+ qp_resp_from_atmeth(qp, pkt);
433	508	access = IB_ACCESS_REMOTE_ATOMIC;
434	509	} else {
435	510	return RESPST_EXECUTE;
436	511	}
437	512
438		− /* A zero-byte op is not required to set an addr or rkey. */
	513	+ /* A zero-byte op is not required to set an addr or rkey. See C9-88 */
439	514	if ((pkt->mask & RXE_READ_OR_WRITE_MASK) &&
440	515	(pkt->mask & RXE_RETH_MASK) &&
441	516	reth_len(pkt) == 0) {
@@ -450,15 +525,14 @@ static enum resp_states check_rkey(struct rxe_qp *qp, · 8 unmodified lines
450	525	if (rkey_is_mw(rkey)) {
451	526	mw = rxe_lookup_mw(qp, access, rkey);
452	527	if (!mw) {
453		− pr_debug("%s: no MW matches rkey %#x\n",
454		− __func__, rkey);
	528	+ rxe_dbg_qp(qp, "no MW matches rkey %#x\n", rkey);
455	529	state = RESPST_ERR_RKEY_VIOLATION;
456	530	goto err;
457	531	}
458	532
459	533	mr = mw->mr;
460	534	if (!mr) {
461		− pr_err("%s: MW doesn't have an MR\n", __func__);
	535	+ rxe_dbg_qp(qp, "MW doesn't have an MR\n");
462	536	state = RESPST_ERR_RKEY_VIOLATION;
463	537	goto err;
464	538	}
@@ -471,19 +545,27 @@ static enum resp_states check_rkey(struct rxe_qp *qp, · 6 unmodified lines
471	545	} else {
472	546	mr = lookup_mr(qp->pd, access, rkey, RXE_LOOKUP_REMOTE);
473	547	if (!mr) {
474		− pr_debug("%s: no MR matches rkey %#x\n",
475		− __func__, rkey);
	548	+ rxe_dbg_qp(qp, "no MR matches rkey %#x\n", rkey);
476	549	state = RESPST_ERR_RKEY_VIOLATION;
477	550	goto err;
478	551	}
479	552	}
480	553
	554	+ if (pkt->mask & RXE_FLUSH_MASK) {
	555	+ /* FLUSH MR may not set va or resid
	556	+ * no need to check range since we will flush whole mr
	557	+ */
	558	+ if (feth_sel(pkt) == IB_FLUSH_MR)
	559	+ goto skip_check_range;
	560	+ }
	561	+
481	562	if (mr_check_range(mr, va + qp->resp.offset, resid)) {
482	563	state = RESPST_ERR_RKEY_VIOLATION;
483	564	goto err;
484	565	}
485	566
486		− if (pkt->mask & RXE_WRITE_MASK) {
	567	+skip_check_range:
	568	+ if (pkt->mask & (RXE_WRITE_MASK \| RXE_ATOMIC_WRITE_MASK)) {
487	569	if (resid > mtu) {
488	570	if (pktlen != mtu \|\| bth_pad(pkt)) {
489	571	state = RESPST_ERR_LENGTH;
@@ -583,15 +665,66 @@ static struct resp_res rxe_prepare_res(struct rxe_qp qp, · 93 unmodified lines
583	665	res->state = rdatm_res_state_new;
584	666	break;
585	667	case RXE_ATOMIC_MASK:
	668	+ case RXE_ATOMIC_WRITE_MASK:
586	669	res->first_psn = pkt->psn;
587	670	res->last_psn = pkt->psn;
588	671	res->cur_psn = pkt->psn;
589	672	break;
	673	+ case RXE_FLUSH_MASK:
	674	+ res->flush.va = qp->resp.va + qp->resp.offset;
	675	+ res->flush.length = qp->resp.length;
	676	+ res->flush.type = feth_plt(pkt);
	677	+ res->flush.level = feth_sel(pkt);
590	678	}
591	679
592	680	return res;
593	681	}
594	682
	683	+static enum resp_states process_flush(struct rxe_qp *qp,
	684	+ struct rxe_pkt_info *pkt)
	685	+{
	686	+ u64 length, start;
	687	+ struct rxe_mr *mr = qp->resp.mr;
	688	+ struct resp_res *res = qp->resp.res;
	689	+
	690	+ /* oA19-14, oA19-15 */
	691	+ if (res && res->replay)
	692	+ return RESPST_ACKNOWLEDGE;
	693	+ else if (!res) {
	694	+ res = rxe_prepare_res(qp, pkt, RXE_FLUSH_MASK);
	695	+ qp->resp.res = res;
	696	+ }
	697	+
	698	+ if (res->flush.level == IB_FLUSH_RANGE) {
	699	+ start = res->flush.va;
	700	+ length = res->flush.length;
	701	+ } else { /* level == IB_FLUSH_MR */
	702	+ start = mr->ibmr.iova;
	703	+ length = mr->ibmr.length;
	704	+ }
	705	+
	706	+ if (res->flush.type & IB_FLUSH_PERSISTENT) {
	707	+ if (rxe_flush_pmem_iova(mr, start, length))
	708	+ return RESPST_ERR_RKEY_VIOLATION;
	709	+ /* Make data persistent. */
	710	+ wmb();
	711	+ } else if (res->flush.type & IB_FLUSH_GLOBAL) {
	712	+ /* Make data global visibility. */
	713	+ wmb();
	714	+ }
	715	+
	716	+ qp->resp.msn++;
	717	+
	718	+ /* next expected psn, read handles this separately */
	719	+ qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
	720	+ qp->resp.ack_psn = qp->resp.psn;
	721	+
	722	+ qp->resp.opcode = pkt->opcode;
	723	+ qp->resp.status = IB_WC_SUCCESS;
	724	+
	725	+ return RESPST_ACKNOWLEDGE;
	726	+}
	727	+
595	728	/* Guarantee atomicity of atomic operations at the machine level. */
596	729	static DEFINE_SPINLOCK(atomic_ops_lock);
597	730
@@ -652,6 +785,55 @@ out: · 54 unmodified lines
652	785	return ret;
653	786	}
654	787
	788	+static enum resp_states atomic_write_reply(struct rxe_qp *qp,
	789	+ struct rxe_pkt_info *pkt)
	790	+{
	791	+ u64 src, *dst;
	792	+ struct resp_res *res = qp->resp.res;
	793	+ struct rxe_mr *mr = qp->resp.mr;
	794	+ int payload = payload_size(pkt);
	795	+
	796	+ if (!res) {
	797	+ res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
	798	+ qp->resp.res = res;
	799	+ }
	800	+
	801	+ if (!res->replay) {
	802	+#ifdef CONFIG_64BIT
	803	+ if (mr->state != RXE_MR_STATE_VALID)
	804	+ return RESPST_ERR_RKEY_VIOLATION;
	805	+
	806	+ memcpy(&src, payload_addr(pkt), payload);
	807	+
	808	+ dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
	809	+ /* check vaddr is 8 bytes aligned. */
	810	+ if (!dst \|\| (uintptr_t)dst & 7)
	811	+ return RESPST_ERR_MISALIGNED_ATOMIC;
	812	+
	813	+ /* Do atomic write after all prior operations have completed */
	814	+ smp_store_release(dst, src);
	815	+
	816	+ /* decrease resp.resid to zero */
	817	+ qp->resp.resid -= sizeof(payload);
	818	+
	819	+ qp->resp.msn++;
	820	+
	821	+ /* next expected psn, read handles this separately */
	822	+ qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
	823	+ qp->resp.ack_psn = qp->resp.psn;
	824	+
	825	+ qp->resp.opcode = pkt->opcode;
	826	+ qp->resp.status = IB_WC_SUCCESS;
	827	+
	828	+ return RESPST_ACKNOWLEDGE;
	829	+#else
	830	+ return RESPST_ERR_UNSUPPORTED_OPCODE;
	831	+#endif /* CONFIG_64BIT */
	832	+ }
	833	+
	834	+ return RESPST_ACKNOWLEDGE;
	835	+}
	836	+
655	837	static struct sk_buff prepare_ack_packet(struct rxe_qp qp,
656	838	struct rxe_pkt_info *ack,
657	839	int opcode,
@@ -807,14 +989,19 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 149 unmodified lines
807	989	skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload,
808	990	res->cur_psn, AETH_ACK_UNLIMITED);
809	991	if (!skb) {
810		− rxe_put(mr);
	992	+ if (mr)
	993	+ rxe_put(mr);
811	994	return RESPST_ERR_RNR;
812	995	}
813	996
814		− rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
815		− payload, RXE_FROM_MR_OBJ);
	997	+ err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
	998	+ payload, RXE_FROM_MR_OBJ);
816	999	if (mr)
817	1000	rxe_put(mr);
	1001	+ if (err) {
	1002	+ kfree_skb(skb);
	1003	+ return RESPST_ERR_RKEY_VIOLATION;
	1004	+ }
818	1005
819	1006	if (bth_pad(&ack_pkt)) {
820	1007	u8 *pad = payload_addr(&ack_pkt) + payload;
@@ -890,6 +1077,10 @@ static enum resp_states execute(struct rxe_qp qp, struct rxe_pkt_info pkt) · 69 unmodified lines
890	1077	return RESPST_READ_REPLY;
891	1078	} else if (pkt->mask & RXE_ATOMIC_MASK) {
892	1079	return RESPST_ATOMIC_REPLY;
	1080	+ } else if (pkt->mask & RXE_ATOMIC_WRITE_MASK) {
	1081	+ return RESPST_ATOMIC_WRITE_REPLY;
	1082	+ } else if (pkt->mask & RXE_FLUSH_MASK) {
	1083	+ return RESPST_PROCESS_FLUSH;
893	1084	} else {
894	1085	/* Unreachable */
895	1086	WARN_ON_ONCE(1);
@@ -1040,7 +1231,7 @@ static int send_common_ack(struct rxe_qp *qp, u8 syndrome, u32 psn, · 144 unmodified lines
1040	1231
1041	1232	err = rxe_xmit_packet(qp, &ack_pkt, skb);
1042	1233	if (err)
1043		− pr_err_ratelimited("Failed sending %s\n", msg);
	1234	+ rxe_dbg_qp(qp, "Failed sending %s\n", msg);
1044	1235
1045	1236	return err;
1046	1237	}
@@ -1063,6 +1254,19 @@ static int send_atomic_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) · 16 unmodified lines
1063	1254	return ret;
1064	1255	}
1065	1256
	1257	+static int send_read_response_ack(struct rxe_qp *qp, u8 syndrome, u32 psn)
	1258	+{
	1259	+ int ret = send_common_ack(qp, syndrome, psn,
	1260	+ IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY,
	1261	+ "RDMA READ response of length zero ACK");
	1262	+
	1263	+ /* have to clear this since it is used to trigger
	1264	+ * long read replies
	1265	+ */
	1266	+ qp->resp.res = NULL;
	1267	+ return ret;
	1268	+}
	1269	+
1066	1270	static enum resp_states acknowledge(struct rxe_qp *qp,
1067	1271	struct rxe_pkt_info *pkt)
1068	1272	{
@@ -1073,6 +1277,8 @@ static enum resp_states acknowledge(struct rxe_qp *qp, · 4 unmodified lines
1073	1277	send_ack(qp, qp->resp.aeth_syndrome, pkt->psn);
1074	1278	else if (pkt->mask & RXE_ATOMIC_MASK)
1075	1279	send_atomic_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
	1280	+ else if (pkt->mask & (RXE_FLUSH_MASK \| RXE_ATOMIC_WRITE_MASK))
	1281	+ send_read_response_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1076	1282	else if (bth_ack(pkt))
1077	1283	send_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1078	1284
@@ -1129,6 +1335,22 @@ static enum resp_states duplicate_request(struct rxe_qp *qp, · 50 unmodified lines
1129	1335	/* SEND. Ack again and cleanup. C9-105. */
1130	1336	send_ack(qp, AETH_ACK_UNLIMITED, prev_psn);
1131	1337	return RESPST_CLEANUP;
	1338	+ } else if (pkt->mask & RXE_FLUSH_MASK) {
	1339	+ struct resp_res *res;
	1340	+
	1341	+ /* Find the operation in our list of responder resources. */
	1342	+ res = find_resource(qp, pkt->psn);
	1343	+ if (res) {
	1344	+ res->replay = 1;
	1345	+ res->cur_psn = pkt->psn;
	1346	+ qp->resp.res = res;
	1347	+ rc = RESPST_PROCESS_FLUSH;
	1348	+ goto out;
	1349	+ }
	1350	+
	1351	+ /* Resource not found. Class D error. Drop the request. */
	1352	+ rc = RESPST_CLEANUP;
	1353	+ goto out;
1132	1354	} else if (pkt->mask & RXE_READ_MASK) {
1133	1355	struct resp_res *res;
1134	1356
@@ -1184,7 +1406,9 @@ static enum resp_states duplicate_request(struct rxe_qp *qp, · 49 unmodified lines
1184	1406	res->replay = 1;
1185	1407	res->cur_psn = pkt->psn;
1186	1408	qp->resp.res = res;
1187		− rc = RESPST_ATOMIC_REPLY;
	1409	+ rc = pkt->mask & RXE_ATOMIC_MASK ?
	1410	+ RESPST_ATOMIC_REPLY :
	1411	+ RESPST_ATOMIC_WRITE_REPLY;
1188	1412	goto out;
1189	1413	}
1190	1414
@@ -1286,8 +1510,7 @@ int rxe_responder(void *arg) · 95 unmodified lines
1286	1510	}
1287	1511
1288	1512	while (1) {
1289		− pr_debug("qp#%d state = %s\n", qp_num(qp),
1290		− resp_state_name[state]);
	1513	+ rxe_dbg_qp(qp, "state = %s\n", resp_state_name[state]);
1291	1514	switch (state) {
1292	1515	case RESPST_GET_REQ:
1293	1516	state = get_req(qp, &pkt);
@@ -1305,7 +1528,7 @@ int rxe_responder(void *arg) · 11 unmodified lines
1305	1528	state = check_resource(qp, pkt);
1306	1529	break;
1307	1530	case RESPST_CHK_LENGTH:
1308		− state = check_length(qp, pkt);
	1531	+ state = rxe_resp_check_length(qp, pkt);
1309	1532	break;
1310	1533	case RESPST_CHK_RKEY:
1311	1534	state = check_rkey(qp, pkt);
@@ -1322,6 +1545,12 @@ int rxe_responder(void *arg) · 10 unmodified lines
1322	1545	case RESPST_ATOMIC_REPLY:
1323	1546	state = atomic_reply(qp, pkt);
1324	1547	break;
	1548	+ case RESPST_ATOMIC_WRITE_REPLY:
	1549	+ state = atomic_write_reply(qp, pkt);
	1550	+ break;
	1551	+ case RESPST_PROCESS_FLUSH:
	1552	+ state = process_flush(qp, pkt);
	1553	+ break;
1325	1554	case RESPST_ACKNOWLEDGE:
1326	1555	state = acknowledge(qp, pkt);
1327	1556	break;
@@ -1444,7 +1673,7 @@ int rxe_responder(void *arg) · 116 unmodified lines
1444	1673
1445	1674	case RESPST_ERROR:
1446	1675	qp->resp.goto_error = 0;
1447		− pr_debug("qp#%d moved to error state\n", qp_num(qp));
	1676	+ rxe_dbg_qp(qp, "moved to error state\n");
1448	1677	rxe_qp_error(qp);
1449	1678	goto exit;
1450	1679

drivers/infiniband/sw/rxe/rxe_srq.c+10−10

@@ -13,13 +13,13 @@ int rxe_srq_chk_init(struct rxe_dev rxe, struct ib_srq_init_attr init) · 12 unmodified lines
13	13	struct ib_srq_attr *attr = &init->attr;
14	14
15	15	if (attr->max_wr > rxe->attr.max_srq_wr) {
16		− pr_warn("max_wr(%d) > max_srq_wr(%d)\n",
	16	+ rxe_dbg(rxe, "max_wr(%d) > max_srq_wr(%d)\n",
17	17	attr->max_wr, rxe->attr.max_srq_wr);
18	18	goto err1;
19	19	}
20	20
21	21	if (attr->max_wr <= 0) {
22		− pr_warn("max_wr(%d) <= 0\n", attr->max_wr);
	22	+ rxe_dbg(rxe, "max_wr(%d) <= 0\n", attr->max_wr);
23	23	goto err1;
24	24	}
25	25
@@ -27,7 +27,7 @@ int rxe_srq_chk_init(struct rxe_dev rxe, struct ib_srq_init_attr init) · 1 unmodified line
27	27	attr->max_wr = RXE_MIN_SRQ_WR;
28	28
29	29	if (attr->max_sge > rxe->attr.max_srq_sge) {
30		− pr_warn("max_sge(%d) > max_srq_sge(%d)\n",
	30	+ rxe_dbg(rxe, "max_sge(%d) > max_srq_sge(%d)\n",
31	31	attr->max_sge, rxe->attr.max_srq_sge);
32	32	goto err1;
33	33	}
@@ -65,7 +65,7 @@ int rxe_srq_from_init(struct rxe_dev rxe, struct rxe_srq srq, · 31 unmodified lines
65	65	type = QUEUE_TYPE_FROM_CLIENT;
66	66	q = rxe_queue_init(rxe, &srq->rq.max_wr, srq_wqe_size, type);
67	67	if (!q) {
68		− pr_warn("unable to allocate queue for srq\n");
	68	+ rxe_dbg_srq(srq, "Unable to allocate queue\n");
69	69	return -ENOMEM;
70	70	}
71	71
@@ -94,24 +94,24 @@ int rxe_srq_chk_attr(struct rxe_dev rxe, struct rxe_srq srq, · 22 unmodified lines
94	94	struct ib_srq_attr *attr, enum ib_srq_attr_mask mask)
95	95	{
96	96	if (srq->error) {
97		− pr_warn("srq in error state\n");
	97	+ rxe_dbg_srq(srq, "in error state\n");
98	98	goto err1;
99	99	}
100	100
101	101	if (mask & IB_SRQ_MAX_WR) {
102	102	if (attr->max_wr > rxe->attr.max_srq_wr) {
103		− pr_warn("max_wr(%d) > max_srq_wr(%d)\n",
	103	+ rxe_dbg_srq(srq, "max_wr(%d) > max_srq_wr(%d)\n",
104	104	attr->max_wr, rxe->attr.max_srq_wr);
105	105	goto err1;
106	106	}
107	107
108	108	if (attr->max_wr <= 0) {
109		− pr_warn("max_wr(%d) <= 0\n", attr->max_wr);
	109	+ rxe_dbg_srq(srq, "max_wr(%d) <= 0\n", attr->max_wr);
110	110	goto err1;
111	111	}
112	112
113	113	if (srq->limit && (attr->max_wr < srq->limit)) {
114		− pr_warn("max_wr (%d) < srq->limit (%d)\n",
	114	+ rxe_dbg_srq(srq, "max_wr (%d) < srq->limit (%d)\n",
115	115	attr->max_wr, srq->limit);
116	116	goto err1;
117	117	}
@@ -122,13 +122,13 @@ int rxe_srq_chk_attr(struct rxe_dev rxe, struct rxe_srq srq, · 4 unmodified lines
122	122
123	123	if (mask & IB_SRQ_LIMIT) {
124	124	if (attr->srq_limit > rxe->attr.max_srq_wr) {
125		− pr_warn("srq_limit(%d) > max_srq_wr(%d)\n",
	125	+ rxe_dbg_srq(srq, "srq_limit(%d) > max_srq_wr(%d)\n",
126	126	attr->srq_limit, rxe->attr.max_srq_wr);
127	127	goto err1;
128	128	}
129	129
130	130	if (attr->srq_limit > srq->rq.queue->buf->index_mask) {
131		− pr_warn("srq_limit (%d) > cur limit(%d)\n",
	131	+ rxe_dbg_srq(srq, "srq_limit (%d) > cur limit(%d)\n",
132	132	attr->srq_limit,
133	133	srq->rq.queue->buf->index_mask);
134	134	goto err1;

drivers/infiniband/sw/rxe/rxe_task.c+26−26

@@ -4,10 +4,6 @@ · 3 unmodified lines
4	4	* Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5	5	*/
6	6
7		−#include <linux/kernel.h>
8		−#include <linux/interrupt.h>
9		−#include <linux/hardirq.h>
10		−
11	7	#include "rxe.h"
12	8
13	9	int __rxe_do_task(struct rxe_task *task)
@@ -28,30 +24,31 @@ int __rxe_do_task(struct rxe_task *task) · 14 unmodified lines
28	24	* a second caller finds the task already running
29	25	* but looks just after the last call to func
30	26	*/
31		−void rxe_do_task(struct tasklet_struct *t)
	27	+static void do_task(struct tasklet_struct *t)
32	28	{
33	29	int cont;
34	30	int ret;
35	31	struct rxe_task *task = from_tasklet(task, t, tasklet);
	32	+ struct rxe_qp qp = (struct rxe_qp )task->arg;
36	33	unsigned int iterations = RXE_MAX_ITERATIONS;
37	34
38		− spin_lock_bh(&task->state_lock);
	35	+ spin_lock_bh(&task->lock);
39	36	switch (task->state) {
40	37	case TASK_STATE_START:
41	38	task->state = TASK_STATE_BUSY;
42		− spin_unlock_bh(&task->state_lock);
	39	+ spin_unlock_bh(&task->lock);
43	40	break;
44	41
45	42	case TASK_STATE_BUSY:
46	43	task->state = TASK_STATE_ARMED;
47	44	fallthrough;
48	45	case TASK_STATE_ARMED:
49		− spin_unlock_bh(&task->state_lock);
	46	+ spin_unlock_bh(&task->lock);
50	47	return;
51	48
52	49	default:
53		− spin_unlock_bh(&task->state_lock);
54		− pr_warn("%s failed with bad state %d\n", __func__, task->state);
	50	+ spin_unlock_bh(&task->lock);
	51	+ rxe_dbg_qp(qp, "failed with bad state %d\n", task->state);
55	52	return;
56	53	}
57	54
@@ -59,7 +56,7 @@ void rxe_do_task(struct tasklet_struct *t) · 1 unmodified line
59	56	cont = 0;
60	57	ret = task->func(task->arg);
61	58
62		− spin_lock_bh(&task->state_lock);
	59	+ spin_lock_bh(&task->lock);
63	60	switch (task->state) {
64	61	case TASK_STATE_BUSY:
65	62	if (ret) {
@@ -85,27 +82,25 @@ void rxe_do_task(struct tasklet_struct *t) · 19 unmodified lines
85	82	break;
86	83
87	84	default:
88		− pr_warn("%s failed with bad state %d\n", __func__,
89		− task->state);
	85	+ rxe_dbg_qp(qp, "failed with bad state %d\n",
	86	+ task->state);
90	87	}
91		− spin_unlock_bh(&task->state_lock);
	88	+ spin_unlock_bh(&task->lock);
92	89	} while (cont);
93	90
94	91	task->ret = ret;
95	92	}
96	93
97		−int rxe_init_task(struct rxe_task *task,
98		− void arg, int (func)(void ), char name)
	94	+int rxe_init_task(struct rxe_task task, void arg, int (func)(void ))
99	95	{
100	96	task->arg = arg;
101	97	task->func = func;
102		− snprintf(task->name, sizeof(task->name), "%s", name);
103	98	task->destroyed = false;
104	99
105		− tasklet_setup(&task->tasklet, rxe_do_task);
	100	+ tasklet_setup(&task->tasklet, do_task);
106	101
107	102	task->state = TASK_STATE_START;
108		− spin_lock_init(&task->state_lock);
	103	+ spin_lock_init(&task->lock);
109	104
110	105	return 0;
111	106	}
@@ -121,23 +116,28 @@ void rxe_cleanup_task(struct rxe_task *task) · 9 unmodified lines
121	116	task->destroyed = true;
122	117
123	118	do {
124		− spin_lock_bh(&task->state_lock);
	119	+ spin_lock_bh(&task->lock);
125	120	idle = (task->state == TASK_STATE_START);
126		− spin_unlock_bh(&task->state_lock);
	121	+ spin_unlock_bh(&task->lock);
127	122	} while (!idle);
128	123
129	124	tasklet_kill(&task->tasklet);
130	125	}
131	126
132		−void rxe_run_task(struct rxe_task *task, int sched)
	127	+void rxe_run_task(struct rxe_task *task)
	128	+{
	129	+ if (task->destroyed)
	130	+ return;
	131	+
	132	+ do_task(&task->tasklet);
	133	+}
	134	+
	135	+void rxe_sched_task(struct rxe_task *task)
133	136	{
134	137	if (task->destroyed)
135	138	return;
136	139
137		− if (sched)
138		− tasklet_schedule(&task->tasklet);
139		− else
140		− rxe_do_task(&task->tasklet);
	140	+ tasklet_schedule(&task->tasklet);
141	141	}
142	142
143	143	void rxe_disable_task(struct rxe_task *task)

drivers/infiniband/sw/rxe/rxe_task.h+4−15

@@ -21,11 +21,10 @@ enum { · 20 unmodified lines
21	21	struct rxe_task {
22	22	struct tasklet_struct tasklet;
23	23	int state;
24		− spinlock_t state_lock; /* spinlock for task state */
	24	+ spinlock_t lock;
25	25	void *arg;
26	26	int (func)(void arg);
27	27	int ret;
28		− char name[16];
29	28	bool destroyed;
30	29	};
31	30
@@ -34,8 +33,7 @@ struct rxe_task { · 2 unmodified lines
34	33	* arg => parameter to pass to fcn
35	34	* func => function to call until it returns != 0
36	35	*/
37		−int rxe_init_task(struct rxe_task *task,
38		− void arg, int (func)(void ), char name);
	36	+int rxe_init_task(struct rxe_task task, void arg, int (func)(void ));
39	37
40	38	/* cleanup task */
41	39	void rxe_cleanup_task(struct rxe_task *task);
@@ -46,18 +44,9 @@ void rxe_cleanup_task(struct rxe_task *task); · 4 unmodified lines
46	44	*/
47	45	int __rxe_do_task(struct rxe_task *task);
48	46
49		−/*
50		− * common function called by any of the main tasklets
51		− * If there is any chance that there is additional
52		− * work to do someone must reschedule the task before
53		− * leaving
54		− */
55		−void rxe_do_task(struct tasklet_struct *t);
	47	+void rxe_run_task(struct rxe_task *task);
56	48
57		−/* run a task, else schedule it to run as a tasklet, The decision
58		− * to run or schedule tasklet is based on the parameter sched.
59		− */
60		−void rxe_run_task(struct rxe_task *task, int sched);
	49	+void rxe_sched_task(struct rxe_task *task);
61	50
62	51	/* keep a task from scheduling */
63	52	void rxe_disable_task(struct rxe_task *task);

drivers/infiniband/sw/rxe/rxe_verbs.c+38−68

@@ -172,10 +172,6 @@ static int rxe_create_ah(struct ib_ah *ibah, · 171 unmodified lines
172	172	ah->is_user = false;
173	173	}
174	174
175		− err = rxe_av_chk_attr(rxe, init_attr->ah_attr);
176		− if (err)
177		− return err;
178		−
179	175	err = rxe_add_to_pool_ah(&rxe->ah_pool, ah,
180	176	init_attr->flags & RDMA_CREATE_AH_SLEEPABLE);
181	177	if (err)
@@ -184,6 +180,12 @@ static int rxe_create_ah(struct ib_ah *ibah, · 2 unmodified lines
184	180	/* create index > 0 */
185	181	ah->ah_num = ah->elem.index;
186	182
	183	+ err = rxe_ah_chk_attr(ah, init_attr->ah_attr);
	184	+ if (err) {
	185	+ rxe_cleanup(ah);
	186	+ return err;
	187	+ }
	188	+
187	189	if (uresp) {
188	190	/* only if new user provider */
189	191	err = copy_to_user(&uresp->ah_num, &ah->ah_num,
@@ -206,10 +208,9 @@ static int rxe_create_ah(struct ib_ah *ibah, · 16 unmodified lines
206	208	static int rxe_modify_ah(struct ib_ah ibah, struct rdma_ah_attr attr)
207	209	{
208	210	int err;
209		− struct rxe_dev *rxe = to_rdev(ibah->device);
210	211	struct rxe_ah *ah = to_rah(ibah);
211	212
212		− err = rxe_av_chk_attr(rxe, attr);
	213	+ err = rxe_ah_chk_attr(ah, attr);
213	214	if (err)
214	215	return err;
215	216
@@ -238,7 +239,6 @@ static int rxe_destroy_ah(struct ib_ah *ibah, u32 flags) · 22 unmodified lines
238	239
239	240	static int post_one_recv(struct rxe_rq rq, const struct ib_recv_wr ibwr)
240	241	{
241		− int err;
242	242	int i;
243	243	u32 length;
244	244	struct rxe_recv_wqe *recv_wqe;
@@ -246,15 +246,11 @@ static int post_one_recv(struct rxe_rq rq, const struct ib_recv_wr ibwr) · 1 unmodified line
246	246	int full;
247	247
248	248	full = queue_full(rq->queue, QUEUE_TYPE_TO_DRIVER);
249		− if (unlikely(full)) {
250		− err = -ENOMEM;
251		− goto err1;
252		− }
	249	+ if (unlikely(full))
	250	+ return -ENOMEM;
253	251
254		− if (unlikely(num_sge > rq->max_sge)) {
255		− err = -EINVAL;
256		− goto err1;
257		− }
	252	+ if (unlikely(num_sge > rq->max_sge))
	253	+ return -EINVAL;
258	254
259	255	length = 0;
260	256	for (i = 0; i < num_sge; i++)
@@ -275,9 +271,6 @@ static int post_one_recv(struct rxe_rq rq, const struct ib_recv_wr ibwr) · 14 unmodified lines
275	271	queue_advance_producer(rq->queue, QUEUE_TYPE_TO_DRIVER);
276	272
277	273	return 0;
278		−
279		−err1:
280		− return err;
281	274	}
282	275
283	276	static int rxe_create_srq(struct ib_srq ibsrq, struct ib_srq_init_attr init,
@@ -343,10 +336,7 @@ static int rxe_modify_srq(struct ib_srq ibsrq, struct ib_srq_attr attr, · 59 unmodified lines
343	336	if (err)
344	337	return err;
345	338
346		− err = rxe_srq_from_attr(rxe, srq, attr, mask, &ucmd, udata);
347		− if (err)
348		− return err;
349		− return 0;
	339	+ return rxe_srq_from_attr(rxe, srq, attr, mask, &ucmd, udata);
350	340	}
351	341
352	342	static int rxe_query_srq(struct ib_srq ibsrq, struct ib_srq_attr attr)
@@ -453,11 +443,11 @@ static int rxe_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr, · 100 unmodified lines
453	443
454	444	err = rxe_qp_chk_attr(rxe, qp, attr, mask);
455	445	if (err)
456		− goto err1;
	446	+ return err;
457	447
458	448	err = rxe_qp_from_attr(qp, attr, mask, udata);
459	449	if (err)
460		− goto err1;
	450	+ return err;
461	451
462	452	if ((mask & IB_QP_AV) && (attr->ah_attr.ah_flags & IB_AH_GRH))
463	453	qp->src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
@@ -465,9 +455,6 @@ static int rxe_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr, · 1 unmodified line
465	455	qp->attr.dest_qp_num);
466	456
467	457	return 0;
468		−
469		−err1:
470		− return err;
471	458	}
472	459
473	460	static int rxe_query_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
@@ -501,24 +488,21 @@ static int validate_send_wr(struct rxe_qp qp, const struct ib_send_wr ibwr, · 27 unmodified lines
501	488	struct rxe_sq *sq = &qp->sq;
502	489
503	490	if (unlikely(num_sge > sq->max_sge))
504		− goto err1;
	491	+ return -EINVAL;
505	492
506	493	if (unlikely(mask & WR_ATOMIC_MASK)) {
507	494	if (length < 8)
508		− goto err1;
	495	+ return -EINVAL;
509	496
510	497	if (atomic_wr(ibwr)->remote_addr & 0x7)
511		− goto err1;
	498	+ return -EINVAL;
512	499	}
513	500
514	501	if (unlikely((ibwr->send_flags & IB_SEND_INLINE) &&
515	502	(length > sq->max_inline)))
516		− goto err1;
	503	+ return -EINVAL;
517	504
518	505	return 0;
519		−
520		−err1:
521		− return -EINVAL;
522	506	}
523	507
524	508	static void init_send_wr(struct rxe_qp qp, struct rxe_send_wr wr,
@@ -695,9 +679,9 @@ static int rxe_post_send_kernel(struct rxe_qp qp, const struct ib_send_wr wr, · 170 unmodified lines
695	679	wr = next;
696	680	}
697	681
698		− rxe_run_task(&qp->req.task, 1);
	682	+ rxe_sched_task(&qp->req.task);
699	683	if (unlikely(qp->req.state == QP_STATE_ERROR))
700		− rxe_run_task(&qp->comp.task, 1);
	684	+ rxe_sched_task(&qp->comp.task);
701	685
702	686	return err;
703	687	}
@@ -719,7 +703,7 @@ static int rxe_post_send(struct ib_qp ibqp, const struct ib_send_wr wr, · 15 unmodified lines
719	703
720	704	if (qp->is_user) {
721	705	/* Utilize process context to do protocol processing */
722		− rxe_run_task(&qp->req.task, 0);
	706	+ rxe_run_task(&qp->req.task);
723	707	return 0;
724	708	} else
725	709	return rxe_post_send_kernel(qp, wr, bad_wr);
@@ -735,14 +719,12 @@ static int rxe_post_recv(struct ib_qp ibqp, const struct ib_recv_wr wr, · 9 unmodified lines
735	719
736	720	if (unlikely((qp_state(qp) < IB_QPS_INIT) \|\| !qp->valid)) {
737	721	*bad_wr = wr;
738		− err = -EINVAL;
739		− goto err1;
	722	+ return -EINVAL;
740	723	}
741	724
742	725	if (unlikely(qp->srq)) {
743	726	*bad_wr = wr;
744		− err = -EINVAL;
745		− goto err1;
	727	+ return -EINVAL;
746	728	}
747	729
748	730	spin_lock_irqsave(&rq->producer_lock, flags);
@@ -759,9 +741,8 @@ static int rxe_post_recv(struct ib_qp ibqp, const struct ib_recv_wr wr, · 10 unmodified lines
759	741	spin_unlock_irqrestore(&rq->producer_lock, flags);
760	742
761	743	if (qp->resp.state == QP_STATE_ERROR)
762		− rxe_run_task(&qp->resp.task, 1);
	744	+ rxe_sched_task(&qp->resp.task);
763	745
764		−err1:
765	746	return err;
766	747	}
767	748
@@ -826,16 +807,9 @@ static int rxe_resize_cq(struct ib_cq ibcq, int cqe, struct ib_udata udata) · 58 unmodified lines
826	807
827	808	err = rxe_cq_chk_attr(rxe, cq, cqe, 0);
828	809	if (err)
829		− goto err1;
830		−
831		− err = rxe_cq_resize_queue(cq, cqe, uresp, udata);
832		− if (err)
833		− goto err1;
834		−
835		− return 0;
	810	+ return err;
836	811
837		−err1:
838		− return err;
	812	+ return rxe_cq_resize_queue(cq, cqe, uresp, udata);
839	813	}
840	814
841	815	static int rxe_poll_cq(struct ib_cq ibcq, int num_entries, struct ib_wc wc)
@@ -902,6 +876,7 @@ static struct ib_mr rxe_get_dma_mr(struct ib_pd ibpd, int access) · 60 unmodified lines
902	876
903	877	rxe_get(pd);
904	878	mr->ibmr.pd = ibpd;
	879	+ mr->ibmr.device = ibpd->device;
905	880
906	881	rxe_mr_init_dma(access, mr);
907	882	rxe_finalize(mr);
@@ -921,26 +896,23 @@ static struct ib_mr rxe_reg_user_mr(struct ib_pd ibpd, · 13 unmodified lines
921	896	struct rxe_mr *mr;
922	897
923	898	mr = rxe_alloc(&rxe->mr_pool);
924		− if (!mr) {
925		− err = -ENOMEM;
926		− goto err2;
927		− }
928		−
	899	+ if (!mr)
	900	+ return ERR_PTR(-ENOMEM);
929	901
930	902	rxe_get(pd);
931	903	mr->ibmr.pd = ibpd;
	904	+ mr->ibmr.device = ibpd->device;
932	905
933	906	err = rxe_mr_init_user(rxe, start, length, iova, access, mr);
934	907	if (err)
935		− goto err3;
	908	+ goto err1;
936	909
937	910	rxe_finalize(mr);
938	911
939	912	return &mr->ibmr;
940	913
941		−err3:
	914	+err1:
942	915	rxe_cleanup(mr);
943		−err2:
944	916	return ERR_PTR(err);
945	917	}
946	918
@@ -956,25 +928,23 @@ static struct ib_mr rxe_alloc_mr(struct ib_pd ibpd, enum ib_mr_type mr_type, · 9 unmodified lines
956	928	return ERR_PTR(-EINVAL);
957	929
958	930	mr = rxe_alloc(&rxe->mr_pool);
959		− if (!mr) {
960		− err = -ENOMEM;
961		− goto err1;
962		− }
	931	+ if (!mr)
	932	+ return ERR_PTR(-ENOMEM);
963	933
964	934	rxe_get(pd);
965	935	mr->ibmr.pd = ibpd;
	936	+ mr->ibmr.device = ibpd->device;
966	937
967	938	err = rxe_mr_init_fast(max_num_sg, mr);
968	939	if (err)
969		− goto err2;
	940	+ goto err1;
970	941
971	942	rxe_finalize(mr);
972	943
973	944	return &mr->ibmr;
974	945
975		−err2:
976		− rxe_cleanup(mr);
977	946	err1:
	947	+ rxe_cleanup(mr);
978	948	return ERR_PTR(err);
979	949	}
980	950
@@ -1134,7 +1104,7 @@ int rxe_register_device(struct rxe_dev rxe, const char ibdev_name) · 153 unmodified lines
1134	1104
1135	1105	err = ib_register_device(dev, ibdev_name, NULL);
1136	1106	if (err)
1137		− pr_warn("%s failed with error %d\n", __func__, err);
	1107	+ rxe_dbg(rxe, "failed with error %d\n", err);
1138	1108
1139	1109	/*
1140	1110	* Note that rxe may be invalid at this point if another thread

drivers/infiniband/sw/rxe/rxe_verbs.h+6−1

@@ -165,6 +165,12 @@ struct resp_res { · 164 unmodified lines
165	165	u64 va;
166	166	u32 resid;
167	167	} read;
	168	+ struct {
	169	+ u32 length;
	170	+ u64 va;
	171	+ u8 type;
	172	+ u8 level;
	173	+ } flush;
168	174	};
169	175	};
170	176
@@ -304,7 +310,6 @@ struct rxe_mr { · 133 unmodified lines
304	310	u32 lkey;
305	311	u32 rkey;
306	312	enum rxe_mr_state state;
307		− enum ib_mr_type type;
308	313	u32 offset;
309	314	int access;
310	315

drivers/infiniband/sw/siw/siw_cq.c+22−2

@@ -56,8 +56,6 @@ int siw_reap_cqe(struct siw_cq cq, struct ib_wc wc) · 55 unmodified lines
56	56	if (READ_ONCE(cqe->flags) & SIW_WQE_VALID) {
57	57	memset(wc, 0, sizeof(*wc));
58	58	wc->wr_id = cqe->id;
59		− wc->status = map_cqe_status[cqe->status].ib;
60		− wc->opcode = map_wc_opcode[cqe->opcode];
61	59	wc->byte_len = cqe->bytes;
62	60
63	61	/*
@@ -71,10 +69,32 @@ int siw_reap_cqe(struct siw_cq cq, struct ib_wc wc) · 7 unmodified lines
71	69	wc->wc_flags = IB_WC_WITH_INVALIDATE;
72	70	}
73	71	wc->qp = cqe->base_qp;
	72	+ wc->opcode = map_wc_opcode[cqe->opcode];
	73	+ wc->status = map_cqe_status[cqe->status].ib;
74	74	siw_dbg_cq(cq,
75	75	"idx %u, type %d, flags %2x, id 0x%pK\n",
76	76	cq->cq_get % cq->num_cqe, cqe->opcode,
77	77	cqe->flags, (void *)(uintptr_t)cqe->id);
	78	+ } else {
	79	+ /*
	80	+ * A malicious user may set invalid opcode or
	81	+ * status in the user mmapped CQE array.
	82	+ * Sanity check and correct values in that case
	83	+ * to avoid out-of-bounds access to global arrays
	84	+ * for opcode and status mapping.
	85	+ */
	86	+ u8 opcode = cqe->opcode;
	87	+ u16 status = cqe->status;
	88	+
	89	+ if (opcode >= SIW_NUM_OPCODES) {
	90	+ opcode = 0;
	91	+ status = SIW_WC_GENERAL_ERR;
	92	+ } else if (status >= SIW_NUM_WC_STATUS) {
	93	+ status = SIW_WC_GENERAL_ERR;
	94	+ }
	95	+ wc->opcode = map_wc_opcode[opcode];
	96	+ wc->status = map_cqe_status[status].ib;
	97	+
78	98	}
79	99	WRITE_ONCE(cqe->flags, 0);
80	100	cq->cq_get++;

drivers/infiniband/sw/siw/siw_verbs.c+36−4

@@ -676,13 +676,45 @@ static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr, · 675 unmodified lines
676	676	static int siw_sq_flush_wr(struct siw_qp qp, const struct ib_send_wr wr,
677	677	const struct ib_send_wr **bad_wr)
678	678	{
679		− struct siw_sqe sqe = {};
680	679	int rv = 0;
681	680
682	681	while (wr) {
683		− sqe.id = wr->wr_id;
684		− sqe.opcode = wr->opcode;
685		− rv = siw_sqe_complete(qp, &sqe, 0, SIW_WC_WR_FLUSH_ERR);
	682	+ struct siw_sqe sqe = {};
	683	+
	684	+ switch (wr->opcode) {
	685	+ case IB_WR_RDMA_WRITE:
	686	+ sqe.opcode = SIW_OP_WRITE;
	687	+ break;
	688	+ case IB_WR_RDMA_READ:
	689	+ sqe.opcode = SIW_OP_READ;
	690	+ break;
	691	+ case IB_WR_RDMA_READ_WITH_INV:
	692	+ sqe.opcode = SIW_OP_READ_LOCAL_INV;
	693	+ break;
	694	+ case IB_WR_SEND:
	695	+ sqe.opcode = SIW_OP_SEND;
	696	+ break;
	697	+ case IB_WR_SEND_WITH_IMM:
	698	+ sqe.opcode = SIW_OP_SEND_WITH_IMM;
	699	+ break;
	700	+ case IB_WR_SEND_WITH_INV:
	701	+ sqe.opcode = SIW_OP_SEND_REMOTE_INV;
	702	+ break;
	703	+ case IB_WR_LOCAL_INV:
	704	+ sqe.opcode = SIW_OP_INVAL_STAG;
	705	+ break;
	706	+ case IB_WR_REG_MR:
	707	+ sqe.opcode = SIW_OP_REG_MR;
	708	+ break;
	709	+ default:
	710	+ rv = -EINVAL;
	711	+ break;
	712	+ }
	713	+ if (!rv) {
	714	+ sqe.id = wr->wr_id;
	715	+ rv = siw_sqe_complete(qp, &sqe, 0,
	716	+ SIW_WC_WR_FLUSH_ERR);
	717	+ }
686	718	if (rv) {
687	719	if (bad_wr)
688	720	*bad_wr = wr;

drivers/infiniband/ulp/ipoib/ipoib_netlink.c+7−0

@@ -41,6 +41,11 @@ static const struct nla_policy ipoib_policy[IFLA_IPOIB_MAX + 1] = { · 40 unmodified lines
41	41	[IFLA_IPOIB_UMCAST] = { .type = NLA_U16 },
42	42	};
43	43
	44	+static unsigned int ipoib_get_max_num_queues(void)
	45	+{
	46	+ return min_t(unsigned int, num_possible_cpus(), 128);
	47	+}
	48	+
44	49	static int ipoib_fill_info(struct sk_buff skb, const struct net_device dev)
45	50	{
46	51	struct ipoib_dev_priv *priv = ipoib_priv(dev);
@@ -172,6 +177,8 @@ static struct rtnl_link_ops ipoib_link_ops __read_mostly = { · 125 unmodified lines
172	177	.changelink = ipoib_changelink,
173	178	.get_size = ipoib_get_size,
174	179	.fill_info = ipoib_fill_info,
	180	+ .get_num_rx_queues = ipoib_get_max_num_queues,
	181	+ .get_num_tx_queues = ipoib_get_max_num_queues,
175	182	};
176	183
177	184	struct rtnl_link_ops *ipoib_get_link_ops(void)

drivers/infiniband/ulp/iser/iser_verbs.c+29−38

@@ -347,22 +347,6 @@ static void iser_device_try_release(struct iser_device *device) · 346 unmodified lines
347	347	mutex_unlock(&ig.device_list_mutex);
348	348	}
349	349
350		−/*
351		− * Called with state mutex held
352		− */
353		−static int iser_conn_state_comp_exch(struct iser_conn *iser_conn,
354		− enum iser_conn_state comp,
355		− enum iser_conn_state exch)
356		−{
357		− int ret;
358		−
359		− ret = (iser_conn->state == comp);
360		− if (ret)
361		− iser_conn->state = exch;
362		−
363		− return ret;
364		−}
365		−
366	350	void iser_release_work(struct work_struct *work)
367	351	{
368	352	struct iser_conn *iser_conn;
@@ -464,11 +448,13 @@ int iser_conn_terminate(struct iser_conn *iser_conn) · 95 unmodified lines
464	448	struct ib_conn *ib_conn = &iser_conn->ib_conn;
465	449	int err = 0;
466	450
	451	+ lockdep_assert_held(&iser_conn->state_mutex);
	452	+
467	453	/* terminate the iser conn only if the conn state is UP */
468		− if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP,
469		− ISER_CONN_TERMINATING))
	454	+ if (iser_conn->state != ISER_CONN_UP)
470	455	return 0;
471	456
	457	+ iser_conn->state = ISER_CONN_TERMINATING;
472	458	iser_info("iser_conn %p state %d\n", iser_conn, iser_conn->state);
473	459
474	460	/* suspend queuing of new iscsi commands */
@@ -498,9 +484,10 @@ int iser_conn_terminate(struct iser_conn *iser_conn) · 23 unmodified lines
498	484	*/
499	485	static void iser_connect_error(struct rdma_cm_id *cma_id)
500	486	{
501		− struct iser_conn *iser_conn;
	487	+ struct iser_conn *iser_conn = cma_id->context;
	488	+
	489	+ lockdep_assert_held(&iser_conn->state_mutex);
502	490
503		− iser_conn = cma_id->context;
504	491	iser_conn->state = ISER_CONN_TERMINATING;
505	492	}
506	493
@@ -542,12 +529,13 @@ static void iser_calc_scsi_params(struct iser_conn *iser_conn, · 35 unmodified lines
542	529	*/
543	530	static void iser_addr_handler(struct rdma_cm_id *cma_id)
544	531	{
	532	+ struct iser_conn *iser_conn = cma_id->context;
545	533	struct iser_device *device;
546		− struct iser_conn *iser_conn;
547	534	struct ib_conn *ib_conn;
548	535	int ret;
549	536
550		− iser_conn = cma_id->context;
	537	+ lockdep_assert_held(&iser_conn->state_mutex);
	538	+
551	539	if (iser_conn->state != ISER_CONN_PENDING)
552	540	/* bailout */
553	541	return;
@@ -597,6 +585,8 @@ static void iser_route_handler(struct rdma_cm_id *cma_id) · 43 unmodified lines
597	585	struct ib_conn *ib_conn = &iser_conn->ib_conn;
598	586	struct ib_device *ib_dev = ib_conn->device->ib_device;
599	587
	588	+ lockdep_assert_held(&iser_conn->state_mutex);
	589	+
600	590	if (iser_conn->state != ISER_CONN_PENDING)
601	591	/* bailout */
602	592	return;
@@ -629,14 +619,18 @@ failure: · 26 unmodified lines
629	619	iser_connect_error(cma_id);
630	620	}
631	621
	622	+/*
	623	+ * Called with state mutex held
	624	+ */
632	625	static void iser_connected_handler(struct rdma_cm_id *cma_id,
633	626	const void *private_data)
634	627	{
635		− struct iser_conn *iser_conn;
	628	+ struct iser_conn *iser_conn = cma_id->context;
636	629	struct ib_qp_attr attr;
637	630	struct ib_qp_init_attr init_attr;
638	631
639		− iser_conn = cma_id->context;
	632	+ lockdep_assert_held(&iser_conn->state_mutex);
	633	+
640	634	if (iser_conn->state != ISER_CONN_PENDING)
641	635	/* bailout */
642	636	return;
@@ -657,30 +651,27 @@ static void iser_connected_handler(struct rdma_cm_id *cma_id, · 14 unmodified lines
657	651	complete(&iser_conn->up_completion);
658	652	}
659	653
660		−static void iser_disconnected_handler(struct rdma_cm_id *cma_id)
661		−{
662		− struct iser_conn *iser_conn = cma_id->context;
663		−
664		− if (iser_conn_terminate(iser_conn)) {
665		− if (iser_conn->iscsi_conn)
666		− iscsi_conn_failure(iser_conn->iscsi_conn,
667		− ISCSI_ERR_CONN_FAILED);
668		− else
669		− iser_err("iscsi_iser connection isn't bound\n");
670		− }
671		−}
672		−
	654	+/*
	655	+ * Called with state mutex held
	656	+ */
673	657	static void iser_cleanup_handler(struct rdma_cm_id *cma_id,
674	658	bool destroy)
675	659	{
676	660	struct iser_conn *iser_conn = cma_id->context;
677	661
	662	+ lockdep_assert_held(&iser_conn->state_mutex);
678	663	/*
679	664	* We are not guaranteed that we visited disconnected_handler
680	665	* by now, call it here to be safe that we handle CM drep
681	666	* and flush errors.
682	667	*/
683		− iser_disconnected_handler(cma_id);
	668	+ if (iser_conn_terminate(iser_conn)) {
	669	+ if (iser_conn->iscsi_conn)
	670	+ iscsi_conn_failure(iser_conn->iscsi_conn,
	671	+ ISCSI_ERR_CONN_FAILED);
	672	+ else
	673	+ iser_err("iscsi_iser connection isn't bound\n");
	674	+ }
684	675	iser_free_ib_conn_res(iser_conn, destroy);
685	676	complete(&iser_conn->ib_completion);
686	677	}

drivers/infiniband/ulp/isert/ib_isert.c+2−3

@@ -993,9 +993,8 @@ isert_rx_login_req(struct isert_conn *isert_conn) · 992 unmodified lines
993	993	* login request PDU.
994	994	*/
995	995	login->leading_connection = (!login_req->tsih) ? 1 : 0;
996		− login->current_stage =
997		− (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
998		− >> 2;
	996	+ login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
	997	+ login_req->flags);
999	998	login->version_min = login_req->min_version;
1000	999	login->version_max = login_req->max_version;
1001	1000	memcpy(login->isid, login_req->isid, 6);

drivers/infiniband/ulp/rtrs/rtrs-clt.c+2−4

@@ -1064,10 +1064,8 @@ static int rtrs_map_sg_fr(struct rtrs_clt_io_req *req, size_t count) · 1063 unmodified lines
1064	1064
1065	1065	/* Align the MR to a 4K page size to match the block virt boundary */
1066	1066	nr = ib_map_mr_sg(req->mr, req->sglist, count, NULL, SZ_4K);
1067		− if (nr < 0)
1068		− return nr;
1069		− if (nr < req->sg_cnt)
1070		− return -EINVAL;
	1067	+ if (nr != count)
	1068	+ return nr < 0 ? nr : -EINVAL;
1071	1069	ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
1072	1070
1073	1071	return nr;

drivers/infiniband/ulp/rtrs/rtrs-pri.h+0−3

@@ -68,10 +68,7 @@ enum { · 67 unmodified lines
68	68	struct rtrs_ib_dev;
69	69
70	70	struct rtrs_rdma_dev_pd_ops {
71		− struct rtrs_ib_dev (alloc)(void);
72		− void (free)(struct rtrs_ib_dev dev);
73	71	int (init)(struct rtrs_ib_dev dev);
74		− void (deinit)(struct rtrs_ib_dev dev);
75	72	};
76	73
77	74	struct rtrs_rdma_dev_pd {

drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c+9−4

@@ -203,7 +203,6 @@ rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_path *srv_path) · 202 unmodified lines
203	203
204	204	mutex_lock(&srv->paths_mutex);
205	205	if (!--srv->dev_ref) {
206		− kobject_del(srv->kobj_paths);
207	206	kobject_put(srv->kobj_paths);
208	207	mutex_unlock(&srv->paths_mutex);
209	208	device_del(&srv->dev);
@@ -304,12 +303,18 @@ destroy_root: · 94 unmodified lines
304	303
305	304	void rtrs_srv_destroy_path_files(struct rtrs_srv_path *srv_path)
306	305	{
307		− if (srv_path->kobj.state_in_sysfs) {
	306	+ if (srv_path->stats->kobj_stats.state_in_sysfs) {
	307	+ sysfs_remove_group(&srv_path->stats->kobj_stats,
	308	+ &rtrs_srv_stats_attr_group);
308	309	kobject_del(&srv_path->stats->kobj_stats);
309	310	kobject_put(&srv_path->stats->kobj_stats);
	311	+ }
	312	+
	313	+ if (srv_path->kobj.state_in_sysfs) {
310	314	sysfs_remove_group(&srv_path->kobj, &rtrs_srv_path_attr_group);
	315	+ kobject_del(&srv_path->kobj);
311	316	kobject_put(&srv_path->kobj);
312		−
313		− rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
314	317	}
	318	+
	319	+ rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
315	320	}

drivers/infiniband/ulp/rtrs/rtrs-srv.c+29−43

@@ -561,9 +561,11 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) · 560 unmodified lines
561	561	{
562	562	struct rtrs_srv_sess *srv = srv_path->srv;
563	563	struct rtrs_path *ss = &srv_path->s;
564		− int i, mri, err, mrs_num;
	564	+ int i, err, mrs_num;
565	565	unsigned int chunk_bits;
566	566	int chunks_per_mr = 1;
	567	+ struct ib_mr *mr;
	568	+ struct sg_table *sgt;
567	569
568	570	/*
569	571	* Here we map queue_depth chunks to MR. Firstly we have to
@@ -586,16 +588,14 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) · 16 unmodified lines
586	588	if (!srv_path->mrs)
587	589	return -ENOMEM;
588	590
589		− srv_path->mrs_num = mrs_num;
590		−
591		− for (mri = 0; mri < mrs_num; mri++) {
592		− struct rtrs_srv_mr *srv_mr = &srv_path->mrs[mri];
593		− struct sg_table *sgt = &srv_mr->sgt;
	591	+ for (srv_path->mrs_num = 0; srv_path->mrs_num < mrs_num;
	592	+ srv_path->mrs_num++) {
	593	+ struct rtrs_srv_mr *srv_mr = &srv_path->mrs[srv_path->mrs_num];
594	594	struct scatterlist *s;
595		− struct ib_mr *mr;
596	595	int nr, nr_sgt, chunks;
597	596
598		− chunks = chunks_per_mr * mri;
	597	+ sgt = &srv_mr->sgt;
	598	+ chunks = chunks_per_mr * srv_path->mrs_num;
599	599	if (!always_invalidate)
600	600	chunks_per_mr = min_t(int, chunks_per_mr,
601	601	srv->queue_depth - chunks);
@@ -622,7 +622,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) · 20 unmodified lines
622	622	}
623	623	nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt,
624	624	NULL, max_chunk_size);
625		− if (nr < 0 \|\| nr < sgt->nents) {
	625	+ if (nr != nr_sgt) {
626	626	err = nr < 0 ? nr : -EINVAL;
627	627	goto dereg_mr;
628	628	}
@@ -644,31 +644,24 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) · 15 unmodified lines
644	644
645	645	ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
646	646	srv_mr->mr = mr;
647		−
648		− continue;
649		−err:
650		− while (mri--) {
651		− srv_mr = &srv_path->mrs[mri];
652		− sgt = &srv_mr->sgt;
653		− mr = srv_mr->mr;
654		− rtrs_iu_free(srv_mr->iu, srv_path->s.dev->ib_dev, 1);
655		−dereg_mr:
656		− ib_dereg_mr(mr);
657		−unmap_sg:
658		− ib_dma_unmap_sg(srv_path->s.dev->ib_dev, sgt->sgl,
659		− sgt->nents, DMA_BIDIRECTIONAL);
660		−free_sg:
661		− sg_free_table(sgt);
662		− }
663		− kfree(srv_path->mrs);
664		−
665		− return err;
666	647	}
667	648
668	649	chunk_bits = ilog2(srv->queue_depth - 1) + 1;
669	650	srv_path->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits);
670	651
671	652	return 0;
	653	+
	654	+dereg_mr:
	655	+ ib_dereg_mr(mr);
	656	+unmap_sg:
	657	+ ib_dma_unmap_sg(srv_path->s.dev->ib_dev, sgt->sgl,
	658	+ sgt->nents, DMA_BIDIRECTIONAL);
	659	+free_sg:
	660	+ sg_free_table(sgt);
	661	+err:
	662	+ unmap_cont_bufs(srv_path);
	663	+
	664	+ return err;
672	665	}
673	666
674	667	static void rtrs_srv_hb_err_handler(struct rtrs_con *c)
@@ -1678,12 +1671,6 @@ static int create_con(struct rtrs_srv_path *srv_path, · 1003 unmodified lines
1678	1671	srv->queue_depth * (1 + 2) + 1);
1679	1672
1680	1673	max_recv_wr = srv->queue_depth + 1;
1681		− /*
1682		− * If we have all receive requests posted and
1683		− * all write requests posted and each read request
1684		− * requires an invalidate request + drain
1685		− * and qp gets into error state.
1686		− */
1687	1674	}
1688	1675	cq_num = max_send_wr + max_recv_wr;
1689	1676	atomic_set(&con->c.sq_wr_avail, max_send_wr);
@@ -1950,22 +1937,21 @@ static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, · 260 unmodified lines
1950	1937	{
1951	1938	struct rtrs_srv_path *srv_path = NULL;
1952	1939	struct rtrs_path *s = NULL;
	1940	+ struct rtrs_con *c = NULL;
1953	1941
1954		− if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
1955		− struct rtrs_con *c = cm_id->context;
1956		−
1957		− s = c->path;
1958		− srv_path = to_srv_path(s);
1959		− }
1960		−
1961		− switch (ev->event) {
1962		− case RDMA_CM_EVENT_CONNECT_REQUEST:
	1942	+ if (ev->event == RDMA_CM_EVENT_CONNECT_REQUEST)
1963	1943	/*
1964	1944	* In case of error cma.c will destroy cm_id,
1965	1945	* see cma_process_remove()
1966	1946	*/
1967	1947	return rtrs_rdma_connect(cm_id, ev->param.conn.private_data,
1968	1948	ev->param.conn.private_data_len);
	1949	+
	1950	+ c = cm_id->context;
	1951	+ s = c->path;
	1952	+ srv_path = to_srv_path(s);
	1953	+
	1954	+ switch (ev->event) {
1969	1955	case RDMA_CM_EVENT_ESTABLISHED:
1970	1956	/* Nothing here */
1971	1957	break;

drivers/infiniband/ulp/rtrs/rtrs.c+4−18

@@ -557,7 +557,6 @@ EXPORT_SYMBOL(rtrs_addr_to_sockaddr); · 556 unmodified lines
557	557	void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,
558	558	struct rtrs_rdma_dev_pd *pool)
559	559	{
560		− WARN_ON(pool->ops && (!pool->ops->alloc ^ !pool->ops->free));
561	560	INIT_LIST_HEAD(&pool->list);
562	561	mutex_init(&pool->mutex);
563	562	pool->pd_flags = pd_flags;
@@ -583,15 +582,8 @@ static void dev_free(struct kref *ref) · 19 unmodified lines
583	582	list_del(&dev->entry);
584	583	mutex_unlock(&pool->mutex);
585	584
586		− if (pool->ops && pool->ops->deinit)
587		− pool->ops->deinit(dev);
588		−
589	585	ib_dealloc_pd(dev->ib_pd);
590		−
591		− if (pool->ops && pool->ops->free)
592		− pool->ops->free(dev);
593		− else
594		− kfree(dev);
	586	+ kfree(dev);
595	587	}
596	588
597	589	int rtrs_ib_dev_put(struct rtrs_ib_dev *dev)
@@ -618,11 +610,8 @@ rtrs_ib_dev_find_or_add(struct ib_device *ib_dev, · 20 unmodified lines
618	610	goto out_unlock;
619	611	}
620	612	mutex_unlock(&pool->mutex);
621		− if (pool->ops && pool->ops->alloc)
622		− dev = pool->ops->alloc();
623		− else
624		− dev = kzalloc(sizeof(*dev), GFP_KERNEL);
625		− if (IS_ERR_OR_NULL(dev))
	613	+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
	614	+ if (!dev)
626	615	goto out_err;
627	616
628	617	kref_init(&dev->ref);
@@ -644,10 +633,7 @@ out_unlock: · 15 unmodified lines
644	633	out_free_pd:
645	634	ib_dealloc_pd(dev->ib_pd);
646	635	out_free_dev:
647		− if (pool->ops && pool->ops->free)
648		− pool->ops->free(dev);
649		− else
650		− kfree(dev);
	636	+ kfree(dev);
651	637	out_err:
652	638	return NULL;
653	639	}

drivers/infiniband/ulp/srp/ib_srp.c+82−14

@@ -3410,7 +3410,8 @@ static int srp_parse_options(struct net net, const char buf, · 3409 unmodified lines
3410	3410	break;
3411	3411
3412	3412	case SRP_OPT_PKEY:
3413		− if (match_hex(args, &token)) {
	3413	+ ret = match_hex(args, &token);
	3414	+ if (ret) {
3414	3415	pr_warn("bad P_Key parameter '%s'\n", p);
3415	3416	goto out;
3416	3417	}
@@ -3470,7 +3471,8 @@ static int srp_parse_options(struct net net, const char buf, · 53 unmodified lines
3470	3471	break;
3471	3472
3472	3473	case SRP_OPT_MAX_SECT:
3473		− if (match_int(args, &token)) {
	3474	+ ret = match_int(args, &token);
	3475	+ if (ret) {
3474	3476	pr_warn("bad max sect parameter '%s'\n", p);
3475	3477	goto out;
3476	3478	}
@@ -3478,8 +3480,15 @@ static int srp_parse_options(struct net net, const char buf, · 1 unmodified line
3478	3480	break;
3479	3481
3480	3482	case SRP_OPT_QUEUE_SIZE:
3481		− if (match_int(args, &token) \|\| token < 1) {
	3483	+ ret = match_int(args, &token);
	3484	+ if (ret) {
	3485	+ pr_warn("match_int() failed for queue_size parameter '%s', Error %d\n",
	3486	+ p, ret);
	3487	+ goto out;
	3488	+ }
	3489	+ if (token < 1) {
3482	3490	pr_warn("bad queue_size parameter '%s'\n", p);
	3491	+ ret = -EINVAL;
3483	3492	goto out;
3484	3493	}
3485	3494	target->scsi_host->can_queue = token;
@@ -3490,25 +3499,40 @@ static int srp_parse_options(struct net net, const char buf, · 4 unmodified lines
3490	3499	break;
3491	3500
3492	3501	case SRP_OPT_MAX_CMD_PER_LUN:
3493		− if (match_int(args, &token) \|\| token < 1) {
	3502	+ ret = match_int(args, &token);
	3503	+ if (ret) {
	3504	+ pr_warn("match_int() failed for max cmd_per_lun parameter '%s', Error %d\n",
	3505	+ p, ret);
	3506	+ goto out;
	3507	+ }
	3508	+ if (token < 1) {
3494	3509	pr_warn("bad max cmd_per_lun parameter '%s'\n",
3495	3510	p);
	3511	+ ret = -EINVAL;
3496	3512	goto out;
3497	3513	}
3498	3514	target->scsi_host->cmd_per_lun = token;
3499	3515	break;
3500	3516
3501	3517	case SRP_OPT_TARGET_CAN_QUEUE:
3502		− if (match_int(args, &token) \|\| token < 1) {
	3518	+ ret = match_int(args, &token);
	3519	+ if (ret) {
	3520	+ pr_warn("match_int() failed for max target_can_queue parameter '%s', Error %d\n",
	3521	+ p, ret);
	3522	+ goto out;
	3523	+ }
	3524	+ if (token < 1) {
3503	3525	pr_warn("bad max target_can_queue parameter '%s'\n",
3504	3526	p);
	3527	+ ret = -EINVAL;
3505	3528	goto out;
3506	3529	}
3507	3530	target->target_can_queue = token;
3508	3531	break;
3509	3532
3510	3533	case SRP_OPT_IO_CLASS:
3511		− if (match_hex(args, &token)) {
	3534	+ ret = match_hex(args, &token);
	3535	+ if (ret) {
3512	3536	pr_warn("bad IO class parameter '%s'\n", p);
3513	3537	goto out;
3514	3538	}
@@ -3517,6 +3541,7 @@ static int srp_parse_options(struct net net, const char buf, · 2 unmodified lines
3517	3541	pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3518	3542	token, SRP_REV10_IB_IO_CLASS,
3519	3543	SRP_REV16A_IB_IO_CLASS);
	3544	+ ret = -EINVAL;
3520	3545	goto out;
3521	3546	}
3522	3547	target->io_class = token;
@@ -3539,16 +3564,24 @@ static int srp_parse_options(struct net net, const char buf, · 16 unmodified lines
3539	3564	break;
3540	3565
3541	3566	case SRP_OPT_CMD_SG_ENTRIES:
3542		− if (match_int(args, &token) \|\| token < 1 \|\| token > 255) {
	3567	+ ret = match_int(args, &token);
	3568	+ if (ret) {
	3569	+ pr_warn("match_int() failed for max cmd_sg_entries parameter '%s', Error %d\n",
	3570	+ p, ret);
	3571	+ goto out;
	3572	+ }
	3573	+ if (token < 1 \|\| token > 255) {
3543	3574	pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3544	3575	p);
	3576	+ ret = -EINVAL;
3545	3577	goto out;
3546	3578	}
3547	3579	target->cmd_sg_cnt = token;
3548	3580	break;
3549	3581
3550	3582	case SRP_OPT_ALLOW_EXT_SG:
3551		− if (match_int(args, &token)) {
	3583	+ ret = match_int(args, &token);
	3584	+ if (ret) {
3552	3585	pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3553	3586	goto out;
3554	3587	}
@@ -3556,43 +3589,77 @@ static int srp_parse_options(struct net net, const char buf, · 1 unmodified line
3556	3589	break;
3557	3590
3558	3591	case SRP_OPT_SG_TABLESIZE:
3559		− if (match_int(args, &token) \|\| token < 1 \|\|
3560		− token > SG_MAX_SEGMENTS) {
	3592	+ ret = match_int(args, &token);
	3593	+ if (ret) {
	3594	+ pr_warn("match_int() failed for max sg_tablesize parameter '%s', Error %d\n",
	3595	+ p, ret);
	3596	+ goto out;
	3597	+ }
	3598	+ if (token < 1 \|\| token > SG_MAX_SEGMENTS) {
3561	3599	pr_warn("bad max sg_tablesize parameter '%s'\n",
3562	3600	p);
	3601	+ ret = -EINVAL;
3563	3602	goto out;
3564	3603	}
3565	3604	target->sg_tablesize = token;
3566	3605	break;
3567	3606
3568	3607	case SRP_OPT_COMP_VECTOR:
3569		− if (match_int(args, &token) \|\| token < 0) {
	3608	+ ret = match_int(args, &token);
	3609	+ if (ret) {
	3610	+ pr_warn("match_int() failed for comp_vector parameter '%s', Error %d\n",
	3611	+ p, ret);
	3612	+ goto out;
	3613	+ }
	3614	+ if (token < 0) {
3570	3615	pr_warn("bad comp_vector parameter '%s'\n", p);
	3616	+ ret = -EINVAL;
3571	3617	goto out;
3572	3618	}
3573	3619	target->comp_vector = token;
3574	3620	break;
3575	3621
3576	3622	case SRP_OPT_TL_RETRY_COUNT:
3577		− if (match_int(args, &token) \|\| token < 2 \|\| token > 7) {
	3623	+ ret = match_int(args, &token);
	3624	+ if (ret) {
	3625	+ pr_warn("match_int() failed for tl_retry_count parameter '%s', Error %d\n",
	3626	+ p, ret);
	3627	+ goto out;
	3628	+ }
	3629	+ if (token < 2 \|\| token > 7) {
3578	3630	pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3579	3631	p);
	3632	+ ret = -EINVAL;
3580	3633	goto out;
3581	3634	}
3582	3635	target->tl_retry_count = token;
3583	3636	break;
3584	3637
3585	3638	case SRP_OPT_MAX_IT_IU_SIZE:
3586		− if (match_int(args, &token) \|\| token < 0) {
	3639	+ ret = match_int(args, &token);
	3640	+ if (ret) {
	3641	+ pr_warn("match_int() failed for max it_iu_size parameter '%s', Error %d\n",
	3642	+ p, ret);
	3643	+ goto out;
	3644	+ }
	3645	+ if (token < 0) {
3587	3646	pr_warn("bad maximum initiator to target IU size '%s'\n", p);
	3647	+ ret = -EINVAL;
3588	3648	goto out;
3589	3649	}
3590	3650	target->max_it_iu_size = token;
3591	3651	break;
3592	3652
3593	3653	case SRP_OPT_CH_COUNT:
3594		− if (match_int(args, &token) \|\| token < 1) {
	3654	+ ret = match_int(args, &token);
	3655	+ if (ret) {
	3656	+ pr_warn("match_int() failed for channel count parameter '%s', Error %d\n",
	3657	+ p, ret);
	3658	+ goto out;
	3659	+ }
	3660	+ if (token < 1) {
3595	3661	pr_warn("bad channel count %s\n", p);
	3662	+ ret = -EINVAL;
3596	3663	goto out;
3597	3664	}
3598	3665	target->ch_count = token;
@@ -3601,6 +3668,7 @@ static int srp_parse_options(struct net net, const char buf, · 2 unmodified lines
3601	3668	default:
3602	3669	pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3603	3670	p);
	3671	+ ret = -EINVAL;
3604	3672	goto out;
3605	3673	}
3606	3674	}

drivers/net/ethernet/microsoft/mana/gdma_main.c+1−2

@@ -671,8 +671,7 @@ free_q: · 670 unmodified lines
671	671	return err;
672	672	}
673	673
674		−int mana_gd_destroy_dma_region(struct gdma_context *gc,
675		− gdma_obj_handle_t dma_region_handle)
	674	+int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle)
676	675	{
677	676	struct gdma_destroy_dma_region_req req = {};
678	677	struct gdma_general_resp resp = {};

include/net/mana/gdma.h+14−17

@@ -65,8 +65,6 @@ enum { · 64 unmodified lines
65	65	GDMA_DEVICE_MANA = 2,
66	66	};
67	67
68		−typedef u64 gdma_obj_handle_t;
69		−
70	68	struct gdma_resource {
71	69	/* Protect the bitmap */
72	70	spinlock_t lock;
@@ -200,7 +198,7 @@ struct gdma_mem_info { · 127 unmodified lines
200	198	u64 length;
201	199
202	200	/* Allocated by the PF driver */
203		− gdma_obj_handle_t dma_region_handle;
	201	+ u64 dma_region_handle;
204	202	};
205	203
206	204	#define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
@@ -632,7 +630,7 @@ struct gdma_create_queue_req { · 425 unmodified lines
632	630	u32 reserved1;
633	631	u32 pdid;
634	632	u32 doolbell_id;
635		− gdma_obj_handle_t gdma_region;
	633	+ u64 gdma_region;
636	634	u32 reserved2;
637	635	u32 queue_size;
638	636	u32 log2_throttle_limit;
@@ -707,14 +705,14 @@ struct gdma_create_dma_region_req { · 68 unmodified lines
707	705
708	706	struct gdma_create_dma_region_resp {
709	707	struct gdma_resp_hdr hdr;
710		− gdma_obj_handle_t dma_region_handle;
	708	+ u64 dma_region_handle;
711	709	}; /* HW DATA */
712	710
713	711	/* GDMA_DMA_REGION_ADD_PAGES */
714	712	struct gdma_dma_region_add_pages_req {
715	713	struct gdma_req_hdr hdr;
716	714
717		− gdma_obj_handle_t dma_region_handle;
	715	+ u64 dma_region_handle;
718	716
719	717	u32 page_addr_list_len;
720	718	u32 reserved3;
@@ -726,7 +724,7 @@ struct gdma_dma_region_add_pages_req { · 5 unmodified lines
726	724	struct gdma_destroy_dma_region_req {
727	725	struct gdma_req_hdr hdr;
728	726
729		− gdma_obj_handle_t dma_region_handle;
	727	+ u64 dma_region_handle;
730	728	}; /* HW DATA */
731	729
732	730	enum gdma_pd_flags {
@@ -741,14 +739,14 @@ struct gdma_create_pd_req { · 8 unmodified lines
741	739
742	740	struct gdma_create_pd_resp {
743	741	struct gdma_resp_hdr hdr;
744		− gdma_obj_handle_t pd_handle;
	742	+ u64 pd_handle;
745	743	u32 pd_id;
746	744	u32 reserved;
747	745	};/* HW DATA */
748	746
749	747	struct gdma_destroy_pd_req {
750	748	struct gdma_req_hdr hdr;
751		− gdma_obj_handle_t pd_handle;
	749	+ u64 pd_handle;
752	750	};/* HW DATA */
753	751
754	752	struct gdma_destory_pd_resp {
@@ -764,11 +762,11 @@ enum gdma_mr_type { · 9 unmodified lines
764	762	};
765	763
766	764	struct gdma_create_mr_params {
767		− gdma_obj_handle_t pd_handle;
	765	+ u64 pd_handle;
768	766	enum gdma_mr_type mr_type;
769	767	union {
770	768	struct {
771		− gdma_obj_handle_t dma_region_handle;
	769	+ u64 dma_region_handle;
772	770	u64 virtual_address;
773	771	enum gdma_mr_access_flags access_flags;
774	772	} gva;
@@ -777,13 +775,13 @@ struct gdma_create_mr_params { · 2 unmodified lines
777	775
778	776	struct gdma_create_mr_request {
779	777	struct gdma_req_hdr hdr;
780		− gdma_obj_handle_t pd_handle;
	778	+ u64 pd_handle;
781	779	enum gdma_mr_type mr_type;
782	780	u32 reserved_1;
783	781
784	782	union {
785	783	struct {
786		− gdma_obj_handle_t dma_region_handle;
	784	+ u64 dma_region_handle;
787	785	u64 virtual_address;
788	786	enum gdma_mr_access_flags access_flags;
789	787	} gva;
@@ -794,14 +792,14 @@ struct gdma_create_mr_request { · 4 unmodified lines
794	792
795	793	struct gdma_create_mr_response {
796	794	struct gdma_resp_hdr hdr;
797		− gdma_obj_handle_t mr_handle;
	795	+ u64 mr_handle;
798	796	u32 lkey;
799	797	u32 rkey;
800	798	};/* HW DATA */
801	799
802	800	struct gdma_destroy_mr_request {
803	801	struct gdma_req_hdr hdr;
804		− gdma_obj_handle_t mr_handle;
	802	+ u64 mr_handle;
805	803	};/* HW DATA */
806	804
807	805	struct gdma_destroy_mr_response {
@@ -835,7 +833,6 @@ void mana_gd_free_memory(struct gdma_mem_info *gmi); · 27 unmodified lines
835	833	int mana_gd_send_request(struct gdma_context gc, u32 req_len, const void req,
836	834	u32 resp_len, void *resp);
837	835
838		−int mana_gd_destroy_dma_region(struct gdma_context *gc,
839		− gdma_obj_handle_t dma_region_handle);
	836	+int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle);
840	837
841	838	#endif /* _GDMA_H */

include/net/mana/mana.h+3−0

@@ -412,6 +412,9 @@ int mana_bpf(struct net_device ndev, struct netdev_bpf bpf); · 411 unmodified lines
412	412
413	413	extern const struct ethtool_ops mana_ethtool_ops;
414	414
	415	+/* A CQ can be created not associated with any EQ */
	416	+#define GDMA_CQ_NO_EQ 0xffff
	417	+
415	418	struct mana_obj_spec {
416	419	u32 queue_index;
417	420	u64 gdma_region;

include/rdma/ib_pack.h+5−0

@@ -84,6 +84,8 @@ enum { · 83 unmodified lines
84	84	/* opcode 0x15 is reserved */
85	85	IB_OPCODE_SEND_LAST_WITH_INVALIDATE = 0x16,
86	86	IB_OPCODE_SEND_ONLY_WITH_INVALIDATE = 0x17,
	87	+ IB_OPCODE_FLUSH = 0x1C,
	88	+ IB_OPCODE_ATOMIC_WRITE = 0x1D,
87	89
88	90	/* real constants follow -- see comment about above IB_OPCODE()
89	91	macro for more details */
@@ -112,6 +114,8 @@ enum { · 22 unmodified lines
112	114	IB_OPCODE(RC, FETCH_ADD),
113	115	IB_OPCODE(RC, SEND_LAST_WITH_INVALIDATE),
114	116	IB_OPCODE(RC, SEND_ONLY_WITH_INVALIDATE),
	117	+ IB_OPCODE(RC, FLUSH),
	118	+ IB_OPCODE(RC, ATOMIC_WRITE),
115	119
116	120	/* UC */
117	121	IB_OPCODE(UC, SEND_FIRST),
@@ -149,6 +153,7 @@ enum { · 31 unmodified lines
149	153	IB_OPCODE(RD, ATOMIC_ACKNOWLEDGE),
150	154	IB_OPCODE(RD, COMPARE_SWAP),
151	155	IB_OPCODE(RD, FETCH_ADD),
	156	+ IB_OPCODE(RD, FLUSH),
152	157
153	158	/* UD */
154	159	IB_OPCODE(UD, SEND_ONLY),

include/rdma/ib_verbs.h+22−2

@@ -270,6 +270,10 @@ enum ib_device_cap_flags { · 269 unmodified lines
270	270	/* The device supports padding incoming writes to cacheline. */
271	271	IB_DEVICE_PCI_WRITE_END_PADDING =
272	272	IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING,
	273	+ /* Placement type attributes */
	274	+ IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL,
	275	+ IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT,
	276	+ IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE,
273	277	};
274	278
275	279	enum ib_kernel_cap_flags {
@@ -982,9 +986,11 @@ enum ib_wc_opcode { · 706 unmodified lines
982	986	IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW,
983	987	IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV,
984	988	IB_WC_LSO = IB_UVERBS_WC_TSO,
	989	+ IB_WC_ATOMIC_WRITE = IB_UVERBS_WC_ATOMIC_WRITE,
985	990	IB_WC_REG_MR,
986	991	IB_WC_MASKED_COMP_SWAP,
987	992	IB_WC_MASKED_FETCH_ADD,
	993	+ IB_WC_FLUSH = IB_UVERBS_WC_FLUSH,
988	994	/*
989	995	* Set value of IB_WC_RECV so consumers can test if a completion is a
990	996	* receive by testing (opcode & IB_WC_RECV).
@@ -1325,6 +1331,8 @@ enum ib_wr_opcode { · 334 unmodified lines
1325	1331	IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP,
1326	1332	IB_WR_MASKED_ATOMIC_FETCH_AND_ADD =
1327	1333	IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD,
	1334	+ IB_WR_FLUSH = IB_UVERBS_WR_FLUSH,
	1335	+ IB_WR_ATOMIC_WRITE = IB_UVERBS_WR_ATOMIC_WRITE,
1328	1336
1329	1337	/* These are kernel only and can not be issued by userspace */
1330	1338	IB_WR_REG_MR = 0x20,
@@ -1458,10 +1466,12 @@ enum ib_access_flags { · 127 unmodified lines
1458	1466	IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND,
1459	1467	IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB,
1460	1468	IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING,
	1469	+ IB_ACCESS_FLUSH_GLOBAL = IB_UVERBS_ACCESS_FLUSH_GLOBAL,
	1470	+ IB_ACCESS_FLUSH_PERSISTENT = IB_UVERBS_ACCESS_FLUSH_PERSISTENT,
1461	1471
1462	1472	IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE,
1463	1473	IB_ACCESS_SUPPORTED =
1464		− ((IB_ACCESS_HUGETLB << 1) - 1) \| IB_ACCESS_OPTIONAL,
	1474	+ ((IB_ACCESS_FLUSH_PERSISTENT << 1) - 1) \| IB_ACCESS_OPTIONAL,
1465	1475	};
1466	1476
1467	1477	/*
@@ -2203,6 +2213,7 @@ struct ib_port_data { · 735 unmodified lines
2203	2213	struct ib_port_cache cache;
2204	2214
2205	2215	struct net_device __rcu *netdev;
	2216	+ netdevice_tracker netdev_tracker;
2206	2217	struct hlist_node ndev_hash_link;
2207	2218	struct rdma_port_counter port_counter;
2208	2219	struct ib_port *sysfs;
@@ -4321,6 +4332,8 @@ int ib_dealloc_xrcd_user(struct ib_xrcd xrcd, struct ib_udata udata); · 2112 unmodified lines
4321	4332	static inline int ib_check_mr_access(struct ib_device *ib_dev,
4322	4333	unsigned int flags)
4323	4334	{
	4335	+ u64 device_cap = ib_dev->attrs.device_cap_flags;
	4336	+
4324	4337	/*
4325	4338	* Local write permission is required if remote write or
4326	4339	* remote atomic permission is also requested.
@@ -4334,7 +4347,14 @@ static inline int ib_check_mr_access(struct ib_device *ib_dev, · 7 unmodified lines
4334	4347
4335	4348	if (flags & IB_ACCESS_ON_DEMAND &&
4336	4349	!(ib_dev->attrs.kernel_cap_flags & IBK_ON_DEMAND_PAGING))
4337		− return -EINVAL;
	4350	+ return -EOPNOTSUPP;
	4351	+
	4352	+ if ((flags & IB_ACCESS_FLUSH_GLOBAL &&
	4353	+ !(device_cap & IB_DEVICE_FLUSH_GLOBAL)) \|\|
	4354	+ (flags & IB_ACCESS_FLUSH_PERSISTENT &&
	4355	+ !(device_cap & IB_DEVICE_FLUSH_PERSISTENT)))
	4356	+ return -EOPNOTSUPP;
	4357	+
4338	4358	return 0;
4339	4359	}
4340	4360

include/rdma/opa_vnic.h+1−1

@@ -51,7 +51,7 @@ static inline void opa_vnic_dev_priv(const struct net_device dev) · 50 unmodified lines
51	51	return oparn->dev_priv;
52	52	}
53	53
54		−/* opa_vnic skb meta data structrue */
	54	+/* opa_vnic skb meta data structure */
55	55	struct opa_vnic_skb_mdata {
56	56	u8 vl;
57	57	u8 entropy;

include/trace/events/ib_mad.h+4−9

@@ -49,7 +49,6 @@ DECLARE_EVENT_CLASS(ib_mad_send_template, · 48 unmodified lines
49	49	__field(int, retries_left)
50	50	__field(int, max_retries)
51	51	__field(int, retry)
52		− __field(u16, pkey)
53	52	),
54	53
55	54	TP_fast_assign(
@@ -89,7 +88,7 @@ DECLARE_EVENT_CLASS(ib_mad_send_template, · 33 unmodified lines
89	88	"hdr : base_ver 0x%x class 0x%x class_ver 0x%x " \
90	89	"method 0x%x status 0x%x class_specific 0x%x tid 0x%llx " \
91	90	"attr_id 0x%x attr_mod 0x%x => dlid 0x%08x sl %d "\
92		− "pkey 0x%x rpqn 0x%x rqpkey 0x%x",
	91	+ "rpqn 0x%x rqpkey 0x%x",
93	92	__entry->dev_index, __entry->port_num, __entry->qp_num,
94	93	__entry->agent_priv, be64_to_cpu(__entry->wrtid),
95	94	__entry->retries_left, __entry->max_retries,
@@ -100,7 +99,7 @@ DECLARE_EVENT_CLASS(ib_mad_send_template, · 4 unmodified lines
100	99	be16_to_cpu(__entry->class_specific),
101	100	be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id),
102	101	be32_to_cpu(__entry->attr_mod),
103		− be32_to_cpu(__entry->dlid), __entry->sl, __entry->pkey,
	102	+ be32_to_cpu(__entry->dlid), __entry->sl,
104	103	__entry->rqpn, __entry->rqkey
105	104	)
106	105	);
@@ -204,7 +203,6 @@ TRACE_EVENT(ib_mad_recv_done_handler, · 97 unmodified lines
204	203	__field(u16, wc_status)
205	204	__field(u32, slid)
206	205	__field(u32, dev_index)
207		− __field(u16, pkey)
208	206	),
209	207
210	208	TP_fast_assign(
@@ -224,9 +222,6 @@ TRACE_EVENT(ib_mad_recv_done_handler, · 13 unmodified lines
224	222	__entry->slid = wc->slid;
225	223	__entry->src_qp = wc->src_qp;
226	224	__entry->sl = wc->sl;
227		− ib_query_pkey(qp_info->port_priv->device,
228		− qp_info->port_priv->port_num,
229		− wc->pkey_index, &__entry->pkey);
230	225	__entry->wc_status = wc->status;
231	226	),
232	227
@@ -234,7 +229,7 @@ TRACE_EVENT(ib_mad_recv_done_handler, · 1 unmodified line
234	229	"base_ver 0x%02x class 0x%02x class_ver 0x%02x " \
235	230	"method 0x%02x status 0x%04x class_specific 0x%04x " \
236	231	"tid 0x%016llx attr_id 0x%04x attr_mod 0x%08x " \
237		− "slid 0x%08x src QP%d, sl %d pkey 0x%04x",
	232	+ "slid 0x%08x src QP%d, sl %d",
238	233	__entry->dev_index, __entry->port_num, __entry->qp_num,
239	234	__entry->wc_status,
240	235	__entry->length,
@@ -244,7 +239,7 @@ TRACE_EVENT(ib_mad_recv_done_handler, · 3 unmodified lines
244	239	be16_to_cpu(__entry->class_specific),
245	240	be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id),
246	241	be32_to_cpu(__entry->attr_mod),
247		− __entry->slid, __entry->src_qp, __entry->sl, __entry->pkey
	242	+ __entry->slid, __entry->src_qp, __entry->sl
248	243	)
249	244	);
250	245

include/uapi/rdma/hns-abi.h+15−0

@@ -85,11 +85,26 @@ struct hns_roce_ib_create_qp_resp { · 84 unmodified lines
85	85	__aligned_u64 dwqe_mmap_key;
86	86	};
87	87
	88	+enum {
	89	+ HNS_ROCE_EXSGE_FLAGS = 1 << 0,
	90	+};
	91	+
	92	+enum {
	93	+ HNS_ROCE_RSP_EXSGE_FLAGS = 1 << 0,
	94	+};
	95	+
88	96	struct hns_roce_ib_alloc_ucontext_resp {
89	97	__u32 qp_tab_size;
90	98	__u32 cqe_size;
91	99	__u32 srq_tab_size;
92	100	__u32 reserved;
	101	+ __u32 config;
	102	+ __u32 max_inline_data;
	103	+};
	104	+
	105	+struct hns_roce_ib_alloc_ucontext {
	106	+ __u32 config;
	107	+ __u32 reserved;
93	108	};
94	109
95	110	struct hns_roce_ib_alloc_pd_resp {

include/uapi/rdma/ib_user_ioctl_verbs.h+3−0

@@ -57,6 +57,8 @@ enum ib_uverbs_access_flags { · 56 unmodified lines
57	57	IB_UVERBS_ACCESS_ZERO_BASED = 1 << 5,
58	58	IB_UVERBS_ACCESS_ON_DEMAND = 1 << 6,
59	59	IB_UVERBS_ACCESS_HUGETLB = 1 << 7,
	60	+ IB_UVERBS_ACCESS_FLUSH_GLOBAL = 1 << 8,
	61	+ IB_UVERBS_ACCESS_FLUSH_PERSISTENT = 1 << 9,
60	62
61	63	IB_UVERBS_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_OPTIONAL_FIRST,
62	64	IB_UVERBS_ACCESS_OPTIONAL_RANGE =
@@ -251,6 +253,7 @@ enum rdma_driver_id { · 188 unmodified lines
251	253	RDMA_DRIVER_EFA,
252	254	RDMA_DRIVER_SIW,
253	255	RDMA_DRIVER_ERDMA,
	256	+ RDMA_DRIVER_MANA,
254	257	};
255	258
256	259	enum ib_uverbs_gid_type {

include/uapi/rdma/ib_user_verbs.h+21−0

@@ -105,6 +105,18 @@ enum { · 104 unmodified lines
105	105	IB_USER_VERBS_EX_CMD_MODIFY_CQ
106	106	};
107	107
	108	+/* see IBA A19.4.1.1 Placement Types */
	109	+enum ib_placement_type {
	110	+ IB_FLUSH_GLOBAL = 1U << 0,
	111	+ IB_FLUSH_PERSISTENT = 1U << 1,
	112	+};
	113	+
	114	+/* see IBA A19.4.1.2 Selectivity Level */
	115	+enum ib_selectivity_level {
	116	+ IB_FLUSH_RANGE = 0,
	117	+ IB_FLUSH_MR,
	118	+};
	119	+
108	120	/*
109	121	* Make sure that all structs defined in this file remain laid out so
110	122	* that they pack the same way on 32-bit and 64-bit architectures (to
@@ -466,6 +478,8 @@ enum ib_uverbs_wc_opcode { · 355 unmodified lines
466	478	IB_UVERBS_WC_BIND_MW = 5,
467	479	IB_UVERBS_WC_LOCAL_INV = 6,
468	480	IB_UVERBS_WC_TSO = 7,
	481	+ IB_UVERBS_WC_FLUSH = 8,
	482	+ IB_UVERBS_WC_ATOMIC_WRITE = 9,
469	483	};
470	484
471	485	struct ib_uverbs_wc {
@@ -784,6 +798,8 @@ enum ib_uverbs_wr_opcode { · 312 unmodified lines
784	798	IB_UVERBS_WR_RDMA_READ_WITH_INV = 11,
785	799	IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP = 12,
786	800	IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD = 13,
	801	+ IB_UVERBS_WR_FLUSH = 14,
	802	+ IB_UVERBS_WR_ATOMIC_WRITE = 15,
787	803	/* Review enum ib_wr_opcode before modifying this */
788	804	};
789	805
@@ -1331,6 +1347,11 @@ enum ib_uverbs_device_cap_flags { · 541 unmodified lines
1331	1347	/* Deprecated. Please use IB_UVERBS_RAW_PACKET_CAP_SCATTER_FCS. */
1332	1348	IB_UVERBS_DEVICE_RAW_SCATTER_FCS = 1ULL << 34,
1333	1349	IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING = 1ULL << 36,
	1350	+ /* Flush placement types */
	1351	+ IB_UVERBS_DEVICE_FLUSH_GLOBAL = 1ULL << 38,
	1352	+ IB_UVERBS_DEVICE_FLUSH_PERSISTENT = 1ULL << 39,
	1353	+ /* Atomic write attributes */
	1354	+ IB_UVERBS_DEVICE_ATOMIC_WRITE = 1ULL << 40,
1334	1355	};
1335	1356
1336	1357	enum ib_uverbs_raw_packet_caps {

include/uapi/rdma/mana-abi.h+66−0

@@ -0,0 +1,66 @@
	1	+/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) */
	2	+/*
	3	+ * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#ifndef MANA_ABI_USER_H
	7	+#define MANA_ABI_USER_H
	8	+
	9	+#include <linux/types.h>
	10	+#include <rdma/ib_user_ioctl_verbs.h>
	11	+
	12	+/*
	13	+ * Increment this value if any changes that break userspace ABI
	14	+ * compatibility are made.
	15	+ */
	16	+
	17	+#define MANA_IB_UVERBS_ABI_VERSION 1
	18	+
	19	+struct mana_ib_create_cq {
	20	+ __aligned_u64 buf_addr;
	21	+};
	22	+
	23	+struct mana_ib_create_qp {
	24	+ __aligned_u64 sq_buf_addr;
	25	+ __u32 sq_buf_size;
	26	+ __u32 port;
	27	+};
	28	+
	29	+struct mana_ib_create_qp_resp {
	30	+ __u32 sqid;
	31	+ __u32 cqid;
	32	+ __u32 tx_vp_offset;
	33	+ __u32 reserved;
	34	+};
	35	+
	36	+struct mana_ib_create_wq {
	37	+ __aligned_u64 wq_buf_addr;
	38	+ __u32 wq_buf_size;
	39	+ __u32 reserved;
	40	+};
	41	+
	42	+/* RX Hash function flags */
	43	+enum mana_ib_rx_hash_function_flags {
	44	+ MANA_IB_RX_HASH_FUNC_TOEPLITZ = 1 << 0,
	45	+};
	46	+
	47	+struct mana_ib_create_qp_rss {
	48	+ __aligned_u64 rx_hash_fields_mask;
	49	+ __u8 rx_hash_function;
	50	+ __u8 reserved[7];
	51	+ __u32 rx_hash_key_len;
	52	+ __u8 rx_hash_key[40];
	53	+ __u32 port;
	54	+};
	55	+
	56	+struct rss_resp_entry {
	57	+ __u32 cqid;
	58	+ __u32 wqid;
	59	+};
	60	+
	61	+struct mana_ib_create_qp_rss_resp {
	62	+ __aligned_u64 num_entries;
	63	+ struct rss_resp_entry entries[64];
	64	+};
	65	+
	66	+#endif

include/uapi/rdma/rdma_user_rxe.h+8−0

@@ -82,6 +82,13 @@ struct rxe_send_wr { · 81 unmodified lines
82	82	__u32 invalidate_rkey;
83	83	} ex;
84	84	union {
	85	+ struct {
	86	+ __aligned_u64 remote_addr;
	87	+ __u32 length;
	88	+ __u32 rkey;
	89	+ __u8 type;
	90	+ __u8 level;
	91	+ } flush;
85	92	struct {
86	93	__aligned_u64 remote_addr;
87	94	__u32 rkey;
@@ -146,6 +153,7 @@ struct rxe_dma_info { · 58 unmodified lines
146	153	__u32 reserved;
147	154	union {
148	155	__DECLARE_FLEX_ARRAY(__u8, inline_data);
	156	+ __DECLARE_FLEX_ARRAY(__u8, atomic_wr);
149	157	__DECLARE_FLEX_ARRAY(struct rxe_sge, sge);
150	158	};
151	159	};

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code