lael-labs/linux

MAINTAINERS+9−0

@@ -13806,6 +13806,15 @@ F: drivers/scsi/smartpqi/smartpqi*.[ch] · 13805 unmodified lines
13806	F: include/linux/cciss*.h	13806	F: include/linux/cciss*.h
13807	F: include/uapi/linux/cciss*.h	13807	F: include/uapi/linux/cciss*.h
13808		13808
		13809	MICROSOFT MANA RDMA DRIVER
		13810	M: Long Li <longli@microsoft.com>
		13811	M: Ajay Sharma <sharmaajay@microsoft.com>
		13812	L: linux-rdma@vger.kernel.org
		13813	S: Supported
		13814	F: drivers/infiniband/hw/mana/
		13815	F: include/net/mana
		13816	F: include/uapi/rdma/mana-abi.h
		13817
13809	MICROSOFT SURFACE AGGREGATOR TABLET-MODE SWITCH	13818	MICROSOFT SURFACE AGGREGATOR TABLET-MODE SWITCH
13810	M: Maximilian Luz <luzmaximilian@gmail.com>	13819	M: Maximilian Luz <luzmaximilian@gmail.com>
13811	L: platform-driver-x86@vger.kernel.org	13820	L: platform-driver-x86@vger.kernel.org

drivers/infiniband/Kconfig+3−0

drivers/infiniband/core/cache.c+1−1

@@ -1422,7 +1422,7 @@ int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr, · 1421 unmodified lines
1422	*vlan_id = vlan_dev_vlan_id(ndev);	1422	*vlan_id = vlan_dev_vlan_id(ndev);
1423	} else {	1423	} else {
1424	/* If the netdev is upper device and if it's lower	1424	/* If the netdev is upper device and if it's lower
1425	* device is vlan device, consider vlan id of the	1425	* device is vlan device, consider vlan id of
1426	* the lower vlan device for this gid entry.	1426	* the lower vlan device for this gid entry.
1427	*/	1427	*/
1428	netdev_walk_all_lower_dev_rcu(attr->ndev,	1428	netdev_walk_all_lower_dev_rcu(attr->ndev,

drivers/infiniband/core/cm.c+11−2

 		*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
 				IB_QP_PKEY_INDEX | IB_QP_PORT;
 		qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
-		if (cm_id_priv->responder_resources)
+		if (cm_id_priv->responder_resources) {
+			struct ib_device *ib_dev = cm_id_priv->id.device;
+			u64 support_flush = ib_dev->attrs.device_cap_flags &
+			  (IB_DEVICE_FLUSH_GLOBAL | IB_DEVICE_FLUSH_PERSISTENT);
+			u32 flushable = support_flush ?
+					(IB_ACCESS_FLUSH_GLOBAL |
+					 IB_ACCESS_FLUSH_PERSISTENT) : 0;
 			qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
-						    IB_ACCESS_REMOTE_ATOMIC;
+						    IB_ACCESS_REMOTE_ATOMIC |
+						    flushable;
+		}
 		qp_attr->pkey_index = cm_id_priv->av.pkey_index;
 		if (cm_id_priv->av.port)
 			qp_attr->port_num = cm_id_priv->av.port->port_num;

drivers/infiniband/core/cma.c+1−1

@@ -47,7 +47,7 @@ MODULE_LICENSE("Dual BSD/GPL"); · 46 unmodified lines
47	#define CMA_CM_RESPONSE_TIMEOUT 20	47	#define CMA_CM_RESPONSE_TIMEOUT 20
48	#define CMA_MAX_CM_RETRIES 15	48	#define CMA_MAX_CM_RETRIES 15
49	#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY \| 24)	49	#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY \| 24)
50	#define CMA_IBOE_PACKET_LIFETIME 18	50	#define CMA_IBOE_PACKET_LIFETIME 16
51	#define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP	51	#define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
52		52
53	static const char * const cma_events[] = {	53	static const char * const cma_events[] = {

drivers/infiniband/core/device.c+6−4

 		return 0;
 	}
+	if (old_ndev)
+		netdev_tracker_free(ndev, &pdata->netdev_tracker);
 	if (ndev)
-		dev_hold(ndev);
+		netdev_hold(ndev, &pdata->netdev_tracker, GFP_ATOMIC);
 	rcu_assign_pointer(pdata->netdev, ndev);
 	spin_unlock_irqrestore(&pdata->netdev_lock, flags);
 	add_ndev_hash(pdata);
 	if (old_ndev)
-		dev_put(old_ndev);
+		__dev_put(old_ndev);
 	return 0;
 }
 			 * comparisons after the put
 			 */
 			rcu_assign_pointer(pdata->netdev, NULL);
-			dev_put(ndev);
+			netdev_put(ndev, &pdata->netdev_tracker);
 		}
 		spin_unlock_irqrestore(&pdata->netdev_lock, flags);
 	}
 static void __exit ib_core_cleanup(void)
 {
 	roce_gid_mgmt_cleanup();
-	nldev_exit();
 	rdma_nl_unregister(RDMA_NL_LS);
+	nldev_exit();
 	unregister_pernet_device(&rdma_dev_net_ops);
 	unregister_blocking_lsm_notifier(&ibdev_lsm_nb);
 	ib_sa_cleanup();

drivers/infiniband/core/mad.c+0−5

 			  struct ib_mad_qp_info *qp_info,
 			  struct trace_event_raw_ib_mad_send_template *entry)
 {
-	u16 pkey;
-	struct ib_device *dev = qp_info->port_priv->device;
-	u32 pnum = qp_info->port_priv->port_num;
 	struct ib_ud_wr *wr = &mad_send_wr->send_wr;
 	struct rdma_ah_attr attr = {};
 	/* These are common */
 	entry->sl = attr.sl;
-	ib_query_pkey(dev, pnum, wr->pkey_index, &pkey);
-	entry->pkey = pkey;
 	entry->rqpn = wr->remote_qpn;
 	entry->rqkey = wr->remote_qkey;
 	entry->dlid = rdma_ah_get_dlid(&attr);

drivers/infiniband/core/nldev.c+40−10

 	/* In create_qp() port is not set yet */
 	if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port))
-		return -EINVAL;
+		return -EMSGSIZE;
 	ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num);
 	if (ret)
 	struct rdma_cm_id *cm_id = &id_priv->id;
 	if (port && port != cm_id->port_num)
-		return 0;
+		return -EAGAIN;
 	if (cm_id->port_num &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id->port_num))
 	int ret = 0;
 	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);
+	if (!table_attr)
+		return -EMSGSIZE;
 	rt = &counter->device->res[RDMA_RESTRACK_QP];
 	xa_lock(&rt->xa);
 	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
 			0, 0);
+	if (!nlh) {
+		err = -EMSGSIZE;
+		goto err_free;
+	}
 	err = fill_dev_info(msg, device);
 	if (err)
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
 			0, NLM_F_MULTI);
-	if (fill_dev_info(skb, device)) {
+	if (!nlh || fill_dev_info(skb, device)) {
 		nlmsg_cancel(skb, nlh);
 		goto out;
 	}
 	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
 			0, 0);
+	if (!nlh) {
+		err = -EMSGSIZE;
+		goto err_free;
+	}
 	err = fill_port_info(msg, device, port, sock_net(skb->sk));
 	if (err)
 						 RDMA_NLDEV_CMD_PORT_GET),
 				0, NLM_F_MULTI);
-		if (fill_port_info(skb, device, p, sock_net(skb->sk))) {
+		if (!nlh || fill_port_info(skb, device, p, sock_net(skb->sk))) {
 			nlmsg_cancel(skb, nlh);
 			goto out;
 		}
 	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
 			0, 0);
+	if (!nlh) {
+		ret = -EMSGSIZE;
+		goto err_free;
+	}
 	ret = fill_res_info(msg, device);
 	if (ret)
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
 			0, NLM_F_MULTI);
-	if (fill_res_info(skb, device)) {
+	if (!nlh || fill_res_info(skb, device)) {
 		nlmsg_cancel(skb, nlh);
 		goto out;
 	}
 					 RDMA_NL_GET_OP(nlh->nlmsg_type)),
 			0, 0);
-	if (fill_nldev_handle(msg, device)) {
+	if (!nlh || fill_nldev_handle(msg, device)) {
 		ret = -EMSGSIZE;
 		goto err_free;
 	}
 					 RDMA_NL_GET_OP(cb->nlh->nlmsg_type)),
 			0, NLM_F_MULTI);
-	if (fill_nldev_handle(skb, device)) {
+	if (!nlh || fill_nldev_handle(skb, device)) {
 		ret = -EMSGSIZE;
 		goto err;
 	}
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
 					 RDMA_NLDEV_CMD_GET_CHARDEV),
 			0, 0);
+	if (!nlh) {
+		err = -EMSGSIZE;
+		goto out_nlmsg;
+	}
 	data.nl_msg = msg;
 	err = ib_get_client_nl_info(ibdev, client_name, &data);
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
 					 RDMA_NLDEV_CMD_SYS_GET),
 			0, 0);
+	if (!nlh) {
+		nlmsg_free(msg);
+		return -EMSGSIZE;
+	}
 	err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_NETNS_MODE,
 			 (u8)ib_devices_shared_netns);
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
 					 RDMA_NLDEV_CMD_STAT_SET),
 			0, 0);
-	if (fill_nldev_handle(msg, device) ||
+	if (!nlh || fill_nldev_handle(msg, device) ||
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
 		ret = -EMSGSIZE;
 		goto err_free_msg;
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
 					 RDMA_NLDEV_CMD_STAT_SET),
 			0, 0);
+	if (!nlh) {
+		ret = -EMSGSIZE;
+		goto err_fill;
+	}
 	cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);
 	qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);
 					 RDMA_NLDEV_CMD_STAT_GET),
 			0, 0);
-	if (fill_nldev_handle(msg, device) ||
+	if (!nlh || fill_nldev_handle(msg, device) ||
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
 		ret = -EMSGSIZE;
 		goto err_msg;
 			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
 					 RDMA_NLDEV_CMD_STAT_GET),
 			0, 0);
+	if (!nlh) {
+		ret = -EMSGSIZE;
+		goto err_msg;
+	}
 	ret = rdma_counter_get_mode(device, port, &mode, &mask);
 	if (ret)
 		0, 0);
 	ret = -EMSGSIZE;
-	if (fill_nldev_handle(msg, device) ||
+	if (!nlh || fill_nldev_handle(msg, device) ||
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
 		goto err_msg;

drivers/infiniband/core/restrack.c+0−2

@@ -343,8 +343,6 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) · 342 unmodified lines
343	rt = &dev->res[res->type];	343	rt = &dev->res[res->type];
344		344
345	old = xa_erase(&rt->xa, res->id);	345	old = xa_erase(&rt->xa, res->id);
346	if (res->type == RDMA_RESTRACK_MR)
347	return;
348	WARN_ON(old != res);	346	WARN_ON(old != res);
349		347
350	out:	348	out:

drivers/infiniband/core/sysfs.c+11−6

 	p->port_num = port_num;
 	kobject_init(&p->kobj, &port_type);
+	if (device->port_data && is_full_dev)
+		device->port_data[port_num].sysfs = p;
 	cur_group = p->groups_list;
 	ret = alloc_port_table_group("gids", &p->groups[0], p->attrs_list,
 				     attr->gid_tbl_len, show_port_gid);
 	}
 	list_add_tail(&p->kobj.entry, &coredev->port_list);
-	if (device->port_data && is_full_dev)
-		device->port_data[port_num].sysfs = p;
 	return p;
 err_groups:
 err_del:
 	kobject_del(&p->kobj);
 err_put:
+	if (device->port_data && is_full_dev)
+		device->port_data[port_num].sysfs = NULL;
 	kobject_put(&p->kobj);
 	return ERR_PTR(ret);
 }
 {
 	bool is_full_dev = &port->ibdev->coredev == coredev;
-	if (port->ibdev->port_data &&
-	    port->ibdev->port_data[port->port_num].sysfs == port)
-		port->ibdev->port_data[port->port_num].sysfs = NULL;
 	list_del(&port->kobj.entry);
 	if (is_full_dev)
 		sysfs_remove_groups(&port->kobj, port->ibdev->ops.port_groups);
 	sysfs_remove_groups(&port->kobj, port->groups_list);
 	kobject_del(&port->kobj);
+	if (port->ibdev->port_data &&
+	    port->ibdev->port_data[port->port_num].sysfs == port)
+		port->ibdev->port_data[port->port_num].sysfs = NULL;
 	kobject_put(&port->kobj);
 }

drivers/infiniband/core/uverbs_std_types_qp.c+1−1

@@ -163,7 +163,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( · 162 unmodified lines
163	UVERBS_ATTR_CREATE_QP_SRQ_HANDLE))	163	UVERBS_ATTR_CREATE_QP_SRQ_HANDLE))
164	return -EINVAL;	164	return -EINVAL;
165		165
166	/* send_cq is optinal */	166	/* send_cq is optional */
167	if (cap.max_send_wr) {	167	if (cap.max_send_wr) {
168	send_cq = uverbs_attr_get_obj(attrs,	168	send_cq = uverbs_attr_get_obj(attrs,
169	UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE);	169	UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE);

drivers/infiniband/hw/Makefile+1−0

@@ -4,6 +4,7 @@ obj-$(CONFIG_INFINIBAND_QIB) += qib/ · 3 unmodified lines
4	obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/	4	obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/
5	obj-$(CONFIG_INFINIBAND_EFA) += efa/	5	obj-$(CONFIG_INFINIBAND_EFA) += efa/
6	obj-$(CONFIG_INFINIBAND_IRDMA) += irdma/	6	obj-$(CONFIG_INFINIBAND_IRDMA) += irdma/
		7	obj-$(CONFIG_MANA_INFINIBAND) += mana/
7	obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/	8	obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/
8	obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/	9	obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/
9	obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/	10	obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/

drivers/infiniband/hw/erdma/erdma.h+3−1

 	u32 fw_version;
 	unsigned char peer_addr[ETH_ALEN];
+	unsigned long cap_flags;
 	int numa_node;
 	enum erdma_cc_alg cc;
 	struct net_device *netdev;
 	struct pci_dev *pdev;
 	struct notifier_block netdev_nb;
+	struct workqueue_struct *reflush_wq;
 	resource_size_t func_bar_addr;
 	resource_size_t func_bar_len;
 	DECLARE_BITMAP(sdb_page, ERDMA_DWQE_TYPE0_CNT);
 	/*
 	 * We provide max 496 uContexts that each has one SQ normal Db,
-	 * and one directWQE db。
+	 * and one directWQE db.
 	 */
 	DECLARE_BITMAP(sdb_entry, ERDMA_DWQE_TYPE1_CNT);

drivers/infiniband/hw/erdma/erdma_cq.c+2−0

@@ -64,6 +64,8 @@ static const enum ib_wc_opcode wc_mapping_table[ERDMA_NUM_OPCODES] = { · 63 unmodified lines
64	[ERDMA_OP_REG_MR] = IB_WC_REG_MR,	64	[ERDMA_OP_REG_MR] = IB_WC_REG_MR,
65	[ERDMA_OP_LOCAL_INV] = IB_WC_LOCAL_INV,	65	[ERDMA_OP_LOCAL_INV] = IB_WC_LOCAL_INV,
66	[ERDMA_OP_READ_WITH_INV] = IB_WC_RDMA_READ,	66	[ERDMA_OP_READ_WITH_INV] = IB_WC_RDMA_READ,
		67	[ERDMA_OP_ATOMIC_CAS] = IB_WC_COMP_SWAP,
		68	[ERDMA_OP_ATOMIC_FAD] = IB_WC_FETCH_ADD,
67	};	69	};
68		70
69	static const struct {	71	static const struct {

drivers/infiniband/hw/erdma/erdma_hw.h+30−7

 	CMDQ_OPCODE_MODIFY_QP = 3,
 	CMDQ_OPCODE_CREATE_CQ = 4,
 	CMDQ_OPCODE_DESTROY_CQ = 5,
+	CMDQ_OPCODE_REFLUSH = 6,
 	CMDQ_OPCODE_REG_MR = 8,
 	CMDQ_OPCODE_DEREG_MR = 9
 };
 /* regmr cfg1 */
 #define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12)
 #define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6)
-#define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 2)
+#define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1)
-#define ERDMA_CMD_REGMR_ACC_MODE_MASK GENMASK(1, 0)
 /* regmr cfg2 */
 #define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)
 	u32 qpn;
 };
+struct erdma_cmdq_reflush_req {
+	u64 hdr;
+	u32 qpn;
+	u32 sq_pi;
+	u32 rq_pi;
+};
 /* cap qword 0 definition */
 #define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)
+#define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
 #define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16)
 #define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0)
 #define ERDMA_NQP_PER_QBLOCK 1024
+enum {
+	ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 << 7,
+};
 #define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0)
 /* CQE hdr */
 };
 struct erdma_sge {
-	__aligned_le64 laddr;
+	__aligned_le64 addr;
 	__le32 length;
-	__le32 lkey;
+	__le32 key;
 };
 /* Receive Queue Element */
 #define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
 /* REG MR attrs */
-#define ERDMA_SQE_MR_MODE_MASK GENMASK(1, 0)
+#define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1)
-#define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 2)
 #define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6)
 #define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12)
 	__le32 rsvd;
 };
+struct erdma_atomic_sqe {
+	__le64 hdr;
+	__le64 rsvd;
+	__le64 fetchadd_swap_data;
+	__le64 cmp_data;
+	struct erdma_sge remote;
+	struct erdma_sge sgl;
+};
 struct erdma_reg_mr_sqe {
 	__le64 hdr;
 	__le64 addr;
 	ERDMA_OP_REG_MR = 14,
 	ERDMA_OP_LOCAL_INV = 15,
 	ERDMA_OP_READ_WITH_INV = 16,
-	ERDMA_NUM_OPCODES = 17,
+	ERDMA_OP_ATOMIC_CAS = 17,
+	ERDMA_OP_ATOMIC_FAD = 18,
+	ERDMA_NUM_OPCODES = 19,
 	ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
 };

drivers/infiniband/hw/erdma/erdma_main.c+13−2

 	dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);
 	dev->attrs.max_mr = dev->attrs.max_qp << 1;
 	dev->attrs.max_cq = dev->attrs.max_qp << 1;
+	dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);
 	dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR;
 	dev->attrs.max_ord = ERDMA_MAX_ORD;
 	u64_to_ether_addr(mac, dev->attrs.peer_addr);
+	dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,
+					  WQ_UNBOUND_MAX_ACTIVE);
+	if (!dev->reflush_wq) {
+		ret = -ENOMEM;
+		goto err_alloc_workqueue;
+	}
 	ret = erdma_device_register(dev);
 	if (ret)
-		goto err_out;
+		goto err_register;
 	return 0;
-err_out:
+err_register:
+	destroy_workqueue(dev->reflush_wq);
+err_alloc_workqueue:
 	xa_destroy(&dev->qp_xa);
 	xa_destroy(&dev->cq_xa);
 	unregister_netdevice_notifier(&dev->netdev_nb);
 	ib_unregister_device(&dev->ibdev);
+	destroy_workqueue(dev->reflush_wq);
 	erdma_res_cb_free(dev);
 	xa_destroy(&dev->qp_xa);
 	xa_destroy(&dev->cq_xa);

drivers/infiniband/hw/erdma/erdma_qp.c+57−16

 int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
 			     enum erdma_qp_attr_mask mask)
 {
+	bool need_reflush = false;
 	int drop_conn, ret = 0;
 	if (!mask)
 			ret = erdma_modify_qp_state_to_rts(qp, attrs, mask);
 		} else if (attrs->state == ERDMA_QP_STATE_ERROR) {
 			qp->attrs.state = ERDMA_QP_STATE_ERROR;
+			need_reflush = true;
 			if (qp->cep) {
 				erdma_cep_put(qp->cep);
 				qp->cep = NULL;
 	case ERDMA_QP_STATE_RTS:
 		drop_conn = 0;
-		if (attrs->state == ERDMA_QP_STATE_CLOSING) {
+		if (attrs->state == ERDMA_QP_STATE_CLOSING ||
+		    attrs->state == ERDMA_QP_STATE_TERMINATE ||
+		    attrs->state == ERDMA_QP_STATE_ERROR) {
 			ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 			drop_conn = 1;
-		} else if (attrs->state == ERDMA_QP_STATE_TERMINATE) {
+			need_reflush = true;
-			qp->attrs.state = ERDMA_QP_STATE_TERMINATE;
-			ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
-			drop_conn = 1;
-		} else if (attrs->state == ERDMA_QP_STATE_ERROR) {
-			ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
-			qp->attrs.state = ERDMA_QP_STATE_ERROR;
-			drop_conn = 1;
 		}
 		if (drop_conn)
 		break;
 	}
+	if (need_reflush && !ret && rdma_is_kernel_res(&qp->ibqp.res)) {
+		qp->flags |= ERDMA_QP_IN_FLUSHING;
+		mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
+				 usecs_to_jiffies(100));
+	}
 	return ret;
 }
 	u32 wqe_size, wqebb_cnt, hw_op, flags, sgl_offset;
 	u32 idx = *pi & (qp->attrs.sq_size - 1);
 	enum ib_wr_opcode op = send_wr->opcode;
+	struct erdma_atomic_sqe *atomic_sqe;
 	struct erdma_readreq_sqe *read_sqe;
 	struct erdma_reg_mr_sqe *regmr_sge;
 	struct erdma_write_sqe *write_sqe;
 	struct erdma_send_sqe *send_sqe;
 	struct ib_rdma_wr *rdma_wr;
-	struct erdma_mr *mr;
+	struct erdma_sge *sge;
 	__le32 *length_field;
+	struct erdma_mr *mr;
 	u64 wqe_hdr, *entry;
-	struct ib_sge *sge;
 	u32 attrs;
 	int ret;
 		sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
 				      qp->attrs.sq_size, SQEBB_SHIFT);
-		sge->addr = rdma_wr->remote_addr;
+		sge->addr = cpu_to_le64(rdma_wr->remote_addr);
-		sge->lkey = rdma_wr->rkey;
+		sge->key = cpu_to_le32(rdma_wr->rkey);
-		sge->length = send_wr->sg_list[0].length;
+		sge->length = cpu_to_le32(send_wr->sg_list[0].length);
 		wqe_size = sizeof(struct erdma_readreq_sqe) +
 			   send_wr->num_sge * sizeof(struct ib_sge);
 		regmr_sge->addr = cpu_to_le64(mr->ibmr.iova);
 		regmr_sge->length = cpu_to_le32(mr->ibmr.length);
 		regmr_sge->stag = cpu_to_le32(reg_wr(send_wr)->key);
-		attrs = FIELD_PREP(ERDMA_SQE_MR_MODE_MASK, 0) |
+		attrs = FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr->access) |
-			FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr->access) |
 			FIELD_PREP(ERDMA_SQE_MR_MTT_CNT_MASK,
 				   mr->mem.mtt_nents);
 		regmr_sge->stag = cpu_to_le32(send_wr->ex.invalidate_rkey);
 		wqe_size = sizeof(struct erdma_reg_mr_sqe);
 		goto out;
+	case IB_WR_ATOMIC_CMP_AND_SWP:
+	case IB_WR_ATOMIC_FETCH_AND_ADD:
+		atomic_sqe = (struct erdma_atomic_sqe *)entry;
+		if (op == IB_WR_ATOMIC_CMP_AND_SWP) {
+			wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
+					      ERDMA_OP_ATOMIC_CAS);
+			atomic_sqe->fetchadd_swap_data =
+				cpu_to_le64(atomic_wr(send_wr)->swap);
+			atomic_sqe->cmp_data =
+				cpu_to_le64(atomic_wr(send_wr)->compare_add);
+		} else {
+			wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
+					      ERDMA_OP_ATOMIC_FAD);
+			atomic_sqe->fetchadd_swap_data =
+				cpu_to_le64(atomic_wr(send_wr)->compare_add);
+		}
+		sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
+				      qp->attrs.sq_size, SQEBB_SHIFT);
+		sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr);
+		sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey);
+		sge++;
+		sge->addr = cpu_to_le64(send_wr->sg_list[0].addr);
+		sge->key = cpu_to_le32(send_wr->sg_list[0].lkey);
+		sge->length = cpu_to_le32(send_wr->sg_list[0].length);
+		wqe_size = sizeof(*atomic_sqe);
+		goto out;
 	default:
 		return -EOPNOTSUPP;
 	}
 	}
 	spin_unlock_irqrestore(&qp->lock, flags);
+	if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING))
+		mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
+				 usecs_to_jiffies(100));
 	return ret;
 }
 	}
 	spin_unlock_irqrestore(&qp->lock, flags);
+	if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING))
+		mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
+				 usecs_to_jiffies(100));
 	return ret;
 }

drivers/infiniband/hw/erdma/erdma_verbs.c+23−2

 		   FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
 	req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
 		   FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
-		   FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access) |
+		   FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
-		   FIELD_PREP(ERDMA_CMD_REGMR_ACC_MODE_MASK, 0);
 	req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
 			      ilog2(mr->mem.page_size)) |
 		   FIELD_PREP(ERDMA_CMD_REGMR_MTT_TYPE_MASK, mr->mem.mtt_type) |
 	attr->max_mw = dev->attrs.max_mw;
 	attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
 	attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
+	if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
+		attr->atomic_cap = IB_ATOMIC_GLOB;
 	attr->fw_ver = dev->attrs.fw_version;
 	if (dev->netdev)
 	return 0;
 }
+static void erdma_flush_worker(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct erdma_qp *qp =
+		container_of(dwork, struct erdma_qp, reflush_dwork);
+	struct erdma_cmdq_reflush_req req;
+	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
+				CMDQ_OPCODE_REFLUSH);
+	req.qpn = QP_ID(qp);
+	req.sq_pi = qp->kern_qp.sq_pi;
+	req.rq_pi = qp->kern_qp.rq_pi;
+	erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL);
+}
 static int erdma_qp_validate_cap(struct erdma_dev *dev,
 				 struct ib_qp_init_attr *attrs)
 {
 	qp->attrs.max_send_sge = attrs->cap.max_send_sge;
 	qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
 	qp->attrs.state = ERDMA_QP_STATE_IDLE;
+	INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
 	ret = create_qp_cmd(dev, qp);
 	if (ret)
 	erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
 	up_write(&qp->state_lock);
+	cancel_delayed_work_sync(&qp->reflush_dwork);
 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
 				CMDQ_OPCODE_DESTROY_QP);
 	req.qpn = QP_ID(qp);

drivers/infiniband/hw/erdma/erdma_verbs.h+14−5

 #define ERDMA_MR_INLINE_MTT 0
 #define ERDMA_MR_INDIRECT_MTT 1
-#define ERDMA_MR_ACC_LR BIT(0)
+#define ERDMA_MR_ACC_RA BIT(0)
-#define ERDMA_MR_ACC_LW BIT(1)
+#define ERDMA_MR_ACC_LR BIT(1)
-#define ERDMA_MR_ACC_RR BIT(2)
+#define ERDMA_MR_ACC_LW BIT(2)
-#define ERDMA_MR_ACC_RW BIT(3)
+#define ERDMA_MR_ACC_RR BIT(3)
+#define ERDMA_MR_ACC_RW BIT(4)
 static inline u8 to_erdma_access_flags(int access)
 {
 	return (access & IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) |
 	       (access & IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) |
-	       (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0);
+	       (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0) |
+	       (access & IB_ACCESS_REMOTE_ATOMIC ? ERDMA_MR_ACC_RA : 0);
 }
 struct erdma_mem {
 	ERDMA_QP_ATTR_MPA = (1 << 7)
 };
+enum erdma_qp_flags {
+	ERDMA_QP_IN_FLUSHING = (1 << 0),
+};
 struct erdma_qp_attrs {
 	enum erdma_qp_state state;
 	enum erdma_cc_alg cc; /* Congestion control algorithm */
 	struct erdma_cep *cep;
 	struct rw_semaphore state_lock;
+	unsigned long flags;
+	struct delayed_work reflush_dwork;
 	union {
 		struct erdma_kqp kern_qp;
 		struct erdma_uqp user_qp;

drivers/infiniband/hw/hfi1/affinity.c+2−0

@@ -177,6 +177,8 @@ out: · 176 unmodified lines
177	for (node = 0; node < node_affinity.num_possible_nodes; node++)	177	for (node = 0; node < node_affinity.num_possible_nodes; node++)
178	hfi1_per_node_cntr[node] = 1;	178	hfi1_per_node_cntr[node] = 1;
179		179
		180	pci_dev_put(dev);
		181
180	return 0;	182	return 0;
181	}	183	}
182		184

drivers/infiniband/hw/hfi1/driver.c+1−1

@@ -112,7 +112,7 @@ static int hfi1_caps_get(char buffer, const struct kernel_param kp) · 111 unmodified lines
112	cap_mask &= ~HFI1_CAP_LOCKED_SMASK;	112	cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
113	cap_mask \|= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);	113	cap_mask \|= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);
114		114
115	return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);	115	return sysfs_emit(buffer, "0x%lx\n", cap_mask);
116	}	116	}
117		117
118	struct pci_dev get_pci_dev(struct rvt_dev_info rdi)	118	struct pci_dev get_pci_dev(struct rvt_dev_info rdi)

drivers/infiniband/hw/hfi1/firmware.c+6−0

 	if (!dd->platform_config.data) {
 		dd_dev_err(dd, "%s: Missing config file\n", __func__);
+		ret = -EINVAL;
 		goto bail;
 	}
 	ptr = (u32 *)dd->platform_config.data;
 	ptr++;
 	if (magic_num != PLATFORM_CONFIG_MAGIC_NUM) {
 		dd_dev_err(dd, "%s: Bad config file\n", __func__);
+		ret = -EINVAL;
 		goto bail;
 	}
 	if (file_length > dd->platform_config.size) {
 		dd_dev_info(dd, "%s:File claims to be larger than read size\n",
 			    __func__);
+		ret = -EINVAL;
 		goto bail;
 	} else if (file_length < dd->platform_config.size) {
 		dd_dev_info(dd,
 			dd_dev_err(dd, "%s: Failed validation at offset %ld\n",
 				   __func__, (ptr - (u32 *)
 					      dd->platform_config.data));
+			ret = -EINVAL;
 			goto bail;
 		}
 					   __func__, table_type,
 					   (ptr - (u32 *)
 					    dd->platform_config.data));
+				ret = -EINVAL;
 				goto bail; /* We don't trust this file now */
 			}
 			pcfgcache->config_tables[table_type].table = ptr;
 					   __func__, table_type,
 					   (ptr -
 					    (u32 *)dd->platform_config.data));
+				ret = -EINVAL;
 				goto bail; /* We don't trust this file now */
 			}
 			pcfgcache->config_tables[table_type].table_metadata =

drivers/infiniband/hw/hfi1/mad.c+11−11

@@ -2437,9 +2437,9 @@ struct opa_port_data_counters_msg { · 2436 unmodified lines
2437	__be64 port_vl_xmit_wait_data;	2437	__be64 port_vl_xmit_wait_data;
2438	__be64 port_vl_rcv_bubble;	2438	__be64 port_vl_rcv_bubble;
2439	__be64 port_vl_mark_fecn;	2439	__be64 port_vl_mark_fecn;
2440	} vls[0];	2440	} vls[];
2441	/* array size defined by #bits set in vl_select_mask*/	2441	/* array size defined by #bits set in vl_select_mask*/
2442	} port[1]; /* array size defined by #ports in attribute modifier */	2442	} port;
2443	};	2443	};
2444		2444
2445	struct opa_port_error_counters64_msg {	2445	struct opa_port_error_counters64_msg {
@@ -2470,9 +2470,9 @@ struct opa_port_error_counters64_msg { · 24 unmodified lines
2470	u8 reserved3[7];	2470	u8 reserved3[7];
2471	struct _vls_ectrs {	2471	struct _vls_ectrs {
2472	__be64 port_vl_xmit_discards;	2472	__be64 port_vl_xmit_discards;
2473	} vls[0];	2473	} vls[];
2474	/* array size defined by #bits set in vl_select_mask */	2474	/* array size defined by #bits set in vl_select_mask */
2475	} port[1]; /* array size defined by #ports in attribute modifier */	2475	} port;
2476	};	2476	};
2477		2477
2478	struct opa_port_error_info_msg {	2478	struct opa_port_error_info_msg {
@@ -2543,7 +2543,7 @@ struct opa_port_error_info_msg { · 64 unmodified lines
2543	u8 error_info;	2543	u8 error_info;
2544	} __packed fm_config_ei;	2544	} __packed fm_config_ei;
2545	__u32 reserved9;	2545	__u32 reserved9;
2546	} port[1]; /* actual array size defined by #ports in attr modifier */	2546	} port;
2547	};	2547	};
2548		2548
2549	/* opa_port_error_info_msg error_info_select_mask bit definitions */	2549	/* opa_port_error_info_msg error_info_select_mask bit definitions */
@@ -2966,7 +2966,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, · 416 unmodified lines
2966	}	2966	}
2967		2967
2968	/* Sanity check */	2968	/* Sanity check */
2969	response_data_size = struct_size(req, port[0].vls, num_vls);	2969	response_data_size = struct_size(req, port.vls, num_vls);
2970		2970
2971	if (response_data_size > sizeof(pmp->data)) {	2971	if (response_data_size > sizeof(pmp->data)) {
2972	pmp->mad_hdr.status \|= IB_SMP_INVALID_FIELD;	2972	pmp->mad_hdr.status \|= IB_SMP_INVALID_FIELD;
@@ -2986,7 +2986,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, · 13 unmodified lines
2986	return reply((struct ib_mad_hdr *)pmp);	2986	return reply((struct ib_mad_hdr *)pmp);
2987	}	2987	}
2988		2988
2989	rsp = &req->port[0];	2989	rsp = &req->port;
2990	memset(rsp, 0, sizeof(*rsp));	2990	memset(rsp, 0, sizeof(*rsp));
2991		2991
2992	rsp->port_number = port;	2992	rsp->port_number = port;
@@ -3182,7 +3182,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, · 189 unmodified lines
3182	return reply((struct ib_mad_hdr *)pmp);	3182	return reply((struct ib_mad_hdr *)pmp);
3183	}	3183	}
3184		3184
3185	response_data_size = struct_size(req, port[0].vls, num_vls);	3185	response_data_size = struct_size(req, port.vls, num_vls);
3186		3186
3187	if (response_data_size > sizeof(pmp->data)) {	3187	if (response_data_size > sizeof(pmp->data)) {
3188	pmp->mad_hdr.status \|= IB_SMP_INVALID_FIELD;	3188	pmp->mad_hdr.status \|= IB_SMP_INVALID_FIELD;
@@ -3201,7 +3201,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, · 12 unmodified lines
3201	return reply((struct ib_mad_hdr *)pmp);	3201	return reply((struct ib_mad_hdr *)pmp);
3202	}	3202	}
3203		3203
3204	rsp = &req->port[0];	3204	rsp = &req->port;
3205		3205
3206	ibp = to_iport(ibdev, port_num);	3206	ibp = to_iport(ibdev, port_num);
3207	ppd = ppd_from_ibp(ibp);	3207	ppd = ppd_from_ibp(ibp);
@@ -3340,7 +3340,7 @@ static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp, · 132 unmodified lines
3340	u64 reg;	3340	u64 reg;
3341		3341
3342	req = (struct opa_port_error_info_msg *)pmp->data;	3342	req = (struct opa_port_error_info_msg *)pmp->data;
3343	rsp = &req->port[0];	3343	rsp = &req->port;
3344		3344
3345	num_ports = OPA_AM_NPORT(be32_to_cpu(pmp->mad_hdr.attr_mod));	3345	num_ports = OPA_AM_NPORT(be32_to_cpu(pmp->mad_hdr.attr_mod));
3346	num_pslm = hweight64(be64_to_cpu(req->port_select_mask[3]));	3346	num_pslm = hweight64(be64_to_cpu(req->port_select_mask[3]));
@@ -3590,7 +3590,7 @@ static int pma_set_opa_errorinfo(struct opa_pma_mad *pmp, · 243 unmodified lines
3590	u32 error_info_select;	3590	u32 error_info_select;
3591		3591
3592	req = (struct opa_port_error_info_msg *)pmp->data;	3592	req = (struct opa_port_error_info_msg *)pmp->data;
3593	rsp = &req->port[0];	3593	rsp = &req->port;
3594		3594
3595	num_ports = OPA_AM_NPORT(be32_to_cpu(pmp->mad_hdr.attr_mod));	3595	num_ports = OPA_AM_NPORT(be32_to_cpu(pmp->mad_hdr.attr_mod));
3596	num_pslm = hweight64(be64_to_cpu(req->port_select_mask[3]));	3596	num_pslm = hweight64(be64_to_cpu(req->port_select_mask[3]));

drivers/infiniband/hw/hfi1/netdev_rx.c+1−1

@@ -216,7 +216,7 @@ static int hfi1_netdev_rxq_init(struct hfi1_netdev_rx *rx) · 215 unmodified lines
216	* right now.	216	* right now.
217	*/	217	*/
218	set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state);	218	set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state);
219	netif_napi_add_weight(dev, &rxq->napi, hfi1_netdev_rx_napi, 64);	219	netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi);
220	rc = msix_netdev_request_rcd_irq(rxq->rcd);	220	rc = msix_netdev_request_rcd_irq(rxq->rcd);
221	if (rc)	221	if (rc)
222	goto bail_context_irq_failure;	222	goto bail_context_irq_failure;

drivers/infiniband/hw/hns/hns_roce_device.h+3−0

 	struct list_head	page_list;
 	struct mutex		page_mutex;
 	struct hns_user_mmap_entry *db_mmap_entry;
+	u32			config;
 };
 struct hns_roce_pd {
 	u32		head;
 	u32		tail;
 	void __iomem	*db_reg;
+	u32		ext_sge_cnt;
 };
 struct hns_roce_sge {
 	struct list_head	rq_node; /* all recv qps are on a list */
 	struct list_head	sq_node; /* all send qps are on a list */
 	struct hns_user_mmap_entry *dwqe_mmap_entry;
+	u32			config;
 };
 struct hns_roce_ib_iboe {

drivers/infiniband/hw/hns/hns_roce_hw_v2.c+168−49

 				 unsigned int *sge_idx, u32 msg_len)
 {
 	struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;
-	unsigned int ext_sge_sz = qp->sq.max_gs * HNS_ROCE_SGE_SIZE;
 	unsigned int left_len_in_pg;
 	unsigned int idx = *sge_idx;
 	unsigned int i = 0;
 	void *addr;
 	void *dseg;
-	if (msg_len > ext_sge_sz) {
+	if (msg_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) {
 		ibdev_err(ibdev,
 			  "no enough extended sge space for inline data.\n");
 		return -EINVAL;
 		hr_dev->cmd.state = HNS_ROCE_CMDQ_STATE_FATAL_ERR;
 }
+static int hns_roce_cmd_err_convert_errno(u16 desc_ret)
+{
+	struct hns_roce_cmd_errcode errcode_table[] = {
+		{CMD_EXEC_SUCCESS, 0},
+		{CMD_NO_AUTH, -EPERM},
+		{CMD_NOT_EXIST, -EOPNOTSUPP},
+		{CMD_CRQ_FULL, -EXFULL},
+		{CMD_NEXT_ERR, -ENOSR},
+		{CMD_NOT_EXEC, -ENOTBLK},
+		{CMD_PARA_ERR, -EINVAL},
+		{CMD_RESULT_ERR, -ERANGE},
+		{CMD_TIMEOUT, -ETIME},
+		{CMD_HILINK_ERR, -ENOLINK},
+		{CMD_INFO_ILLEGAL, -ENXIO},
+		{CMD_INVALID, -EBADR},
+	};
+	u16 i;
+	for (i = 0; i < ARRAY_SIZE(errcode_table); i++)
+		if (desc_ret == errcode_table[i].return_status)
+			return errcode_table[i].errno;
+	return -EIO;
+}
 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
 			       struct hns_roce_cmq_desc *desc, int num)
 {
 			dev_err_ratelimited(hr_dev->dev,
 					    "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n",
 					    desc->opcode, desc_ret);
-			ret = -EIO;
+			ret = hns_roce_cmd_err_convert_errno(desc_ret);
 		}
 	} else {
 		/* FW/HW reset or incorrect number of desc */
 	caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW |
 		       HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR |
-		       HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL | HNS_ROCE_CAP_FLAG_XRC;
+		       HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
 	caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
 	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
 		caps->flags |= HNS_ROCE_CAP_FLAG_STASH |
-			       HNS_ROCE_CAP_FLAG_DIRECT_WQE;
+			       HNS_ROCE_CAP_FLAG_DIRECT_WQE |
+			       HNS_ROCE_CAP_FLAG_XRC;
 		caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE;
 	} else {
 		caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
 	caps->wqe_sge_hop_num = hr_reg_read(resp_d, PF_CAPS_D_EX_SGE_HOP_NUM);
 	caps->wqe_rq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_RQWQE_HOP_NUM);
+	if (!(caps->page_size_cap & PAGE_SIZE))
+		caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
 	return 0;
 }
 	spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags);
 }
-static void free_mr_exit(struct hns_roce_dev *hr_dev)
+static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
+	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct hns_roce_pd *hr_pd;
+	struct ib_pd *pd;
+	hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL);
+	if (ZERO_OR_NULL_PTR(hr_pd))
+		return NULL;
+	pd = &hr_pd->ibpd;
+	pd->device = ibdev;
+	if (hns_roce_alloc_pd(pd, NULL)) {
+		ibdev_err(ibdev, "failed to create pd for free mr.\n");
+		kfree(hr_pd);
+		return NULL;
+	}
+	free_mr->rsv_pd = to_hr_pd(pd);
+	free_mr->rsv_pd->ibpd.device = &hr_dev->ib_dev;
+	free_mr->rsv_pd->ibpd.uobject = NULL;
+	free_mr->rsv_pd->ibpd.__internal_mr = NULL;
+	atomic_set(&free_mr->rsv_pd->ibpd.usecnt, 0);
+	return pd;
+}
+static struct ib_cq *free_mr_init_cq(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct ib_cq_init_attr cq_init_attr = {};
+	struct hns_roce_cq *hr_cq;
+	struct ib_cq *cq;
+	cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;
+	hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);
+	if (ZERO_OR_NULL_PTR(hr_cq))
+		return NULL;
+	cq = &hr_cq->ib_cq;
+	cq->device = ibdev;
+	if (hns_roce_create_cq(cq, &cq_init_attr, NULL)) {
+		ibdev_err(ibdev, "failed to create cq for free mr.\n");
+		kfree(hr_cq);
+		return NULL;
+	}
+	free_mr->rsv_cq = to_hr_cq(cq);
+	free_mr->rsv_cq->ib_cq.device = &hr_dev->ib_dev;
+	free_mr->rsv_cq->ib_cq.uobject = NULL;
+	free_mr->rsv_cq->ib_cq.comp_handler = NULL;
+	free_mr->rsv_cq->ib_cq.event_handler = NULL;
+	free_mr->rsv_cq->ib_cq.cq_context = NULL;
+	atomic_set(&free_mr->rsv_cq->ib_cq.usecnt, 0);
+	return cq;
+}
+static int free_mr_init_qp(struct hns_roce_dev *hr_dev, struct ib_cq *cq,
+			   struct ib_qp_init_attr *init_attr, int i)
+{
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
+	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct hns_roce_qp *hr_qp;
+	struct ib_qp *qp;
 	int ret;
+	hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
+	if (ZERO_OR_NULL_PTR(hr_qp))
+		return -ENOMEM;
+	qp = &hr_qp->ibqp;
+	qp->device = ibdev;
+	ret = hns_roce_create_qp(qp, init_attr, NULL);
+	if (ret) {
+		ibdev_err(ibdev, "failed to create qp for free mr.\n");
+		kfree(hr_qp);
+		return ret;
+	}
+	free_mr->rsv_qp[i] = hr_qp;
+	free_mr->rsv_qp[i]->ibqp.recv_cq = cq;
+	free_mr->rsv_qp[i]->ibqp.send_cq = cq;
+	return 0;
+}
+static void free_mr_exit(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
+	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
+	struct ib_qp *qp;
 	int i;
 	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
 		if (free_mr->rsv_qp[i]) {
-			ret = ib_destroy_qp(free_mr->rsv_qp[i]);
+			qp = &free_mr->rsv_qp[i]->ibqp;
-			if (ret)
+			hns_roce_v2_destroy_qp(qp, NULL);
-				ibdev_err(&hr_dev->ib_dev,
+			kfree(free_mr->rsv_qp[i]);
-					  "failed to destroy qp in free mr.\n");
 			free_mr->rsv_qp[i] = NULL;
 		}
 	}
 	if (free_mr->rsv_cq) {
-		ib_destroy_cq(free_mr->rsv_cq);
+		hns_roce_destroy_cq(&free_mr->rsv_cq->ib_cq, NULL);
+		kfree(free_mr->rsv_cq);
 		free_mr->rsv_cq = NULL;
 	}
 	if (free_mr->rsv_pd) {
-		ib_dealloc_pd(free_mr->rsv_pd);
+		hns_roce_dealloc_pd(&free_mr->rsv_pd->ibpd, NULL);
+		kfree(free_mr->rsv_pd);
 		free_mr->rsv_pd = NULL;
 	}
 }
 {
 	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
-	struct ib_device *ibdev = &hr_dev->ib_dev;
-	struct ib_cq_init_attr cq_init_attr = {};
 	struct ib_qp_init_attr qp_init_attr = {};
 	struct ib_pd *pd;
 	struct ib_cq *cq;
-	struct ib_qp *qp;
 	int ret;
 	int i;
-	pd = ib_alloc_pd(ibdev, 0);
+	pd = free_mr_init_pd(hr_dev);
-	if (IS_ERR(pd)) {
+	if (!pd)
-		ibdev_err(ibdev, "failed to create pd for free mr.\n");
+		return -ENOMEM;
-		return PTR_ERR(pd);
-	}
-	free_mr->rsv_pd = pd;
-	cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;
+	cq = free_mr_init_cq(hr_dev);
-	cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_init_attr);
+	if (!cq) {
-	if (IS_ERR(cq)) {
+		ret = -ENOMEM;
-		ibdev_err(ibdev, "failed to create cq for free mr.\n");
+		goto create_failed_cq;
-		ret = PTR_ERR(cq);
-		goto create_failed;
 	}
-	free_mr->rsv_cq = cq;
 	qp_init_attr.qp_type = IB_QPT_RC;
 	qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
-	qp_init_attr.send_cq = free_mr->rsv_cq;
+	qp_init_attr.send_cq = cq;
-	qp_init_attr.recv_cq = free_mr->rsv_cq;
+	qp_init_attr.recv_cq = cq;
 	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
 		qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM;
 		qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM;
 		qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM;
 		qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM;
-		qp = ib_create_qp(free_mr->rsv_pd, &qp_init_attr);
+		ret = free_mr_init_qp(hr_dev, cq, &qp_init_attr, i);
-		if (IS_ERR(qp)) {
+		if (ret)
-			ibdev_err(ibdev, "failed to create qp for free mr.\n");
+			goto create_failed_qp;
-			ret = PTR_ERR(qp);
-			goto create_failed;
-		}
-		free_mr->rsv_qp[i] = qp;
 	}
 	return 0;
-create_failed:
+create_failed_qp:
-	free_mr_exit(hr_dev);
+	hns_roce_destroy_cq(cq, NULL);
+	kfree(cq);
+create_failed_cq:
+	hns_roce_dealloc_pd(pd, NULL);
+	kfree(pd);
 	return ret;
 }
 	int mask;
 	int ret;
-	hr_qp = to_hr_qp(free_mr->rsv_qp[sl_num]);
+	hr_qp = to_hr_qp(&free_mr->rsv_qp[sl_num]->ibqp);
 	hr_qp->free_mr_en = 1;
+	hr_qp->ibqp.device = ibdev;
+	hr_qp->ibqp.qp_type = IB_QPT_RC;
 	mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS;
 	attr->qp_state = IB_QPS_INIT;
 	attr->port_num = 1;
 	attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
-	ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
+	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
+				    IB_QPS_INIT);
 	if (ret) {
 		ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n",
 			  ret);
 	rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num);
-	ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
+	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
+				    IB_QPS_RTR);
 	hr_dev->loop_idc = loopback;
 	if (ret) {
 		ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",
 	attr->sq_psn = HNS_ROCE_FREE_MR_USED_PSN;
 	attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;
 	attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;
-	ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
+	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR,
+				    IB_QPS_RTS);
 	if (ret)
 		ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",
 			  ret);
 	int i, count;
 	count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
-				  ARRAY_SIZE(pages), &pbl_ba);
+				  min_t(int, ARRAY_SIZE(pages), mr->npages),
+				  &pbl_ba);
 	if (count < 1) {
 		ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
 			  count);
 	mutex_lock(&free_mr->mutex);
 	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
-		hr_qp = to_hr_qp(free_mr->rsv_qp[i]);
+		hr_qp = free_mr->rsv_qp[i];
 		ret = free_mr_post_send_lp_wqe(hr_qp);
 		if (ret) {
 	end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies;
 	while (cqe_cnt) {
-		npolled = hns_roce_v2_poll_cq(free_mr->rsv_cq, cqe_cnt, wc);
+		npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc);
 		if (npolled < 0) {
 			ibdev_err(ibdev,
 				  "failed to poll cqe for free mr, remain %d cqe.\n",
 		rdma_ah_set_sl(&qp_attr->ah_attr,
 			       hr_reg_read(&context, QPC_SL));
+		rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1);
+		rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH);
 		grh->flow_label = hr_reg_read(&context, QPC_FL);
 		grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX);
 		grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT);
 	return ret;
 }
-static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
+int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);

drivers/infiniband/hw/hns/hns_roce_hw_v2.h+10−3

 	CMD_OTHER_ERR = 0xff
 };
+struct hns_roce_cmd_errcode {
+	enum hns_roce_cmd_return_status return_status;
+	int errno;
+};
 enum hns_roce_sgid_type {
 	GID_TYPE_FLAG_ROCE_V1 = 0,
 	GID_TYPE_FLAG_ROCE_V2_IPV4,
 #define HNS_ROCE_EXT_LLM_MIN_PAGES(que_num) ((que_num) * 4 + 2)
 struct hns_roce_v2_free_mr {
-	struct ib_qp *rsv_qp[HNS_ROCE_FREE_MR_USED_QP_NUM];
+	struct hns_roce_qp *rsv_qp[HNS_ROCE_FREE_MR_USED_QP_NUM];
-	struct ib_cq *rsv_cq;
+	struct hns_roce_cq *rsv_cq;
-	struct ib_pd *rsv_pd;
+	struct hns_roce_pd *rsv_pd;
 	struct mutex mutex;
 };
 	__le32 rsv[5];
 };
+int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],
 				    void __iomem *dest)
 {

drivers/infiniband/hw/hns/hns_roce_main.c+16−2

 static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
 				   struct ib_udata *udata)
 {
-	int ret;
 	struct hns_roce_ucontext *context = to_hr_ucontext(uctx);
-	struct hns_roce_ib_alloc_ucontext_resp resp = {};
 	struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device);
+	struct hns_roce_ib_alloc_ucontext_resp resp = {};
+	struct hns_roce_ib_alloc_ucontext ucmd = {};
+	int ret;
 	if (!hr_dev->active)
 		return -EAGAIN;
 	resp.qp_tab_size = hr_dev->caps.num_qps;
 	resp.srq_tab_size = hr_dev->caps.num_srqs;
+	ret = ib_copy_from_udata(&ucmd, udata,
+				 min(udata->inlen, sizeof(ucmd)));
+	if (ret)
+		return ret;
+	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
+		context->config = ucmd.config & HNS_ROCE_EXSGE_FLAGS;
+	if (context->config & HNS_ROCE_EXSGE_FLAGS) {
+		resp.config |= HNS_ROCE_RSP_EXSGE_FLAGS;
+		resp.max_inline_data = hr_dev->caps.max_sq_inline;
+	}
 	ret = hns_roce_uar_alloc(hr_dev, &context->uar);
 	if (ret)
 		goto error_fail_uar_alloc;

drivers/infiniband/hw/hns/hns_roce_mr.c+2−2

 	return &mr->ibmr;
-err_key:
-	free_mr_key(hr_dev, mr);
 err_pbl:
 	free_mr_pbl(hr_dev, mr);
+err_key:
+	free_mr_key(hr_dev, mr);
 err_free:
 	kfree(mr);
 	return ERR_PTR(ret);

drivers/infiniband/hw/hns/hns_roce_qp.c+90−17

@@ -476,38 +476,109 @@ static int set_rq_size(struct hns_roce_dev hr_dev, struct ib_qp_cap cap, · 475 unmodified lines
476	return 0;	476	return 0;
477	}	477	}
478		478
479	static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp)	479	static u32 get_max_inline_data(struct hns_roce_dev *hr_dev,
		480	struct ib_qp_cap *cap)
480	{	481	{
481	/* GSI/UD QP only has extended sge */	482	if (cap->max_inline_data) {
482	if (qp->ibqp.qp_type == IB_QPT_GSI \|\| qp->ibqp.qp_type == IB_QPT_UD)	483	cap->max_inline_data = roundup_pow_of_two(cap->max_inline_data);
483	return qp->sq.max_gs;	484	return min(cap->max_inline_data,
484		485	hr_dev->caps.max_sq_inline);
485	if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE)	486	}
486	return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE;
487		487
488	return 0;	488	return 0;
489	}	489	}
490		490
		491	static void update_inline_data(struct hns_roce_qp *hr_qp,
		492	struct ib_qp_cap *cap)
		493	{
		494	u32 sge_num = hr_qp->sq.ext_sge_cnt;
		495
		496	if (hr_qp->config & HNS_ROCE_EXSGE_FLAGS) {
		497	if (!(hr_qp->ibqp.qp_type == IB_QPT_GSI \|\|
		498	hr_qp->ibqp.qp_type == IB_QPT_UD))
		499	sge_num = max((u32)HNS_ROCE_SGE_IN_WQE, sge_num);
		500
		501	cap->max_inline_data = max(cap->max_inline_data,
		502	sge_num * HNS_ROCE_SGE_SIZE);
		503	}
		504
		505	hr_qp->max_inline_data = cap->max_inline_data;
		506	}
		507
		508	static u32 get_sge_num_from_max_send_sge(bool is_ud_or_gsi,
		509	u32 max_send_sge)
		510	{
		511	unsigned int std_sge_num;
		512	unsigned int min_sge;
		513
		514	std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;
		515	min_sge = is_ud_or_gsi ? 1 : 0;
		516	return max_send_sge > std_sge_num ? (max_send_sge - std_sge_num) :
		517	min_sge;
		518	}
		519
		520	static unsigned int get_sge_num_from_max_inl_data(bool is_ud_or_gsi,
		521	u32 max_inline_data)
		522	{
		523	unsigned int inline_sge;
		524
		525	inline_sge = roundup_pow_of_two(max_inline_data) / HNS_ROCE_SGE_SIZE;
		526
		527	/*
		528	* if max_inline_data less than
		529	* HNS_ROCE_SGE_IN_WQE * HNS_ROCE_SGE_SIZE,
		530	* In addition to ud's mode, no need to extend sge.
		531	*/
		532	if (!is_ud_or_gsi && inline_sge <= HNS_ROCE_SGE_IN_WQE)
		533	inline_sge = 0;
		534
		535	return inline_sge;
		536	}
		537
491	static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,	538	static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
492	struct hns_roce_qp hr_qp, struct ib_qp_cap cap)	539	struct hns_roce_qp hr_qp, struct ib_qp_cap cap)
493	{	540	{
		541	bool is_ud_or_gsi = (hr_qp->ibqp.qp_type == IB_QPT_GSI \|\|
		542	hr_qp->ibqp.qp_type == IB_QPT_UD);
		543	unsigned int std_sge_num;
		544	u32 inline_ext_sge = 0;
		545	u32 ext_wqe_sge_cnt;
494	u32 total_sge_cnt;	546	u32 total_sge_cnt;
495	u32 wqe_sge_cnt;	547
		548	cap->max_inline_data = get_max_inline_data(hr_dev, cap);
496		549
497	hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;	550	hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
		551	std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;
		552	ext_wqe_sge_cnt = get_sge_num_from_max_send_sge(is_ud_or_gsi,
		553	cap->max_send_sge);
498		554
499	hr_qp->sq.max_gs = max(1U, cap->max_send_sge);	555	if (hr_qp->config & HNS_ROCE_EXSGE_FLAGS) {
		556	inline_ext_sge = max(ext_wqe_sge_cnt,
		557	get_sge_num_from_max_inl_data(is_ud_or_gsi,
		558	cap->max_inline_data));
		559	hr_qp->sq.ext_sge_cnt = inline_ext_sge ?
		560	roundup_pow_of_two(inline_ext_sge) : 0;
500		561
501	wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp);	562	hr_qp->sq.max_gs = max(1U, (hr_qp->sq.ext_sge_cnt + std_sge_num));
		563	hr_qp->sq.max_gs = min(hr_qp->sq.max_gs, hr_dev->caps.max_sq_sg);
		564
		565	ext_wqe_sge_cnt = hr_qp->sq.ext_sge_cnt;
		566	} else {
		567	hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
		568	hr_qp->sq.max_gs = min(hr_qp->sq.max_gs, hr_dev->caps.max_sq_sg);
		569	hr_qp->sq.ext_sge_cnt = hr_qp->sq.max_gs;
		570	}
502		571
503	/* If the number of extended sge is not zero, they MUST use the	572	/* If the number of extended sge is not zero, they MUST use the
504	* space of HNS_HW_PAGE_SIZE at least.	573	* space of HNS_HW_PAGE_SIZE at least.
505	*/	574	*/
506	if (wqe_sge_cnt) {	575	if (ext_wqe_sge_cnt) {
507	total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt);	576	total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * ext_wqe_sge_cnt);
508	hr_qp->sge.sge_cnt = max(total_sge_cnt,	577	hr_qp->sge.sge_cnt = max(total_sge_cnt,
509	(u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);	578	(u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);
510	}	579	}
		580
		581	update_inline_data(hr_qp, cap);
511	}	582	}
512		583
513	static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,	584	static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
@@ -556,6 +627,7 @@ static int set_user_sq_size(struct hns_roce_dev *hr_dev, · 42 unmodified lines
556		627
557	hr_qp->sq.wqe_shift = ucmd->log_sq_stride;	628	hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
558	hr_qp->sq.wqe_cnt = cnt;	629	hr_qp->sq.wqe_cnt = cnt;
		630	cap->max_send_sge = hr_qp->sq.max_gs;
559		631
560	return 0;	632	return 0;
561	}	633	}
@@ -986,13 +1058,9 @@ static int set_qp_param(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp, · 424 unmodified lines
986	struct hns_roce_ib_create_qp *ucmd)	1058	struct hns_roce_ib_create_qp *ucmd)
987	{	1059	{
988	struct ib_device *ibdev = &hr_dev->ib_dev;	1060	struct ib_device *ibdev = &hr_dev->ib_dev;
		1061	struct hns_roce_ucontext *uctx;
989	int ret;	1062	int ret;
990		1063
991	if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline)
992	init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline;
993
994	hr_qp->max_inline_data = init_attr->cap.max_inline_data;
995
996	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)	1064	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
997	hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;	1065	hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
998	else	1066	else
@@ -1015,12 +1083,17 @@ static int set_qp_param(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp, · 16 unmodified lines
1015	return ret;	1083	return ret;
1016	}	1084	}
1017		1085
		1086	uctx = rdma_udata_to_drv_context(udata, struct hns_roce_ucontext,
		1087	ibucontext);
		1088	hr_qp->config = uctx->config;
1018	ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);	1089	ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);
1019	if (ret)	1090	if (ret)
1020	ibdev_err(ibdev,	1091	ibdev_err(ibdev,
1021	"failed to set user SQ size, ret = %d.\n",	1092	"failed to set user SQ size, ret = %d.\n",
1022	ret);	1093	ret);
1023	} else {	1094	} else {
		1095	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
		1096	hr_qp->config = HNS_ROCE_EXSGE_FLAGS;
1024	ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);	1097	ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);
1025	if (ret)	1098	if (ret)
1026	ibdev_err(ibdev,	1099	ibdev_err(ibdev,

drivers/infiniband/hw/irdma/uk.c+107−63

@@ -566,21 +566,37 @@ static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe, · 565 unmodified lines
566		566
567	/**	567	/**
568	* irdma_copy_inline_data_gen_1 - Copy inline data to wqe	568	* irdma_copy_inline_data_gen_1 - Copy inline data to wqe
569	* @dest: pointer to wqe	569	* @wqe: pointer to wqe
570	* @src: pointer to inline data	570	* @sge_list: table of pointers to inline data
571	* @len: length of inline data to copy	571	* @num_sges: Total inline data length
572	* @polarity: compatibility parameter	572	* @polarity: compatibility parameter
573	*/	573	*/
574	static void irdma_copy_inline_data_gen_1(u8 dest, u8 src, u32 len,	574	static void irdma_copy_inline_data_gen_1(u8 wqe, struct ib_sge sge_list,
575	u8 polarity)	575	u32 num_sges, u8 polarity)
576	{	576	{
577	if (len <= 16) {	577	u32 quanta_bytes_remaining = 16;
578	memcpy(dest, src, len);	578	int i;
579	} else {	579
580	memcpy(dest, src, 16);	580	for (i = 0; i < num_sges; i++) {
581	src += 16;	581	u8 cur_sge = (u8 )(uintptr_t)sge_list[i].addr;
582	dest = dest + 32;	582	u32 sge_len = sge_list[i].length;
583	memcpy(dest, src, len - 16);	583
		584	while (sge_len) {
		585	u32 bytes_copied;
		586
		587	bytes_copied = min(sge_len, quanta_bytes_remaining);
		588	memcpy(wqe, cur_sge, bytes_copied);
		589	wqe += bytes_copied;
		590	cur_sge += bytes_copied;
		591	quanta_bytes_remaining -= bytes_copied;
		592	sge_len -= bytes_copied;
		593
		594	if (!quanta_bytes_remaining) {
		595	/* Remaining inline bytes reside after hdr */
		596	wqe += 16;
		597	quanta_bytes_remaining = 32;
		598	}
		599	}
584	}	600	}
585	}	601	}
586		602
@@ -612,35 +628,51 @@ static void irdma_set_mw_bind_wqe(__le64 *wqe, · 25 unmodified lines
612		628
613	/**	629	/**
614	* irdma_copy_inline_data - Copy inline data to wqe	630	* irdma_copy_inline_data - Copy inline data to wqe
615	* @dest: pointer to wqe	631	* @wqe: pointer to wqe
616	* @src: pointer to inline data	632	* @sge_list: table of pointers to inline data
617	* @len: length of inline data to copy	633	* @num_sges: number of SGE's
618	* @polarity: polarity of wqe valid bit	634	* @polarity: polarity of wqe valid bit
619	*/	635	*/
620	static void irdma_copy_inline_data(u8 dest, u8 src, u32 len, u8 polarity)	636	static void irdma_copy_inline_data(u8 wqe, struct ib_sge sge_list,
		637	u32 num_sges, u8 polarity)
621	{	638	{
622	u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;	639	u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
623	u32 copy_size;	640	u32 quanta_bytes_remaining = 8;
624		641	bool first_quanta = true;
625	dest += 8;	642	int i;
626	if (len <= 8) {	643
627	memcpy(dest, src, len);	644	wqe += 8;
628	return;	645
629	}	646	for (i = 0; i < num_sges; i++) {
630		647	u8 cur_sge = (u8 )(uintptr_t)sge_list[i].addr;
631	((u64 )dest) = ((u64 )src);	648	u32 sge_len = sge_list[i].length;
632	len -= 8;	649
633	src += 8;	650	while (sge_len) {
634	dest += 24; /* point to additional 32 byte quanta */	651	u32 bytes_copied;
635		652
636	while (len) {	653	bytes_copied = min(sge_len, quanta_bytes_remaining);
637	copy_size = len < 31 ? len : 31;	654	memcpy(wqe, cur_sge, bytes_copied);
638	memcpy(dest, src, copy_size);	655	wqe += bytes_copied;
639	*(dest + 31) = inline_valid;	656	cur_sge += bytes_copied;
640	len -= copy_size;	657	quanta_bytes_remaining -= bytes_copied;
641	dest += 32;	658	sge_len -= bytes_copied;
642	src += copy_size;	659
		660	if (!quanta_bytes_remaining) {
		661	quanta_bytes_remaining = 31;
		662
		663	/* Remaining inline bytes reside after hdr */
		664	if (first_quanta) {
		665	first_quanta = false;
		666	wqe += 16;
		667	} else {
		668	*wqe = inline_valid;
		669	wqe++;
		670	}
		671	}
		672	}
643	}	673	}
		674	if (!first_quanta && quanta_bytes_remaining < 31)
		675	*(wqe + quanta_bytes_remaining) = inline_valid;
644	}	676	}
645		677
646	/**	678	/**
@@ -679,20 +711,27 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, · 32 unmodified lines
679	struct irdma_post_sq_info *info, bool post_sq)	711	struct irdma_post_sq_info *info, bool post_sq)
680	{	712	{
681	__le64 *wqe;	713	__le64 *wqe;
682	struct irdma_inline_rdma_write *op_info;	714	struct irdma_rdma_write *op_info;
683	u64 hdr = 0;	715	u64 hdr = 0;
684	u32 wqe_idx;	716	u32 wqe_idx;
685	bool read_fence = false;	717	bool read_fence = false;
		718	u32 i, total_size = 0;
686	u16 quanta;	719	u16 quanta;
687		720
688	info->push_wqe = qp->push_db ? true : false;	721	info->push_wqe = qp->push_db ? true : false;
689	op_info = &info->op.inline_rdma_write;	722	op_info = &info->op.rdma_write;
		723
		724	if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
		725	return -EINVAL;
		726
		727	for (i = 0; i < op_info->num_lo_sges; i++)
		728	total_size += op_info->lo_sg_list[i].length;
690		729
691	if (op_info->len > qp->max_inline_data)	730	if (unlikely(total_size > qp->max_inline_data))
692	return -EINVAL;	731	return -EINVAL;
693		732
694	quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);	733	quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
695	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,	734	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
696	info);	735	info);
697	if (!wqe)	736	if (!wqe)
698	return -ENOMEM;	737	return -ENOMEM;
@@ -705,7 +744,7 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, · 6 unmodified lines
705		744
706	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) \|	745	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) \|
707	FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) \|	746	FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) \|
708	FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, op_info->len) \|	747	FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) \|
709	FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) \|	748	FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) \|
710	FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) \|	749	FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) \|
711	FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) \|	750	FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) \|
@@ -719,7 +758,8 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, · 7 unmodified lines
719	set_64bit_val(wqe, 0,	758	set_64bit_val(wqe, 0,
720	FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));	759	FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
721		760
722	qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,	761	qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
		762	op_info->num_lo_sges,
723	qp->swqe_polarity);	763	qp->swqe_polarity);
724	dma_wmb(); /* make sure WQE is populated before valid bit is set */	764	dma_wmb(); /* make sure WQE is populated before valid bit is set */
725		765
@@ -745,20 +785,27 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp, · 19 unmodified lines
745	struct irdma_post_sq_info *info, bool post_sq)	785	struct irdma_post_sq_info *info, bool post_sq)
746	{	786	{
747	__le64 *wqe;	787	__le64 *wqe;
748	struct irdma_post_inline_send *op_info;	788	struct irdma_post_send *op_info;
749	u64 hdr;	789	u64 hdr;
750	u32 wqe_idx;	790	u32 wqe_idx;
751	bool read_fence = false;	791	bool read_fence = false;
		792	u32 i, total_size = 0;
752	u16 quanta;	793	u16 quanta;
753		794
754	info->push_wqe = qp->push_db ? true : false;	795	info->push_wqe = qp->push_db ? true : false;
755	op_info = &info->op.inline_send;	796	op_info = &info->op.send;
		797
		798	if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
		799	return -EINVAL;
756		800
757	if (op_info->len > qp->max_inline_data)	801	for (i = 0; i < op_info->num_sges; i++)
		802	total_size += op_info->sg_list[i].length;
		803
		804	if (unlikely(total_size > qp->max_inline_data))
758	return -EINVAL;	805	return -EINVAL;
759		806
760	quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);	807	quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
761	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,	808	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
762	info);	809	info);
763	if (!wqe)	810	if (!wqe)
764	return -ENOMEM;	811	return -ENOMEM;
@@ -773,7 +820,7 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp, · 8 unmodified lines
773	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) \|	820	hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) \|
774	FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) \|	821	FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) \|
775	FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) \|	822	FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) \|
776	FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, op_info->len) \|	823	FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) \|
777	FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,	824	FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
778	(info->imm_data_valid ? 1 : 0)) \|	825	(info->imm_data_valid ? 1 : 0)) \|
779	FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) \|	826	FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) \|
@@ -789,8 +836,8 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp, · 9 unmodified lines
789	if (info->imm_data_valid)	836	if (info->imm_data_valid)
790	set_64bit_val(wqe, 0,	837	set_64bit_val(wqe, 0,
791	FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));	838	FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
792	qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,	839	qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
793	qp->swqe_polarity);	840	op_info->num_sges, qp->swqe_polarity);
794		841
795	dma_wmb(); /* make sure WQE is populated before valid bit is set */	842	dma_wmb(); /* make sure WQE is populated before valid bit is set */
796		843
@@ -1002,11 +1049,10 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 205 unmodified lines
1002	__le64 *cqe;	1049	__le64 *cqe;
1003	struct irdma_qp_uk *qp;	1050	struct irdma_qp_uk *qp;
1004	struct irdma_ring *pring = NULL;	1051	struct irdma_ring *pring = NULL;
1005	u32 wqe_idx, q_type;	1052	u32 wqe_idx;
1006	int ret_code;	1053	int ret_code;
1007	bool move_cq_head = true;	1054	bool move_cq_head = true;
1008	u8 polarity;	1055	u8 polarity;
1009	u8 op_type;
1010	bool ext_valid;	1056	bool ext_valid;
1011	__le64 *ext_cqe;	1057	__le64 *ext_cqe;
1012		1058
@@ -1074,7 +1120,7 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 61 unmodified lines
1074	info->ud_vlan_valid = false;	1120	info->ud_vlan_valid = false;
1075	}	1121	}
1076		1122
1077	q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);	1123	info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
1078	info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);	1124	info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
1079	info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3);	1125	info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3);
1080	info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);	1126	info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
@@ -1113,8 +1159,9 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 32 unmodified lines
1113	}	1159	}
1114	wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);	1160	wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
1115	info->qp_handle = (irdma_qp_handle)(unsigned long)qp;	1161	info->qp_handle = (irdma_qp_handle)(unsigned long)qp;
		1162	info->op_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
1116		1163
1117	if (q_type == IRDMA_CQE_QTYPE_RQ) {	1164	if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
1118	u32 array_idx;	1165	u32 array_idx;
1119		1166
1120	array_idx = wqe_idx / qp->rq_wqe_size_multiplier;	1167	array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
@@ -1134,10 +1181,6 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 13 unmodified lines
1134		1181
1135	info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);	1182	info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
1136		1183
1137	if (info->imm_valid)
1138	info->op_type = IRDMA_OP_TYPE_REC_IMM;
1139	else
1140	info->op_type = IRDMA_OP_TYPE_REC;
1141	if (qword3 & IRDMACQ_STAG) {	1184	if (qword3 & IRDMACQ_STAG) {
1142	info->stag_invalid_set = true;	1185	info->stag_invalid_set = true;
1143	info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);	1186	info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);
@@ -1195,17 +1238,18 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, · 51 unmodified lines
1195	sw_wqe = qp->sq_base[tail].elem;	1238	sw_wqe = qp->sq_base[tail].elem;
1196	get_64bit_val(sw_wqe, 24,	1239	get_64bit_val(sw_wqe, 24,
1197	&wqe_qword);	1240	&wqe_qword);
1198	op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);	1241	info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE,
1199	info->op_type = op_type;	1242	wqe_qword);
1200	IRDMA_RING_SET_TAIL(qp->sq_ring,	1243	IRDMA_RING_SET_TAIL(qp->sq_ring,
1201	tail + qp->sq_wrtrk_array[tail].quanta);	1244	tail + qp->sq_wrtrk_array[tail].quanta);
1202	if (op_type != IRDMAQP_OP_NOP) {	1245	if (info->op_type != IRDMAQP_OP_NOP) {
1203	info->wr_id = qp->sq_wrtrk_array[tail].wrid;	1246	info->wr_id = qp->sq_wrtrk_array[tail].wrid;
1204	info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;	1247	info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
1205	break;	1248	break;
1206	}	1249	}
1207	} while (1);	1250	} while (1);
1208	if (op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR)	1251	if (info->op_type == IRDMA_OP_TYPE_BIND_MW &&
		1252	info->minor_err == FLUSH_PROT_ERR)
1209	info->minor_err = FLUSH_MW_BIND_ERR;	1253	info->minor_err = FLUSH_MW_BIND_ERR;
1210	qp->sq_flush_seen = true;	1254	qp->sq_flush_seen = true;
1211	if (!IRDMA_RING_MORE_WORK(qp->sq_ring))	1255	if (!IRDMA_RING_MORE_WORK(qp->sq_ring))

drivers/infiniband/hw/irdma/user.h+3−17

 	u32 ah_id;
 };
-struct irdma_post_inline_send {
-	void *data;
-	u32 len;
-	u32 qkey;
-	u32 dest_qp;
-	u32 ah_id;
-};
 struct irdma_post_rq_info {
 	u64 wr_id;
 	struct ib_sge *sg_list;
 	struct ib_sge rem_addr;
 };
-struct irdma_inline_rdma_write {
-	void *data;
-	u32 len;
-	struct ib_sge rem_addr;
-};
 struct irdma_rdma_read {
 	struct ib_sge *lo_sg_list;
 	u32 num_lo_sges;
 		struct irdma_rdma_read rdma_read;
 		struct irdma_bind_window bind_window;
 		struct irdma_inv_local_stag inv_local_stag;
-		struct irdma_inline_rdma_write inline_rdma_write;
-		struct irdma_post_inline_send inline_send;
 	} op;
 };
 	u16 ud_vlan;
 	u8 ud_smac[6];
 	u8 op_type;
+	u8 q_type;
 	bool stag_invalid_set:1; /* or L_R_Key set */
 	bool push_dropped:1;
 	bool error:1;
 				   bool post_sq);
 struct irdma_wqe_uk_ops {
-	void (*iw_copy_inline_data)(u8 *dest, u8 *src, u32 len, u8 polarity);
+	void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list,
+				    u32 num_sges, u8 polarity);
 	u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
 	void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
 				u8 valid);

drivers/infiniband/hw/irdma/utils.c+2−0

 			sw_wqe = qp->sq_base[wqe_idx].elem;
 			get_64bit_val(sw_wqe, 24, &wqe_qword);
 			cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
+			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
 			/* remove the SQ WR by moving SQ tail*/
 			IRDMA_RING_SET_TAIL(*sq_ring,
 				sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
 			cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
 			cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
+			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
 			/* remove the RQ WR by moving RQ tail */
 			IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
 			ibdev_dbg(iwqp->iwrcq->ibcq.device,

drivers/infiniband/hw/irdma/verbs.c+34−111

@@ -63,36 +63,6 @@ static int irdma_query_device(struct ib_device *ibdev, · 62 unmodified lines
63	return 0;	63	return 0;
64	}	64	}
65		65
66	/**
67	* irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed
68	* @link_speed: netdev phy link speed
69	* @active_speed: IB port speed
70	* @active_width: IB port width
71	*/
72	static void irdma_get_eth_speed_and_width(u32 link_speed, u16 *active_speed,
73	u8 *active_width)
74	{
75	if (link_speed <= SPEED_1000) {
76	*active_width = IB_WIDTH_1X;
77	*active_speed = IB_SPEED_SDR;
78	} else if (link_speed <= SPEED_10000) {
79	*active_width = IB_WIDTH_1X;
80	*active_speed = IB_SPEED_FDR10;
81	} else if (link_speed <= SPEED_20000) {
82	*active_width = IB_WIDTH_4X;
83	*active_speed = IB_SPEED_DDR;
84	} else if (link_speed <= SPEED_25000) {
85	*active_width = IB_WIDTH_1X;
86	*active_speed = IB_SPEED_EDR;
87	} else if (link_speed <= SPEED_40000) {
88	*active_width = IB_WIDTH_4X;
89	*active_speed = IB_SPEED_FDR10;
90	} else {
91	*active_width = IB_WIDTH_4X;
92	*active_speed = IB_SPEED_EDR;
93	}
94	}
95
96	/**	66	/**
97	* irdma_query_port - get port attributes	67	* irdma_query_port - get port attributes
98	* @ibdev: device pointer from stack	68	* @ibdev: device pointer from stack
@@ -120,8 +90,9 @@ static int irdma_query_port(struct ib_device *ibdev, u32 port, · 21 unmodified lines
120	props->state = IB_PORT_DOWN;	90	props->state = IB_PORT_DOWN;
121	props->phys_state = IB_PORT_PHYS_STATE_DISABLED;	91	props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
122	}	92	}
123	irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed,	93
124	&props->active_width);	94	ib_get_eth_speed(ibdev, port, &props->active_speed,
		95	&props->active_width);
125		96
126	if (rdma_protocol_roce(ibdev, 1)) {	97	if (rdma_protocol_roce(ibdev, 1)) {
127	props->gid_tbl_len = 32;	98	props->gid_tbl_len = 32;
@@ -1242,6 +1213,7 @@ int irdma_modify_qp_roce(struct ib_qp ibqp, struct ib_qp_attr attr, · 1114 unmodified lines
1242	av->attrs = attr->ah_attr;	1213	av->attrs = attr->ah_attr;
1243	rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid);	1214	rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid);
1244	rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid);	1215	rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid);
		1216	av->net_type = rdma_gid_attr_network_type(sgid_attr);
1245	if (av->net_type == RDMA_NETWORK_IPV6) {	1217	if (av->net_type == RDMA_NETWORK_IPV6) {
1246	__be32 *daddr =	1218	__be32 *daddr =
1247	av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;	1219	av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;
@@ -2358,9 +2330,10 @@ static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, · 1110 unmodified lines
2358	* @rf: RDMA PCI function	2330	* @rf: RDMA PCI function
2359	* @iwmr: mr pointer for this memory registration	2331	* @iwmr: mr pointer for this memory registration
2360	* @use_pbles: flag if to use pble's	2332	* @use_pbles: flag if to use pble's
		2333	* @lvl_1_only: request only level 1 pble if true
2361	*/	2334	*/
2362	static int irdma_setup_pbles(struct irdma_pci_f rf, struct irdma_mr iwmr,	2335	static int irdma_setup_pbles(struct irdma_pci_f rf, struct irdma_mr iwmr,
2363	bool use_pbles)	2336	bool use_pbles, bool lvl_1_only)
2364	{	2337	{
2365	struct irdma_pbl *iwpbl = &iwmr->iwpbl;	2338	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2366	struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;	2339	struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
@@ -2371,7 +2344,7 @@ static int irdma_setup_pbles(struct irdma_pci_f rf, struct irdma_mr iwmr, · 4 unmodified lines
2371		2344
2372	if (use_pbles) {	2345	if (use_pbles) {
2373	status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,	2346	status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,
2374	false);	2347	lvl_1_only);
2375	if (status)	2348	if (status)
2376	return status;	2349	return status;
2377		2350
@@ -2414,16 +2387,10 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev, · 36 unmodified lines
2414	bool ret = true;	2387	bool ret = true;
2415		2388
2416	pg_size = iwmr->page_size;	2389	pg_size = iwmr->page_size;
2417	err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles);	2390	err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, true);
2418	if (err)	2391	if (err)
2419	return err;	2392	return err;
2420		2393
2421	if (use_pbles && palloc->level != PBLE_LEVEL_1) {
2422	irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
2423	iwpbl->pbl_allocated = false;
2424	return -ENOMEM;
2425	}
2426
2427	if (use_pbles)	2394	if (use_pbles)
2428	arr = palloc->level1.addr;	2395	arr = palloc->level1.addr;
2429		2396
@@ -2899,7 +2866,7 @@ static struct ib_mr irdma_reg_user_mr(struct ib_pd pd, u64 start, u64 len, · 469 unmodified lines
2899	case IRDMA_MEMREG_TYPE_MEM:	2866	case IRDMA_MEMREG_TYPE_MEM:
2900	use_pbles = (iwmr->page_cnt != 1);	2867	use_pbles = (iwmr->page_cnt != 1);
2901		2868
2902	err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles);	2869	err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
2903	if (err)	2870	if (err)
2904	goto error;	2871	goto error;
2905		2872
@@ -3165,30 +3132,20 @@ static int irdma_post_send(struct ib_qp *ibqp, · 259 unmodified lines
3165	info.stag_to_inv = ib_wr->ex.invalidate_rkey;	3132	info.stag_to_inv = ib_wr->ex.invalidate_rkey;
3166	}	3133	}
3167		3134
3168	if (ib_wr->send_flags & IB_SEND_INLINE) {	3135	info.op.send.num_sges = ib_wr->num_sge;
3169	info.op.inline_send.data = (void *)(unsigned long)	3136	info.op.send.sg_list = ib_wr->sg_list;
3170	ib_wr->sg_list[0].addr;	3137	if (iwqp->ibqp.qp_type == IB_QPT_UD \|\|
3171	info.op.inline_send.len = ib_wr->sg_list[0].length;	3138	iwqp->ibqp.qp_type == IB_QPT_GSI) {
3172	if (iwqp->ibqp.qp_type == IB_QPT_UD \|\|	3139	ah = to_iwah(ud_wr(ib_wr)->ah);
3173	iwqp->ibqp.qp_type == IB_QPT_GSI) {	3140	info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
3174	ah = to_iwah(ud_wr(ib_wr)->ah);	3141	info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
3175	info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx;	3142	info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
3176	info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey;	3143	}
3177	info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn;	3144
3178	}	3145	if (ib_wr->send_flags & IB_SEND_INLINE)
3179	err = irdma_uk_inline_send(ukqp, &info, false);	3146	err = irdma_uk_inline_send(ukqp, &info, false);
3180	} else {	3147	else
3181	info.op.send.num_sges = ib_wr->num_sge;
3182	info.op.send.sg_list = ib_wr->sg_list;
3183	if (iwqp->ibqp.qp_type == IB_QPT_UD \|\|
3184	iwqp->ibqp.qp_type == IB_QPT_GSI) {
3185	ah = to_iwah(ud_wr(ib_wr)->ah);
3186	info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
3187	info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
3188	info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
3189	}
3190	err = irdma_uk_send(ukqp, &info, false);	3148	err = irdma_uk_send(ukqp, &info, false);
3191	}
3192	break;	3149	break;
3193	case IB_WR_RDMA_WRITE_WITH_IMM:	3150	case IB_WR_RDMA_WRITE_WITH_IMM:
3194	if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) {	3151	if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) {
@@ -3205,22 +3162,15 @@ static int irdma_post_send(struct ib_qp *ibqp, · 10 unmodified lines
3205	else	3162	else
3206	info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;	3163	info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
3207		3164
3208	if (ib_wr->send_flags & IB_SEND_INLINE) {	3165	info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
3209	info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;	3166	info.op.rdma_write.lo_sg_list = ib_wr->sg_list;
3210	info.op.inline_rdma_write.len =	3167	info.op.rdma_write.rem_addr.addr =
3211	ib_wr->sg_list[0].length;	3168	rdma_wr(ib_wr)->remote_addr;
3212	info.op.inline_rdma_write.rem_addr.addr =	3169	info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
3213	rdma_wr(ib_wr)->remote_addr;	3170	if (ib_wr->send_flags & IB_SEND_INLINE)
3214	info.op.inline_rdma_write.rem_addr.lkey =
3215	rdma_wr(ib_wr)->rkey;
3216	err = irdma_uk_inline_rdma_write(ukqp, &info, false);	3171	err = irdma_uk_inline_rdma_write(ukqp, &info, false);
3217	} else {	3172	else
3218	info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
3219	info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
3220	info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
3221	info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
3222	err = irdma_uk_rdma_write(ukqp, &info, false);	3173	err = irdma_uk_rdma_write(ukqp, &info, false);
3223	}
3224	break;	3174	break;
3225	case IB_WR_RDMA_READ_WITH_INV:	3175	case IB_WR_RDMA_READ_WITH_INV:
3226	inv_stag = true;	3176	inv_stag = true;
@@ -3380,7 +3330,6 @@ static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode · 153 unmodified lines
3380	static void irdma_process_cqe(struct ib_wc *entry,	3330	static void irdma_process_cqe(struct ib_wc *entry,
3381	struct irdma_cq_poll_info *cq_poll_info)	3331	struct irdma_cq_poll_info *cq_poll_info)
3382	{	3332	{
3383	struct irdma_qp *iwqp;
3384	struct irdma_sc_qp *qp;	3333	struct irdma_sc_qp *qp;
3385		3334
3386	entry->wc_flags = 0;	3335	entry->wc_flags = 0;
@@ -3388,7 +3337,6 @@ static void irdma_process_cqe(struct ib_wc *entry, · 1 unmodified line
3388	entry->wr_id = cq_poll_info->wr_id;	3337	entry->wr_id = cq_poll_info->wr_id;
3389		3338
3390	qp = cq_poll_info->qp_handle;	3339	qp = cq_poll_info->qp_handle;
3391	iwqp = qp->qp_uk.back_qp;
3392	entry->qp = qp->qp_uk.back_qp;	3340	entry->qp = qp->qp_uk.back_qp;
3393		3341
3394	if (cq_poll_info->error) {	3342	if (cq_poll_info->error) {
@@ -3421,42 +3369,17 @@ static void irdma_process_cqe(struct ib_wc *entry, · 26 unmodified lines
3421	}	3369	}
3422	}	3370	}
3423		3371
3424	switch (cq_poll_info->op_type) {	3372	if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) {
3425	case IRDMA_OP_TYPE_RDMA_WRITE:	3373	set_ib_wc_op_sq(cq_poll_info, entry);
3426	case IRDMA_OP_TYPE_RDMA_WRITE_SOL:	3374	} else {
3427	entry->opcode = IB_WC_RDMA_WRITE;	3375	set_ib_wc_op_rq(cq_poll_info, entry,
3428	break;	3376	qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM ?
3429	case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:	3377	true : false);
3430	case IRDMA_OP_TYPE_RDMA_READ:
3431	entry->opcode = IB_WC_RDMA_READ;
3432	break;
3433	case IRDMA_OP_TYPE_SEND_INV:
3434	case IRDMA_OP_TYPE_SEND_SOL:
3435	case IRDMA_OP_TYPE_SEND_SOL_INV:
3436	case IRDMA_OP_TYPE_SEND:
3437	entry->opcode = IB_WC_SEND;
3438	break;
3439	case IRDMA_OP_TYPE_FAST_REG_NSMR:
3440	entry->opcode = IB_WC_REG_MR;
3441	break;
3442	case IRDMA_OP_TYPE_INV_STAG:
3443	entry->opcode = IB_WC_LOCAL_INV;
3444	break;
3445	case IRDMA_OP_TYPE_REC_IMM:
3446	case IRDMA_OP_TYPE_REC:
3447	entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ?
3448	IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
3449	if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD &&	3378	if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD &&
3450	cq_poll_info->stag_invalid_set) {	3379	cq_poll_info->stag_invalid_set) {
3451	entry->ex.invalidate_rkey = cq_poll_info->inv_stag;	3380	entry->ex.invalidate_rkey = cq_poll_info->inv_stag;
3452	entry->wc_flags \|= IB_WC_WITH_INVALIDATE;	3381	entry->wc_flags \|= IB_WC_WITH_INVALIDATE;
3453	}	3382	}
3454	break;
3455	default:
3456	ibdev_err(&iwqp->iwdev->ibdev,
3457	"Invalid opcode = %d in CQE\n", cq_poll_info->op_type);
3458	entry->status = IB_WC_GENERAL_ERR;
3459	return;
3460	}	3383	}
3461		3384
3462	if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) {	3385	if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) {

drivers/infiniband/hw/irdma/verbs.h+53−0

@@ -232,6 +232,59 @@ static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev) · 231 unmodified lines
232	return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);	232	return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
233	}	233	}
234		234
		235	static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
		236	struct ib_wc *entry)
		237	{
		238	switch (cq_poll_info->op_type) {
		239	case IRDMA_OP_TYPE_RDMA_WRITE:
		240	case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
		241	entry->opcode = IB_WC_RDMA_WRITE;
		242	break;
		243	case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
		244	case IRDMA_OP_TYPE_RDMA_READ:
		245	entry->opcode = IB_WC_RDMA_READ;
		246	break;
		247	case IRDMA_OP_TYPE_SEND_SOL:
		248	case IRDMA_OP_TYPE_SEND_SOL_INV:
		249	case IRDMA_OP_TYPE_SEND_INV:
		250	case IRDMA_OP_TYPE_SEND:
		251	entry->opcode = IB_WC_SEND;
		252	break;
		253	case IRDMA_OP_TYPE_FAST_REG_NSMR:
		254	entry->opcode = IB_WC_REG_MR;
		255	break;
		256	case IRDMA_OP_TYPE_INV_STAG:
		257	entry->opcode = IB_WC_LOCAL_INV;
		258	break;
		259	default:
		260	entry->status = IB_WC_GENERAL_ERR;
		261	}
		262	}
		263
		264	static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
		265	struct ib_wc *entry, bool send_imm_support)
		266	{
		267	/**
		268	* iWARP does not support sendImm, so the presence of Imm data
		269	* must be WriteImm.
		270	*/
		271	if (!send_imm_support) {
		272	entry->opcode = cq_poll_info->imm_valid ?
		273	IB_WC_RECV_RDMA_WITH_IMM :
		274	IB_WC_RECV;
		275	return;
		276	}
		277
		278	switch (cq_poll_info->op_type) {
		279	case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
		280	case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
		281	entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
		282	break;
		283	default:
		284	entry->opcode = IB_WC_RECV;
		285	}
		286	}
		287
235	void irdma_mcast_mac(u32 ip_addr, u8 mac, bool ipv4);	288	void irdma_mcast_mac(u32 ip_addr, u8 mac, bool ipv4);
236	int irdma_ib_register_device(struct irdma_device *iwdev);	289	int irdma_ib_register_device(struct irdma_device *iwdev);
237	void irdma_ib_unregister_device(struct irdma_device *iwdev);	290	void irdma_ib_unregister_device(struct irdma_device *iwdev);

drivers/infiniband/hw/mana/Kconfig+10−0

@@ -0,0 +1,10 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	config MANA_INFINIBAND
	3	tristate "Microsoft Azure Network Adapter support"
	4	depends on NETDEVICES && ETHERNET && PCI && MICROSOFT_MANA
	5	help
	6	This driver provides low-level RDMA support for Microsoft Azure
	7	Network Adapter (MANA). MANA supports RDMA features that can be used
	8	for workloads (e.g. DPDK, MPI etc) that uses RDMA verbs to directly
	9	access hardware from user-mode processes in Microsoft Azure cloud
	10	environment.

drivers/infiniband/hw/mana/Makefile+4−0

@@ -0,0 +1,4 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	obj-$(CONFIG_MANA_INFINIBAND) += mana_ib.o
	3
	4	mana_ib-y := device.o main.o wq.o qp.o cq.o mr.o

drivers/infiniband/hw/mana/cq.c+79−0

@@ -0,0 +1,79 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	*/
	5
	6	#include "mana_ib.h"
	7
	8	int mana_ib_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
	9	struct ib_udata *udata)
	10	{
	11	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
	12	struct ib_device *ibdev = ibcq->device;
	13	struct mana_ib_create_cq ucmd = {};
	14	struct mana_ib_dev *mdev;
	15	int err;
	16
	17	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	18
	19	if (udata->inlen < sizeof(ucmd))
	20	return -EINVAL;
	21
	22	err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
	23	if (err) {
	24	ibdev_dbg(ibdev,
	25	"Failed to copy from udata for create cq, %d\n", err);
	26	return err;
	27	}
	28
	29	if (attr->cqe > MAX_SEND_BUFFERS_PER_QUEUE) {
	30	ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
	31	return -EINVAL;
	32	}
	33
	34	cq->cqe = attr->cqe;
	35	cq->umem = ib_umem_get(ibdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
	36	IB_ACCESS_LOCAL_WRITE);
	37	if (IS_ERR(cq->umem)) {
	38	err = PTR_ERR(cq->umem);
	39	ibdev_dbg(ibdev, "Failed to get umem for create cq, err %d\n",
	40	err);
	41	return err;
	42	}
	43
	44	err = mana_ib_gd_create_dma_region(mdev, cq->umem, &cq->gdma_region);
	45	if (err) {
	46	ibdev_dbg(ibdev,
	47	"Failed to create dma region for create cq, %d\n",
	48	err);
	49	goto err_release_umem;
	50	}
	51
	52	ibdev_dbg(ibdev,
	53	"mana_ib_gd_create_dma_region ret %d gdma_region 0x%llx\n",
	54	err, cq->gdma_region);
	55
	56	/*
	57	* The CQ ID is not known at this time. The ID is generated at create_qp
	58	*/
	59
	60	return 0;
	61
	62	err_release_umem:
	63	ib_umem_release(cq->umem);
	64	return err;
	65	}
	66
	67	int mana_ib_destroy_cq(struct ib_cq ibcq, struct ib_udata udata)
	68	{
	69	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
	70	struct ib_device *ibdev = ibcq->device;
	71	struct mana_ib_dev *mdev;
	72
	73	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	74
	75	mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
	76	ib_umem_release(cq->umem);
	77
	78	return 0;
	79	}

drivers/infiniband/hw/mana/device.c+117−0

@@ -0,0 +1,117 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	*/
	5
	6	#include "mana_ib.h"
	7	#include <net/mana/mana_auxiliary.h>
	8
	9	MODULE_DESCRIPTION("Microsoft Azure Network Adapter IB driver");
	10	MODULE_LICENSE("GPL");
	11	MODULE_IMPORT_NS(NET_MANA);
	12
	13	static const struct ib_device_ops mana_ib_dev_ops = {
	14	.owner = THIS_MODULE,
	15	.driver_id = RDMA_DRIVER_MANA,
	16	.uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
	17
	18	.alloc_pd = mana_ib_alloc_pd,
	19	.alloc_ucontext = mana_ib_alloc_ucontext,
	20	.create_cq = mana_ib_create_cq,
	21	.create_qp = mana_ib_create_qp,
	22	.create_rwq_ind_table = mana_ib_create_rwq_ind_table,
	23	.create_wq = mana_ib_create_wq,
	24	.dealloc_pd = mana_ib_dealloc_pd,
	25	.dealloc_ucontext = mana_ib_dealloc_ucontext,
	26	.dereg_mr = mana_ib_dereg_mr,
	27	.destroy_cq = mana_ib_destroy_cq,
	28	.destroy_qp = mana_ib_destroy_qp,
	29	.destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table,
	30	.destroy_wq = mana_ib_destroy_wq,
	31	.disassociate_ucontext = mana_ib_disassociate_ucontext,
	32	.get_port_immutable = mana_ib_get_port_immutable,
	33	.mmap = mana_ib_mmap,
	34	.modify_qp = mana_ib_modify_qp,
	35	.modify_wq = mana_ib_modify_wq,
	36	.query_device = mana_ib_query_device,
	37	.query_gid = mana_ib_query_gid,
	38	.query_port = mana_ib_query_port,
	39	.reg_user_mr = mana_ib_reg_user_mr,
	40
	41	INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
	42	INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
	43	INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
	44	INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
	45	INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mana_ib_rwq_ind_table,
	46	ib_ind_table),
	47	};
	48
	49	static int mana_ib_probe(struct auxiliary_device *adev,
	50	const struct auxiliary_device_id *id)
	51	{
	52	struct mana_adev *madev = container_of(adev, struct mana_adev, adev);
	53	struct gdma_dev *mdev = madev->mdev;
	54	struct mana_context *mc;
	55	struct mana_ib_dev *dev;
	56	int ret;
	57
	58	mc = mdev->driver_data;
	59
	60	dev = ib_alloc_device(mana_ib_dev, ib_dev);
	61	if (!dev)
	62	return -ENOMEM;
	63
	64	ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_ops);
	65
	66	dev->ib_dev.phys_port_cnt = mc->num_ports;
	67
	68	ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
	69	mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt);
	70
	71	dev->gdma_dev = mdev;
	72	dev->ib_dev.node_type = RDMA_NODE_IB_CA;
	73
	74	/*
	75	* num_comp_vectors needs to set to the max MSIX index
	76	* when interrupts and event queues are implemented
	77	*/
	78	dev->ib_dev.num_comp_vectors = 1;
	79	dev->ib_dev.dev.parent = mdev->gdma_context->dev;
	80
	81	ret = ib_register_device(&dev->ib_dev, "mana_%d",
	82	mdev->gdma_context->dev);
	83	if (ret) {
	84	ib_dealloc_device(&dev->ib_dev);
	85	return ret;
	86	}
	87
	88	dev_set_drvdata(&adev->dev, dev);
	89
	90	return 0;
	91	}
	92
	93	static void mana_ib_remove(struct auxiliary_device *adev)
	94	{
	95	struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev);
	96
	97	ib_unregister_device(&dev->ib_dev);
	98	ib_dealloc_device(&dev->ib_dev);
	99	}
	100
	101	static const struct auxiliary_device_id mana_id_table[] = {
	102	{
	103	.name = "mana.rdma",
	104	},
	105	{},
	106	};
	107
	108	MODULE_DEVICE_TABLE(auxiliary, mana_id_table);
	109
	110	static struct auxiliary_driver mana_driver = {
	111	.name = "rdma",
	112	.probe = mana_ib_probe,
	113	.remove = mana_ib_remove,
	114	.id_table = mana_id_table,
	115	};
	116
	117	module_auxiliary_driver(mana_driver);

drivers/infiniband/hw/mana/main.c+521−0

@@ -0,0 +1,521 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	*/
	5
	6	#include "mana_ib.h"
	7
	8	void mana_ib_uncfg_vport(struct mana_ib_dev dev, struct mana_ib_pd pd,
	9	u32 port)
	10	{
	11	struct gdma_dev *gd = dev->gdma_dev;
	12	struct mana_port_context *mpc;
	13	struct net_device *ndev;
	14	struct mana_context *mc;
	15
	16	mc = gd->driver_data;
	17	ndev = mc->ports[port];
	18	mpc = netdev_priv(ndev);
	19
	20	mutex_lock(&pd->vport_mutex);
	21
	22	pd->vport_use_count--;
	23	WARN_ON(pd->vport_use_count < 0);
	24
	25	if (!pd->vport_use_count)
	26	mana_uncfg_vport(mpc);
	27
	28	mutex_unlock(&pd->vport_mutex);
	29	}
	30
	31	int mana_ib_cfg_vport(struct mana_ib_dev dev, u32 port, struct mana_ib_pd pd,
	32	u32 doorbell_id)
	33	{
	34	struct gdma_dev *mdev = dev->gdma_dev;
	35	struct mana_port_context *mpc;
	36	struct mana_context *mc;
	37	struct net_device *ndev;
	38	int err;
	39
	40	mc = mdev->driver_data;
	41	ndev = mc->ports[port];
	42	mpc = netdev_priv(ndev);
	43
	44	mutex_lock(&pd->vport_mutex);
	45
	46	pd->vport_use_count++;
	47	if (pd->vport_use_count > 1) {
	48	ibdev_dbg(&dev->ib_dev,
	49	"Skip as this PD is already configured vport\n");
	50	mutex_unlock(&pd->vport_mutex);
	51	return 0;
	52	}
	53
	54	err = mana_cfg_vport(mpc, pd->pdn, doorbell_id);
	55	if (err) {
	56	pd->vport_use_count--;
	57	mutex_unlock(&pd->vport_mutex);
	58
	59	ibdev_dbg(&dev->ib_dev, "Failed to configure vPort %d\n", err);
	60	return err;
	61	}
	62
	63	mutex_unlock(&pd->vport_mutex);
	64
	65	pd->tx_shortform_allowed = mpc->tx_shortform_allowed;
	66	pd->tx_vp_offset = mpc->tx_vp_offset;
	67
	68	ibdev_dbg(&dev->ib_dev, "vport handle %llx pdid %x doorbell_id %x\n",
	69	mpc->port_handle, pd->pdn, doorbell_id);
	70
	71	return 0;
	72	}
	73
	74	int mana_ib_alloc_pd(struct ib_pd ibpd, struct ib_udata udata)
	75	{
	76	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	77	struct ib_device *ibdev = ibpd->device;
	78	struct gdma_create_pd_resp resp = {};
	79	struct gdma_create_pd_req req = {};
	80	enum gdma_pd_flags flags = 0;
	81	struct mana_ib_dev *dev;
	82	struct gdma_dev *mdev;
	83	int err;
	84
	85	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	86	mdev = dev->gdma_dev;
	87
	88	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
	89	sizeof(resp));
	90
	91	req.flags = flags;
	92	err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
	93	sizeof(resp), &resp);
	94
	95	if (err \|\| resp.hdr.status) {
	96	ibdev_dbg(&dev->ib_dev,
	97	"Failed to get pd_id err %d status %u\n", err,
	98	resp.hdr.status);
	99	if (!err)
	100	err = -EPROTO;
	101
	102	return err;
	103	}
	104
	105	pd->pd_handle = resp.pd_handle;
	106	pd->pdn = resp.pd_id;
	107	ibdev_dbg(&dev->ib_dev, "pd_handle 0x%llx pd_id %d\n",
	108	pd->pd_handle, pd->pdn);
	109
	110	mutex_init(&pd->vport_mutex);
	111	pd->vport_use_count = 0;
	112	return 0;
	113	}
	114
	115	int mana_ib_dealloc_pd(struct ib_pd ibpd, struct ib_udata udata)
	116	{
	117	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	118	struct ib_device *ibdev = ibpd->device;
	119	struct gdma_destory_pd_resp resp = {};
	120	struct gdma_destroy_pd_req req = {};
	121	struct mana_ib_dev *dev;
	122	struct gdma_dev *mdev;
	123	int err;
	124
	125	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	126	mdev = dev->gdma_dev;
	127
	128	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
	129	sizeof(resp));
	130
	131	req.pd_handle = pd->pd_handle;
	132	err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
	133	sizeof(resp), &resp);
	134
	135	if (err \|\| resp.hdr.status) {
	136	ibdev_dbg(&dev->ib_dev,
	137	"Failed to destroy pd_handle 0x%llx err %d status %u",
	138	pd->pd_handle, err, resp.hdr.status);
	139	if (!err)
	140	err = -EPROTO;
	141	}
	142
	143	return err;
	144	}
	145
	146	static int mana_gd_destroy_doorbell_page(struct gdma_context *gc,
	147	int doorbell_page)
	148	{
	149	struct gdma_destroy_resource_range_req req = {};
	150	struct gdma_resp_hdr resp = {};
	151	int err;
	152
	153	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_RESOURCE_RANGE,
	154	sizeof(req), sizeof(resp));
	155
	156	req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;
	157	req.num_resources = 1;
	158	req.allocated_resources = doorbell_page;
	159
	160	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
	161	if (err \|\| resp.status) {
	162	dev_err(gc->dev,
	163	"Failed to destroy doorbell page: ret %d, 0x%x\n",
	164	err, resp.status);
	165	return err ?: -EPROTO;
	166	}
	167
	168	return 0;
	169	}
	170
	171	static int mana_gd_allocate_doorbell_page(struct gdma_context *gc,
	172	int *doorbell_page)
	173	{
	174	struct gdma_allocate_resource_range_req req = {};
	175	struct gdma_allocate_resource_range_resp resp = {};
	176	int err;
	177
	178	mana_gd_init_req_hdr(&req.hdr, GDMA_ALLOCATE_RESOURCE_RANGE,
	179	sizeof(req), sizeof(resp));
	180
	181	req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;
	182	req.num_resources = 1;
	183	req.alignment = 1;
	184
	185	/* Have GDMA start searching from 0 */
	186	req.allocated_resources = 0;
	187
	188	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
	189	if (err \|\| resp.hdr.status) {
	190	dev_err(gc->dev,
	191	"Failed to allocate doorbell page: ret %d, 0x%x\n",
	192	err, resp.hdr.status);
	193	return err ?: -EPROTO;
	194	}
	195
	196	*doorbell_page = resp.allocated_resources;
	197
	198	return 0;
	199	}
	200
	201	int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
	202	struct ib_udata *udata)
	203	{
	204	struct mana_ib_ucontext *ucontext =
	205	container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
	206	struct ib_device *ibdev = ibcontext->device;
	207	struct mana_ib_dev *mdev;
	208	struct gdma_context *gc;
	209	struct gdma_dev *dev;
	210	int doorbell_page;
	211	int ret;
	212
	213	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	214	dev = mdev->gdma_dev;
	215	gc = dev->gdma_context;
	216
	217	/* Allocate a doorbell page index */
	218	ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
	219	if (ret) {
	220	ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret);
	221	return ret;
	222	}
	223
	224	ibdev_dbg(ibdev, "Doorbell page allocated %d\n", doorbell_page);
	225
	226	ucontext->doorbell = doorbell_page;
	227
	228	return 0;
	229	}
	230
	231	void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
	232	{
	233	struct mana_ib_ucontext *mana_ucontext =
	234	container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
	235	struct ib_device *ibdev = ibcontext->device;
	236	struct mana_ib_dev *mdev;
	237	struct gdma_context *gc;
	238	int ret;
	239
	240	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	241	gc = mdev->gdma_dev->gdma_context;
	242
	243	ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
	244	if (ret)
	245	ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);
	246	}
	247
	248	static int
	249	mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
	250	struct gdma_context *gc,
	251	struct gdma_create_dma_region_req *create_req,
	252	size_t num_pages, mana_handle_t *gdma_region)
	253	{
	254	struct gdma_create_dma_region_resp create_resp = {};
	255	unsigned int create_req_msg_size;
	256	int err;
	257
	258	create_req_msg_size =
	259	struct_size(create_req, page_addr_list, num_pages);
	260	create_req->page_addr_list_len = num_pages;
	261
	262	err = mana_gd_send_request(gc, create_req_msg_size, create_req,
	263	sizeof(create_resp), &create_resp);
	264	if (err \|\| create_resp.hdr.status) {
	265	ibdev_dbg(&dev->ib_dev,
	266	"Failed to create DMA region: %d, 0x%x\n",
	267	err, create_resp.hdr.status);
	268	if (!err)
	269	err = -EPROTO;
	270
	271	return err;
	272	}
	273
	274	*gdma_region = create_resp.dma_region_handle;
	275	ibdev_dbg(&dev->ib_dev, "Created DMA region handle 0x%llx\n",
	276	*gdma_region);
	277
	278	return 0;
	279	}
	280
	281	static int
	282	mana_ib_gd_add_dma_region(struct mana_ib_dev dev, struct gdma_context gc,
	283	struct gdma_dma_region_add_pages_req *add_req,
	284	unsigned int num_pages, u32 expected_status)
	285	{
	286	unsigned int add_req_msg_size =
	287	struct_size(add_req, page_addr_list, num_pages);
	288	struct gdma_general_resp add_resp = {};
	289	int err;
	290
	291	mana_gd_init_req_hdr(&add_req->hdr, GDMA_DMA_REGION_ADD_PAGES,
	292	add_req_msg_size, sizeof(add_resp));
	293	add_req->page_addr_list_len = num_pages;
	294
	295	err = mana_gd_send_request(gc, add_req_msg_size, add_req,
	296	sizeof(add_resp), &add_resp);
	297	if (err \|\| add_resp.hdr.status != expected_status) {
	298	ibdev_dbg(&dev->ib_dev,
	299	"Failed to create DMA region: %d, 0x%x\n",
	300	err, add_resp.hdr.status);
	301
	302	if (!err)
	303	err = -EPROTO;
	304
	305	return err;
	306	}
	307
	308	return 0;
	309	}
	310
	311	int mana_ib_gd_create_dma_region(struct mana_ib_dev dev, struct ib_umem umem,
	312	mana_handle_t *gdma_region)
	313	{
	314	struct gdma_dma_region_add_pages_req *add_req = NULL;
	315	size_t num_pages_processed = 0, num_pages_to_handle;
	316	struct gdma_create_dma_region_req *create_req;
	317	unsigned int create_req_msg_size;
	318	struct hw_channel_context *hwc;
	319	struct ib_block_iter biter;
	320	size_t max_pgs_add_cmd = 0;
	321	size_t max_pgs_create_cmd;
	322	struct gdma_context *gc;
	323	size_t num_pages_total;
	324	struct gdma_dev *mdev;
	325	unsigned long page_sz;
	326	unsigned int tail = 0;
	327	u64 *page_addr_list;
	328	void *request_buf;
	329	int err;
	330
	331	mdev = dev->gdma_dev;
	332	gc = mdev->gdma_context;
	333	hwc = gc->hwc.driver_data;
	334
	335	/* Hardware requires dma region to align to chosen page size */
	336	page_sz = ib_umem_find_best_pgsz(umem, PAGE_SZ_BM, 0);
	337	if (!page_sz) {
	338	ibdev_dbg(&dev->ib_dev, "failed to find page size.\n");
	339	return -ENOMEM;
	340	}
	341	num_pages_total = ib_umem_num_dma_blocks(umem, page_sz);
	342
	343	max_pgs_create_cmd =
	344	(hwc->max_req_msg_size - sizeof(*create_req)) / sizeof(u64);
	345	num_pages_to_handle =
	346	min_t(size_t, num_pages_total, max_pgs_create_cmd);
	347	create_req_msg_size =
	348	struct_size(create_req, page_addr_list, num_pages_to_handle);
	349
	350	request_buf = kzalloc(hwc->max_req_msg_size, GFP_KERNEL);
	351	if (!request_buf)
	352	return -ENOMEM;
	353
	354	create_req = request_buf;
	355	mana_gd_init_req_hdr(&create_req->hdr, GDMA_CREATE_DMA_REGION,
	356	create_req_msg_size,
	357	sizeof(struct gdma_create_dma_region_resp));
	358
	359	create_req->length = umem->length;
	360	create_req->offset_in_page = umem->address & (page_sz - 1);
	361	create_req->gdma_page_type = order_base_2(page_sz) - PAGE_SHIFT;
	362	create_req->page_count = num_pages_total;
	363
	364	ibdev_dbg(&dev->ib_dev, "size_dma_region %lu num_pages_total %lu\n",
	365	umem->length, num_pages_total);
	366
	367	ibdev_dbg(&dev->ib_dev, "page_sz %lu offset_in_page %u\n",
	368	page_sz, create_req->offset_in_page);
	369
	370	ibdev_dbg(&dev->ib_dev, "num_pages_to_handle %lu, gdma_page_type %u",
	371	num_pages_to_handle, create_req->gdma_page_type);
	372
	373	page_addr_list = create_req->page_addr_list;
	374	rdma_umem_for_each_dma_block(umem, &biter, page_sz) {
	375	page_addr_list[tail++] = rdma_block_iter_dma_address(&biter);
	376	if (tail < num_pages_to_handle)
	377	continue;
	378
	379	if (!num_pages_processed) {
	380	/* First create message */
	381	err = mana_ib_gd_first_dma_region(dev, gc, create_req,
	382	tail, gdma_region);
	383	if (err)
	384	goto out;
	385
	386	max_pgs_add_cmd = (hwc->max_req_msg_size -
	387	sizeof(*add_req)) / sizeof(u64);
	388
	389	add_req = request_buf;
	390	add_req->dma_region_handle = *gdma_region;
	391	add_req->reserved3 = 0;
	392	page_addr_list = add_req->page_addr_list;
	393	} else {
	394	/* Subsequent create messages */
	395	u32 expected_s = 0;
	396
	397	if (num_pages_processed + num_pages_to_handle <
	398	num_pages_total)
	399	expected_s = GDMA_STATUS_MORE_ENTRIES;
	400
	401	err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail,
	402	expected_s);
	403	if (err)
	404	break;
	405	}
	406
	407	num_pages_processed += tail;
	408	tail = 0;
	409
	410	/* The remaining pages to create */
	411	num_pages_to_handle =
	412	min_t(size_t,
	413	num_pages_total - num_pages_processed,
	414	max_pgs_add_cmd);
	415	}
	416
	417	if (err)
	418	mana_ib_gd_destroy_dma_region(dev, *gdma_region);
	419
	420	out:
	421	kfree(request_buf);
	422	return err;
	423	}
	424
	425	int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
	426	{
	427	struct gdma_dev *mdev = dev->gdma_dev;
	428	struct gdma_context *gc;
	429
	430	gc = mdev->gdma_context;
	431	ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
	432
	433	return mana_gd_destroy_dma_region(gc, gdma_region);
	434	}
	435
	436	int mana_ib_mmap(struct ib_ucontext ibcontext, struct vm_area_struct vma)
	437	{
	438	struct mana_ib_ucontext *mana_ucontext =
	439	container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
	440	struct ib_device *ibdev = ibcontext->device;
	441	struct mana_ib_dev *mdev;
	442	struct gdma_context *gc;
	443	phys_addr_t pfn;
	444	pgprot_t prot;
	445	int ret;
	446
	447	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	448	gc = mdev->gdma_dev->gdma_context;
	449
	450	if (vma->vm_pgoff != 0) {
	451	ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
	452	return -EINVAL;
	453	}
	454
	455	/* Map to the page indexed by ucontext->doorbell */
	456	pfn = (gc->phys_db_page_base +
	457	gc->db_page_size * mana_ucontext->doorbell) >>
	458	PAGE_SHIFT;
	459	prot = pgprot_writecombine(vma->vm_page_prot);
	460
	461	ret = rdma_user_mmap_io(ibcontext, vma, pfn, gc->db_page_size, prot,
	462	NULL);
	463	if (ret)
	464	ibdev_dbg(ibdev, "can't rdma_user_mmap_io ret %d\n", ret);
	465	else
	466	ibdev_dbg(ibdev, "mapped I/O pfn 0x%llx page_size %u, ret %d\n",
	467	pfn, gc->db_page_size, ret);
	468
	469	return ret;
	470	}
	471
	472	int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
	473	struct ib_port_immutable *immutable)
	474	{
	475	/*
	476	* This version only support RAW_PACKET
	477	* other values need to be filled for other types
	478	*/
	479	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
	480
	481	return 0;
	482	}
	483
	484	int mana_ib_query_device(struct ib_device ibdev, struct ib_device_attr props,
	485	struct ib_udata *uhw)
	486	{
	487	props->max_qp = MANA_MAX_NUM_QUEUES;
	488	props->max_qp_wr = MAX_SEND_BUFFERS_PER_QUEUE;
	489
	490	/*
	491	* max_cqe could be potentially much bigger.
	492	* As this version of driver only support RAW QP, set it to the same
	493	* value as max_qp_wr
	494	*/
	495	props->max_cqe = MAX_SEND_BUFFERS_PER_QUEUE;
	496
	497	props->max_mr_size = MANA_IB_MAX_MR_SIZE;
	498	props->max_mr = MANA_IB_MAX_MR;
	499	props->max_send_sge = MAX_TX_WQE_SGL_ENTRIES;
	500	props->max_recv_sge = MAX_RX_WQE_SGL_ENTRIES;
	501
	502	return 0;
	503	}
	504
	505	int mana_ib_query_port(struct ib_device *ibdev, u32 port,
	506	struct ib_port_attr *props)
	507	{
	508	/* This version doesn't return port properties */
	509	return 0;
	510	}
	511
	512	int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
	513	union ib_gid *gid)
	514	{
	515	/* This version doesn't return GID properties */
	516	return 0;
	517	}
	518
	519	void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
	520	{
	521	}

drivers/infiniband/hw/mana/mana_ib.h+162−0

@@ -0,0 +1,162 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/*
	3	* Copyright (c) 2022 Microsoft Corporation. All rights reserved.
	4	*/
	5
	6	#ifndef _MANA_IB_H_
	7	#define _MANA_IB_H_
	8
	9	#include <rdma/ib_verbs.h>
	10	#include <rdma/ib_mad.h>
	11	#include <rdma/ib_umem.h>
	12	#include <rdma/mana-abi.h>
	13	#include <rdma/uverbs_ioctl.h>
	14
	15	#include <net/mana/mana.h>
	16
	17	#define PAGE_SZ_BM \
	18	(SZ_4K \| SZ_8K \| SZ_16K \| SZ_32K \| SZ_64K \| SZ_128K \| SZ_256K \| \
	19	SZ_512K \| SZ_1M \| SZ_2M)
	20
	21	/* MANA doesn't have any limit for MR size */
	22	#define MANA_IB_MAX_MR_SIZE U64_MAX
	23
	24	/*
	25	* The hardware limit of number of MRs is greater than maximum number of MRs
	26	* that can possibly represent in 24 bits
	27	*/
	28	#define MANA_IB_MAX_MR 0xFFFFFFu
	29
	30	struct mana_ib_dev {
	31	struct ib_device ib_dev;
	32	struct gdma_dev *gdma_dev;
	33	};
	34
	35	struct mana_ib_wq {
	36	struct ib_wq ibwq;
	37	struct ib_umem *umem;
	38	int wqe;
	39	u32 wq_buf_size;
	40	u64 gdma_region;
	41	u64 id;
	42	mana_handle_t rx_object;
	43	};
	44
	45	struct mana_ib_pd {
	46	struct ib_pd ibpd;
	47	u32 pdn;
	48	mana_handle_t pd_handle;
	49
	50	/* Mutex for sharing access to vport_use_count */
	51	struct mutex vport_mutex;
	52	int vport_use_count;
	53
	54	bool tx_shortform_allowed;
	55	u32 tx_vp_offset;
	56	};
	57
	58	struct mana_ib_mr {
	59	struct ib_mr ibmr;
	60	struct ib_umem *umem;
	61	mana_handle_t mr_handle;
	62	};
	63
	64	struct mana_ib_cq {
	65	struct ib_cq ibcq;
	66	struct ib_umem *umem;
	67	int cqe;
	68	u64 gdma_region;
	69	u64 id;
	70	};
	71
	72	struct mana_ib_qp {
	73	struct ib_qp ibqp;
	74
	75	/* Work queue info */
	76	struct ib_umem *sq_umem;
	77	int sqe;
	78	u64 sq_gdma_region;
	79	u64 sq_id;
	80	mana_handle_t tx_object;
	81
	82	/* The port on the IB device, starting with 1 */
	83	u32 port;
	84	};
	85
	86	struct mana_ib_ucontext {
	87	struct ib_ucontext ibucontext;
	88	u32 doorbell;
	89	};
	90
	91	struct mana_ib_rwq_ind_table {
	92	struct ib_rwq_ind_table ib_ind_table;
	93	};
	94
	95	int mana_ib_gd_create_dma_region(struct mana_ib_dev dev, struct ib_umem umem,
	96	mana_handle_t *gdma_region);
	97
	98	int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
	99	mana_handle_t gdma_region);
	100
	101	struct ib_wq mana_ib_create_wq(struct ib_pd pd,
	102	struct ib_wq_init_attr *init_attr,
	103	struct ib_udata *udata);
	104
	105	int mana_ib_modify_wq(struct ib_wq wq, struct ib_wq_attr wq_attr,
	106	u32 wq_attr_mask, struct ib_udata *udata);
	107
	108	int mana_ib_destroy_wq(struct ib_wq ibwq, struct ib_udata udata);
	109
	110	int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
	111	struct ib_rwq_ind_table_init_attr *init_attr,
	112	struct ib_udata *udata);
	113
	114	int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl);
	115
	116	struct ib_mr mana_ib_get_dma_mr(struct ib_pd ibpd, int access_flags);
	117
	118	struct ib_mr mana_ib_reg_user_mr(struct ib_pd pd, u64 start, u64 length,
	119	u64 iova, int access_flags,
	120	struct ib_udata *udata);
	121
	122	int mana_ib_dereg_mr(struct ib_mr ibmr, struct ib_udata udata);
	123
	124	int mana_ib_create_qp(struct ib_qp qp, struct ib_qp_init_attr qp_init_attr,
	125	struct ib_udata *udata);
	126
	127	int mana_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
	128	int attr_mask, struct ib_udata *udata);
	129
	130	int mana_ib_destroy_qp(struct ib_qp ibqp, struct ib_udata udata);
	131
	132	int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id,
	133	struct mana_ib_pd *pd, u32 doorbell_id);
	134	void mana_ib_uncfg_vport(struct mana_ib_dev dev, struct mana_ib_pd pd,
	135	u32 port);
	136
	137	int mana_ib_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
	138	struct ib_udata *udata);
	139
	140	int mana_ib_destroy_cq(struct ib_cq ibcq, struct ib_udata udata);
	141
	142	int mana_ib_alloc_pd(struct ib_pd ibpd, struct ib_udata udata);
	143	int mana_ib_dealloc_pd(struct ib_pd ibpd, struct ib_udata udata);
	144
	145	int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
	146	struct ib_udata *udata);
	147	void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);
	148
	149	int mana_ib_mmap(struct ib_ucontext ibcontext, struct vm_area_struct vma);
	150
	151	int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
	152	struct ib_port_immutable *immutable);
	153	int mana_ib_query_device(struct ib_device ibdev, struct ib_device_attr props,
	154	struct ib_udata *uhw);
	155	int mana_ib_query_port(struct ib_device *ibdev, u32 port,
	156	struct ib_port_attr *props);
	157	int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
	158	union ib_gid *gid);
	159
	160	void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);
	161
	162	#endif

drivers/infiniband/hw/mana/mr.c+197−0

@@ -0,0 +1,197 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	*/
	5
	6	#include "mana_ib.h"
	7
	8	#define VALID_MR_FLAGS \
	9	(IB_ACCESS_LOCAL_WRITE \| IB_ACCESS_REMOTE_WRITE \| IB_ACCESS_REMOTE_READ)
	10
	11	static enum gdma_mr_access_flags
	12	mana_ib_verbs_to_gdma_access_flags(int access_flags)
	13	{
	14	enum gdma_mr_access_flags flags = GDMA_ACCESS_FLAG_LOCAL_READ;
	15
	16	if (access_flags & IB_ACCESS_LOCAL_WRITE)
	17	flags \|= GDMA_ACCESS_FLAG_LOCAL_WRITE;
	18
	19	if (access_flags & IB_ACCESS_REMOTE_WRITE)
	20	flags \|= GDMA_ACCESS_FLAG_REMOTE_WRITE;
	21
	22	if (access_flags & IB_ACCESS_REMOTE_READ)
	23	flags \|= GDMA_ACCESS_FLAG_REMOTE_READ;
	24
	25	return flags;
	26	}
	27
	28	static int mana_ib_gd_create_mr(struct mana_ib_dev dev, struct mana_ib_mr mr,
	29	struct gdma_create_mr_params *mr_params)
	30	{
	31	struct gdma_create_mr_response resp = {};
	32	struct gdma_create_mr_request req = {};
	33	struct gdma_dev *mdev = dev->gdma_dev;
	34	struct gdma_context *gc;
	35	int err;
	36
	37	gc = mdev->gdma_context;
	38
	39	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
	40	sizeof(resp));
	41	req.pd_handle = mr_params->pd_handle;
	42	req.mr_type = mr_params->mr_type;
	43
	44	switch (mr_params->mr_type) {
	45	case GDMA_MR_TYPE_GVA:
	46	req.gva.dma_region_handle = mr_params->gva.dma_region_handle;
	47	req.gva.virtual_address = mr_params->gva.virtual_address;
	48	req.gva.access_flags = mr_params->gva.access_flags;
	49	break;
	50
	51	default:
	52	ibdev_dbg(&dev->ib_dev,
	53	"invalid param (GDMA_MR_TYPE) passed, type %d\n",
	54	req.mr_type);
	55	return -EINVAL;
	56	}
	57
	58	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
	59
	60	if (err \|\| resp.hdr.status) {
	61	ibdev_dbg(&dev->ib_dev, "Failed to create mr %d, %u", err,
	62	resp.hdr.status);
	63	if (!err)
	64	err = -EPROTO;
	65
	66	return err;
	67	}
	68
	69	mr->ibmr.lkey = resp.lkey;
	70	mr->ibmr.rkey = resp.rkey;
	71	mr->mr_handle = resp.mr_handle;
	72
	73	return 0;
	74	}
	75
	76	static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
	77	{
	78	struct gdma_destroy_mr_response resp = {};
	79	struct gdma_destroy_mr_request req = {};
	80	struct gdma_dev *mdev = dev->gdma_dev;
	81	struct gdma_context *gc;
	82	int err;
	83
	84	gc = mdev->gdma_context;
	85
	86	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
	87	sizeof(resp));
	88
	89	req.mr_handle = mr_handle;
	90
	91	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
	92	if (err \|\| resp.hdr.status) {
	93	dev_err(gc->dev, "Failed to destroy MR: %d, 0x%x\n", err,
	94	resp.hdr.status);
	95	if (!err)
	96	err = -EPROTO;
	97	return err;
	98	}
	99
	100	return 0;
	101	}
	102
	103	struct ib_mr mana_ib_reg_user_mr(struct ib_pd ibpd, u64 start, u64 length,
	104	u64 iova, int access_flags,
	105	struct ib_udata *udata)
	106	{
	107	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	108	struct gdma_create_mr_params mr_params = {};
	109	struct ib_device *ibdev = ibpd->device;
	110	struct mana_ib_dev *dev;
	111	struct mana_ib_mr *mr;
	112	u64 dma_region_handle;
	113	int err;
	114
	115	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	116
	117	ibdev_dbg(ibdev,
	118	"start 0x%llx, iova 0x%llx length 0x%llx access_flags 0x%x",
	119	start, iova, length, access_flags);
	120
	121	if (access_flags & ~VALID_MR_FLAGS)
	122	return ERR_PTR(-EINVAL);
	123
	124	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
	125	if (!mr)
	126	return ERR_PTR(-ENOMEM);
	127
	128	mr->umem = ib_umem_get(ibdev, start, length, access_flags);
	129	if (IS_ERR(mr->umem)) {
	130	err = PTR_ERR(mr->umem);
	131	ibdev_dbg(ibdev,
	132	"Failed to get umem for register user-mr, %d\n", err);
	133	goto err_free;
	134	}
	135
	136	err = mana_ib_gd_create_dma_region(dev, mr->umem, &dma_region_handle);
	137	if (err) {
	138	ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n",
	139	err);
	140	goto err_umem;
	141	}
	142
	143	ibdev_dbg(ibdev,
	144	"mana_ib_gd_create_dma_region ret %d gdma_region %llx\n", err,
	145	dma_region_handle);
	146
	147	mr_params.pd_handle = pd->pd_handle;
	148	mr_params.mr_type = GDMA_MR_TYPE_GVA;
	149	mr_params.gva.dma_region_handle = dma_region_handle;
	150	mr_params.gva.virtual_address = iova;
	151	mr_params.gva.access_flags =
	152	mana_ib_verbs_to_gdma_access_flags(access_flags);
	153
	154	err = mana_ib_gd_create_mr(dev, mr, &mr_params);
	155	if (err)
	156	goto err_dma_region;
	157
	158	/*
	159	* There is no need to keep track of dma_region_handle after MR is
	160	* successfully created. The dma_region_handle is tracked in the PF
	161	* as part of the lifecycle of this MR.
	162	*/
	163
	164	return &mr->ibmr;
	165
	166	err_dma_region:
	167	mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
	168	dma_region_handle);
	169
	170	err_umem:
	171	ib_umem_release(mr->umem);
	172
	173	err_free:
	174	kfree(mr);
	175	return ERR_PTR(err);
	176	}
	177
	178	int mana_ib_dereg_mr(struct ib_mr ibmr, struct ib_udata udata)
	179	{
	180	struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);
	181	struct ib_device *ibdev = ibmr->device;
	182	struct mana_ib_dev *dev;
	183	int err;
	184
	185	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
	186
	187	err = mana_ib_gd_destroy_mr(dev, mr->mr_handle);
	188	if (err)
	189	return err;
	190
	191	if (mr->umem)
	192	ib_umem_release(mr->umem);
	193
	194	kfree(mr);
	195
	196	return 0;
	197	}

drivers/infiniband/hw/mana/qp.c+506−0

@@ -0,0 +1,506 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	*/
	5
	6	#include "mana_ib.h"
	7
	8	static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
	9	struct net_device *ndev,
	10	mana_handle_t default_rxobj,
	11	mana_handle_t ind_table[],
	12	u32 log_ind_tbl_size, u32 rx_hash_key_len,
	13	u8 *rx_hash_key)
	14	{
	15	struct mana_port_context *mpc = netdev_priv(ndev);
	16	struct mana_cfg_rx_steer_req *req = NULL;
	17	struct mana_cfg_rx_steer_resp resp = {};
	18	mana_handle_t *req_indir_tab;
	19	struct gdma_context *gc;
	20	struct gdma_dev *mdev;
	21	u32 req_buf_size;
	22	int i, err;
	23
	24	mdev = dev->gdma_dev;
	25	gc = mdev->gdma_context;
	26
	27	req_buf_size =
	28	sizeof(req) + sizeof(mana_handle_t) MANA_INDIRECT_TABLE_SIZE;
	29	req = kzalloc(req_buf_size, GFP_KERNEL);
	30	if (!req)
	31	return -ENOMEM;
	32
	33	mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
	34	sizeof(resp));
	35
	36	req->vport = mpc->port_handle;
	37	req->rx_enable = 1;
	38	req->update_default_rxobj = 1;
	39	req->default_rxobj = default_rxobj;
	40	req->hdr.dev_id = mdev->dev_id;
	41
	42	/* If there are more than 1 entries in indirection table, enable RSS */
	43	if (log_ind_tbl_size)
	44	req->rss_enable = true;
	45
	46	req->num_indir_entries = MANA_INDIRECT_TABLE_SIZE;
	47	req->indir_tab_offset = sizeof(*req);
	48	req->update_indir_tab = true;
	49
	50	req_indir_tab = (mana_handle_t *)(req + 1);
	51	/* The ind table passed to the hardware must have
	52	* MANA_INDIRECT_TABLE_SIZE entries. Adjust the verb
	53	* ind_table to MANA_INDIRECT_TABLE_SIZE if required
	54	*/
	55	ibdev_dbg(&dev->ib_dev, "ind table size %u\n", 1 << log_ind_tbl_size);
	56	for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) {
	57	req_indir_tab[i] = ind_table[i % (1 << log_ind_tbl_size)];
	58	ibdev_dbg(&dev->ib_dev, "index %u handle 0x%llx\n", i,
	59	req_indir_tab[i]);
	60	}
	61
	62	req->update_hashkey = true;
	63	if (rx_hash_key_len)
	64	memcpy(req->hashkey, rx_hash_key, rx_hash_key_len);
	65	else
	66	netdev_rss_key_fill(req->hashkey, MANA_HASH_KEY_SIZE);
	67
	68	ibdev_dbg(&dev->ib_dev, "vport handle %llu default_rxobj 0x%llx\n",
	69	req->vport, default_rxobj);
	70
	71	err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &resp);
	72	if (err) {
	73	netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);
	74	goto out;
	75	}
	76
	77	if (resp.hdr.status) {
	78	netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
	79	resp.hdr.status);
	80	err = -EPROTO;
	81	goto out;
	82	}
	83
	84	netdev_info(ndev, "Configured steering vPort %llu log_entries %u\n",
	85	mpc->port_handle, log_ind_tbl_size);
	86
	87	out:
	88	kfree(req);
	89	return err;
	90	}
	91
	92	static int mana_ib_create_qp_rss(struct ib_qp ibqp, struct ib_pd pd,
	93	struct ib_qp_init_attr *attr,
	94	struct ib_udata *udata)
	95	{
	96	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
	97	struct mana_ib_dev *mdev =
	98	container_of(pd->device, struct mana_ib_dev, ib_dev);
	99	struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl;
	100	struct mana_ib_create_qp_rss_resp resp = {};
	101	struct mana_ib_create_qp_rss ucmd = {};
	102	struct gdma_dev *gd = mdev->gdma_dev;
	103	mana_handle_t *mana_ind_table;
	104	struct mana_port_context *mpc;
	105	struct mana_context *mc;
	106	struct net_device *ndev;
	107	struct mana_ib_cq *cq;
	108	struct mana_ib_wq *wq;
	109	unsigned int ind_tbl_size;
	110	struct ib_cq *ibcq;
	111	struct ib_wq *ibwq;
	112	int i = 0;
	113	u32 port;
	114	int ret;
	115
	116	mc = gd->driver_data;
	117
	118	if (!udata \|\| udata->inlen < sizeof(ucmd))
	119	return -EINVAL;
	120
	121	ret = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
	122	if (ret) {
	123	ibdev_dbg(&mdev->ib_dev,
	124	"Failed copy from udata for create rss-qp, err %d\n",
	125	ret);
	126	return ret;
	127	}
	128
	129	if (attr->cap.max_recv_wr > MAX_SEND_BUFFERS_PER_QUEUE) {
	130	ibdev_dbg(&mdev->ib_dev,
	131	"Requested max_recv_wr %d exceeding limit\n",
	132	attr->cap.max_recv_wr);
	133	return -EINVAL;
	134	}
	135
	136	if (attr->cap.max_recv_sge > MAX_RX_WQE_SGL_ENTRIES) {
	137	ibdev_dbg(&mdev->ib_dev,
	138	"Requested max_recv_sge %d exceeding limit\n",
	139	attr->cap.max_recv_sge);
	140	return -EINVAL;
	141	}
	142
	143	ind_tbl_size = 1 << ind_tbl->log_ind_tbl_size;
	144	if (ind_tbl_size > MANA_INDIRECT_TABLE_SIZE) {
	145	ibdev_dbg(&mdev->ib_dev,
	146	"Indirect table size %d exceeding limit\n",
	147	ind_tbl_size);
	148	return -EINVAL;
	149	}
	150
	151	if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) {
	152	ibdev_dbg(&mdev->ib_dev,
	153	"RX Hash function is not supported, %d\n",
	154	ucmd.rx_hash_function);
	155	return -EINVAL;
	156	}
	157
	158	/* IB ports start with 1, MANA start with 0 */
	159	port = ucmd.port;
	160	if (port < 1 \|\| port > mc->num_ports) {
	161	ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n",
	162	port);
	163	return -EINVAL;
	164	}
	165	ndev = mc->ports[port - 1];
	166	mpc = netdev_priv(ndev);
	167
	168	ibdev_dbg(&mdev->ib_dev, "rx_hash_function %d port %d\n",
	169	ucmd.rx_hash_function, port);
	170
	171	mana_ind_table = kcalloc(ind_tbl_size, sizeof(mana_handle_t),
	172	GFP_KERNEL);
	173	if (!mana_ind_table) {
	174	ret = -ENOMEM;
	175	goto fail;
	176	}
	177
	178	qp->port = port;
	179
	180	for (i = 0; i < ind_tbl_size; i++) {
	181	struct mana_obj_spec wq_spec = {};
	182	struct mana_obj_spec cq_spec = {};
	183
	184	ibwq = ind_tbl->ind_tbl[i];
	185	wq = container_of(ibwq, struct mana_ib_wq, ibwq);
	186
	187	ibcq = ibwq->cq;
	188	cq = container_of(ibcq, struct mana_ib_cq, ibcq);
	189
	190	wq_spec.gdma_region = wq->gdma_region;
	191	wq_spec.queue_size = wq->wq_buf_size;
	192
	193	cq_spec.gdma_region = cq->gdma_region;
	194	cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE;
	195	cq_spec.modr_ctx_id = 0;
	196	cq_spec.attached_eq = GDMA_CQ_NO_EQ;
	197
	198	ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ,
	199	&wq_spec, &cq_spec, &wq->rx_object);
	200	if (ret)
	201	goto fail;
	202
	203	/* The GDMA regions are now owned by the WQ object */
	204	wq->gdma_region = GDMA_INVALID_DMA_REGION;
	205	cq->gdma_region = GDMA_INVALID_DMA_REGION;
	206
	207	wq->id = wq_spec.queue_index;
	208	cq->id = cq_spec.queue_index;
	209
	210	ibdev_dbg(&mdev->ib_dev,
	211	"ret %d rx_object 0x%llx wq id %llu cq id %llu\n",
	212	ret, wq->rx_object, wq->id, cq->id);
	213
	214	resp.entries[i].cqid = cq->id;
	215	resp.entries[i].wqid = wq->id;
	216
	217	mana_ind_table[i] = wq->rx_object;
	218	}
	219	resp.num_entries = i;
	220
	221	ret = mana_ib_cfg_vport_steering(mdev, ndev, wq->rx_object,
	222	mana_ind_table,
	223	ind_tbl->log_ind_tbl_size,
	224	ucmd.rx_hash_key_len,
	225	ucmd.rx_hash_key);
	226	if (ret)
	227	goto fail;
	228
	229	ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
	230	if (ret) {
	231	ibdev_dbg(&mdev->ib_dev,
	232	"Failed to copy to udata create rss-qp, %d\n",
	233	ret);
	234	goto fail;
	235	}
	236
	237	kfree(mana_ind_table);
	238
	239	return 0;
	240
	241	fail:
	242	while (i-- > 0) {
	243	ibwq = ind_tbl->ind_tbl[i];
	244	wq = container_of(ibwq, struct mana_ib_wq, ibwq);
	245	mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
	246	}
	247
	248	kfree(mana_ind_table);
	249
	250	return ret;
	251	}
	252
	253	static int mana_ib_create_qp_raw(struct ib_qp ibqp, struct ib_pd ibpd,
	254	struct ib_qp_init_attr *attr,
	255	struct ib_udata *udata)
	256	{
	257	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	258	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
	259	struct mana_ib_dev *mdev =
	260	container_of(ibpd->device, struct mana_ib_dev, ib_dev);
	261	struct mana_ib_cq *send_cq =
	262	container_of(attr->send_cq, struct mana_ib_cq, ibcq);
	263	struct mana_ib_ucontext *mana_ucontext =
	264	rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
	265	ibucontext);
	266	struct mana_ib_create_qp_resp resp = {};
	267	struct gdma_dev *gd = mdev->gdma_dev;
	268	struct mana_ib_create_qp ucmd = {};
	269	struct mana_obj_spec wq_spec = {};
	270	struct mana_obj_spec cq_spec = {};
	271	struct mana_port_context *mpc;
	272	struct mana_context *mc;
	273	struct net_device *ndev;
	274	struct ib_umem *umem;
	275	int err;
	276	u32 port;
	277
	278	mc = gd->driver_data;
	279
	280	if (!mana_ucontext \|\| udata->inlen < sizeof(ucmd))
	281	return -EINVAL;
	282
	283	err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
	284	if (err) {
	285	ibdev_dbg(&mdev->ib_dev,
	286	"Failed to copy from udata create qp-raw, %d\n", err);
	287	return err;
	288	}
	289
	290	/* IB ports start with 1, MANA Ethernet ports start with 0 */
	291	port = ucmd.port;
	292	if (ucmd.port > mc->num_ports)
	293	return -EINVAL;
	294
	295	if (attr->cap.max_send_wr > MAX_SEND_BUFFERS_PER_QUEUE) {
	296	ibdev_dbg(&mdev->ib_dev,
	297	"Requested max_send_wr %d exceeding limit\n",
	298	attr->cap.max_send_wr);
	299	return -EINVAL;
	300	}
	301
	302	if (attr->cap.max_send_sge > MAX_TX_WQE_SGL_ENTRIES) {
	303	ibdev_dbg(&mdev->ib_dev,
	304	"Requested max_send_sge %d exceeding limit\n",
	305	attr->cap.max_send_sge);
	306	return -EINVAL;
	307	}
	308
	309	ndev = mc->ports[port - 1];
	310	mpc = netdev_priv(ndev);
	311	ibdev_dbg(&mdev->ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc);
	312
	313	err = mana_ib_cfg_vport(mdev, port - 1, pd, mana_ucontext->doorbell);
	314	if (err)
	315	return -ENODEV;
	316
	317	qp->port = port;
	318
	319	ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",
	320	ucmd.sq_buf_addr, ucmd.port);
	321
	322	umem = ib_umem_get(ibpd->device, ucmd.sq_buf_addr, ucmd.sq_buf_size,
	323	IB_ACCESS_LOCAL_WRITE);
	324	if (IS_ERR(umem)) {
	325	err = PTR_ERR(umem);
	326	ibdev_dbg(&mdev->ib_dev,
	327	"Failed to get umem for create qp-raw, err %d\n",
	328	err);
	329	goto err_free_vport;
	330	}
	331	qp->sq_umem = umem;
	332
	333	err = mana_ib_gd_create_dma_region(mdev, qp->sq_umem,
	334	&qp->sq_gdma_region);
	335	if (err) {
	336	ibdev_dbg(&mdev->ib_dev,
	337	"Failed to create dma region for create qp-raw, %d\n",
	338	err);
	339	goto err_release_umem;
	340	}
	341
	342	ibdev_dbg(&mdev->ib_dev,
	343	"mana_ib_gd_create_dma_region ret %d gdma_region 0x%llx\n",
	344	err, qp->sq_gdma_region);
	345
	346	/* Create a WQ on the same port handle used by the Ethernet */
	347	wq_spec.gdma_region = qp->sq_gdma_region;
	348	wq_spec.queue_size = ucmd.sq_buf_size;
	349
	350	cq_spec.gdma_region = send_cq->gdma_region;
	351	cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE;
	352	cq_spec.modr_ctx_id = 0;
	353	cq_spec.attached_eq = GDMA_CQ_NO_EQ;
	354
	355	err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec,
	356	&cq_spec, &qp->tx_object);
	357	if (err) {
	358	ibdev_dbg(&mdev->ib_dev,
	359	"Failed to create wq for create raw-qp, err %d\n",
	360	err);
	361	goto err_destroy_dma_region;
	362	}
	363
	364	/* The GDMA regions are now owned by the WQ object */
	365	qp->sq_gdma_region = GDMA_INVALID_DMA_REGION;
	366	send_cq->gdma_region = GDMA_INVALID_DMA_REGION;
	367
	368	qp->sq_id = wq_spec.queue_index;
	369	send_cq->id = cq_spec.queue_index;
	370
	371	ibdev_dbg(&mdev->ib_dev,
	372	"ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
	373	qp->tx_object, qp->sq_id, send_cq->id);
	374
	375	resp.sqid = qp->sq_id;
	376	resp.cqid = send_cq->id;
	377	resp.tx_vp_offset = pd->tx_vp_offset;
	378
	379	err = ib_copy_to_udata(udata, &resp, sizeof(resp));
	380	if (err) {
	381	ibdev_dbg(&mdev->ib_dev,
	382	"Failed copy udata for create qp-raw, %d\n",
	383	err);
	384	goto err_destroy_wq_obj;
	385	}
	386
	387	return 0;
	388
	389	err_destroy_wq_obj:
	390	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
	391
	392	err_destroy_dma_region:
	393	mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region);
	394
	395	err_release_umem:
	396	ib_umem_release(umem);
	397
	398	err_free_vport:
	399	mana_ib_uncfg_vport(mdev, pd, port - 1);
	400
	401	return err;
	402	}
	403
	404	int mana_ib_create_qp(struct ib_qp ibqp, struct ib_qp_init_attr attr,
	405	struct ib_udata *udata)
	406	{
	407	switch (attr->qp_type) {
	408	case IB_QPT_RAW_PACKET:
	409	/* When rwq_ind_tbl is used, it's for creating WQs for RSS */
	410	if (attr->rwq_ind_tbl)
	411	return mana_ib_create_qp_rss(ibqp, ibqp->pd, attr,
	412	udata);
	413
	414	return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
	415	default:
	416	/* Creating QP other than IB_QPT_RAW_PACKET is not supported */
	417	ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n",
	418	attr->qp_type);
	419	}
	420
	421	return -EINVAL;
	422	}
	423
	424	int mana_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
	425	int attr_mask, struct ib_udata *udata)
	426	{
	427	/* modify_qp is not supported by this version of the driver */
	428	return -EOPNOTSUPP;
	429	}
	430
	431	static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
	432	struct ib_rwq_ind_table *ind_tbl,
	433	struct ib_udata *udata)
	434	{
	435	struct mana_ib_dev *mdev =
	436	container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
	437	struct gdma_dev *gd = mdev->gdma_dev;
	438	struct mana_port_context *mpc;
	439	struct mana_context *mc;
	440	struct net_device *ndev;
	441	struct mana_ib_wq *wq;
	442	struct ib_wq *ibwq;
	443	int i;
	444
	445	mc = gd->driver_data;
	446	ndev = mc->ports[qp->port - 1];
	447	mpc = netdev_priv(ndev);
	448
	449	for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
	450	ibwq = ind_tbl->ind_tbl[i];
	451	wq = container_of(ibwq, struct mana_ib_wq, ibwq);
	452	ibdev_dbg(&mdev->ib_dev, "destroying wq->rx_object %llu\n",
	453	wq->rx_object);
	454	mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
	455	}
	456
	457	return 0;
	458	}
	459
	460	static int mana_ib_destroy_qp_raw(struct mana_ib_qp qp, struct ib_udata udata)
	461	{
	462	struct mana_ib_dev *mdev =
	463	container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
	464	struct gdma_dev *gd = mdev->gdma_dev;
	465	struct ib_pd *ibpd = qp->ibqp.pd;
	466	struct mana_port_context *mpc;
	467	struct mana_context *mc;
	468	struct net_device *ndev;
	469	struct mana_ib_pd *pd;
	470
	471	mc = gd->driver_data;
	472	ndev = mc->ports[qp->port - 1];
	473	mpc = netdev_priv(ndev);
	474	pd = container_of(ibpd, struct mana_ib_pd, ibpd);
	475
	476	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
	477
	478	if (qp->sq_umem) {
	479	mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region);
	480	ib_umem_release(qp->sq_umem);
	481	}
	482
	483	mana_ib_uncfg_vport(mdev, pd, qp->port - 1);
	484
	485	return 0;
	486	}
	487
	488	int mana_ib_destroy_qp(struct ib_qp ibqp, struct ib_udata udata)
	489	{
	490	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
	491
	492	switch (ibqp->qp_type) {
	493	case IB_QPT_RAW_PACKET:
	494	if (ibqp->rwq_ind_tbl)
	495	return mana_ib_destroy_qp_rss(qp, ibqp->rwq_ind_tbl,
	496	udata);
	497
	498	return mana_ib_destroy_qp_raw(qp, udata);
	499
	500	default:
	501	ibdev_dbg(ibqp->device, "Unexpected QP type %u\n",
	502	ibqp->qp_type);
	503	}
	504
	505	return -ENOENT;
	506	}

drivers/infiniband/hw/mana/wq.c+115−0

@@ -0,0 +1,115 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	*/
	5
	6	#include "mana_ib.h"
	7
	8	struct ib_wq mana_ib_create_wq(struct ib_pd pd,
	9	struct ib_wq_init_attr *init_attr,
	10	struct ib_udata *udata)
	11	{
	12	struct mana_ib_dev *mdev =
	13	container_of(pd->device, struct mana_ib_dev, ib_dev);
	14	struct mana_ib_create_wq ucmd = {};
	15	struct mana_ib_wq *wq;
	16	struct ib_umem *umem;
	17	int err;
	18
	19	if (udata->inlen < sizeof(ucmd))
	20	return ERR_PTR(-EINVAL);
	21
	22	err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
	23	if (err) {
	24	ibdev_dbg(&mdev->ib_dev,
	25	"Failed to copy from udata for create wq, %d\n", err);
	26	return ERR_PTR(err);
	27	}
	28
	29	wq = kzalloc(sizeof(*wq), GFP_KERNEL);
	30	if (!wq)
	31	return ERR_PTR(-ENOMEM);
	32
	33	ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr);
	34
	35	umem = ib_umem_get(pd->device, ucmd.wq_buf_addr, ucmd.wq_buf_size,
	36	IB_ACCESS_LOCAL_WRITE);
	37	if (IS_ERR(umem)) {
	38	err = PTR_ERR(umem);
	39	ibdev_dbg(&mdev->ib_dev,
	40	"Failed to get umem for create wq, err %d\n", err);
	41	goto err_free_wq;
	42	}
	43
	44	wq->umem = umem;
	45	wq->wqe = init_attr->max_wr;
	46	wq->wq_buf_size = ucmd.wq_buf_size;
	47	wq->rx_object = INVALID_MANA_HANDLE;
	48
	49	err = mana_ib_gd_create_dma_region(mdev, wq->umem, &wq->gdma_region);
	50	if (err) {
	51	ibdev_dbg(&mdev->ib_dev,
	52	"Failed to create dma region for create wq, %d\n",
	53	err);
	54	goto err_release_umem;
	55	}
	56
	57	ibdev_dbg(&mdev->ib_dev,
	58	"mana_ib_gd_create_dma_region ret %d gdma_region 0x%llx\n",
	59	err, wq->gdma_region);
	60
	61	/* WQ ID is returned at wq_create time, doesn't know the value yet */
	62
	63	return &wq->ibwq;
	64
	65	err_release_umem:
	66	ib_umem_release(umem);
	67
	68	err_free_wq:
	69	kfree(wq);
	70
	71	return ERR_PTR(err);
	72	}
	73
	74	int mana_ib_modify_wq(struct ib_wq wq, struct ib_wq_attr wq_attr,
	75	u32 wq_attr_mask, struct ib_udata *udata)
	76	{
	77	/* modify_wq is not supported by this version of the driver */
	78	return -EOPNOTSUPP;
	79	}
	80
	81	int mana_ib_destroy_wq(struct ib_wq ibwq, struct ib_udata udata)
	82	{
	83	struct mana_ib_wq *wq = container_of(ibwq, struct mana_ib_wq, ibwq);
	84	struct ib_device *ib_dev = ibwq->device;
	85	struct mana_ib_dev *mdev;
	86
	87	mdev = container_of(ib_dev, struct mana_ib_dev, ib_dev);
	88
	89	mana_ib_gd_destroy_dma_region(mdev, wq->gdma_region);
	90	ib_umem_release(wq->umem);
	91
	92	kfree(wq);
	93
	94	return 0;
	95	}
	96
	97	int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
	98	struct ib_rwq_ind_table_init_attr *init_attr,
	99	struct ib_udata *udata)
	100	{
	101	/*
	102	* There is no additional data in ind_table to be maintained by this
	103	* driver, do nothing
	104	*/
	105	return 0;
	106	}
	107
	108	int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
	109	{
	110	/*
	111	* There is no additional data in ind_table to be maintained by this
	112	* driver, do nothing
	113	*/
	114	return 0;
	115	}

drivers/infiniband/hw/mlx4/main.c+4−8

 			}
 		}
 	}
-	if (dev)
+	dev_hold(dev);
-		dev_hold(dev);
 	rcu_read_unlock();
 	return dev;
 	spin_lock_bh(&mdev->iboe.lock);
 	ndev = mdev->iboe.netdevs[mqp->port - 1];
-	if (ndev)
+	dev_hold(ndev);
-		dev_hold(ndev);
 	spin_unlock_bh(&mdev->iboe.lock);
 	if (ndev) {
 	if (ge) {
 		spin_lock_bh(&mdev->iboe.lock);
 		ndev = ge->added ? mdev->iboe.netdevs[ge->port - 1] : NULL;
-		if (ndev)
+		dev_hold(ndev);
-			dev_hold(ndev);
 		spin_unlock_bh(&mdev->iboe.lock);
-		if (ndev)
+		dev_put(ndev);
-			dev_put(ndev);
 		list_del(&ge->list);
 		kfree(ge);
 	} else

drivers/infiniband/hw/mlx5/cq.c+15−12

 	wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
 }
-static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe)
+static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe,
+		     struct ib_wc *wc, const char *level)
 {
-	mlx5_ib_warn(dev, "dump error cqe\n");
+	mlx5_ib_log(level, dev, "WC error: %d, Message: %s\n", wc->status,
-	mlx5_dump_err_cqe(dev->mdev, cqe);
+		    ib_wc_status_msg(wc->status));
+	print_hex_dump(level, "cqe_dump: ", DUMP_PREFIX_OFFSET, 16, 1,
+		       cqe, sizeof(*cqe), false);
 }
 static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,
 				  struct mlx5_err_cqe *cqe,
 				  struct ib_wc *wc)
 {
-	int dump = 1;
+	const char *dump = KERN_WARNING;
 	switch (cqe->syndrome) {
 	case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR:
 		wc->status = IB_WC_LOC_QP_OP_ERR;
 		break;
 	case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR:
+		dump = KERN_DEBUG;
 		wc->status = IB_WC_LOC_PROT_ERR;
 		break;
 	case MLX5_CQE_SYNDROME_WR_FLUSH_ERR:
-		dump = 0;
+		dump = NULL;
 		wc->status = IB_WC_WR_FLUSH_ERR;
 		break;
 	case MLX5_CQE_SYNDROME_MW_BIND_ERR:
 		wc->status = IB_WC_REM_INV_REQ_ERR;
 		break;
 	case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR:
+		dump = KERN_DEBUG;
 		wc->status = IB_WC_REM_ACCESS_ERR;
 		break;
 	case MLX5_CQE_SYNDROME_REMOTE_OP_ERR:
+		dump = KERN_DEBUG;
 		wc->status = IB_WC_REM_OP_ERR;
 		break;
 	case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
+		dump = NULL;
 		wc->status = IB_WC_RETRY_EXC_ERR;
-		dump = 0;
 		break;
 	case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
+		dump = NULL;
 		wc->status = IB_WC_RNR_RETRY_EXC_ERR;
-		dump = 0;
 		break;
 	case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR:
 		wc->status = IB_WC_REM_ABORT_ERR;
 	}
 	wc->vendor_err = cqe->vendor_err_synd;
-	if (dump) {
+	if (dump)
-		mlx5_ib_warn(dev, "WC error: %d, Message: %s\n", wc->status,
+		dump_cqe(dev, cqe, wc, dump);
-			     ib_wc_status_msg(wc->status));
-		dump_cqe(dev, cqe);
-	}
 }
 static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,

drivers/infiniband/hw/mlx5/fs.c+0−1

@@ -127,7 +127,6 @@ static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) · 126 unmodified lines
127	}	127	}
128		128
129	#define LAST_ETH_FIELD vlan_tag	129	#define LAST_ETH_FIELD vlan_tag
130	#define LAST_IB_FIELD sl
131	#define LAST_IPV4_FIELD tos	130	#define LAST_IPV4_FIELD tos
132	#define LAST_IPV6_FIELD traffic_class	131	#define LAST_IPV6_FIELD traffic_class
133	#define LAST_TCP_UDP_FIELD src_port	132	#define LAST_TCP_UDP_FIELD src_port

drivers/infiniband/hw/mlx5/mlx5_ib.h+4−0

@@ -38,6 +38,10 @@ · 37 unmodified lines
38	dev_warn(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__, \	38	dev_warn(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__, \
39	__LINE__, current->pid, ##arg)	39	__LINE__, current->pid, ##arg)
40		40
		41	#define mlx5_ib_log(lvl, _dev, format, arg...) \
		42	dev_printk(lvl, &(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, \
		43	__func__, __LINE__, current->pid, ##arg)
		44
41	#define MLX5_IB_DEFAULT_UIDX 0xffffff	45	#define MLX5_IB_DEFAULT_UIDX 0xffffff
42	#define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)	46	#define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
43		47

drivers/infiniband/hw/mlx5/mr.c+2−4

@@ -1929,10 +1929,8 @@ int mlx5_ib_alloc_mw(struct ib_mw ibmw, struct ib_udata udata) · 1928 unmodified lines
1929	ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);	1929	ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
1930		1930
1931	in = kzalloc(inlen, GFP_KERNEL);	1931	in = kzalloc(inlen, GFP_KERNEL);
1932	if (!in) {	1932	if (!in)
1933	err = -ENOMEM;	1933	return -ENOMEM;
1934	goto free;
1935	}
1936		1934
1937	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);	1935	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1938		1936

drivers/infiniband/hw/qedr/main.c+1−1

@@ -479,7 +479,7 @@ static irqreturn_t qedr_irq_handler(int irq, void *handle) · 478 unmodified lines
479	/* The CQ's CNQ notification counter is checked before	479	/* The CQ's CNQ notification counter is checked before
480	* destroying the CQ in a busy-wait loop that waits for all of	480	* destroying the CQ in a busy-wait loop that waits for all of
481	* the CQ's CNQ interrupts to be processed. It is increased	481	* the CQ's CNQ interrupts to be processed. It is increased
482	* here, only after the completion handler, to ensure that the	482	* here, only after the completion handler, to ensure that
483	* the handler is not running when the CQ is destroyed.	483	* the handler is not running when the CQ is destroyed.
484	*/	484	*/
485	cq->cnq_notif++;	485	cq->cnq_notif++;

drivers/infiniband/hw/qib/qib_iba6120.c+2−5

 			hwerrs &= ~TXE_PIO_PARITY;
 		}
-		if (!hwerrs) {
+		if (!hwerrs)
-			static u32 freeze_cnt;
-			freeze_cnt++;
 			qib_6120_clear_freeze(dd);
-		} else
+		else
 			isfatal = 1;
 	}

drivers/infiniband/hw/qib/qib_tx.c+1−4

 	struct qib_devdata *dd = rcd->dd;
 	unsigned i;
 	unsigned last;
-	unsigned n = 0;
 	last = rcd->pio_base + rcd->piocnt;
 	/*
 	}
 	spin_lock_irq(&dd->pioavail_lock);
 	for (i = rcd->pio_base; i < last; i++) {
-		if (__test_and_clear_bit(i, dd->pio_need_disarm)) {
+		if (__test_and_clear_bit(i, dd->pio_need_disarm))
-			n++;
 			dd->f_sendctrl(rcd->ppd, QIB_SENDCTRL_DISARM_BUF(i));
-		}
 	}
 	spin_unlock_irq(&dd->pioavail_lock);
 	return 0;

drivers/infiniband/hw/qib/qib_user_sdma.c+1−1

@@ -851,7 +851,7 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd, · 850 unmodified lines
851	}	851	}
852		852
853	/*	853	/*
854	* This assignment is a bit strange. it's because the	854	* This assignment is a bit strange. it's because
855	* the pbc counts the number of 32 bit words in the full	855	* the pbc counts the number of 32 bit words in the full
856	* packet _except_ the first word of the pbc itself...	856	* packet _except_ the first word of the pbc itself...
857	*/	857	*/

drivers/infiniband/sw/rxe/rxe.c+2−2

@@ -187,14 +187,14 @@ static int rxe_newlink(const char ibdev_name, struct net_device ndev) · 186 unmodified lines
187	exists = rxe_get_dev_from_net(ndev);	187	exists = rxe_get_dev_from_net(ndev);
188	if (exists) {	188	if (exists) {
189	ib_device_put(&exists->ib_dev);	189	ib_device_put(&exists->ib_dev);
190	pr_err("already configured on %s\n", ndev->name);	190	rxe_dbg(exists, "already configured on %s\n", ndev->name);
191	err = -EEXIST;	191	err = -EEXIST;
192	goto err;	192	goto err;
193	}	193	}
194		194
195	err = rxe_net_add(ibdev_name, ndev);	195	err = rxe_net_add(ibdev_name, ndev);
196	if (err) {	196	if (err) {
197	pr_err("failed to add %s\n", ndev->name);	197	rxe_dbg(exists, "failed to add %s\n", ndev->name);
198	goto err;	198	goto err;
199	}	199	}
200	err:	200	err:

drivers/infiniband/sw/rxe/rxe.h+19−0

@@ -38,6 +38,25 @@ · 37 unmodified lines
38		38
39	#define RXE_ROCE_V2_SPORT (0xc000)	39	#define RXE_ROCE_V2_SPORT (0xc000)
40		40
		41	#define rxe_dbg(rxe, fmt, ...) ibdev_dbg(&(rxe)->ib_dev, \
		42	"%s: " fmt, __func__, ##__VA_ARGS__)
		43	#define rxe_dbg_uc(uc, fmt, ...) ibdev_dbg((uc)->ibuc.device, \
		44	"uc#%d %s: " fmt, (uc)->elem.index, __func__, ##__VA_ARGS__)
		45	#define rxe_dbg_pd(pd, fmt, ...) ibdev_dbg((pd)->ibpd.device, \
		46	"pd#%d %s: " fmt, (pd)->elem.index, __func__, ##__VA_ARGS__)
		47	#define rxe_dbg_ah(ah, fmt, ...) ibdev_dbg((ah)->ibah.device, \
		48	"ah#%d %s: " fmt, (ah)->elem.index, __func__, ##__VA_ARGS__)
		49	#define rxe_dbg_srq(srq, fmt, ...) ibdev_dbg((srq)->ibsrq.device, \
		50	"srq#%d %s: " fmt, (srq)->elem.index, __func__, ##__VA_ARGS__)
		51	#define rxe_dbg_qp(qp, fmt, ...) ibdev_dbg((qp)->ibqp.device, \
		52	"qp#%d %s: " fmt, (qp)->elem.index, __func__, ##__VA_ARGS__)
		53	#define rxe_dbg_cq(cq, fmt, ...) ibdev_dbg((cq)->ibcq.device, \
		54	"cq#%d %s: " fmt, (cq)->elem.index, __func__, ##__VA_ARGS__)
		55	#define rxe_dbg_mr(mr, fmt, ...) ibdev_dbg((mr)->ibmr.device, \
		56	"mr#%d %s: " fmt, (mr)->elem.index, __func__, ##__VA_ARGS__)
		57	#define rxe_dbg_mw(mw, fmt, ...) ibdev_dbg((mw)->ibmw.device, \
		58	"mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
		59
41	void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);	60	void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
42		61
43	int rxe_add(struct rxe_dev rxe, unsigned int mtu, const char ibdev_name);	62	int rxe_add(struct rxe_dev rxe, unsigned int mtu, const char ibdev_name);

drivers/infiniband/sw/rxe/rxe_av.c+36−7

 	memcpy(av->dmac, attr->roce.dmac, ETH_ALEN);
 }
-int rxe_av_chk_attr(struct rxe_dev *rxe, struct rdma_ah_attr *attr)
+static int chk_attr(void *obj, struct rdma_ah_attr *attr, bool obj_is_ah)
 {
 	const struct ib_global_route *grh = rdma_ah_read_grh(attr);
 	struct rxe_port *port;
+	struct rxe_dev *rxe;
+	struct rxe_qp *qp;
+	struct rxe_ah *ah;
 	int type;
+	if (obj_is_ah) {
+		ah = obj;
+		rxe = to_rdev(ah->ibah.device);
+	} else {
+		qp = obj;
+		rxe = to_rdev(qp->ibqp.device);
+	}
 	port = &rxe->port;
 	if (rdma_ah_get_ah_flags(attr) & IB_AH_GRH) {
 		if (grh->sgid_index > port->attr.gid_tbl_len) {
-			pr_warn("invalid sgid index = %d\n",
+			if (obj_is_ah)
-					grh->sgid_index);
+				rxe_dbg_ah(ah, "invalid sgid index = %d\n",
+						grh->sgid_index);
+			else
+				rxe_dbg_qp(qp, "invalid sgid index = %d\n",
+						grh->sgid_index);
 			return -EINVAL;
 		}
 		type = rdma_gid_attr_network_type(grh->sgid_attr);
 		if (type < RDMA_NETWORK_IPV4 ||
 		    type > RDMA_NETWORK_IPV6) {
-			pr_warn("invalid network type for rdma_rxe = %d\n",
+			if (obj_is_ah)
-					type);
+				rxe_dbg_ah(ah, "invalid network type for rdma_rxe = %d\n",
+						type);
+			else
+				rxe_dbg_qp(qp, "invalid network type for rdma_rxe = %d\n",
+						type);
 			return -EINVAL;
 		}
 	}
 	return 0;
 }
+int rxe_av_chk_attr(struct rxe_qp *qp, struct rdma_ah_attr *attr)
+{
+	return chk_attr(qp, attr, false);
+}
+int rxe_ah_chk_attr(struct rxe_ah *ah, struct rdma_ah_attr *attr)
+{
+	return chk_attr(ah, attr, true);
+}
 void rxe_av_from_attr(u8 port_num, struct rxe_av *av,
 		     struct rdma_ah_attr *attr)
 {
 		/* only new user provider or kernel client */
 		ah = rxe_pool_get_index(&pkt->rxe->ah_pool, ah_num);
 		if (!ah) {
-			pr_warn("Unable to find AH matching ah_num\n");
+			rxe_dbg_qp(pkt->qp, "Unable to find AH matching ah_num\n");
 			return NULL;
 		}
 		if (rxe_ah_pd(ah) != pkt->qp->pd) {
-			pr_warn("PDs don't match for AH and QP\n");
+			rxe_dbg_qp(pkt->qp, "PDs don't match for AH and QP\n");
 			rxe_put(ah);
 			return NULL;
 		}

drivers/infiniband/sw/rxe/rxe_comp.c+31−16

 	case IB_WR_LOCAL_INV:			return IB_WC_LOCAL_INV;
 	case IB_WR_REG_MR:			return IB_WC_REG_MR;
 	case IB_WR_BIND_MW:			return IB_WC_BIND_MW;
+	case IB_WR_ATOMIC_WRITE:		return IB_WC_ATOMIC_WRITE;
+	case IB_WR_FLUSH:			return IB_WC_FLUSH;
 	default:
 		return 0xff;
 {
 	struct rxe_qp *qp = from_timer(qp, t, retrans_timer);
-	pr_debug("%s: fired for qp#%d\n", __func__, qp->elem.index);
+	rxe_dbg_qp(qp, "retransmit timer fired\n");
 	if (qp->valid) {
 		qp->comp.timeout = 1;
-		rxe_run_task(&qp->comp.task, 1);
+		rxe_sched_task(&qp->comp.task);
 	}
 }
 	if (must_sched != 0)
 		rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED);
-	rxe_run_task(&qp->comp.task, must_sched);
+	if (must_sched)
+		rxe_sched_task(&qp->comp.task);
+	else
+		rxe_run_task(&qp->comp.task);
 }
 static inline enum comp_state get_wqe(struct rxe_qp *qp,
 		 */
 		if (pkt->psn == wqe->last_psn)
 			return COMPST_COMP_ACK;
+		else if (pkt->opcode == IB_OPCODE_RC_ACKNOWLEDGE &&
+			 (qp->comp.opcode == IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST ||
+			  qp->comp.opcode == IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE))
+			return COMPST_CHECK_ACK;
 		else
 			return COMPST_DONE;
 	} else if ((diff > 0) && (wqe->mask & WR_ATOMIC_OR_READ_MASK)) {
 	case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
 	case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+		/* Check NAK code to handle a remote error */
+		if (pkt->opcode == IB_OPCODE_RC_ACKNOWLEDGE)
+			break;
 		if (pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE &&
 		    pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST) {
 			/* read retries of partial data may restart from
 		if ((syn & AETH_TYPE_MASK) != AETH_ACK)
 			return COMPST_ERROR;
+		if (wqe->wr.opcode == IB_WR_ATOMIC_WRITE)
+			return COMPST_WRITE_SEND;
 		fallthrough;
 		/* (IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE doesn't have an AETH)
 		 */
 	case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
 		if (wqe->wr.opcode != IB_WR_RDMA_READ &&
-		    wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV) {
+		    wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV &&
+		    wqe->wr.opcode != IB_WR_FLUSH) {
 			wqe->status = IB_WC_FATAL_ERR;
 			return COMPST_ERROR;
 		}
 					qp->comp.psn = pkt->psn;
 					if (qp->req.wait_psn) {
 						qp->req.wait_psn = 0;
-						rxe_run_task(&qp->req.task, 0);
+						rxe_run_task(&qp->req.task);
 					}
 				}
 				return COMPST_ERROR_RETRY;
 				return COMPST_ERROR;
 			default:
-				pr_warn("unexpected nak %x\n", syn);
+				rxe_dbg_qp(qp, "unexpected nak %x\n", syn);
 				wqe->status = IB_WC_REM_OP_ERR;
 				return COMPST_ERROR;
 			}
 		break;
 	default:
-		pr_warn("unexpected opcode\n");
+		rxe_dbg_qp(qp, "unexpected opcode\n");
 	}
 	return COMPST_ERROR;
 	 */
 	if (qp->req.wait_fence) {
 		qp->req.wait_fence = 0;
-		rxe_run_task(&qp->req.task, 0);
+		rxe_run_task(&qp->req.task);
 	}
 }
 		if (qp->req.need_rd_atomic) {
 			qp->comp.timeout_retry = 0;
 			qp->req.need_rd_atomic = 0;
-			rxe_run_task(&qp->req.task, 0);
+			rxe_run_task(&qp->req.task);
 		}
 	}
 		if (qp->req.wait_psn) {
 			qp->req.wait_psn = 0;
-			rxe_run_task(&qp->req.task, 1);
+			rxe_sched_task(&qp->req.task);
 		}
 	}
 	state = COMPST_GET_ACK;
 	while (1) {
-		pr_debug("qp#%d state = %s\n", qp_num(qp),
+		rxe_dbg_qp(qp, "state = %s\n", comp_state_name[state]);
-			 comp_state_name[state]);
 		switch (state) {
 		case COMPST_GET_ACK:
 			skb = skb_dequeue(&qp->resp_pkts);
 			if (qp->req.wait_psn) {
 				qp->req.wait_psn = 0;
-				rxe_run_task(&qp->req.task, 1);
+				rxe_sched_task(&qp->req.task);
 			}
 			state = COMPST_DONE;
 							RXE_CNT_COMP_RETRY);
 					qp->req.need_retry = 1;
 					qp->comp.started_retry = 1;
-					rxe_run_task(&qp->req.task, 0);
+					rxe_run_task(&qp->req.task);
 				}
 				goto done;
 				 * rnr timer has fired
 				 */
 				qp->req.wait_for_rnr_timer = 1;
-				pr_debug("qp#%d set rnr nak timer\n",
+				rxe_dbg_qp(qp, "set rnr nak timer\n");
-					 qp_num(qp));
 				mod_timer(&qp->rnr_nak_timer,
 					  jiffies + rnrnak_jiffies(aeth_syn(pkt)
 						& ~AETH_TYPE_MASK));

drivers/infiniband/sw/rxe/rxe_cq.c+4−4

@@ -14,12 +14,12 @@ int rxe_cq_chk_attr(struct rxe_dev rxe, struct rxe_cq cq, · 13 unmodified lines
14	int count;	14	int count;
15		15
16	if (cqe <= 0) {	16	if (cqe <= 0) {
17	pr_warn("cqe(%d) <= 0\n", cqe);	17	rxe_dbg(rxe, "cqe(%d) <= 0\n", cqe);
18	goto err1;	18	goto err1;
19	}	19	}
20		20
21	if (cqe > rxe->attr.max_cqe) {	21	if (cqe > rxe->attr.max_cqe) {
22	pr_debug("cqe(%d) > max_cqe(%d)\n",	22	rxe_dbg(rxe, "cqe(%d) > max_cqe(%d)\n",
23	cqe, rxe->attr.max_cqe);	23	cqe, rxe->attr.max_cqe);
24	goto err1;	24	goto err1;
25	}	25	}
@@ -27,7 +27,7 @@ int rxe_cq_chk_attr(struct rxe_dev rxe, struct rxe_cq cq, · 1 unmodified line
27	if (cq) {	27	if (cq) {
28	count = queue_count(cq->queue, QUEUE_TYPE_TO_CLIENT);	28	count = queue_count(cq->queue, QUEUE_TYPE_TO_CLIENT);
29	if (cqe < count) {	29	if (cqe < count) {
30	pr_debug("cqe(%d) < current # elements in queue (%d)",	30	rxe_dbg_cq(cq, "cqe(%d) < current # elements in queue (%d)",
31	cqe, count);	31	cqe, count);
32	goto err1;	32	goto err1;
33	}	33	}
@@ -65,7 +65,7 @@ int rxe_cq_from_init(struct rxe_dev rxe, struct rxe_cq cq, int cqe, · 31 unmodified lines
65	cq->queue = rxe_queue_init(rxe, &cqe,	65	cq->queue = rxe_queue_init(rxe, &cqe,
66	sizeof(struct rxe_cqe), type);	66	sizeof(struct rxe_cqe), type);
67	if (!cq->queue) {	67	if (!cq->queue) {
68	pr_warn("unable to create cq\n");	68	rxe_dbg(rxe, "unable to create cq\n");
69	return -ENOMEM;	69	return -ENOMEM;
70	}	70	}
71		71

drivers/infiniband/sw/rxe/rxe_hdr.h+47−1

@@ -607,6 +607,52 @@ static inline void reth_set_len(struct rxe_pkt_info *pkt, u32 len) · 606 unmodified lines
607	rxe_opcode[pkt->opcode].offset[RXE_RETH], len);	607	rxe_opcode[pkt->opcode].offset[RXE_RETH], len);
608	}	608	}
609		609
		610	/******************************************************************************
		611	* FLUSH Extended Transport Header
		612	******************************************************************************/
		613
		614	struct rxe_feth {
		615	__be32 bits;
		616	};
		617
		618	#define FETH_PLT_MASK (0x0000000f) /* bits 3-0 */
		619	#define FETH_SEL_MASK (0x00000030) /* bits 5-4 */
		620	#define FETH_SEL_SHIFT (4U)
		621
		622	static inline u32 __feth_plt(void *arg)
		623	{
		624	struct rxe_feth *feth = arg;
		625
		626	return be32_to_cpu(feth->bits) & FETH_PLT_MASK;
		627	}
		628
		629	static inline u32 __feth_sel(void *arg)
		630	{
		631	struct rxe_feth *feth = arg;
		632
		633	return (be32_to_cpu(feth->bits) & FETH_SEL_MASK) >> FETH_SEL_SHIFT;
		634	}
		635
		636	static inline u32 feth_plt(struct rxe_pkt_info *pkt)
		637	{
		638	return __feth_plt(pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
		639	}
		640
		641	static inline u32 feth_sel(struct rxe_pkt_info *pkt)
		642	{
		643	return __feth_sel(pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
		644	}
		645
		646	static inline void feth_init(struct rxe_pkt_info *pkt, u8 type, u8 level)
		647	{
		648	struct rxe_feth feth = (struct rxe_feth )
		649	(pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
		650	u32 bits = ((level << FETH_SEL_SHIFT) & FETH_SEL_MASK) \|
		651	(type & FETH_PLT_MASK);
		652
		653	feth->bits = cpu_to_be32(bits);
		654	}
		655
610	/******************************************************************************	656	/******************************************************************************
611	* Atomic Extended Transport Header	657	* Atomic Extended Transport Header
612	******************************************************************************/	658	******************************************************************************/
@@ -742,7 +788,6 @@ enum aeth_syndrome { · 129 unmodified lines
742	AETH_NAK_INVALID_REQ = 0x61,	788	AETH_NAK_INVALID_REQ = 0x61,
743	AETH_NAK_REM_ACC_ERR = 0x62,	789	AETH_NAK_REM_ACC_ERR = 0x62,
744	AETH_NAK_REM_OP_ERR = 0x63,	790	AETH_NAK_REM_OP_ERR = 0x63,
745	AETH_NAK_INV_RD_REQ = 0x64,
746	};	791	};
747		792
748	static inline u8 __aeth_syn(void *arg)	793	static inline u8 __aeth_syn(void *arg)
@@ -910,6 +955,7 @@ enum rxe_hdr_length { · 161 unmodified lines
910	RXE_ATMETH_BYTES = sizeof(struct rxe_atmeth),	955	RXE_ATMETH_BYTES = sizeof(struct rxe_atmeth),
911	RXE_IETH_BYTES = sizeof(struct rxe_ieth),	956	RXE_IETH_BYTES = sizeof(struct rxe_ieth),
912	RXE_RDETH_BYTES = sizeof(struct rxe_rdeth),	957	RXE_RDETH_BYTES = sizeof(struct rxe_rdeth),
		958	RXE_FETH_BYTES = sizeof(struct rxe_feth),
913	};	959	};
914		960
915	static inline size_t header_size(struct rxe_pkt_info *pkt)	961	static inline size_t header_size(struct rxe_pkt_info *pkt)

drivers/infiniband/sw/rxe/rxe_icrc.c+2−2

@@ -21,7 +21,7 @@ int rxe_icrc_init(struct rxe_dev *rxe) · 20 unmodified lines
21		21
22	tfm = crypto_alloc_shash("crc32", 0, 0);	22	tfm = crypto_alloc_shash("crc32", 0, 0);
23	if (IS_ERR(tfm)) {	23	if (IS_ERR(tfm)) {
24	pr_warn("failed to init crc32 algorithm err:%ld\n",	24	rxe_dbg(rxe, "failed to init crc32 algorithm err: %ld\n",
25	PTR_ERR(tfm));	25	PTR_ERR(tfm));
26	return PTR_ERR(tfm);	26	return PTR_ERR(tfm);
27	}	27	}
@@ -51,7 +51,7 @@ static __be32 rxe_crc32(struct rxe_dev rxe, __be32 crc, void next, size_t len) · 23 unmodified lines
51	(__be32 )shash_desc_ctx(shash) = crc;	51	(__be32 )shash_desc_ctx(shash) = crc;
52	err = crypto_shash_update(shash, next, len);	52	err = crypto_shash_update(shash, next, len);
53	if (unlikely(err)) {	53	if (unlikely(err)) {
54	pr_warn_ratelimited("failed crc calculation, err: %d\n", err);	54	rxe_dbg(rxe, "failed crc calculation, err: %d\n", err);
55	return (__force __be32)crc32_le((__force u32)crc, next, len);	55	return (__force __be32)crc32_le((__force u32)crc, next, len);
56	}	56	}
57		57

drivers/infiniband/sw/rxe/rxe_loc.h+3−6

@@ -9,16 +9,12 @@ · 8 unmodified lines
9		9
10	/* rxe_av.c */	10	/* rxe_av.c */
11	void rxe_init_av(struct rdma_ah_attr attr, struct rxe_av av);	11	void rxe_init_av(struct rdma_ah_attr attr, struct rxe_av av);
12		12	int rxe_av_chk_attr(struct rxe_qp qp, struct rdma_ah_attr attr);
13	int rxe_av_chk_attr(struct rxe_dev rxe, struct rdma_ah_attr attr);	13	int rxe_ah_chk_attr(struct rxe_ah ah, struct rdma_ah_attr attr);
14
15	void rxe_av_from_attr(u8 port_num, struct rxe_av *av,	14	void rxe_av_from_attr(u8 port_num, struct rxe_av *av,
16	struct rdma_ah_attr *attr);	15	struct rdma_ah_attr *attr);
17
18	void rxe_av_to_attr(struct rxe_av av, struct rdma_ah_attr attr);	16	void rxe_av_to_attr(struct rxe_av av, struct rdma_ah_attr attr);
19
20	void rxe_av_fill_ip_info(struct rxe_av av, struct rdma_ah_attr attr);	17	void rxe_av_fill_ip_info(struct rxe_av av, struct rdma_ah_attr attr);
21
22	struct rxe_av rxe_get_av(struct rxe_pkt_info pkt, struct rxe_ah **ahp);	18	struct rxe_av rxe_get_av(struct rxe_pkt_info pkt, struct rxe_ah **ahp);
23		19
24	/* rxe_cq.c */	20	/* rxe_cq.c */
@@ -68,6 +64,7 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr); · 43 unmodified lines
68	int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,	64	int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
69	int access, struct rxe_mr *mr);	65	int access, struct rxe_mr *mr);
70	int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);	66	int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
		67	int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length);
71	int rxe_mr_copy(struct rxe_mr mr, u64 iova, void addr, int length,	68	int rxe_mr_copy(struct rxe_mr mr, u64 iova, void addr, int length,
72	enum rxe_mr_copy_dir dir);	69	enum rxe_mr_copy_dir dir);
73	int copy_data(struct rxe_pd pd, int access, struct rxe_dma_info dma,	70	int copy_data(struct rxe_pd pd, int access, struct rxe_dma_info dma,

drivers/infiniband/sw/rxe/rxe_mmap.c+3−3

@@ -79,7 +79,7 @@ int rxe_mmap(struct ib_ucontext context, struct vm_area_struct vma) · 78 unmodified lines
79		79
80	/* Don't allow a mmap larger than the object. */	80	/* Don't allow a mmap larger than the object. */
81	if (size > ip->info.size) {	81	if (size > ip->info.size) {
82	pr_err("mmap region is larger than the object!\n");	82	rxe_dbg(rxe, "mmap region is larger than the object!\n");
83	spin_unlock_bh(&rxe->pending_lock);	83	spin_unlock_bh(&rxe->pending_lock);
84	ret = -EINVAL;	84	ret = -EINVAL;
85	goto done;	85	goto done;
@@ -87,7 +87,7 @@ int rxe_mmap(struct ib_ucontext context, struct vm_area_struct vma) · 1 unmodified line
87		87
88	goto found_it;	88	goto found_it;
89	}	89	}
90	pr_warn("unable to find pending mmap info\n");	90	rxe_dbg(rxe, "unable to find pending mmap info\n");
91	spin_unlock_bh(&rxe->pending_lock);	91	spin_unlock_bh(&rxe->pending_lock);
92	ret = -EINVAL;	92	ret = -EINVAL;
93	goto done;	93	goto done;
@@ -98,7 +98,7 @@ found_it: · 4 unmodified lines
98		98
99	ret = remap_vmalloc_range(vma, ip->obj, 0);	99	ret = remap_vmalloc_range(vma, ip->obj, 0);
100	if (ret) {	100	if (ret) {
101	pr_err("err %d from remap_vmalloc_range\n", ret);	101	rxe_dbg(rxe, "err %d from remap_vmalloc_range\n", ret);
102	goto done;	102	goto done;
103	}	103	}
104		104

drivers/infiniband/sw/rxe/rxe_mr.c+82−40

  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
  */
+#include <linux/libnvdimm.h>
 #include "rxe.h"
 #include "rxe_loc.h"
 {
-	switch (mr->type) {
+	switch (mr->ibmr.type) {
 	case IB_MR_TYPE_DMA:
 		return 0;
 		return 0;
 	default:
-		pr_warn("%s: mr type (%d) not supported\n",
+		rxe_dbg_mr(mr, "type (%d) not supported\n", mr->ibmr.type);
-			__func__, mr->type);
 		return -EFAULT;
 	}
 }
 	mr->rkey = mr->ibmr.rkey = rkey;
 	mr->state = RXE_MR_STATE_INVALID;
-	mr->map_shift = ilog2(RXE_BUF_PER_MAP);
 }
 static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf)
 		kfree(mr->map[i]);
 	kfree(mr->map);
+	mr->map = NULL;
 err1:
 	return -ENOMEM;
 }
 	mr->access = access;
 	mr->state = RXE_MR_STATE_VALID;
-	mr->type = IB_MR_TYPE_DMA;
+	mr->ibmr.type = IB_MR_TYPE_DMA;
+}
+static bool is_pmem_page(struct page *pg)
+{
+	unsigned long paddr = page_to_phys(pg);
+	return REGION_INTERSECTS ==
+	       region_intersects(paddr, PAGE_SIZE, IORESOURCE_MEM,
+				 IORES_DESC_PERSISTENT_MEMORY);
 }
 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 	int			num_buf;
 	void			*vaddr;
 	int err;
-	int i;
 	umem = ib_umem_get(&rxe->ib_dev, start, length, access);
 	if (IS_ERR(umem)) {
-		pr_warn("%s: Unable to pin memory region err = %d\n",
+		rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
-			__func__, (int)PTR_ERR(umem));
+			(int)PTR_ERR(umem));
 		err = PTR_ERR(umem);
 		goto err_out;
 	}
 	err = rxe_mr_alloc(mr, num_buf);
 	if (err) {
-		pr_warn("%s: Unable to allocate memory for map\n",
+		rxe_dbg_mr(mr, "Unable to allocate memory for map\n");
-				__func__);
 		goto err_release_umem;
 	}
 	num_buf			= 0;
 	map = mr->map;
 	if (length > 0) {
-		buf = map[0]->buf;
+		bool persistent_access = access & IB_ACCESS_FLUSH_PERSISTENT;
+		buf = map[0]->buf;
 		for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {
+			struct page *pg = sg_page_iter_page(&sg_iter);
+			if (persistent_access && !is_pmem_page(pg)) {
+				rxe_dbg_mr(mr, "Unable to register persistent access to non-pmem device\n");
+				err = -EINVAL;
+				goto err_release_umem;
+			}
 			if (num_buf >= RXE_BUF_PER_MAP) {
 				map++;
 				buf = map[0]->buf;
 				num_buf = 0;
 			}
-			vaddr = page_address(sg_page_iter_page(&sg_iter));
+			vaddr = page_address(pg);
 			if (!vaddr) {
-				pr_warn("%s: Unable to get virtual address\n",
+				rxe_dbg_mr(mr, "Unable to get virtual address\n");
-						__func__);
 				err = -ENOMEM;
-				goto err_cleanup_map;
+				goto err_release_umem;
 			}
 			buf->addr = (uintptr_t)vaddr;
 			buf->size = PAGE_SIZE;
 			num_buf++;
 	mr->access = access;
 	mr->offset = ib_umem_offset(umem);
 	mr->state = RXE_MR_STATE_VALID;
-	mr->type = IB_MR_TYPE_USER;
+	mr->ibmr.type = IB_MR_TYPE_USER;
+	mr->ibmr.page_size = PAGE_SIZE;
 	return 0;
-err_cleanup_map:
-	for (i = 0; i < mr->num_map; i++)
-		kfree(mr->map[i]);
-	kfree(mr->map);
 err_release_umem:
 	ib_umem_release(umem);
 err_out:
 	mr->max_buf = max_pages;
 	mr->state = RXE_MR_STATE_FREE;
-	mr->type = IB_MR_TYPE_MEM_REG;
+	mr->ibmr.type = IB_MR_TYPE_MEM_REG;
 	return 0;
 	void *addr;
 	if (mr->state != RXE_MR_STATE_VALID) {
-		pr_warn("mr not in valid state\n");
+		rxe_dbg_mr(mr, "Not in valid state\n");
 		addr = NULL;
 		goto out;
 	}
 	}
 	if (mr_check_range(mr, iova, length)) {
-		pr_warn("range violation\n");
+		rxe_dbg_mr(mr, "Range violation\n");
 		addr = NULL;
 		goto out;
 	}
 	lookup_iova(mr, iova, &m, &n, &offset);
 	if (offset + length > mr->map[m]->buf[n].size) {
-		pr_warn("crosses page boundary\n");
+		rxe_dbg_mr(mr, "Crosses page boundary\n");
 		addr = NULL;
 		goto out;
 	}
 	return addr;
 }
+int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)
+{
+	size_t offset;
+	if (length == 0)
+		return 0;
+	if (mr->ibmr.type == IB_MR_TYPE_DMA)
+		return -EFAULT;
+	offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask;
+	while (length > 0) {
+		u8 *va;
+		int bytes;
+		bytes = mr->ibmr.page_size - offset;
+		if (bytes > length)
+			bytes = length;
+		va = iova_to_vaddr(mr, iova, length);
+		if (!va)
+			return -EFAULT;
+		arch_wb_cache_pmem(va, bytes);
+		length -= bytes;
+		iova += bytes;
+		offset = 0;
+	}
+	return 0;
+}
 /* copy data from a range (vaddr, vaddr+length-1) to or from
  * a mr object starting at iova.
  */
 	if (length == 0)
 		return 0;
-	if (mr->type == IB_MR_TYPE_DMA) {
+	if (mr->ibmr.type == IB_MR_TYPE_DMA) {
 		u8 *src, *dest;
 		src = (dir == RXE_TO_MR_OBJ) ? addr : ((void *)(uintptr_t)iova);
 	if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) ||
 		     (type == RXE_LOOKUP_REMOTE && mr->rkey != key) ||
-		     mr_pd(mr) != pd || (access && !(access & mr->access)) ||
+		     mr_pd(mr) != pd || ((access & mr->access) != access) ||
 		     mr->state != RXE_MR_STATE_VALID)) {
 		rxe_put(mr);
 		mr = NULL;
 	mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
 	if (!mr) {
-		pr_err("%s: No MR for key %#x\n", __func__, key);
+		rxe_dbg_qp(qp, "No MR for key %#x\n", key);
 		ret = -EINVAL;
 		goto err;
 	}
 	if (mr->rkey ? (key != mr->rkey) : (key != mr->lkey)) {
-		pr_err("%s: wr key (%#x) doesn't match mr key (%#x)\n",
+		rxe_dbg_mr(mr, "wr key (%#x) doesn't match mr key (%#x)\n",
-			__func__, key, (mr->rkey ? mr->rkey : mr->lkey));
+			key, (mr->rkey ? mr->rkey : mr->lkey));
 		ret = -EINVAL;
 		goto err_drop_ref;
 	}
 	if (atomic_read(&mr->num_mw) > 0) {
-		pr_warn("%s: Attempt to invalidate an MR while bound to MWs\n",
+		rxe_dbg_mr(mr, "Attempt to invalidate an MR while bound to MWs\n");
-			__func__);
 		ret = -EINVAL;
 		goto err_drop_ref;
 	}
-	if (unlikely(mr->type != IB_MR_TYPE_MEM_REG)) {
+	if (unlikely(mr->ibmr.type != IB_MR_TYPE_MEM_REG)) {
-		pr_warn("%s: mr->type (%d) is wrong type\n", __func__, mr->type);
+		rxe_dbg_mr(mr, "Type (%d) is wrong\n", mr->ibmr.type);
 		ret = -EINVAL;
 		goto err_drop_ref;
 	}
 	/* user can only register MR in free state */
 	if (unlikely(mr->state != RXE_MR_STATE_FREE)) {
-		pr_warn("%s: mr->lkey = 0x%x not free\n",
+		rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n", mr->lkey);
-			__func__, mr->lkey);
 		return -EINVAL;
 	}
 	/* user can only register mr with qp in same protection domain */
 	if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {
-		pr_warn("%s: qp->pd and mr->pd don't match\n",
+		rxe_dbg_mr(mr, "qp->pd and mr->pd don't match\n");
-			__func__);
 		return -EINVAL;
 	}
 	/* user is only allowed to change key portion of l/rkey */
 	if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {
-		pr_warn("%s: key = 0x%x has wrong index mr->lkey = 0x%x\n",
+		rxe_dbg_mr(mr, "key = 0x%x has wrong index mr->lkey = 0x%x\n",
-			__func__, key, mr->lkey);
+			key, mr->lkey);
 		return -EINVAL;
 	}

drivers/infiniband/sw/rxe/rxe_mw.c+11−12

 {
 	if (mw->ibmw.type == IB_MW_TYPE_1) {
 		if (unlikely(mw->state != RXE_MW_STATE_VALID)) {
-			pr_err_once(
+			rxe_dbg_mw(mw,
 				"attempt to bind a type 1 MW not in the valid state\n");
 			return -EINVAL;
 		}
 		/* o10-36.2.2 */
 		if (unlikely((mw->access & IB_ZERO_BASED))) {
-			pr_err_once("attempt to bind a zero based type 1 MW\n");
+			rxe_dbg_mw(mw, "attempt to bind a zero based type 1 MW\n");
 			return -EINVAL;
 		}
 	}
 	if (mw->ibmw.type == IB_MW_TYPE_2) {
 		/* o10-37.2.30 */
 		if (unlikely(mw->state != RXE_MW_STATE_FREE)) {
-			pr_err_once(
+			rxe_dbg_mw(mw,
 				"attempt to bind a type 2 MW not in the free state\n");
 			return -EINVAL;
 		}
 		/* C10-72 */
 		if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) {
-			pr_err_once(
+			rxe_dbg_mw(mw,
 				"attempt to bind type 2 MW with qp with different PD\n");
 			return -EINVAL;
 		}
 		/* o10-37.2.40 */
 		if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) {
-			pr_err_once(
+			rxe_dbg_mw(mw,
 				"attempt to invalidate type 2 MW by binding with NULL or zero length MR\n");
 			return -EINVAL;
 		}
 		return 0;
 	if (unlikely(mr->access & IB_ZERO_BASED)) {
-		pr_err_once("attempt to bind MW to zero based MR\n");
+		rxe_dbg_mw(mw, "attempt to bind MW to zero based MR\n");
 		return -EINVAL;
 	}
 	/* C10-73 */
 	if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) {
-		pr_err_once(
+		rxe_dbg_mw(mw,
 			"attempt to bind an MW to an MR without bind access\n");
 		return -EINVAL;
 	}
 	if (unlikely((mw->access &
 		      (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC)) &&
 		     !(mr->access & IB_ACCESS_LOCAL_WRITE))) {
-		pr_err_once(
+		rxe_dbg_mw(mw,
 			"attempt to bind an Writable MW to an MR without local write access\n");
 		return -EINVAL;
 	}
 	/* C10-75 */
 	if (mw->access & IB_ZERO_BASED) {
 		if (unlikely(wqe->wr.wr.mw.length > mr->ibmr.length)) {
-			pr_err_once(
+			rxe_dbg_mw(mw,
 				"attempt to bind a ZB MW outside of the MR\n");
 			return -EINVAL;
 		}
 		if (unlikely((wqe->wr.wr.mw.addr < mr->ibmr.iova) ||
 			     ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) >
 			      (mr->ibmr.iova + mr->ibmr.length)))) {
-			pr_err_once(
+			rxe_dbg_mw(mw,
 				"attempt to bind a VA MW outside of the MR\n");
 			return -EINVAL;
 		}
 	if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd ||
 		     (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) ||
-		     (mw->length == 0) ||
+		     (mw->length == 0) || ((access & mw->access) != access) ||
-		     (access && !(access & mw->access)) ||
 		     mw->state != RXE_MW_STATE_VALID)) {
 		rxe_put(mw);
 		return NULL;

drivers/infiniband/sw/rxe/rxe_net.c+23−19

 static struct rxe_recv_sockets recv_sockets;
-static struct dst_entry *rxe_find_route4(struct net_device *ndev,
+static struct dst_entry *rxe_find_route4(struct rxe_qp *qp,
-				  struct in_addr *saddr,
+					 struct net_device *ndev,
-				  struct in_addr *daddr)
+					 struct in_addr *saddr,
+					 struct in_addr *daddr)
 {
 	struct rtable *rt;
 	struct flowi4 fl = { { 0 } };
 	rt = ip_route_output_key(&init_net, &fl);
 	if (IS_ERR(rt)) {
-		pr_err_ratelimited("no route to %pI4\n", &daddr->s_addr);
+		rxe_dbg_qp(qp, "no route to %pI4\n", &daddr->s_addr);
 		return NULL;
 	}
 }
 #if IS_ENABLED(CONFIG_IPV6)
-static struct dst_entry *rxe_find_route6(struct net_device *ndev,
+static struct dst_entry *rxe_find_route6(struct rxe_qp *qp,
+					 struct net_device *ndev,
 					 struct in6_addr *saddr,
 					 struct in6_addr *daddr)
 {
 					       recv_sockets.sk6->sk, &fl6,
 					       NULL);
 	if (IS_ERR(ndst)) {
-		pr_err_ratelimited("no route to %pI6\n", daddr);
+		rxe_dbg_qp(qp, "no route to %pI6\n", daddr);
 		return NULL;
 	}
 	if (unlikely(ndst->error)) {
-		pr_err("no route to %pI6\n", daddr);
+		rxe_dbg_qp(qp, "no route to %pI6\n", daddr);
 		goto put;
 	}
 #else
-static struct dst_entry *rxe_find_route6(struct net_device *ndev,
+static struct dst_entry *rxe_find_route6(struct rxe_qp *qp,
+					 struct net_device *ndev,
 					 struct in6_addr *saddr,
 					 struct in6_addr *daddr)
 {
 			saddr = &av->sgid_addr._sockaddr_in.sin_addr;
 			daddr = &av->dgid_addr._sockaddr_in.sin_addr;
-			dst = rxe_find_route4(ndev, saddr, daddr);
+			dst = rxe_find_route4(qp, ndev, saddr, daddr);
 		} else if (av->network_type == RXE_NETWORK_TYPE_IPV6) {
 			struct in6_addr *saddr6;
 			struct in6_addr *daddr6;
 			saddr6 = &av->sgid_addr._sockaddr_in6.sin6_addr;
 			daddr6 = &av->dgid_addr._sockaddr_in6.sin6_addr;
-			dst = rxe_find_route6(ndev, saddr6, daddr6);
+			dst = rxe_find_route6(qp, ndev, saddr6, daddr6);
 #if IS_ENABLED(CONFIG_IPV6)
 			if (dst)
 				qp->dst_cookie =
 	dst = rxe_find_route(skb->dev, qp, av);
 	if (!dst) {
-		pr_err("Host not reachable\n");
+		rxe_dbg_qp(qp, "Host not reachable\n");
 		return -EHOSTUNREACH;
 	}
 	dst = rxe_find_route(skb->dev, qp, av);
 	if (!dst) {
-		pr_err("Host not reachable\n");
+		rxe_dbg_qp(qp, "Host not reachable\n");
 		return -EHOSTUNREACH;
 	}
 	if (unlikely(qp->need_req_skb &&
 		     skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW))
-		rxe_run_task(&qp->req.task, 1);
+		rxe_sched_task(&qp->req.task);
 	rxe_put(qp);
 }
 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
 		err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
 	} else {
-		pr_err("Unknown layer 3 protocol: %d\n", skb->protocol);
+		rxe_dbg_qp(pkt->qp, "Unknown layer 3 protocol: %d\n",
+				skb->protocol);
 		atomic_dec(&pkt->qp->skb_out);
 		rxe_put(pkt->qp);
 		kfree_skb(skb);
 	}
 	if (unlikely(net_xmit_eval(err))) {
-		pr_debug("error sending packet: %d\n", err);
+		rxe_dbg_qp(pkt->qp, "error sending packet: %d\n", err);
 		return -EAGAIN;
 	}
 	if ((is_request && (qp->req.state != QP_STATE_READY)) ||
 	    (!is_request && (qp->resp.state != QP_STATE_READY))) {
-		pr_info("Packet dropped. QP is not in ready state\n");
+		rxe_dbg_qp(qp, "Packet dropped. QP is not in ready state\n");
 		goto drop;
 	}
 	if ((qp_type(qp) != IB_QPT_RC) &&
 	    (pkt->mask & RXE_END_MASK)) {
 		pkt->wqe->state = wqe_state_done;
-		rxe_run_task(&qp->comp.task, 1);
+		rxe_sched_task(&qp->comp.task);
 	}
 	rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
 		rxe_port_down(rxe);
 		break;
 	case NETDEV_CHANGEMTU:
-		pr_info("%s changed mtu to %d\n", ndev->name, ndev->mtu);
+		rxe_dbg(rxe, "%s changed mtu to %d\n", ndev->name, ndev->mtu);
 		rxe_set_mtu(rxe, ndev->mtu);
 		break;
 	case NETDEV_CHANGE:
 	case NETDEV_CHANGENAME:
 	case NETDEV_FEAT_CHANGE:
 	default:
-		pr_info("ignoring netdev event = %ld for %s\n",
+		rxe_dbg(rxe, "ignoring netdev event = %ld for %s\n",
 			event, ndev->name);
 		break;
 	}

drivers/infiniband/sw/rxe/rxe_opcode.c+35−0

@@ -101,6 +101,18 @@ struct rxe_wr_opcode_info rxe_wr_opcode_info[] = { · 100 unmodified lines
101	[IB_QPT_UC] = WR_LOCAL_OP_MASK,	101	[IB_QPT_UC] = WR_LOCAL_OP_MASK,
102	},	102	},
103	},	103	},
		104	[IB_WR_FLUSH] = {
		105	.name = "IB_WR_FLUSH",
		106	.mask = {
		107	[IB_QPT_RC] = WR_FLUSH_MASK,
		108	},
		109	},
		110	[IB_WR_ATOMIC_WRITE] = {
		111	.name = "IB_WR_ATOMIC_WRITE",
		112	.mask = {
		113	[IB_QPT_RC] = WR_ATOMIC_WRITE_MASK,
		114	},
		115	},
104	};	116	};
105		117
106	struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = {	118	struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = {
@@ -378,6 +390,29 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 271 unmodified lines
378	RXE_IETH_BYTES,	390	RXE_IETH_BYTES,
379	}	391	}
380	},	392	},
		393	[IB_OPCODE_RC_FLUSH] = {
		394	.name = "IB_OPCODE_RC_FLUSH",
		395	.mask = RXE_FETH_MASK \| RXE_RETH_MASK \| RXE_FLUSH_MASK \|
		396	RXE_START_MASK \| RXE_END_MASK \| RXE_REQ_MASK,
		397	.length = RXE_BTH_BYTES + RXE_FETH_BYTES + RXE_RETH_BYTES,
		398	.offset = {
		399	[RXE_BTH] = 0,
		400	[RXE_FETH] = RXE_BTH_BYTES,
		401	[RXE_RETH] = RXE_BTH_BYTES + RXE_FETH_BYTES,
		402	}
		403	},
		404	[IB_OPCODE_RC_ATOMIC_WRITE] = {
		405	.name = "IB_OPCODE_RC_ATOMIC_WRITE",
		406	.mask = RXE_RETH_MASK \| RXE_PAYLOAD_MASK \| RXE_REQ_MASK \|
		407	RXE_ATOMIC_WRITE_MASK \| RXE_START_MASK \|
		408	RXE_END_MASK,
		409	.length = RXE_BTH_BYTES + RXE_RETH_BYTES,
		410	.offset = {
		411	[RXE_BTH] = 0,
		412	[RXE_RETH] = RXE_BTH_BYTES,
		413	[RXE_PAYLOAD] = RXE_BTH_BYTES + RXE_RETH_BYTES,
		414	}
		415	},
381		416
382	/* UC */	417	/* UC */
383	[IB_OPCODE_UC_SEND_FIRST] = {	418	[IB_OPCODE_UC_SEND_FIRST] = {

drivers/infiniband/sw/rxe/rxe_opcode.h+12−5

 	WR_READ_MASK			= BIT(3),
 	WR_WRITE_MASK			= BIT(4),
 	WR_LOCAL_OP_MASK		= BIT(5),
+	WR_FLUSH_MASK			= BIT(6),
+	WR_ATOMIC_WRITE_MASK		= BIT(7),
 	WR_READ_OR_WRITE_MASK		= WR_READ_MASK | WR_WRITE_MASK,
 	WR_WRITE_OR_SEND_MASK		= WR_WRITE_MASK | WR_SEND_MASK,
 	RXE_RDETH,
 	RXE_DETH,
 	RXE_IMMDT,
+	RXE_FETH,
 	RXE_PAYLOAD,
 	NUM_HDR_TYPES
 };
 	RXE_IETH_MASK		= BIT(RXE_IETH),
 	RXE_RDETH_MASK		= BIT(RXE_RDETH),
 	RXE_DETH_MASK		= BIT(RXE_DETH),
+	RXE_FETH_MASK		= BIT(RXE_FETH),
 	RXE_PAYLOAD_MASK	= BIT(RXE_PAYLOAD),
 	RXE_REQ_MASK		= BIT(NUM_HDR_TYPES + 0),
 	RXE_WRITE_MASK		= BIT(NUM_HDR_TYPES + 3),
 	RXE_READ_MASK		= BIT(NUM_HDR_TYPES + 4),
 	RXE_ATOMIC_MASK		= BIT(NUM_HDR_TYPES + 5),
+	RXE_FLUSH_MASK		= BIT(NUM_HDR_TYPES + 6),
-	RXE_RWR_MASK		= BIT(NUM_HDR_TYPES + 6),
+	RXE_RWR_MASK		= BIT(NUM_HDR_TYPES + 7),
-	RXE_COMP_MASK		= BIT(NUM_HDR_TYPES + 7),
+	RXE_COMP_MASK		= BIT(NUM_HDR_TYPES + 8),
-	RXE_START_MASK		= BIT(NUM_HDR_TYPES + 8),
+	RXE_START_MASK		= BIT(NUM_HDR_TYPES + 9),
-	RXE_MIDDLE_MASK		= BIT(NUM_HDR_TYPES + 9),
+	RXE_MIDDLE_MASK		= BIT(NUM_HDR_TYPES + 10),
-	RXE_END_MASK		= BIT(NUM_HDR_TYPES + 10),
+	RXE_END_MASK		= BIT(NUM_HDR_TYPES + 11),
 	RXE_LOOPBACK_MASK	= BIT(NUM_HDR_TYPES + 12),
+	RXE_ATOMIC_WRITE_MASK   = BIT(NUM_HDR_TYPES + 14),
 	RXE_READ_OR_ATOMIC_MASK	= (RXE_READ_MASK | RXE_ATOMIC_MASK),
 	RXE_WRITE_OR_SEND_MASK	= (RXE_WRITE_MASK | RXE_SEND_MASK),
 	RXE_READ_OR_WRITE_MASK	= (RXE_READ_MASK | RXE_WRITE_MASK),

drivers/infiniband/sw/rxe/rxe_param.h+7−0

 					| IB_DEVICE_SRQ_RESIZE
 					| IB_DEVICE_MEM_MGT_EXTENSIONS
 					| IB_DEVICE_MEM_WINDOW
+					| IB_DEVICE_FLUSH_GLOBAL
+					| IB_DEVICE_FLUSH_PERSISTENT
+#ifdef CONFIG_64BIT
+					| IB_DEVICE_MEM_WINDOW_TYPE_2B
+					| IB_DEVICE_ATOMIC_WRITE,
+#else
 					| IB_DEVICE_MEM_WINDOW_TYPE_2B,
+#endif /* CONFIG_64BIT */
 	RXE_MAX_SGE			= 32,
 	RXE_MAX_WQE_SIZE		= sizeof(struct rxe_send_wqe) +
 					  sizeof(struct ib_sge) * RXE_MAX_SGE,

drivers/infiniband/sw/rxe/rxe_qp.c+43−55

 			  int has_srq)
 {
 	if (cap->max_send_wr > rxe->attr.max_qp_wr) {
-		pr_debug("invalid send wr = %u > %d\n",
+		rxe_dbg(rxe, "invalid send wr = %u > %d\n",
 			 cap->max_send_wr, rxe->attr.max_qp_wr);
 		goto err1;
 	}
 	if (cap->max_send_sge > rxe->attr.max_send_sge) {
-		pr_debug("invalid send sge = %u > %d\n",
+		rxe_dbg(rxe, "invalid send sge = %u > %d\n",
 			 cap->max_send_sge, rxe->attr.max_send_sge);
 		goto err1;
 	}
 	if (!has_srq) {
 		if (cap->max_recv_wr > rxe->attr.max_qp_wr) {
-			pr_debug("invalid recv wr = %u > %d\n",
+			rxe_dbg(rxe, "invalid recv wr = %u > %d\n",
 				 cap->max_recv_wr, rxe->attr.max_qp_wr);
 			goto err1;
 		}
 		if (cap->max_recv_sge > rxe->attr.max_recv_sge) {
-			pr_debug("invalid recv sge = %u > %d\n",
+			rxe_dbg(rxe, "invalid recv sge = %u > %d\n",
 				 cap->max_recv_sge, rxe->attr.max_recv_sge);
 			goto err1;
 		}
 	}
 	if (cap->max_inline_data > rxe->max_inline_data) {
-		pr_debug("invalid max inline data = %u > %d\n",
+		rxe_dbg(rxe, "invalid max inline data = %u > %d\n",
 			 cap->max_inline_data, rxe->max_inline_data);
 		goto err1;
 	}
 	}
 	if (!init->recv_cq || !init->send_cq) {
-		pr_debug("missing cq\n");
+		rxe_dbg(rxe, "missing cq\n");
 		goto err1;
 	}
 	if (init->qp_type == IB_QPT_GSI) {
 		if (!rdma_is_port_valid(&rxe->ib_dev, port_num)) {
-			pr_debug("invalid port = %d\n", port_num);
+			rxe_dbg(rxe, "invalid port = %d\n", port_num);
 			goto err1;
 		}
 		port = &rxe->port;
 		if (init->qp_type == IB_QPT_GSI && port->qp_gsi_index) {
-			pr_debug("GSI QP exists for port %d\n", port_num);
+			rxe_dbg(rxe, "GSI QP exists for port %d\n", port_num);
 			goto err1;
 		}
 	}
 	spin_lock_init(&qp->state_lock);
-	spin_lock_init(&qp->req.task.state_lock);
-	spin_lock_init(&qp->resp.task.state_lock);
-	spin_lock_init(&qp->comp.task.state_lock);
 	spin_lock_init(&qp->sq.sq_lock);
 	spin_lock_init(&qp->rq.producer_lock);
 	spin_lock_init(&qp->rq.consumer_lock);
 	skb_queue_head_init(&qp->req_pkts);
-	rxe_init_task(&qp->req.task, qp,
+	rxe_init_task(&qp->req.task, qp, rxe_requester);
-		      rxe_requester, "req");
+	rxe_init_task(&qp->comp.task, qp, rxe_completer);
-	rxe_init_task(&qp->comp.task, qp,
-		      rxe_completer, "comp");
 	qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */
 	if (init->qp_type == IB_QPT_RC) {
 		wqe_size = rcv_wqe_size(qp->rq.max_sge);
-		pr_debug("qp#%d max_wr = %d, max_sge = %d, wqe_size = %d\n",
-			 qp_num(qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size);
 		type = QUEUE_TYPE_FROM_CLIENT;
 		qp->rq.queue = rxe_queue_init(rxe, &qp->rq.max_wr,
 					wqe_size, type);
 	skb_queue_head_init(&qp->resp_pkts);
-	rxe_init_task(&qp->resp.task, qp,
+	rxe_init_task(&qp->resp.task, qp, rxe_responder);
-		      rxe_responder, "resp");
 	qp->resp.opcode		= OPCODE_NONE;
 	qp->resp.msn		= 0;
 					attr->qp_state : cur_state;
 	if (!ib_modify_qp_is_ok(cur_state, new_state, qp_type(qp), mask)) {
-		pr_debug("invalid mask or state for qp\n");
+		rxe_dbg_qp(qp, "invalid mask or state\n");
 		goto err1;
 	}
 	if (mask & IB_QP_PORT) {
 		if (!rdma_is_port_valid(&rxe->ib_dev, attr->port_num)) {
-			pr_debug("invalid port %d\n", attr->port_num);
+			rxe_dbg_qp(qp, "invalid port %d\n", attr->port_num);
 			goto err1;
 		}
 	}
 	if (mask & IB_QP_CAP && rxe_qp_chk_cap(rxe, &attr->cap, !!qp->srq))
 		goto err1;
-	if (mask & IB_QP_AV && rxe_av_chk_attr(rxe, &attr->ah_attr))
+	if (mask & IB_QP_AV && rxe_av_chk_attr(qp, &attr->ah_attr))
 		goto err1;
 	if (mask & IB_QP_ALT_PATH) {
-		if (rxe_av_chk_attr(rxe, &attr->alt_ah_attr))
+		if (rxe_av_chk_attr(qp, &attr->alt_ah_attr))
 			goto err1;
 		if (!rdma_is_port_valid(&rxe->ib_dev, attr->alt_port_num))  {
-			pr_debug("invalid alt port %d\n", attr->alt_port_num);
+			rxe_dbg_qp(qp, "invalid alt port %d\n", attr->alt_port_num);
 			goto err1;
 		}
 		if (attr->alt_timeout > 31) {
-			pr_debug("invalid QP alt timeout %d > 31\n",
+			rxe_dbg_qp(qp, "invalid alt timeout %d > 31\n",
 				 attr->alt_timeout);
 			goto err1;
 		}
 		enum ib_mtu mtu = attr->path_mtu;
 		if (mtu > max_mtu) {
-			pr_debug("invalid mtu (%d) > (%d)\n",
+			rxe_dbg_qp(qp, "invalid mtu (%d) > (%d)\n",
 				 ib_mtu_enum_to_int(mtu),
 				 ib_mtu_enum_to_int(max_mtu));
 			goto err1;
 	if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
 		if (attr->max_rd_atomic > rxe->attr.max_qp_rd_atom) {
-			pr_debug("invalid max_rd_atomic %d > %d\n",
+			rxe_dbg_qp(qp, "invalid max_rd_atomic %d > %d\n",
 				 attr->max_rd_atomic,
 				 rxe->attr.max_qp_rd_atom);
 			goto err1;
 	if (mask & IB_QP_TIMEOUT) {
 		if (attr->timeout > 31) {
-			pr_debug("invalid QP timeout %d > 31\n", attr->timeout);
+			rxe_dbg_qp(qp, "invalid timeout %d > 31\n",
+					attr->timeout);
 			goto err1;
 		}
 	}
 		if (qp->req.state != QP_STATE_DRAINED) {
 			qp->req.state = QP_STATE_DRAIN;
 			if (qp_type(qp) == IB_QPT_RC)
-				rxe_run_task(&qp->comp.task, 1);
+				rxe_sched_task(&qp->comp.task);
 			else
 				__rxe_do_task(&qp->comp.task);
-			rxe_run_task(&qp->req.task, 1);
+			rxe_sched_task(&qp->req.task);
 		}
 	}
 }
 	qp->attr.qp_state = IB_QPS_ERR;
 	/* drain work and packet queues */
-	rxe_run_task(&qp->resp.task, 1);
+	rxe_sched_task(&qp->resp.task);
 	if (qp_type(qp) == IB_QPT_RC)
-		rxe_run_task(&qp->comp.task, 1);
+		rxe_sched_task(&qp->comp.task);
 	else
 		__rxe_do_task(&qp->comp.task);
-	rxe_run_task(&qp->req.task, 1);
+	rxe_sched_task(&qp->req.task);
 }
 /* called by the modify qp verb */
 	if (mask & IB_QP_RETRY_CNT) {
 		qp->attr.retry_cnt = attr->retry_cnt;
 		qp->comp.retry_cnt = attr->retry_cnt;
-		pr_debug("qp#%d set retry count = %d\n", qp_num(qp),
+		rxe_dbg_qp(qp, "set retry count = %d\n", attr->retry_cnt);
-			 attr->retry_cnt);
 	}
 	if (mask & IB_QP_RNR_RETRY) {
 		qp->attr.rnr_retry = attr->rnr_retry;
 		qp->comp.rnr_retry = attr->rnr_retry;
-		pr_debug("qp#%d set rnr retry count = %d\n", qp_num(qp),
+		rxe_dbg_qp(qp, "set rnr retry count = %d\n", attr->rnr_retry);
-			 attr->rnr_retry);
 	}
 	if (mask & IB_QP_RQ_PSN) {
 		qp->attr.rq_psn = (attr->rq_psn & BTH_PSN_MASK);
 		qp->resp.psn = qp->attr.rq_psn;
-		pr_debug("qp#%d set resp psn = 0x%x\n", qp_num(qp),
+		rxe_dbg_qp(qp, "set resp psn = 0x%x\n", qp->resp.psn);
-			 qp->resp.psn);
 	}
 	if (mask & IB_QP_MIN_RNR_TIMER) {
 		qp->attr.min_rnr_timer = attr->min_rnr_timer;
-		pr_debug("qp#%d set min rnr timer = 0x%x\n", qp_num(qp),
+		rxe_dbg_qp(qp, "set min rnr timer = 0x%x\n",
 			 attr->min_rnr_timer);
 	}
 		qp->attr.sq_psn = (attr->sq_psn & BTH_PSN_MASK);
 		qp->req.psn = qp->attr.sq_psn;
 		qp->comp.psn = qp->attr.sq_psn;
-		pr_debug("qp#%d set req psn = 0x%x\n", qp_num(qp), qp->req.psn);
+		rxe_dbg_qp(qp, "set req psn = 0x%x\n", qp->req.psn);
 	}
 	if (mask & IB_QP_PATH_MIG_STATE)
 		switch (attr->qp_state) {
 		case IB_QPS_RESET:
-			pr_debug("qp#%d state -> RESET\n", qp_num(qp));
+			rxe_dbg_qp(qp, "state -> RESET\n");
 			rxe_qp_reset(qp);
 			break;
 		case IB_QPS_INIT:
-			pr_debug("qp#%d state -> INIT\n", qp_num(qp));
+			rxe_dbg_qp(qp, "state -> INIT\n");
 			qp->req.state = QP_STATE_INIT;
 			qp->resp.state = QP_STATE_INIT;
 			qp->comp.state = QP_STATE_INIT;
 			break;
 		case IB_QPS_RTR:
-			pr_debug("qp#%d state -> RTR\n", qp_num(qp));
+			rxe_dbg_qp(qp, "state -> RTR\n");
 			qp->resp.state = QP_STATE_READY;
 			break;
 		case IB_QPS_RTS:
-			pr_debug("qp#%d state -> RTS\n", qp_num(qp));
+			rxe_dbg_qp(qp, "state -> RTS\n");
 			qp->req.state = QP_STATE_READY;
 			qp->comp.state = QP_STATE_READY;
 			break;
 		case IB_QPS_SQD:
-			pr_debug("qp#%d state -> SQD\n", qp_num(qp));
+			rxe_dbg_qp(qp, "state -> SQD\n");
 			rxe_qp_drain(qp);
 			break;
 		case IB_QPS_SQE:
-			pr_warn("qp#%d state -> SQE !!?\n", qp_num(qp));
+			rxe_dbg_qp(qp, "state -> SQE !!?\n");
 			/* Not possible from modify_qp. */
 			break;
 		case IB_QPS_ERR:
-			pr_debug("qp#%d state -> ERR\n", qp_num(qp));
+			rxe_dbg_qp(qp, "state -> ERR\n");
 			rxe_qp_error(qp);
 			break;
 		}
 		attr->sq_draining = 0;
 	}
-	pr_debug("attr->sq_draining = %d\n", attr->sq_draining);
+	rxe_dbg_qp(qp, "attr->sq_draining = %d\n", attr->sq_draining);
 	return 0;
 }
 	 * will fail immediately.
 	 */
 	if (atomic_read(&qp->mcg_num)) {
-		pr_debug("Attempt to destroy QP while attached to multicast group\n");
+		rxe_dbg_qp(qp, "Attempt to destroy while attached to multicast group\n");
 		return -EBUSY;
 	}
 	if (qp->resp.mr)
 		rxe_put(qp->resp.mr);
-	if (qp_type(qp) == IB_QPT_RC)
-		sk_dst_reset(qp->sk->sk);
 	free_rd_atomic_resources(qp);
 	if (qp->sk) {
+		if (qp_type(qp) == IB_QPT_RC)
+			sk_dst_reset(qp->sk->sk);
 		kernel_sock_shutdown(qp->sk, SHUT_RDWR);
 		sock_release(qp->sk);
 	}

drivers/infiniband/sw/rxe/rxe_req.c+37−13

 {
 	struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
-	pr_debug("%s: fired for qp#%d\n", __func__, qp_num(qp));
+	rxe_dbg_qp(qp, "nak timer fired\n");
 	/* request a send queue retry */
 	qp->req.need_retry = 1;
 	qp->req.wait_for_rnr_timer = 0;
-	rxe_run_task(&qp->req.task, 1);
+	rxe_sched_task(&qp->req.task);
 }
 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
 				IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE :
 				IB_OPCODE_RC_SEND_FIRST;
+	case IB_WR_FLUSH:
+		return IB_OPCODE_RC_FLUSH;
 	case IB_WR_RDMA_READ:
 		return IB_OPCODE_RC_RDMA_READ_REQUEST;
 		else
 			return fits ? IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE :
 				IB_OPCODE_RC_SEND_FIRST;
+	case IB_WR_ATOMIC_WRITE:
+		return IB_OPCODE_RC_ATOMIC_WRITE;
 	case IB_WR_REG_MR:
 	case IB_WR_LOCAL_INV:
 		return opcode;
 	/* init optional headers */
 	if (pkt->mask & RXE_RETH_MASK) {
-		reth_set_rkey(pkt, ibwr->wr.rdma.rkey);
+		if (pkt->mask & RXE_FETH_MASK)
+			reth_set_rkey(pkt, ibwr->wr.flush.rkey);
+		else
+			reth_set_rkey(pkt, ibwr->wr.rdma.rkey);
 		reth_set_va(pkt, wqe->iova);
 		reth_set_len(pkt, wqe->dma.resid);
 	}
+	/* Fill Flush Extension Transport Header */
+	if (pkt->mask & RXE_FETH_MASK)
+		feth_init(pkt, ibwr->wr.flush.type, ibwr->wr.flush.level);
 	if (pkt->mask & RXE_IMMDT_MASK)
 		immdt_set_imm(pkt, ibwr->ex.imm_data);
 			memset(pad, 0, bth_pad(pkt));
 		}
+	} else if (pkt->mask & RXE_FLUSH_MASK) {
+		/* oA19-2: shall have no payload. */
+		wqe->dma.resid = 0;
+	}
+	if (pkt->mask & RXE_ATOMIC_WRITE_MASK) {
+		memcpy(payload_addr(pkt), wqe->dma.atomic_wr, payload);
+		wqe->dma.resid -= payload;
 	}
 	return 0;
 		}
 		break;
 	default:
-		pr_err("Unexpected send wqe opcode %d\n", opcode);
+		rxe_dbg_qp(qp, "Unexpected send wqe opcode %d\n", opcode);
 		wqe->status = IB_WC_LOC_QP_OP_ERR;
 		return -EINVAL;
 	}
 	 * which can lead to a deadlock. So go ahead and complete
 	 * it now.
 	 */
-	rxe_run_task(&qp->comp.task, 1);
+	rxe_sched_task(&qp->comp.task);
 	return 0;
 }
 	}
 	mask = rxe_opcode[opcode].mask;
-	if (unlikely(mask & RXE_READ_OR_ATOMIC_MASK)) {
+	if (unlikely(mask & (RXE_READ_OR_ATOMIC_MASK |
+			RXE_ATOMIC_WRITE_MASK))) {
 		if (check_init_depth(qp, wqe))
 			goto exit;
 	}
 	mtu = get_mtu(qp);
-	payload = (mask & RXE_WRITE_OR_SEND_MASK) ? wqe->dma.resid : 0;
+	payload = (mask & (RXE_WRITE_OR_SEND_MASK | RXE_ATOMIC_WRITE_MASK)) ?
+			wqe->dma.resid : 0;
 	if (payload > mtu) {
 		if (qp_type(qp) == IB_QPT_UD) {
 			/* C10-93.1.1: If the total sum of all the buffer lengths specified for a
 						       qp->req.wqe_index);
 			wqe->state = wqe_state_done;
 			wqe->status = IB_WC_SUCCESS;
-			rxe_run_task(&qp->comp.task, 0);
+			rxe_run_task(&qp->comp.task);
 			goto done;
 		}
 		payload = mtu;
 	av = rxe_get_av(&pkt, &ah);
 	if (unlikely(!av)) {
-		pr_err("qp#%d Failed no address vector\n", qp_num(qp));
+		rxe_dbg_qp(qp, "Failed no address vector\n");
 		wqe->status = IB_WC_LOC_QP_OP_ERR;
 		goto err;
 	}
 	skb = init_req_packet(qp, av, wqe, opcode, payload, &pkt);
 	if (unlikely(!skb)) {
-		pr_err("qp#%d Failed allocating skb\n", qp_num(qp));
+		rxe_dbg_qp(qp, "Failed allocating skb\n");
 		wqe->status = IB_WC_LOC_QP_OP_ERR;
 		if (ah)
 			rxe_put(ah);
 	err = finish_packet(qp, av, wqe, &pkt, skb, payload);
 	if (unlikely(err)) {
-		pr_debug("qp#%d Error during finish packet\n", qp_num(qp));
+		rxe_dbg_qp(qp, "Error during finish packet\n");
 		if (err == -EFAULT)
 			wqe->status = IB_WC_LOC_PROT_ERR;
 		else
 		rollback_state(wqe, qp, &rollback_wqe, rollback_psn);
 		if (err == -EAGAIN) {
-			rxe_run_task(&qp->req.task, 1);
+			rxe_sched_task(&qp->req.task);
 			goto exit;
 		}
 	qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index);
 	wqe->state = wqe_state_error;
 	qp->req.state = QP_STATE_ERROR;
-	rxe_run_task(&qp->comp.task, 0);
+	rxe_run_task(&qp->comp.task);
 exit:
 	ret = -EAGAIN;
 out:

drivers/infiniband/sw/rxe/rxe_resp.c+279−50

@@ -22,6 +22,8 @@ enum resp_states { · 21 unmodified lines
22	RESPST_EXECUTE,	22	RESPST_EXECUTE,
23	RESPST_READ_REPLY,	23	RESPST_READ_REPLY,
24	RESPST_ATOMIC_REPLY,	24	RESPST_ATOMIC_REPLY,
		25	RESPST_ATOMIC_WRITE_REPLY,
		26	RESPST_PROCESS_FLUSH,
25	RESPST_COMPLETE,	27	RESPST_COMPLETE,
26	RESPST_ACKNOWLEDGE,	28	RESPST_ACKNOWLEDGE,
27	RESPST_CLEANUP,	29	RESPST_CLEANUP,
@@ -57,6 +59,8 @@ static char *resp_state_name[] = { · 29 unmodified lines
57	[RESPST_EXECUTE] = "EXECUTE",	59	[RESPST_EXECUTE] = "EXECUTE",
58	[RESPST_READ_REPLY] = "READ_REPLY",	60	[RESPST_READ_REPLY] = "READ_REPLY",
59	[RESPST_ATOMIC_REPLY] = "ATOMIC_REPLY",	61	[RESPST_ATOMIC_REPLY] = "ATOMIC_REPLY",
		62	[RESPST_ATOMIC_WRITE_REPLY] = "ATOMIC_WRITE_REPLY",
		63	[RESPST_PROCESS_FLUSH] = "PROCESS_FLUSH",
60	[RESPST_COMPLETE] = "COMPLETE",	64	[RESPST_COMPLETE] = "COMPLETE",
61	[RESPST_ACKNOWLEDGE] = "ACKNOWLEDGE",	65	[RESPST_ACKNOWLEDGE] = "ACKNOWLEDGE",
62	[RESPST_CLEANUP] = "CLEANUP",	66	[RESPST_CLEANUP] = "CLEANUP",
@@ -91,7 +95,10 @@ void rxe_resp_queue_pkt(struct rxe_qp qp, struct sk_buff skb) · 28 unmodified lines
91	must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) \|\|	95	must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) \|\|
92	(skb_queue_len(&qp->req_pkts) > 1);	96	(skb_queue_len(&qp->req_pkts) > 1);
93		97
94	rxe_run_task(&qp->resp.task, must_sched);	98	if (must_sched)
		99	rxe_sched_task(&qp->resp.task);
		100	else
		101	rxe_run_task(&qp->resp.task);
95	}	102	}
96		103
97	static inline enum resp_states get_req(struct rxe_qp *qp,	104	static inline enum resp_states get_req(struct rxe_qp *qp,
@@ -253,19 +260,37 @@ static enum resp_states check_op_seq(struct rxe_qp *qp, · 155 unmodified lines
253	}	260	}
254	}	261	}
255		262
		263	static bool check_qp_attr_access(struct rxe_qp *qp,
		264	struct rxe_pkt_info *pkt)
		265	{
		266	if (((pkt->mask & RXE_READ_MASK) &&
		267	!(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) \|\|
		268	((pkt->mask & (RXE_WRITE_MASK \| RXE_ATOMIC_WRITE_MASK)) &&
		269	!(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) \|\|
		270	((pkt->mask & RXE_ATOMIC_MASK) &&
		271	!(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
		272	return false;
		273
		274	if (pkt->mask & RXE_FLUSH_MASK) {
		275	u32 flush_type = feth_plt(pkt);
		276
		277	if ((flush_type & IB_FLUSH_GLOBAL &&
		278	!(qp->attr.qp_access_flags & IB_ACCESS_FLUSH_GLOBAL)) \|\|
		279	(flush_type & IB_FLUSH_PERSISTENT &&
		280	!(qp->attr.qp_access_flags & IB_ACCESS_FLUSH_PERSISTENT)))
		281	return false;
		282	}
		283
		284	return true;
		285	}
		286
256	static enum resp_states check_op_valid(struct rxe_qp *qp,	287	static enum resp_states check_op_valid(struct rxe_qp *qp,
257	struct rxe_pkt_info *pkt)	288	struct rxe_pkt_info *pkt)
258	{	289	{
259	switch (qp_type(qp)) {	290	switch (qp_type(qp)) {
260	case IB_QPT_RC:	291	case IB_QPT_RC:
261	if (((pkt->mask & RXE_READ_MASK) &&	292	if (!check_qp_attr_access(qp, pkt))
262	!(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) \|\|
263	((pkt->mask & RXE_WRITE_MASK) &&
264	!(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) \|\|
265	((pkt->mask & RXE_ATOMIC_MASK) &&
266	!(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) {
267	return RESPST_ERR_UNSUPPORTED_OPCODE;	293	return RESPST_ERR_UNSUPPORTED_OPCODE;
268	}
269		294
270	break;	295	break;
271		296
@@ -314,7 +339,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) · 42 unmodified lines
314	/* don't trust user space data */	339	/* don't trust user space data */
315	if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) {	340	if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) {
316	spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);	341	spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
317	pr_warn("%s: invalid num_sge in SRQ entry\n", __func__);	342	rxe_dbg_qp(qp, "invalid num_sge in SRQ entry\n");
318	return RESPST_ERR_MALFORMED_WQE;	343	return RESPST_ERR_MALFORMED_WQE;
319	}	344	}
320	size = sizeof(wqe) + wqe->dma.num_sgesizeof(struct rxe_sge);	345	size = sizeof(wqe) + wqe->dma.num_sgesizeof(struct rxe_sge);
@@ -364,7 +389,7 @@ static enum resp_states check_resource(struct rxe_qp *qp, · 43 unmodified lines
364	}	389	}
365	}	390	}
366		391
367	if (pkt->mask & RXE_READ_OR_ATOMIC_MASK) {	392	if (pkt->mask & (RXE_READ_OR_ATOMIC_MASK \| RXE_ATOMIC_WRITE_MASK)) {
368	/* it is the requesters job to not send	393	/* it is the requesters job to not send
369	* too many read/atomic ops, we just	394	* too many read/atomic ops, we just
370	* recycle the responder resource queue	395	* recycle the responder resource queue
@@ -387,19 +412,66 @@ static enum resp_states check_resource(struct rxe_qp *qp, · 16 unmodified lines
387	return RESPST_CHK_LENGTH;	412	return RESPST_CHK_LENGTH;
388	}	413	}
389		414
390	static enum resp_states check_length(struct rxe_qp *qp,	415	static enum resp_states rxe_resp_check_length(struct rxe_qp *qp,
391	struct rxe_pkt_info *pkt)	416	struct rxe_pkt_info *pkt)
392	{	417	{
393	switch (qp_type(qp)) {	418	/*
394	case IB_QPT_RC:	419	* See IBA C9-92
395	return RESPST_CHK_RKEY;	420	* For UD QPs we only check if the packet will fit in the
396		421	* receive buffer later. For rmda operations additional
397	case IB_QPT_UC:	422	* length checks are performed in check_rkey.
398	return RESPST_CHK_RKEY;	423	*/
		424	if (pkt->mask & RXE_PAYLOAD_MASK && ((qp_type(qp) == IB_QPT_RC) \|\|
		425	(qp_type(qp) == IB_QPT_UC))) {
		426	unsigned int mtu = qp->mtu;
		427	unsigned int payload = payload_size(pkt);
		428
		429	if ((pkt->mask & RXE_START_MASK) &&
		430	(pkt->mask & RXE_END_MASK)) {
		431	if (unlikely(payload > mtu)) {
		432	rxe_dbg_qp(qp, "only packet too long");
		433	return RESPST_ERR_LENGTH;
		434	}
		435	} else if ((pkt->mask & RXE_START_MASK) \|\|
		436	(pkt->mask & RXE_MIDDLE_MASK)) {
		437	if (unlikely(payload != mtu)) {
		438	rxe_dbg_qp(qp, "first or middle packet not mtu");
		439	return RESPST_ERR_LENGTH;
		440	}
		441	} else if (pkt->mask & RXE_END_MASK) {
		442	if (unlikely((payload == 0) \|\| (payload > mtu))) {
		443	rxe_dbg_qp(qp, "last packet zero or too long");
		444	return RESPST_ERR_LENGTH;
		445	}
		446	}
		447	}
399		448
400	default:	449	/* See IBA C9-94 */
401	return RESPST_CHK_RKEY;	450	if (pkt->mask & RXE_RETH_MASK) {
		451	if (reth_len(pkt) > (1U << 31)) {
		452	rxe_dbg_qp(qp, "dma length too long");
		453	return RESPST_ERR_LENGTH;
		454	}
402	}	455	}
		456
		457	return RESPST_CHK_RKEY;
		458	}
		459
		460	static void qp_resp_from_reth(struct rxe_qp qp, struct rxe_pkt_info pkt)
		461	{
		462	qp->resp.va = reth_va(pkt);
		463	qp->resp.offset = 0;
		464	qp->resp.rkey = reth_rkey(pkt);
		465	qp->resp.resid = reth_len(pkt);
		466	qp->resp.length = reth_len(pkt);
		467	}
		468
		469	static void qp_resp_from_atmeth(struct rxe_qp qp, struct rxe_pkt_info pkt)
		470	{
		471	qp->resp.va = atmeth_va(pkt);
		472	qp->resp.offset = 0;
		473	qp->resp.rkey = atmeth_rkey(pkt);
		474	qp->resp.resid = sizeof(u64);
403	}	475	}
404		476
405	static enum resp_states check_rkey(struct rxe_qp *qp,	477	static enum resp_states check_rkey(struct rxe_qp *qp,
@@ -413,29 +485,32 @@ static enum resp_states check_rkey(struct rxe_qp *qp, · 7 unmodified lines
413	u32 pktlen;	485	u32 pktlen;
414	int mtu = qp->mtu;	486	int mtu = qp->mtu;
415	enum resp_states state;	487	enum resp_states state;
416	int access;	488	int access = 0;
417		489
418	if (pkt->mask & RXE_READ_OR_WRITE_MASK) {	490	if (pkt->mask & (RXE_READ_OR_WRITE_MASK \| RXE_ATOMIC_WRITE_MASK)) {
419	if (pkt->mask & RXE_RETH_MASK) {	491	if (pkt->mask & RXE_RETH_MASK)
420	qp->resp.va = reth_va(pkt);	492	qp_resp_from_reth(qp, pkt);
421	qp->resp.offset = 0;	493
422	qp->resp.rkey = reth_rkey(pkt);
423	qp->resp.resid = reth_len(pkt);
424	qp->resp.length = reth_len(pkt);
425	}
426	access = (pkt->mask & RXE_READ_MASK) ? IB_ACCESS_REMOTE_READ	494	access = (pkt->mask & RXE_READ_MASK) ? IB_ACCESS_REMOTE_READ
427	: IB_ACCESS_REMOTE_WRITE;	495	: IB_ACCESS_REMOTE_WRITE;
		496	} else if (pkt->mask & RXE_FLUSH_MASK) {
		497	u32 flush_type = feth_plt(pkt);
		498
		499	if (pkt->mask & RXE_RETH_MASK)
		500	qp_resp_from_reth(qp, pkt);
		501
		502	if (flush_type & IB_FLUSH_GLOBAL)
		503	access \|= IB_ACCESS_FLUSH_GLOBAL;
		504	if (flush_type & IB_FLUSH_PERSISTENT)
		505	access \|= IB_ACCESS_FLUSH_PERSISTENT;
428	} else if (pkt->mask & RXE_ATOMIC_MASK) {	506	} else if (pkt->mask & RXE_ATOMIC_MASK) {
429	qp->resp.va = atmeth_va(pkt);	507	qp_resp_from_atmeth(qp, pkt);
430	qp->resp.offset = 0;
431	qp->resp.rkey = atmeth_rkey(pkt);
432	qp->resp.resid = sizeof(u64);
433	access = IB_ACCESS_REMOTE_ATOMIC;	508	access = IB_ACCESS_REMOTE_ATOMIC;
434	} else {	509	} else {
435	return RESPST_EXECUTE;	510	return RESPST_EXECUTE;
436	}	511	}
437		512
438	/* A zero-byte op is not required to set an addr or rkey. */	513	/* A zero-byte op is not required to set an addr or rkey. See C9-88 */
439	if ((pkt->mask & RXE_READ_OR_WRITE_MASK) &&	514	if ((pkt->mask & RXE_READ_OR_WRITE_MASK) &&
440	(pkt->mask & RXE_RETH_MASK) &&	515	(pkt->mask & RXE_RETH_MASK) &&
441	reth_len(pkt) == 0) {	516	reth_len(pkt) == 0) {
@@ -450,15 +525,14 @@ static enum resp_states check_rkey(struct rxe_qp *qp, · 8 unmodified lines
450	if (rkey_is_mw(rkey)) {	525	if (rkey_is_mw(rkey)) {
451	mw = rxe_lookup_mw(qp, access, rkey);	526	mw = rxe_lookup_mw(qp, access, rkey);
452	if (!mw) {	527	if (!mw) {
453	pr_debug("%s: no MW matches rkey %#x\n",	528	rxe_dbg_qp(qp, "no MW matches rkey %#x\n", rkey);
454	__func__, rkey);
455	state = RESPST_ERR_RKEY_VIOLATION;	529	state = RESPST_ERR_RKEY_VIOLATION;
456	goto err;	530	goto err;
457	}	531	}
458		532
459	mr = mw->mr;	533	mr = mw->mr;
460	if (!mr) {	534	if (!mr) {
461	pr_err("%s: MW doesn't have an MR\n", __func__);	535	rxe_dbg_qp(qp, "MW doesn't have an MR\n");
462	state = RESPST_ERR_RKEY_VIOLATION;	536	state = RESPST_ERR_RKEY_VIOLATION;
463	goto err;	537	goto err;
464	}	538	}
@@ -471,19 +545,27 @@ static enum resp_states check_rkey(struct rxe_qp *qp, · 6 unmodified lines
471	} else {	545	} else {
472	mr = lookup_mr(qp->pd, access, rkey, RXE_LOOKUP_REMOTE);	546	mr = lookup_mr(qp->pd, access, rkey, RXE_LOOKUP_REMOTE);
473	if (!mr) {	547	if (!mr) {
474	pr_debug("%s: no MR matches rkey %#x\n",	548	rxe_dbg_qp(qp, "no MR matches rkey %#x\n", rkey);
475	__func__, rkey);
476	state = RESPST_ERR_RKEY_VIOLATION;	549	state = RESPST_ERR_RKEY_VIOLATION;
477	goto err;	550	goto err;
478	}	551	}
479	}	552	}
480		553
		554	if (pkt->mask & RXE_FLUSH_MASK) {
		555	/* FLUSH MR may not set va or resid
		556	* no need to check range since we will flush whole mr
		557	*/
		558	if (feth_sel(pkt) == IB_FLUSH_MR)
		559	goto skip_check_range;
		560	}
		561
481	if (mr_check_range(mr, va + qp->resp.offset, resid)) {	562	if (mr_check_range(mr, va + qp->resp.offset, resid)) {
482	state = RESPST_ERR_RKEY_VIOLATION;	563	state = RESPST_ERR_RKEY_VIOLATION;
483	goto err;	564	goto err;
484	}	565	}
485		566
486	if (pkt->mask & RXE_WRITE_MASK) {	567	skip_check_range:
		568	if (pkt->mask & (RXE_WRITE_MASK \| RXE_ATOMIC_WRITE_MASK)) {
487	if (resid > mtu) {	569	if (resid > mtu) {
488	if (pktlen != mtu \|\| bth_pad(pkt)) {	570	if (pktlen != mtu \|\| bth_pad(pkt)) {
489	state = RESPST_ERR_LENGTH;	571	state = RESPST_ERR_LENGTH;
@@ -583,15 +665,66 @@ static struct resp_res rxe_prepare_res(struct rxe_qp qp, · 93 unmodified lines
583	res->state = rdatm_res_state_new;	665	res->state = rdatm_res_state_new;
584	break;	666	break;
585	case RXE_ATOMIC_MASK:	667	case RXE_ATOMIC_MASK:
		668	case RXE_ATOMIC_WRITE_MASK:
586	res->first_psn = pkt->psn;	669	res->first_psn = pkt->psn;
587	res->last_psn = pkt->psn;	670	res->last_psn = pkt->psn;
588	res->cur_psn = pkt->psn;	671	res->cur_psn = pkt->psn;
589	break;	672	break;
		673	case RXE_FLUSH_MASK:
		674	res->flush.va = qp->resp.va + qp->resp.offset;
		675	res->flush.length = qp->resp.length;
		676	res->flush.type = feth_plt(pkt);
		677	res->flush.level = feth_sel(pkt);
590	}	678	}
591		679
592	return res;	680	return res;
593	}	681	}
594		682
		683	static enum resp_states process_flush(struct rxe_qp *qp,
		684	struct rxe_pkt_info *pkt)
		685	{
		686	u64 length, start;
		687	struct rxe_mr *mr = qp->resp.mr;
		688	struct resp_res *res = qp->resp.res;
		689
		690	/* oA19-14, oA19-15 */
		691	if (res && res->replay)
		692	return RESPST_ACKNOWLEDGE;
		693	else if (!res) {
		694	res = rxe_prepare_res(qp, pkt, RXE_FLUSH_MASK);
		695	qp->resp.res = res;
		696	}
		697
		698	if (res->flush.level == IB_FLUSH_RANGE) {
		699	start = res->flush.va;
		700	length = res->flush.length;
		701	} else { /* level == IB_FLUSH_MR */
		702	start = mr->ibmr.iova;
		703	length = mr->ibmr.length;
		704	}
		705
		706	if (res->flush.type & IB_FLUSH_PERSISTENT) {
		707	if (rxe_flush_pmem_iova(mr, start, length))
		708	return RESPST_ERR_RKEY_VIOLATION;
		709	/* Make data persistent. */
		710	wmb();
		711	} else if (res->flush.type & IB_FLUSH_GLOBAL) {
		712	/* Make data global visibility. */
		713	wmb();
		714	}
		715
		716	qp->resp.msn++;
		717
		718	/* next expected psn, read handles this separately */
		719	qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
		720	qp->resp.ack_psn = qp->resp.psn;
		721
		722	qp->resp.opcode = pkt->opcode;
		723	qp->resp.status = IB_WC_SUCCESS;
		724
		725	return RESPST_ACKNOWLEDGE;
		726	}
		727
595	/* Guarantee atomicity of atomic operations at the machine level. */	728	/* Guarantee atomicity of atomic operations at the machine level. */
596	static DEFINE_SPINLOCK(atomic_ops_lock);	729	static DEFINE_SPINLOCK(atomic_ops_lock);
597		730
@@ -652,6 +785,55 @@ out: · 54 unmodified lines
652	return ret;	785	return ret;
653	}	786	}
654		787
		788	static enum resp_states atomic_write_reply(struct rxe_qp *qp,
		789	struct rxe_pkt_info *pkt)
		790	{
		791	u64 src, *dst;
		792	struct resp_res *res = qp->resp.res;
		793	struct rxe_mr *mr = qp->resp.mr;
		794	int payload = payload_size(pkt);
		795
		796	if (!res) {
		797	res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
		798	qp->resp.res = res;
		799	}
		800
		801	if (!res->replay) {
		802	#ifdef CONFIG_64BIT
		803	if (mr->state != RXE_MR_STATE_VALID)
		804	return RESPST_ERR_RKEY_VIOLATION;
		805
		806	memcpy(&src, payload_addr(pkt), payload);
		807
		808	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
		809	/* check vaddr is 8 bytes aligned. */
		810	if (!dst \|\| (uintptr_t)dst & 7)
		811	return RESPST_ERR_MISALIGNED_ATOMIC;
		812
		813	/* Do atomic write after all prior operations have completed */
		814	smp_store_release(dst, src);
		815
		816	/* decrease resp.resid to zero */
		817	qp->resp.resid -= sizeof(payload);
		818
		819	qp->resp.msn++;
		820
		821	/* next expected psn, read handles this separately */
		822	qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
		823	qp->resp.ack_psn = qp->resp.psn;
		824
		825	qp->resp.opcode = pkt->opcode;
		826	qp->resp.status = IB_WC_SUCCESS;
		827
		828	return RESPST_ACKNOWLEDGE;
		829	#else
		830	return RESPST_ERR_UNSUPPORTED_OPCODE;
		831	#endif /* CONFIG_64BIT */
		832	}
		833
		834	return RESPST_ACKNOWLEDGE;
		835	}
		836
655	static struct sk_buff prepare_ack_packet(struct rxe_qp qp,	837	static struct sk_buff prepare_ack_packet(struct rxe_qp qp,
656	struct rxe_pkt_info *ack,	838	struct rxe_pkt_info *ack,
657	int opcode,	839	int opcode,
@@ -807,14 +989,19 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 149 unmodified lines
807	skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload,	989	skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload,
808	res->cur_psn, AETH_ACK_UNLIMITED);	990	res->cur_psn, AETH_ACK_UNLIMITED);
809	if (!skb) {	991	if (!skb) {
810	rxe_put(mr);	992	if (mr)
		993	rxe_put(mr);
811	return RESPST_ERR_RNR;	994	return RESPST_ERR_RNR;
812	}	995	}
813		996
814	rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),	997	err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
815	payload, RXE_FROM_MR_OBJ);	998	payload, RXE_FROM_MR_OBJ);
816	if (mr)	999	if (mr)
817	rxe_put(mr);	1000	rxe_put(mr);
		1001	if (err) {
		1002	kfree_skb(skb);
		1003	return RESPST_ERR_RKEY_VIOLATION;
		1004	}
818		1005
819	if (bth_pad(&ack_pkt)) {	1006	if (bth_pad(&ack_pkt)) {
820	u8 *pad = payload_addr(&ack_pkt) + payload;	1007	u8 *pad = payload_addr(&ack_pkt) + payload;
@@ -890,6 +1077,10 @@ static enum resp_states execute(struct rxe_qp qp, struct rxe_pkt_info pkt) · 69 unmodified lines
890	return RESPST_READ_REPLY;	1077	return RESPST_READ_REPLY;
891	} else if (pkt->mask & RXE_ATOMIC_MASK) {	1078	} else if (pkt->mask & RXE_ATOMIC_MASK) {
892	return RESPST_ATOMIC_REPLY;	1079	return RESPST_ATOMIC_REPLY;
		1080	} else if (pkt->mask & RXE_ATOMIC_WRITE_MASK) {
		1081	return RESPST_ATOMIC_WRITE_REPLY;
		1082	} else if (pkt->mask & RXE_FLUSH_MASK) {
		1083	return RESPST_PROCESS_FLUSH;
893	} else {	1084	} else {
894	/* Unreachable */	1085	/* Unreachable */
895	WARN_ON_ONCE(1);	1086	WARN_ON_ONCE(1);
@@ -1040,7 +1231,7 @@ static int send_common_ack(struct rxe_qp *qp, u8 syndrome, u32 psn, · 144 unmodified lines
1040		1231
1041	err = rxe_xmit_packet(qp, &ack_pkt, skb);	1232	err = rxe_xmit_packet(qp, &ack_pkt, skb);
1042	if (err)	1233	if (err)
1043	pr_err_ratelimited("Failed sending %s\n", msg);	1234	rxe_dbg_qp(qp, "Failed sending %s\n", msg);
1044		1235
1045	return err;	1236	return err;
1046	}	1237	}
@@ -1063,6 +1254,19 @@ static int send_atomic_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) · 16 unmodified lines
1063	return ret;	1254	return ret;
1064	}	1255	}
1065		1256
		1257	static int send_read_response_ack(struct rxe_qp *qp, u8 syndrome, u32 psn)
		1258	{
		1259	int ret = send_common_ack(qp, syndrome, psn,
		1260	IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY,
		1261	"RDMA READ response of length zero ACK");
		1262
		1263	/* have to clear this since it is used to trigger
		1264	* long read replies
		1265	*/
		1266	qp->resp.res = NULL;
		1267	return ret;
		1268	}
		1269
1066	static enum resp_states acknowledge(struct rxe_qp *qp,	1270	static enum resp_states acknowledge(struct rxe_qp *qp,
1067	struct rxe_pkt_info *pkt)	1271	struct rxe_pkt_info *pkt)
1068	{	1272	{
@@ -1073,6 +1277,8 @@ static enum resp_states acknowledge(struct rxe_qp *qp, · 4 unmodified lines
1073	send_ack(qp, qp->resp.aeth_syndrome, pkt->psn);	1277	send_ack(qp, qp->resp.aeth_syndrome, pkt->psn);
1074	else if (pkt->mask & RXE_ATOMIC_MASK)	1278	else if (pkt->mask & RXE_ATOMIC_MASK)
1075	send_atomic_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);	1279	send_atomic_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
		1280	else if (pkt->mask & (RXE_FLUSH_MASK \| RXE_ATOMIC_WRITE_MASK))
		1281	send_read_response_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1076	else if (bth_ack(pkt))	1282	else if (bth_ack(pkt))
1077	send_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);	1283	send_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1078		1284
@@ -1129,6 +1335,22 @@ static enum resp_states duplicate_request(struct rxe_qp *qp, · 50 unmodified lines
1129	/* SEND. Ack again and cleanup. C9-105. */	1335	/* SEND. Ack again and cleanup. C9-105. */
1130	send_ack(qp, AETH_ACK_UNLIMITED, prev_psn);	1336	send_ack(qp, AETH_ACK_UNLIMITED, prev_psn);
1131	return RESPST_CLEANUP;	1337	return RESPST_CLEANUP;
		1338	} else if (pkt->mask & RXE_FLUSH_MASK) {
		1339	struct resp_res *res;
		1340
		1341	/* Find the operation in our list of responder resources. */
		1342	res = find_resource(qp, pkt->psn);
		1343	if (res) {
		1344	res->replay = 1;
		1345	res->cur_psn = pkt->psn;
		1346	qp->resp.res = res;
		1347	rc = RESPST_PROCESS_FLUSH;
		1348	goto out;
		1349	}
		1350
		1351	/* Resource not found. Class D error. Drop the request. */
		1352	rc = RESPST_CLEANUP;
		1353	goto out;
1132	} else if (pkt->mask & RXE_READ_MASK) {	1354	} else if (pkt->mask & RXE_READ_MASK) {
1133	struct resp_res *res;	1355	struct resp_res *res;
1134		1356
@@ -1184,7 +1406,9 @@ static enum resp_states duplicate_request(struct rxe_qp *qp, · 49 unmodified lines
1184	res->replay = 1;	1406	res->replay = 1;
1185	res->cur_psn = pkt->psn;	1407	res->cur_psn = pkt->psn;
1186	qp->resp.res = res;	1408	qp->resp.res = res;
1187	rc = RESPST_ATOMIC_REPLY;	1409	rc = pkt->mask & RXE_ATOMIC_MASK ?
		1410	RESPST_ATOMIC_REPLY :
		1411	RESPST_ATOMIC_WRITE_REPLY;
1188	goto out;	1412	goto out;
1189	}	1413	}
1190		1414
@@ -1286,8 +1510,7 @@ int rxe_responder(void *arg) · 95 unmodified lines
1286	}	1510	}
1287		1511
1288	while (1) {	1512	while (1) {
1289	pr_debug("qp#%d state = %s\n", qp_num(qp),	1513	rxe_dbg_qp(qp, "state = %s\n", resp_state_name[state]);
1290	resp_state_name[state]);
1291	switch (state) {	1514	switch (state) {
1292	case RESPST_GET_REQ:	1515	case RESPST_GET_REQ:
1293	state = get_req(qp, &pkt);	1516	state = get_req(qp, &pkt);
@@ -1305,7 +1528,7 @@ int rxe_responder(void *arg) · 11 unmodified lines
1305	state = check_resource(qp, pkt);	1528	state = check_resource(qp, pkt);
1306	break;	1529	break;
1307	case RESPST_CHK_LENGTH:	1530	case RESPST_CHK_LENGTH:
1308	state = check_length(qp, pkt);	1531	state = rxe_resp_check_length(qp, pkt);
1309	break;	1532	break;
1310	case RESPST_CHK_RKEY:	1533	case RESPST_CHK_RKEY:
1311	state = check_rkey(qp, pkt);	1534	state = check_rkey(qp, pkt);
@@ -1322,6 +1545,12 @@ int rxe_responder(void *arg) · 10 unmodified lines
1322	case RESPST_ATOMIC_REPLY:	1545	case RESPST_ATOMIC_REPLY:
1323	state = atomic_reply(qp, pkt);	1546	state = atomic_reply(qp, pkt);
1324	break;	1547	break;
		1548	case RESPST_ATOMIC_WRITE_REPLY:
		1549	state = atomic_write_reply(qp, pkt);
		1550	break;
		1551	case RESPST_PROCESS_FLUSH:
		1552	state = process_flush(qp, pkt);
		1553	break;
1325	case RESPST_ACKNOWLEDGE:	1554	case RESPST_ACKNOWLEDGE:
1326	state = acknowledge(qp, pkt);	1555	state = acknowledge(qp, pkt);
1327	break;	1556	break;
@@ -1444,7 +1673,7 @@ int rxe_responder(void *arg) · 116 unmodified lines
1444		1673
1445	case RESPST_ERROR:	1674	case RESPST_ERROR:
1446	qp->resp.goto_error = 0;	1675	qp->resp.goto_error = 0;
1447	pr_debug("qp#%d moved to error state\n", qp_num(qp));	1676	rxe_dbg_qp(qp, "moved to error state\n");
1448	rxe_qp_error(qp);	1677	rxe_qp_error(qp);
1449	goto exit;	1678	goto exit;
1450		1679

drivers/infiniband/sw/rxe/rxe_srq.c+10−10

@@ -13,13 +13,13 @@ int rxe_srq_chk_init(struct rxe_dev rxe, struct ib_srq_init_attr init) · 12 unmodified lines
13	struct ib_srq_attr *attr = &init->attr;	13	struct ib_srq_attr *attr = &init->attr;
14		14
15	if (attr->max_wr > rxe->attr.max_srq_wr) {	15	if (attr->max_wr > rxe->attr.max_srq_wr) {
16	pr_warn("max_wr(%d) > max_srq_wr(%d)\n",	16	rxe_dbg(rxe, "max_wr(%d) > max_srq_wr(%d)\n",
17	attr->max_wr, rxe->attr.max_srq_wr);	17	attr->max_wr, rxe->attr.max_srq_wr);
18	goto err1;	18	goto err1;
19	}	19	}
20		20
21	if (attr->max_wr <= 0) {	21	if (attr->max_wr <= 0) {
22	pr_warn("max_wr(%d) <= 0\n", attr->max_wr);	22	rxe_dbg(rxe, "max_wr(%d) <= 0\n", attr->max_wr);
23	goto err1;	23	goto err1;
24	}	24	}
25		25
@@ -27,7 +27,7 @@ int rxe_srq_chk_init(struct rxe_dev rxe, struct ib_srq_init_attr init) · 1 unmodified line
27	attr->max_wr = RXE_MIN_SRQ_WR;	27	attr->max_wr = RXE_MIN_SRQ_WR;
28		28
29	if (attr->max_sge > rxe->attr.max_srq_sge) {	29	if (attr->max_sge > rxe->attr.max_srq_sge) {
30	pr_warn("max_sge(%d) > max_srq_sge(%d)\n",	30	rxe_dbg(rxe, "max_sge(%d) > max_srq_sge(%d)\n",
31	attr->max_sge, rxe->attr.max_srq_sge);	31	attr->max_sge, rxe->attr.max_srq_sge);
32	goto err1;	32	goto err1;
33	}	33	}
@@ -65,7 +65,7 @@ int rxe_srq_from_init(struct rxe_dev rxe, struct rxe_srq srq, · 31 unmodified lines
65	type = QUEUE_TYPE_FROM_CLIENT;	65	type = QUEUE_TYPE_FROM_CLIENT;
66	q = rxe_queue_init(rxe, &srq->rq.max_wr, srq_wqe_size, type);	66	q = rxe_queue_init(rxe, &srq->rq.max_wr, srq_wqe_size, type);
67	if (!q) {	67	if (!q) {
68	pr_warn("unable to allocate queue for srq\n");	68	rxe_dbg_srq(srq, "Unable to allocate queue\n");
69	return -ENOMEM;	69	return -ENOMEM;
70	}	70	}
71		71
@@ -94,24 +94,24 @@ int rxe_srq_chk_attr(struct rxe_dev rxe, struct rxe_srq srq, · 22 unmodified lines
94	struct ib_srq_attr *attr, enum ib_srq_attr_mask mask)	94	struct ib_srq_attr *attr, enum ib_srq_attr_mask mask)
95	{	95	{
96	if (srq->error) {	96	if (srq->error) {
97	pr_warn("srq in error state\n");	97	rxe_dbg_srq(srq, "in error state\n");
98	goto err1;	98	goto err1;
99	}	99	}
100		100
101	if (mask & IB_SRQ_MAX_WR) {	101	if (mask & IB_SRQ_MAX_WR) {
102	if (attr->max_wr > rxe->attr.max_srq_wr) {	102	if (attr->max_wr > rxe->attr.max_srq_wr) {
103	pr_warn("max_wr(%d) > max_srq_wr(%d)\n",	103	rxe_dbg_srq(srq, "max_wr(%d) > max_srq_wr(%d)\n",
104	attr->max_wr, rxe->attr.max_srq_wr);	104	attr->max_wr, rxe->attr.max_srq_wr);
105	goto err1;	105	goto err1;
106	}	106	}
107		107
108	if (attr->max_wr <= 0) {	108	if (attr->max_wr <= 0) {
109	pr_warn("max_wr(%d) <= 0\n", attr->max_wr);	109	rxe_dbg_srq(srq, "max_wr(%d) <= 0\n", attr->max_wr);
110	goto err1;	110	goto err1;
111	}	111	}
112		112
113	if (srq->limit && (attr->max_wr < srq->limit)) {	113	if (srq->limit && (attr->max_wr < srq->limit)) {
114	pr_warn("max_wr (%d) < srq->limit (%d)\n",	114	rxe_dbg_srq(srq, "max_wr (%d) < srq->limit (%d)\n",
115	attr->max_wr, srq->limit);	115	attr->max_wr, srq->limit);
116	goto err1;	116	goto err1;
117	}	117	}
@@ -122,13 +122,13 @@ int rxe_srq_chk_attr(struct rxe_dev rxe, struct rxe_srq srq, · 4 unmodified lines
122		122
123	if (mask & IB_SRQ_LIMIT) {	123	if (mask & IB_SRQ_LIMIT) {
124	if (attr->srq_limit > rxe->attr.max_srq_wr) {	124	if (attr->srq_limit > rxe->attr.max_srq_wr) {
125	pr_warn("srq_limit(%d) > max_srq_wr(%d)\n",	125	rxe_dbg_srq(srq, "srq_limit(%d) > max_srq_wr(%d)\n",
126	attr->srq_limit, rxe->attr.max_srq_wr);	126	attr->srq_limit, rxe->attr.max_srq_wr);
127	goto err1;	127	goto err1;
128	}	128	}
129		129
130	if (attr->srq_limit > srq->rq.queue->buf->index_mask) {	130	if (attr->srq_limit > srq->rq.queue->buf->index_mask) {
131	pr_warn("srq_limit (%d) > cur limit(%d)\n",	131	rxe_dbg_srq(srq, "srq_limit (%d) > cur limit(%d)\n",
132	attr->srq_limit,	132	attr->srq_limit,
133	srq->rq.queue->buf->index_mask);	133	srq->rq.queue->buf->index_mask);
134	goto err1;	134	goto err1;

drivers/infiniband/sw/rxe/rxe_task.c+26−26

  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
  */
-#include <linux/kernel.h>
-#include <linux/interrupt.h>
-#include <linux/hardirq.h>
 #include "rxe.h"
 int __rxe_do_task(struct rxe_task *task)
  * a second caller finds the task already running
  * but looks just after the last call to func
  */
-void rxe_do_task(struct tasklet_struct *t)
+static void do_task(struct tasklet_struct *t)
 {
 	int cont;
 	int ret;
 	struct rxe_task *task = from_tasklet(task, t, tasklet);
+	struct rxe_qp *qp = (struct rxe_qp *)task->arg;
 	unsigned int iterations = RXE_MAX_ITERATIONS;
-	spin_lock_bh(&task->state_lock);
+	spin_lock_bh(&task->lock);
 	switch (task->state) {
 	case TASK_STATE_START:
 		task->state = TASK_STATE_BUSY;
-		spin_unlock_bh(&task->state_lock);
+		spin_unlock_bh(&task->lock);
 		break;
 	case TASK_STATE_BUSY:
 		task->state = TASK_STATE_ARMED;
 		fallthrough;
 	case TASK_STATE_ARMED:
-		spin_unlock_bh(&task->state_lock);
+		spin_unlock_bh(&task->lock);
 		return;
 	default:
-		spin_unlock_bh(&task->state_lock);
+		spin_unlock_bh(&task->lock);
-		pr_warn("%s failed with bad state %d\n", __func__, task->state);
+		rxe_dbg_qp(qp, "failed with bad state %d\n", task->state);
 		return;
 	}
 		cont = 0;
 		ret = task->func(task->arg);
-		spin_lock_bh(&task->state_lock);
+		spin_lock_bh(&task->lock);
 		switch (task->state) {
 		case TASK_STATE_BUSY:
 			if (ret) {
 			break;
 		default:
-			pr_warn("%s failed with bad state %d\n", __func__,
+			rxe_dbg_qp(qp, "failed with bad state %d\n",
-				task->state);
+					task->state);
 		}
-		spin_unlock_bh(&task->state_lock);
+		spin_unlock_bh(&task->lock);
 	} while (cont);
 	task->ret = ret;
 }
-int rxe_init_task(struct rxe_task *task,
+int rxe_init_task(struct rxe_task *task, void *arg, int (*func)(void *))
-		  void *arg, int (*func)(void *), char *name)
 {
 	task->arg	= arg;
 	task->func	= func;
-	snprintf(task->name, sizeof(task->name), "%s", name);
 	task->destroyed	= false;
-	tasklet_setup(&task->tasklet, rxe_do_task);
+	tasklet_setup(&task->tasklet, do_task);
 	task->state = TASK_STATE_START;
-	spin_lock_init(&task->state_lock);
+	spin_lock_init(&task->lock);
 	return 0;
 }
 	task->destroyed = true;
 	do {
-		spin_lock_bh(&task->state_lock);
+		spin_lock_bh(&task->lock);
 		idle = (task->state == TASK_STATE_START);
-		spin_unlock_bh(&task->state_lock);
+		spin_unlock_bh(&task->lock);
 	} while (!idle);
 	tasklet_kill(&task->tasklet);
 }
-void rxe_run_task(struct rxe_task *task, int sched)
+void rxe_run_task(struct rxe_task *task)
+{
+	if (task->destroyed)
+		return;
+	do_task(&task->tasklet);
+}
+void rxe_sched_task(struct rxe_task *task)
 {
 	if (task->destroyed)
 		return;
-	if (sched)
+	tasklet_schedule(&task->tasklet);
-		tasklet_schedule(&task->tasklet);
-	else
-		rxe_do_task(&task->tasklet);
 }
 void rxe_disable_task(struct rxe_task *task)

drivers/infiniband/sw/rxe/rxe_task.h+4−15

 struct rxe_task {
 	struct tasklet_struct	tasklet;
 	int			state;
-	spinlock_t		state_lock; /* spinlock for task state */
+	spinlock_t		lock;
 	void			*arg;
 	int			(*func)(void *arg);
 	int			ret;
-	char			name[16];
 	bool			destroyed;
 };
  *	arg  => parameter to pass to fcn
  *	func => function to call until it returns != 0
  */
-int rxe_init_task(struct rxe_task *task,
+int rxe_init_task(struct rxe_task *task, void *arg, int (*func)(void *));
-		  void *arg, int (*func)(void *), char *name);
 /* cleanup task */
 void rxe_cleanup_task(struct rxe_task *task);
  */
 int __rxe_do_task(struct rxe_task *task);
-/*
+void rxe_run_task(struct rxe_task *task);
- * common function called by any of the main tasklets
- * If there is any chance that there is additional
- * work to do someone must reschedule the task before
- * leaving
- */
-void rxe_do_task(struct tasklet_struct *t);
-/* run a task, else schedule it to run as a tasklet, The decision
+void rxe_sched_task(struct rxe_task *task);
- * to run or schedule tasklet is based on the parameter sched.
- */
-void rxe_run_task(struct rxe_task *task, int sched);
 /* keep a task from scheduling */
 void rxe_disable_task(struct rxe_task *task);

drivers/infiniband/sw/rxe/rxe_verbs.c+38−68

 		ah->is_user = false;
 	}
-	err = rxe_av_chk_attr(rxe, init_attr->ah_attr);
-	if (err)
-		return err;
 	err = rxe_add_to_pool_ah(&rxe->ah_pool, ah,
 			init_attr->flags & RDMA_CREATE_AH_SLEEPABLE);
 	if (err)
 	/* create index > 0 */
 	ah->ah_num = ah->elem.index;
+	err = rxe_ah_chk_attr(ah, init_attr->ah_attr);
+	if (err) {
+		rxe_cleanup(ah);
+		return err;
+	}
 	if (uresp) {
 		/* only if new user provider */
 		err = copy_to_user(&uresp->ah_num, &ah->ah_num,
 static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
 {
 	int err;
-	struct rxe_dev *rxe = to_rdev(ibah->device);
 	struct rxe_ah *ah = to_rah(ibah);
-	err = rxe_av_chk_attr(rxe, attr);
+	err = rxe_ah_chk_attr(ah, attr);
 	if (err)
 		return err;
 static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr)
 {
-	int err;
 	int i;
 	u32 length;
 	struct rxe_recv_wqe *recv_wqe;
 	int full;
 	full = queue_full(rq->queue, QUEUE_TYPE_TO_DRIVER);
-	if (unlikely(full)) {
+	if (unlikely(full))
-		err = -ENOMEM;
+		return -ENOMEM;
-		goto err1;
-	}
-	if (unlikely(num_sge > rq->max_sge)) {
+	if (unlikely(num_sge > rq->max_sge))
-		err = -EINVAL;
+		return -EINVAL;
-		goto err1;
-	}
 	length = 0;
 	for (i = 0; i < num_sge; i++)
 	queue_advance_producer(rq->queue, QUEUE_TYPE_TO_DRIVER);
 	return 0;
-err1:
-	return err;
 }
 static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
 	if (err)
 		return err;
-	err = rxe_srq_from_attr(rxe, srq, attr, mask, &ucmd, udata);
+	return rxe_srq_from_attr(rxe, srq, attr, mask, &ucmd, udata);
-	if (err)
-		return err;
-	return 0;
 }
 static int rxe_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 	err = rxe_qp_chk_attr(rxe, qp, attr, mask);
 	if (err)
-		goto err1;
+		return err;
 	err = rxe_qp_from_attr(qp, attr, mask, udata);
 	if (err)
-		goto err1;
+		return err;
 	if ((mask & IB_QP_AV) && (attr->ah_attr.ah_flags & IB_AH_GRH))
 		qp->src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
 						  qp->attr.dest_qp_num);
 	return 0;
-err1:
-	return err;
 }
 static int rxe_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	struct rxe_sq *sq = &qp->sq;
 	if (unlikely(num_sge > sq->max_sge))
-		goto err1;
+		return -EINVAL;
 	if (unlikely(mask & WR_ATOMIC_MASK)) {
 		if (length < 8)
-			goto err1;
+			return -EINVAL;
 		if (atomic_wr(ibwr)->remote_addr & 0x7)
-			goto err1;
+			return -EINVAL;
 	}
 	if (unlikely((ibwr->send_flags & IB_SEND_INLINE) &&
 		     (length > sq->max_inline)))
-		goto err1;
+		return -EINVAL;
 	return 0;
-err1:
-	return -EINVAL;
 }
 static void init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr,
 		wr = next;
 	}
-	rxe_run_task(&qp->req.task, 1);
+	rxe_sched_task(&qp->req.task);
 	if (unlikely(qp->req.state == QP_STATE_ERROR))
-		rxe_run_task(&qp->comp.task, 1);
+		rxe_sched_task(&qp->comp.task);
 	return err;
 }
 	if (qp->is_user) {
 		/* Utilize process context to do protocol processing */
-		rxe_run_task(&qp->req.task, 0);
+		rxe_run_task(&qp->req.task);
 		return 0;
 	} else
 		return rxe_post_send_kernel(qp, wr, bad_wr);
 	if (unlikely((qp_state(qp) < IB_QPS_INIT) || !qp->valid)) {
 		*bad_wr = wr;
-		err = -EINVAL;
+		return -EINVAL;
-		goto err1;
 	}
 	if (unlikely(qp->srq)) {
 		*bad_wr = wr;
-		err = -EINVAL;
+		return -EINVAL;
-		goto err1;
 	}
 	spin_lock_irqsave(&rq->producer_lock, flags);
 	spin_unlock_irqrestore(&rq->producer_lock, flags);
 	if (qp->resp.state == QP_STATE_ERROR)
-		rxe_run_task(&qp->resp.task, 1);
+		rxe_sched_task(&qp->resp.task);
-err1:
 	return err;
 }
 	err = rxe_cq_chk_attr(rxe, cq, cqe, 0);
 	if (err)
-		goto err1;
+		return err;
-	err = rxe_cq_resize_queue(cq, cqe, uresp, udata);
-	if (err)
-		goto err1;
-	return 0;
-err1:
+	return rxe_cq_resize_queue(cq, cqe, uresp, udata);
-	return err;
 }
 static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 	rxe_get(pd);
 	mr->ibmr.pd = ibpd;
+	mr->ibmr.device = ibpd->device;
 	rxe_mr_init_dma(access, mr);
 	rxe_finalize(mr);
 	struct rxe_mr *mr;
 	mr = rxe_alloc(&rxe->mr_pool);
-	if (!mr) {
+	if (!mr)
-		err = -ENOMEM;
+		return ERR_PTR(-ENOMEM);
-		goto err2;
-	}
 	rxe_get(pd);
 	mr->ibmr.pd = ibpd;
+	mr->ibmr.device = ibpd->device;
 	err = rxe_mr_init_user(rxe, start, length, iova, access, mr);
 	if (err)
-		goto err3;
+		goto err1;
 	rxe_finalize(mr);
 	return &mr->ibmr;
-err3:
+err1:
 	rxe_cleanup(mr);
-err2:
 	return ERR_PTR(err);
 }
 		return ERR_PTR(-EINVAL);
 	mr = rxe_alloc(&rxe->mr_pool);
-	if (!mr) {
+	if (!mr)
-		err = -ENOMEM;
+		return ERR_PTR(-ENOMEM);
-		goto err1;
-	}
 	rxe_get(pd);
 	mr->ibmr.pd = ibpd;
+	mr->ibmr.device = ibpd->device;
 	err = rxe_mr_init_fast(max_num_sg, mr);
 	if (err)
-		goto err2;
+		goto err1;
 	rxe_finalize(mr);
 	return &mr->ibmr;
-err2:
-	rxe_cleanup(mr);
 err1:
+	rxe_cleanup(mr);
 	return ERR_PTR(err);
 }
 	err = ib_register_device(dev, ibdev_name, NULL);
 	if (err)
-		pr_warn("%s failed with error %d\n", __func__, err);
+		rxe_dbg(rxe, "failed with error %d\n", err);
 	/*
 	 * Note that rxe may be invalid at this point if another thread

drivers/infiniband/sw/rxe/rxe_verbs.h+6−1

 			u64		va;
 			u32		resid;
 		} read;
+		struct {
+			u32		length;
+			u64		va;
+			u8		type;
+			u8		level;
+		} flush;
 	};
 };
 	u32			lkey;
 	u32			rkey;
 	enum rxe_mr_state	state;
-	enum ib_mr_type		type;
 	u32			offset;
 	int			access;

drivers/infiniband/sw/siw/siw_cq.c+22−2

@@ -56,8 +56,6 @@ int siw_reap_cqe(struct siw_cq cq, struct ib_wc wc) · 55 unmodified lines
56	if (READ_ONCE(cqe->flags) & SIW_WQE_VALID) {	56	if (READ_ONCE(cqe->flags) & SIW_WQE_VALID) {
57	memset(wc, 0, sizeof(*wc));	57	memset(wc, 0, sizeof(*wc));
58	wc->wr_id = cqe->id;	58	wc->wr_id = cqe->id;
59	wc->status = map_cqe_status[cqe->status].ib;
60	wc->opcode = map_wc_opcode[cqe->opcode];
61	wc->byte_len = cqe->bytes;	59	wc->byte_len = cqe->bytes;
62		60
63	/*	61	/*
@@ -71,10 +69,32 @@ int siw_reap_cqe(struct siw_cq cq, struct ib_wc wc) · 7 unmodified lines
71	wc->wc_flags = IB_WC_WITH_INVALIDATE;	69	wc->wc_flags = IB_WC_WITH_INVALIDATE;
72	}	70	}
73	wc->qp = cqe->base_qp;	71	wc->qp = cqe->base_qp;
		72	wc->opcode = map_wc_opcode[cqe->opcode];
		73	wc->status = map_cqe_status[cqe->status].ib;
74	siw_dbg_cq(cq,	74	siw_dbg_cq(cq,
75	"idx %u, type %d, flags %2x, id 0x%pK\n",	75	"idx %u, type %d, flags %2x, id 0x%pK\n",
76	cq->cq_get % cq->num_cqe, cqe->opcode,	76	cq->cq_get % cq->num_cqe, cqe->opcode,
77	cqe->flags, (void *)(uintptr_t)cqe->id);	77	cqe->flags, (void *)(uintptr_t)cqe->id);
		78	} else {
		79	/*
		80	* A malicious user may set invalid opcode or
		81	* status in the user mmapped CQE array.
		82	* Sanity check and correct values in that case
		83	* to avoid out-of-bounds access to global arrays
		84	* for opcode and status mapping.
		85	*/
		86	u8 opcode = cqe->opcode;
		87	u16 status = cqe->status;
		88
		89	if (opcode >= SIW_NUM_OPCODES) {
		90	opcode = 0;
		91	status = SIW_WC_GENERAL_ERR;
		92	} else if (status >= SIW_NUM_WC_STATUS) {
		93	status = SIW_WC_GENERAL_ERR;
		94	}
		95	wc->opcode = map_wc_opcode[opcode];
		96	wc->status = map_cqe_status[status].ib;
		97
78	}	98	}
79	WRITE_ONCE(cqe->flags, 0);	99	WRITE_ONCE(cqe->flags, 0);
80	cq->cq_get++;	100	cq->cq_get++;

drivers/infiniband/sw/siw/siw_verbs.c+36−4

@@ -676,13 +676,45 @@ static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr, · 675 unmodified lines
676	static int siw_sq_flush_wr(struct siw_qp qp, const struct ib_send_wr wr,	676	static int siw_sq_flush_wr(struct siw_qp qp, const struct ib_send_wr wr,
677	const struct ib_send_wr **bad_wr)	677	const struct ib_send_wr **bad_wr)
678	{	678	{
679	struct siw_sqe sqe = {};
680	int rv = 0;	679	int rv = 0;
681		680
682	while (wr) {	681	while (wr) {
683	sqe.id = wr->wr_id;	682	struct siw_sqe sqe = {};
684	sqe.opcode = wr->opcode;	683
685	rv = siw_sqe_complete(qp, &sqe, 0, SIW_WC_WR_FLUSH_ERR);	684	switch (wr->opcode) {
		685	case IB_WR_RDMA_WRITE:
		686	sqe.opcode = SIW_OP_WRITE;
		687	break;
		688	case IB_WR_RDMA_READ:
		689	sqe.opcode = SIW_OP_READ;
		690	break;
		691	case IB_WR_RDMA_READ_WITH_INV:
		692	sqe.opcode = SIW_OP_READ_LOCAL_INV;
		693	break;
		694	case IB_WR_SEND:
		695	sqe.opcode = SIW_OP_SEND;
		696	break;
		697	case IB_WR_SEND_WITH_IMM:
		698	sqe.opcode = SIW_OP_SEND_WITH_IMM;
		699	break;
		700	case IB_WR_SEND_WITH_INV:
		701	sqe.opcode = SIW_OP_SEND_REMOTE_INV;
		702	break;
		703	case IB_WR_LOCAL_INV:
		704	sqe.opcode = SIW_OP_INVAL_STAG;
		705	break;
		706	case IB_WR_REG_MR:
		707	sqe.opcode = SIW_OP_REG_MR;
		708	break;
		709	default:
		710	rv = -EINVAL;
		711	break;
		712	}
		713	if (!rv) {
		714	sqe.id = wr->wr_id;
		715	rv = siw_sqe_complete(qp, &sqe, 0,
		716	SIW_WC_WR_FLUSH_ERR);
		717	}
686	if (rv) {	718	if (rv) {
687	if (bad_wr)	719	if (bad_wr)
688	*bad_wr = wr;	720	*bad_wr = wr;

drivers/infiniband/ulp/ipoib/ipoib_netlink.c+7−0

 	[IFLA_IPOIB_UMCAST]	= { .type = NLA_U16 },
 };
+static unsigned int ipoib_get_max_num_queues(void)
+{
+	return min_t(unsigned int, num_possible_cpus(), 128);
+}
 static int ipoib_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 	.changelink	= ipoib_changelink,
 	.get_size	= ipoib_get_size,
 	.fill_info	= ipoib_fill_info,
+	.get_num_rx_queues = ipoib_get_max_num_queues,
+	.get_num_tx_queues = ipoib_get_max_num_queues,
 };
 struct rtnl_link_ops *ipoib_get_link_ops(void)

drivers/infiniband/ulp/iser/iser_verbs.c+29−38

 	mutex_unlock(&ig.device_list_mutex);
 }
-/*
- * Called with state mutex held
- */
-static int iser_conn_state_comp_exch(struct iser_conn *iser_conn,
-				     enum iser_conn_state comp,
-				     enum iser_conn_state exch)
-{
-	int ret;
-	ret = (iser_conn->state == comp);
-	if (ret)
-		iser_conn->state = exch;
-	return ret;
-}
 void iser_release_work(struct work_struct *work)
 {
 	struct iser_conn *iser_conn;
 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
 	int err = 0;
+	lockdep_assert_held(&iser_conn->state_mutex);
 	/* terminate the iser conn only if the conn state is UP */
-	if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP,
+	if (iser_conn->state != ISER_CONN_UP)
-				       ISER_CONN_TERMINATING))
 		return 0;
+	iser_conn->state = ISER_CONN_TERMINATING;
 	iser_info("iser_conn %p state %d\n", iser_conn, iser_conn->state);
 	/* suspend queuing of new iscsi commands */
  */
 static void iser_connect_error(struct rdma_cm_id *cma_id)
 {
-	struct iser_conn *iser_conn;
+	struct iser_conn *iser_conn = cma_id->context;
+	lockdep_assert_held(&iser_conn->state_mutex);
-	iser_conn = cma_id->context;
 	iser_conn->state = ISER_CONN_TERMINATING;
 }
  */
 static void iser_addr_handler(struct rdma_cm_id *cma_id)
 {
+	struct iser_conn *iser_conn = cma_id->context;
 	struct iser_device *device;
-	struct iser_conn *iser_conn;
 	struct ib_conn *ib_conn;
 	int    ret;
-	iser_conn = cma_id->context;
+	lockdep_assert_held(&iser_conn->state_mutex);
 	if (iser_conn->state != ISER_CONN_PENDING)
 		/* bailout */
 		return;
 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
 	struct ib_device *ib_dev = ib_conn->device->ib_device;
+	lockdep_assert_held(&iser_conn->state_mutex);
 	if (iser_conn->state != ISER_CONN_PENDING)
 		/* bailout */
 		return;
 	iser_connect_error(cma_id);
 }
+/*
+ * Called with state mutex held
+ */
 static void iser_connected_handler(struct rdma_cm_id *cma_id,
 				   const void *private_data)
 {
-	struct iser_conn *iser_conn;
+	struct iser_conn *iser_conn = cma_id->context;
 	struct ib_qp_attr attr;
 	struct ib_qp_init_attr init_attr;
-	iser_conn = cma_id->context;
+	lockdep_assert_held(&iser_conn->state_mutex);
 	if (iser_conn->state != ISER_CONN_PENDING)
 		/* bailout */
 		return;
 	complete(&iser_conn->up_completion);
 }
-static void iser_disconnected_handler(struct rdma_cm_id *cma_id)
+/*
-{
+ * Called with state mutex held
-	struct iser_conn *iser_conn = cma_id->context;
+ */
-	if (iser_conn_terminate(iser_conn)) {
-		if (iser_conn->iscsi_conn)
-			iscsi_conn_failure(iser_conn->iscsi_conn,
-					   ISCSI_ERR_CONN_FAILED);
-		else
-			iser_err("iscsi_iser connection isn't bound\n");
-	}
-}
 static void iser_cleanup_handler(struct rdma_cm_id *cma_id,
 				 bool destroy)
 {
 	struct iser_conn *iser_conn = cma_id->context;
+	lockdep_assert_held(&iser_conn->state_mutex);
 	/*
 	 * We are not guaranteed that we visited disconnected_handler
 	 * by now, call it here to be safe that we handle CM drep
 	 * and flush errors.
 	 */
-	iser_disconnected_handler(cma_id);
+	if (iser_conn_terminate(iser_conn)) {
+		if (iser_conn->iscsi_conn)
+			iscsi_conn_failure(iser_conn->iscsi_conn,
+					   ISCSI_ERR_CONN_FAILED);
+		else
+			iser_err("iscsi_iser connection isn't bound\n");
+	}
 	iser_free_ib_conn_res(iser_conn, destroy);
 	complete(&iser_conn->ib_completion);
 }

drivers/infiniband/ulp/isert/ib_isert.c+2−3

 		 * login request PDU.
 		 */
 		login->leading_connection = (!login_req->tsih) ? 1 : 0;
-		login->current_stage =
+		login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
-			(login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
+				login_req->flags);
-			 >> 2;
 		login->version_min	= login_req->min_version;
 		login->version_max	= login_req->max_version;
 		memcpy(login->isid, login_req->isid, 6);

drivers/infiniband/ulp/rtrs/rtrs-clt.c+2−4

 	/* Align the MR to a 4K page size to match the block virt boundary */
 	nr = ib_map_mr_sg(req->mr, req->sglist, count, NULL, SZ_4K);
-	if (nr < 0)
+	if (nr != count)
-		return nr;
+		return nr < 0 ? nr : -EINVAL;
-	if (nr < req->sg_cnt)
-		return -EINVAL;
 	ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
 	return nr;

drivers/infiniband/ulp/rtrs/rtrs-pri.h+0−3

 struct rtrs_ib_dev;
 struct rtrs_rdma_dev_pd_ops {
-	struct rtrs_ib_dev *(*alloc)(void);
-	void (*free)(struct rtrs_ib_dev *dev);
 	int (*init)(struct rtrs_ib_dev *dev);
-	void (*deinit)(struct rtrs_ib_dev *dev);
 };
 struct rtrs_rdma_dev_pd {

drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c+9−4

 	mutex_lock(&srv->paths_mutex);
 	if (!--srv->dev_ref) {
-		kobject_del(srv->kobj_paths);
 		kobject_put(srv->kobj_paths);
 		mutex_unlock(&srv->paths_mutex);
 		device_del(&srv->dev);
 void rtrs_srv_destroy_path_files(struct rtrs_srv_path *srv_path)
 {
-	if (srv_path->kobj.state_in_sysfs) {
+	if (srv_path->stats->kobj_stats.state_in_sysfs) {
+		sysfs_remove_group(&srv_path->stats->kobj_stats,
+				   &rtrs_srv_stats_attr_group);
 		kobject_del(&srv_path->stats->kobj_stats);
 		kobject_put(&srv_path->stats->kobj_stats);
+	}
+	if (srv_path->kobj.state_in_sysfs) {
 		sysfs_remove_group(&srv_path->kobj, &rtrs_srv_path_attr_group);
+		kobject_del(&srv_path->kobj);
 		kobject_put(&srv_path->kobj);
-		rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
 	}
+	rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
 }

drivers/infiniband/ulp/rtrs/rtrs-srv.c+29−43

@@ -561,9 +561,11 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) · 560 unmodified lines
561	{	561	{
562	struct rtrs_srv_sess *srv = srv_path->srv;	562	struct rtrs_srv_sess *srv = srv_path->srv;
563	struct rtrs_path *ss = &srv_path->s;	563	struct rtrs_path *ss = &srv_path->s;
564	int i, mri, err, mrs_num;	564	int i, err, mrs_num;
565	unsigned int chunk_bits;	565	unsigned int chunk_bits;
566	int chunks_per_mr = 1;	566	int chunks_per_mr = 1;
		567	struct ib_mr *mr;
		568	struct sg_table *sgt;
567		569
568	/*	570	/*
569	* Here we map queue_depth chunks to MR. Firstly we have to	571	* Here we map queue_depth chunks to MR. Firstly we have to
@@ -586,16 +588,14 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) · 16 unmodified lines
586	if (!srv_path->mrs)	588	if (!srv_path->mrs)
587	return -ENOMEM;	589	return -ENOMEM;
588		590
589	srv_path->mrs_num = mrs_num;	591	for (srv_path->mrs_num = 0; srv_path->mrs_num < mrs_num;
590		592	srv_path->mrs_num++) {
591	for (mri = 0; mri < mrs_num; mri++) {	593	struct rtrs_srv_mr *srv_mr = &srv_path->mrs[srv_path->mrs_num];
592	struct rtrs_srv_mr *srv_mr = &srv_path->mrs[mri];
593	struct sg_table *sgt = &srv_mr->sgt;
594	struct scatterlist *s;	594	struct scatterlist *s;
595	struct ib_mr *mr;
596	int nr, nr_sgt, chunks;	595	int nr, nr_sgt, chunks;
597		596
598	chunks = chunks_per_mr * mri;	597	sgt = &srv_mr->sgt;
		598	chunks = chunks_per_mr * srv_path->mrs_num;
599	if (!always_invalidate)	599	if (!always_invalidate)
600	chunks_per_mr = min_t(int, chunks_per_mr,	600	chunks_per_mr = min_t(int, chunks_per_mr,
601	srv->queue_depth - chunks);	601	srv->queue_depth - chunks);
@@ -622,7 +622,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) · 20 unmodified lines
622	}	622	}
623	nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt,	623	nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt,
624	NULL, max_chunk_size);	624	NULL, max_chunk_size);
625	if (nr < 0 \|\| nr < sgt->nents) {	625	if (nr != nr_sgt) {
626	err = nr < 0 ? nr : -EINVAL;	626	err = nr < 0 ? nr : -EINVAL;
627	goto dereg_mr;	627	goto dereg_mr;
628	}	628	}
@@ -644,31 +644,24 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) · 15 unmodified lines
644		644
645	ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));	645	ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
646	srv_mr->mr = mr;	646	srv_mr->mr = mr;
647
648	continue;
649	err:
650	while (mri--) {
651	srv_mr = &srv_path->mrs[mri];
652	sgt = &srv_mr->sgt;
653	mr = srv_mr->mr;
654	rtrs_iu_free(srv_mr->iu, srv_path->s.dev->ib_dev, 1);
655	dereg_mr:
656	ib_dereg_mr(mr);
657	unmap_sg:
658	ib_dma_unmap_sg(srv_path->s.dev->ib_dev, sgt->sgl,
659	sgt->nents, DMA_BIDIRECTIONAL);
660	free_sg:
661	sg_free_table(sgt);
662	}
663	kfree(srv_path->mrs);
664
665	return err;
666	}	647	}
667		648
668	chunk_bits = ilog2(srv->queue_depth - 1) + 1;	649	chunk_bits = ilog2(srv->queue_depth - 1) + 1;
669	srv_path->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits);	650	srv_path->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits);
670		651
671	return 0;	652	return 0;
		653
		654	dereg_mr:
		655	ib_dereg_mr(mr);
		656	unmap_sg:
		657	ib_dma_unmap_sg(srv_path->s.dev->ib_dev, sgt->sgl,
		658	sgt->nents, DMA_BIDIRECTIONAL);
		659	free_sg:
		660	sg_free_table(sgt);
		661	err:
		662	unmap_cont_bufs(srv_path);
		663
		664	return err;
672	}	665	}
673		666
674	static void rtrs_srv_hb_err_handler(struct rtrs_con *c)	667	static void rtrs_srv_hb_err_handler(struct rtrs_con *c)
@@ -1678,12 +1671,6 @@ static int create_con(struct rtrs_srv_path *srv_path, · 1003 unmodified lines
1678	srv->queue_depth * (1 + 2) + 1);	1671	srv->queue_depth * (1 + 2) + 1);
1679		1672
1680	max_recv_wr = srv->queue_depth + 1;	1673	max_recv_wr = srv->queue_depth + 1;
1681	/*
1682	* If we have all receive requests posted and
1683	* all write requests posted and each read request
1684	* requires an invalidate request + drain
1685	* and qp gets into error state.
1686	*/
1687	}	1674	}
1688	cq_num = max_send_wr + max_recv_wr;	1675	cq_num = max_send_wr + max_recv_wr;
1689	atomic_set(&con->c.sq_wr_avail, max_send_wr);	1676	atomic_set(&con->c.sq_wr_avail, max_send_wr);
@@ -1950,22 +1937,21 @@ static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, · 260 unmodified lines
1950	{	1937	{
1951	struct rtrs_srv_path *srv_path = NULL;	1938	struct rtrs_srv_path *srv_path = NULL;
1952	struct rtrs_path *s = NULL;	1939	struct rtrs_path *s = NULL;
		1940	struct rtrs_con *c = NULL;
1953		1941
1954	if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) {	1942	if (ev->event == RDMA_CM_EVENT_CONNECT_REQUEST)
1955	struct rtrs_con *c = cm_id->context;
1956
1957	s = c->path;
1958	srv_path = to_srv_path(s);
1959	}
1960
1961	switch (ev->event) {
1962	case RDMA_CM_EVENT_CONNECT_REQUEST:
1963	/*	1943	/*
1964	* In case of error cma.c will destroy cm_id,	1944	* In case of error cma.c will destroy cm_id,
1965	* see cma_process_remove()	1945	* see cma_process_remove()
1966	*/	1946	*/
1967	return rtrs_rdma_connect(cm_id, ev->param.conn.private_data,	1947	return rtrs_rdma_connect(cm_id, ev->param.conn.private_data,
1968	ev->param.conn.private_data_len);	1948	ev->param.conn.private_data_len);
		1949
		1950	c = cm_id->context;
		1951	s = c->path;
		1952	srv_path = to_srv_path(s);
		1953
		1954	switch (ev->event) {
1969	case RDMA_CM_EVENT_ESTABLISHED:	1955	case RDMA_CM_EVENT_ESTABLISHED:
1970	/* Nothing here */	1956	/* Nothing here */
1971	break;	1957	break;

drivers/infiniband/ulp/rtrs/rtrs.c+4−18

@@ -557,7 +557,6 @@ EXPORT_SYMBOL(rtrs_addr_to_sockaddr); · 556 unmodified lines
557	void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,	557	void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,
558	struct rtrs_rdma_dev_pd *pool)	558	struct rtrs_rdma_dev_pd *pool)
559	{	559	{
560	WARN_ON(pool->ops && (!pool->ops->alloc ^ !pool->ops->free));
561	INIT_LIST_HEAD(&pool->list);	560	INIT_LIST_HEAD(&pool->list);
562	mutex_init(&pool->mutex);	561	mutex_init(&pool->mutex);
563	pool->pd_flags = pd_flags;	562	pool->pd_flags = pd_flags;
@@ -583,15 +582,8 @@ static void dev_free(struct kref *ref) · 19 unmodified lines
583	list_del(&dev->entry);	582	list_del(&dev->entry);
584	mutex_unlock(&pool->mutex);	583	mutex_unlock(&pool->mutex);
585		584
586	if (pool->ops && pool->ops->deinit)
587	pool->ops->deinit(dev);
588
589	ib_dealloc_pd(dev->ib_pd);	585	ib_dealloc_pd(dev->ib_pd);
590		586	kfree(dev);
591	if (pool->ops && pool->ops->free)
592	pool->ops->free(dev);
593	else
594	kfree(dev);
595	}	587	}
596		588
597	int rtrs_ib_dev_put(struct rtrs_ib_dev *dev)	589	int rtrs_ib_dev_put(struct rtrs_ib_dev *dev)
@@ -618,11 +610,8 @@ rtrs_ib_dev_find_or_add(struct ib_device *ib_dev, · 20 unmodified lines
618	goto out_unlock;	610	goto out_unlock;
619	}	611	}
620	mutex_unlock(&pool->mutex);	612	mutex_unlock(&pool->mutex);
621	if (pool->ops && pool->ops->alloc)	613	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
622	dev = pool->ops->alloc();	614	if (!dev)
623	else
624	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
625	if (IS_ERR_OR_NULL(dev))
626	goto out_err;	615	goto out_err;
627		616
628	kref_init(&dev->ref);	617	kref_init(&dev->ref);
@@ -644,10 +633,7 @@ out_unlock: · 15 unmodified lines
644	out_free_pd:	633	out_free_pd:
645	ib_dealloc_pd(dev->ib_pd);	634	ib_dealloc_pd(dev->ib_pd);
646	out_free_dev:	635	out_free_dev:
647	if (pool->ops && pool->ops->free)	636	kfree(dev);
648	pool->ops->free(dev);
649	else
650	kfree(dev);
651	out_err:	637	out_err:
652	return NULL;	638	return NULL;
653	}	639	}

drivers/infiniband/ulp/srp/ib_srp.c+82−14

 			break;
 		case SRP_OPT_PKEY:
-			if (match_hex(args, &token)) {
+			ret = match_hex(args, &token);
+			if (ret) {
 				pr_warn("bad P_Key parameter '%s'\n", p);
 				goto out;
 			}
 			break;
 		case SRP_OPT_MAX_SECT:
-			if (match_int(args, &token)) {
+			ret = match_int(args, &token);
+			if (ret) {
 				pr_warn("bad max sect parameter '%s'\n", p);
 				goto out;
 			}
 			break;
 		case SRP_OPT_QUEUE_SIZE:
-			if (match_int(args, &token) || token < 1) {
+			ret = match_int(args, &token);
+			if (ret) {
+				pr_warn("match_int() failed for queue_size parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 1) {
 				pr_warn("bad queue_size parameter '%s'\n", p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->scsi_host->can_queue = token;
 			break;
 		case SRP_OPT_MAX_CMD_PER_LUN:
-			if (match_int(args, &token) || token < 1) {
+			ret = match_int(args, &token);
+			if (ret) {
+				pr_warn("match_int() failed for max cmd_per_lun parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 1) {
 				pr_warn("bad max cmd_per_lun parameter '%s'\n",
 					p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->scsi_host->cmd_per_lun = token;
 			break;
 		case SRP_OPT_TARGET_CAN_QUEUE:
-			if (match_int(args, &token) || token < 1) {
+			ret = match_int(args, &token);
+			if (ret) {
+				pr_warn("match_int() failed for max target_can_queue parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 1) {
 				pr_warn("bad max target_can_queue parameter '%s'\n",
 					p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->target_can_queue = token;
 			break;
 		case SRP_OPT_IO_CLASS:
-			if (match_hex(args, &token)) {
+			ret = match_hex(args, &token);
+			if (ret) {
 				pr_warn("bad IO class parameter '%s'\n", p);
 				goto out;
 			}
 				pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
 					token, SRP_REV10_IB_IO_CLASS,
 					SRP_REV16A_IB_IO_CLASS);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->io_class = token;
 			break;
 		case SRP_OPT_CMD_SG_ENTRIES:
-			if (match_int(args, &token) || token < 1 || token > 255) {
+			ret = match_int(args, &token);
+			if (ret) {
+				pr_warn("match_int() failed for max cmd_sg_entries parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 1 || token > 255) {
 				pr_warn("bad max cmd_sg_entries parameter '%s'\n",
 					p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->cmd_sg_cnt = token;
 			break;
 		case SRP_OPT_ALLOW_EXT_SG:
-			if (match_int(args, &token)) {
+			ret = match_int(args, &token);
+			if (ret) {
 				pr_warn("bad allow_ext_sg parameter '%s'\n", p);
 				goto out;
 			}
 			break;
 		case SRP_OPT_SG_TABLESIZE:
-			if (match_int(args, &token) || token < 1 ||
+			ret = match_int(args, &token);
-					token > SG_MAX_SEGMENTS) {
+			if (ret) {
+				pr_warn("match_int() failed for max sg_tablesize parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 1 || token > SG_MAX_SEGMENTS) {
 				pr_warn("bad max sg_tablesize parameter '%s'\n",
 					p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->sg_tablesize = token;
 			break;
 		case SRP_OPT_COMP_VECTOR:
-			if (match_int(args, &token) || token < 0) {
+			ret = match_int(args, &token);
+			if (ret) {
+				pr_warn("match_int() failed for comp_vector parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 0) {
 				pr_warn("bad comp_vector parameter '%s'\n", p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->comp_vector = token;
 			break;
 		case SRP_OPT_TL_RETRY_COUNT:
-			if (match_int(args, &token) || token < 2 || token > 7) {
+			ret = match_int(args, &token);
+			if (ret) {
+				pr_warn("match_int() failed for tl_retry_count parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 2 || token > 7) {
 				pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
 					p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->tl_retry_count = token;
 			break;
 		case SRP_OPT_MAX_IT_IU_SIZE:
-			if (match_int(args, &token) || token < 0) {
+			ret = match_int(args, &token);
+			if (ret) {
+				pr_warn("match_int() failed for max it_iu_size parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 0) {
 				pr_warn("bad maximum initiator to target IU size '%s'\n", p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->max_it_iu_size = token;
 			break;
 		case SRP_OPT_CH_COUNT:
-			if (match_int(args, &token) || token < 1) {
+			ret = match_int(args, &token);
+			if (ret) {
+				pr_warn("match_int() failed for channel count parameter '%s', Error %d\n",
+					p, ret);
+				goto out;
+			}
+			if (token < 1) {
 				pr_warn("bad channel count %s\n", p);
+				ret = -EINVAL;
 				goto out;
 			}
 			target->ch_count = token;
 		default:
 			pr_warn("unknown parameter or missing value '%s' in target creation request\n",
 				p);
+			ret = -EINVAL;
 			goto out;
 		}
 	}

drivers/net/ethernet/microsoft/mana/gdma_main.c+1−2

@@ -671,8 +671,7 @@ free_q: · 670 unmodified lines
671	return err;	671	return err;
672	}	672	}
673		673
674	int mana_gd_destroy_dma_region(struct gdma_context *gc,	674	int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle)
675	gdma_obj_handle_t dma_region_handle)
676	{	675	{
677	struct gdma_destroy_dma_region_req req = {};	676	struct gdma_destroy_dma_region_req req = {};
678	struct gdma_general_resp resp = {};	677	struct gdma_general_resp resp = {};

include/net/mana/gdma.h+14−17

 	GDMA_DEVICE_MANA	= 2,
 };
-typedef u64 gdma_obj_handle_t;
 struct gdma_resource {
 	/* Protect the bitmap */
 	spinlock_t lock;
 	u64 length;
 	/* Allocated by the PF driver */
-	gdma_obj_handle_t dma_region_handle;
+	u64 dma_region_handle;
 };
 #define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
 	u32 reserved1;
 	u32 pdid;
 	u32 doolbell_id;
-	gdma_obj_handle_t gdma_region;
+	u64 gdma_region;
 	u32 reserved2;
 	u32 queue_size;
 	u32 log2_throttle_limit;
 struct gdma_create_dma_region_resp {
 	struct gdma_resp_hdr hdr;
-	gdma_obj_handle_t dma_region_handle;
+	u64 dma_region_handle;
 }; /* HW DATA */
 /* GDMA_DMA_REGION_ADD_PAGES */
 struct gdma_dma_region_add_pages_req {
 	struct gdma_req_hdr hdr;
-	gdma_obj_handle_t dma_region_handle;
+	u64 dma_region_handle;
 	u32 page_addr_list_len;
 	u32 reserved3;
 struct gdma_destroy_dma_region_req {
 	struct gdma_req_hdr hdr;
-	gdma_obj_handle_t dma_region_handle;
+	u64 dma_region_handle;
 }; /* HW DATA */
 enum gdma_pd_flags {
 struct gdma_create_pd_resp {
 	struct gdma_resp_hdr hdr;
-	gdma_obj_handle_t pd_handle;
+	u64 pd_handle;
 	u32 pd_id;
 	u32 reserved;
 };/* HW DATA */
 struct gdma_destroy_pd_req {
 	struct gdma_req_hdr hdr;
-	gdma_obj_handle_t pd_handle;
+	u64 pd_handle;
 };/* HW DATA */
 struct gdma_destory_pd_resp {
 };
 struct gdma_create_mr_params {
-	gdma_obj_handle_t pd_handle;
+	u64 pd_handle;
 	enum gdma_mr_type mr_type;
 	union {
 		struct {
-			gdma_obj_handle_t dma_region_handle;
+			u64 dma_region_handle;
 			u64 virtual_address;
 			enum gdma_mr_access_flags access_flags;
 		} gva;
 struct gdma_create_mr_request {
 	struct gdma_req_hdr hdr;
-	gdma_obj_handle_t pd_handle;
+	u64 pd_handle;
 	enum gdma_mr_type mr_type;
 	u32 reserved_1;
 	union {
 		struct {
-			gdma_obj_handle_t dma_region_handle;
+			u64 dma_region_handle;
 			u64 virtual_address;
 			enum gdma_mr_access_flags access_flags;
 		} gva;
 struct gdma_create_mr_response {
 	struct gdma_resp_hdr hdr;
-	gdma_obj_handle_t mr_handle;
+	u64 mr_handle;
 	u32 lkey;
 	u32 rkey;
 };/* HW DATA */
 struct gdma_destroy_mr_request {
 	struct gdma_req_hdr hdr;
-	gdma_obj_handle_t mr_handle;
+	u64 mr_handle;
 };/* HW DATA */
 struct gdma_destroy_mr_response {
 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
 			 u32 resp_len, void *resp);
-int mana_gd_destroy_dma_region(struct gdma_context *gc,
+int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle);
-			       gdma_obj_handle_t dma_region_handle);
 #endif /* _GDMA_H */

include/net/mana/mana.h+3−0

@@ -412,6 +412,9 @@ int mana_bpf(struct net_device ndev, struct netdev_bpf bpf); · 411 unmodified lines
412		412
413	extern const struct ethtool_ops mana_ethtool_ops;	413	extern const struct ethtool_ops mana_ethtool_ops;
414		414
		415	/* A CQ can be created not associated with any EQ */
		416	#define GDMA_CQ_NO_EQ 0xffff
		417
415	struct mana_obj_spec {	418	struct mana_obj_spec {
416	u32 queue_index;	419	u32 queue_index;
417	u64 gdma_region;	420	u64 gdma_region;

include/rdma/ib_pack.h+5−0

 	/* opcode 0x15 is reserved */
 	IB_OPCODE_SEND_LAST_WITH_INVALIDATE         = 0x16,
 	IB_OPCODE_SEND_ONLY_WITH_INVALIDATE         = 0x17,
+	IB_OPCODE_FLUSH                             = 0x1C,
+	IB_OPCODE_ATOMIC_WRITE                      = 0x1D,
 	/* real constants follow -- see comment about above IB_OPCODE()
 	   macro for more details */
 	IB_OPCODE(RC, FETCH_ADD),
 	IB_OPCODE(RC, SEND_LAST_WITH_INVALIDATE),
 	IB_OPCODE(RC, SEND_ONLY_WITH_INVALIDATE),
+	IB_OPCODE(RC, FLUSH),
+	IB_OPCODE(RC, ATOMIC_WRITE),
 	/* UC */
 	IB_OPCODE(UC, SEND_FIRST),
 	IB_OPCODE(RD, ATOMIC_ACKNOWLEDGE),
 	IB_OPCODE(RD, COMPARE_SWAP),
 	IB_OPCODE(RD, FETCH_ADD),
+	IB_OPCODE(RD, FLUSH),
 	/* UD */
 	IB_OPCODE(UD, SEND_ONLY),

include/rdma/ib_verbs.h+22−2

 	/* The device supports padding incoming writes to cacheline. */
 	IB_DEVICE_PCI_WRITE_END_PADDING =
 		IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING,
+	/* Placement type attributes */
+	IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL,
+	IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT,
+	IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE,
 };
 enum ib_kernel_cap_flags {
 	IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW,
 	IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV,
 	IB_WC_LSO = IB_UVERBS_WC_TSO,
+	IB_WC_ATOMIC_WRITE = IB_UVERBS_WC_ATOMIC_WRITE,
 	IB_WC_REG_MR,
 	IB_WC_MASKED_COMP_SWAP,
 	IB_WC_MASKED_FETCH_ADD,
+	IB_WC_FLUSH = IB_UVERBS_WC_FLUSH,
 /*
  * Set value of IB_WC_RECV so consumers can test if a completion is a
  * receive by testing (opcode & IB_WC_RECV).
 		IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP,
 	IB_WR_MASKED_ATOMIC_FETCH_AND_ADD =
 		IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD,
+	IB_WR_FLUSH = IB_UVERBS_WR_FLUSH,
+	IB_WR_ATOMIC_WRITE = IB_UVERBS_WR_ATOMIC_WRITE,
 	/* These are kernel only and can not be issued by userspace */
 	IB_WR_REG_MR = 0x20,
 	IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND,
 	IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB,
 	IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING,
+	IB_ACCESS_FLUSH_GLOBAL = IB_UVERBS_ACCESS_FLUSH_GLOBAL,
+	IB_ACCESS_FLUSH_PERSISTENT = IB_UVERBS_ACCESS_FLUSH_PERSISTENT,
 	IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE,
 	IB_ACCESS_SUPPORTED =
-		((IB_ACCESS_HUGETLB << 1) - 1) | IB_ACCESS_OPTIONAL,
+		((IB_ACCESS_FLUSH_PERSISTENT << 1) - 1) | IB_ACCESS_OPTIONAL,
 };
 /*
 	struct ib_port_cache cache;
 	struct net_device __rcu *netdev;
+	netdevice_tracker netdev_tracker;
 	struct hlist_node ndev_hash_link;
 	struct rdma_port_counter port_counter;
 	struct ib_port *sysfs;
 static inline int ib_check_mr_access(struct ib_device *ib_dev,
 				     unsigned int flags)
 {
+	u64 device_cap = ib_dev->attrs.device_cap_flags;
 	/*
 	 * Local write permission is required if remote write or
 	 * remote atomic permission is also requested.
 	if (flags & IB_ACCESS_ON_DEMAND &&
 	    !(ib_dev->attrs.kernel_cap_flags & IBK_ON_DEMAND_PAGING))
-		return -EINVAL;
+		return -EOPNOTSUPP;
+	if ((flags & IB_ACCESS_FLUSH_GLOBAL &&
+	    !(device_cap & IB_DEVICE_FLUSH_GLOBAL)) ||
+	    (flags & IB_ACCESS_FLUSH_PERSISTENT &&
+	    !(device_cap & IB_DEVICE_FLUSH_PERSISTENT)))
+		return -EOPNOTSUPP;
 	return 0;
 }

include/rdma/opa_vnic.h+1−1

@@ -51,7 +51,7 @@ static inline void opa_vnic_dev_priv(const struct net_device dev) · 50 unmodified lines
51	return oparn->dev_priv;	51	return oparn->dev_priv;
52	}	52	}
53		53
54	/* opa_vnic skb meta data structrue */	54	/* opa_vnic skb meta data structure */
55	struct opa_vnic_skb_mdata {	55	struct opa_vnic_skb_mdata {
56	u8 vl;	56	u8 vl;
57	u8 entropy;	57	u8 entropy;

include/trace/events/ib_mad.h+4−9

 		__field(int,            retries_left)
 		__field(int,            max_retries)
 		__field(int,            retry)
-		__field(u16,            pkey)
 	),
 	TP_fast_assign(
 		  "hdr : base_ver 0x%x class 0x%x class_ver 0x%x " \
 		  "method 0x%x status 0x%x class_specific 0x%x tid 0x%llx " \
 		  "attr_id 0x%x attr_mod 0x%x  => dlid 0x%08x sl %d "\
-		  "pkey 0x%x rpqn 0x%x rqpkey 0x%x",
+		  "rpqn 0x%x rqpkey 0x%x",
 		__entry->dev_index, __entry->port_num, __entry->qp_num,
 		__entry->agent_priv, be64_to_cpu(__entry->wrtid),
 		__entry->retries_left, __entry->max_retries,
 		be16_to_cpu(__entry->class_specific),
 		be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id),
 		be32_to_cpu(__entry->attr_mod),
-		be32_to_cpu(__entry->dlid), __entry->sl, __entry->pkey,
+		be32_to_cpu(__entry->dlid), __entry->sl,
 		__entry->rqpn, __entry->rqkey
 	)
 );
 		__field(u16,            wc_status)
 		__field(u32,            slid)
 		__field(u32,            dev_index)
-		__field(u16,            pkey)
 	),
 	TP_fast_assign(
 		__entry->slid = wc->slid;
 		__entry->src_qp = wc->src_qp;
 		__entry->sl = wc->sl;
-		ib_query_pkey(qp_info->port_priv->device,
-			      qp_info->port_priv->port_num,
-			      wc->pkey_index, &__entry->pkey);
 		__entry->wc_status = wc->status;
 	),
 		  "base_ver 0x%02x class 0x%02x class_ver 0x%02x " \
 		  "method 0x%02x status 0x%04x class_specific 0x%04x " \
 		  "tid 0x%016llx attr_id 0x%04x attr_mod 0x%08x " \
-		  "slid 0x%08x src QP%d, sl %d pkey 0x%04x",
+		  "slid 0x%08x src QP%d, sl %d",
 		__entry->dev_index, __entry->port_num, __entry->qp_num,
 		__entry->wc_status,
 		__entry->length,
 		be16_to_cpu(__entry->class_specific),
 		be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id),
 		be32_to_cpu(__entry->attr_mod),
-		__entry->slid, __entry->src_qp, __entry->sl, __entry->pkey
+		__entry->slid, __entry->src_qp, __entry->sl
 	)
 );

include/uapi/rdma/hns-abi.h+15−0

@@ -85,11 +85,26 @@ struct hns_roce_ib_create_qp_resp { · 84 unmodified lines
85	__aligned_u64 dwqe_mmap_key;	85	__aligned_u64 dwqe_mmap_key;
86	};	86	};
87		87
		88	enum {
		89	HNS_ROCE_EXSGE_FLAGS = 1 << 0,
		90	};
		91
		92	enum {
		93	HNS_ROCE_RSP_EXSGE_FLAGS = 1 << 0,
		94	};
		95
88	struct hns_roce_ib_alloc_ucontext_resp {	96	struct hns_roce_ib_alloc_ucontext_resp {
89	__u32 qp_tab_size;	97	__u32 qp_tab_size;
90	__u32 cqe_size;	98	__u32 cqe_size;
91	__u32 srq_tab_size;	99	__u32 srq_tab_size;
92	__u32 reserved;	100	__u32 reserved;
		101	__u32 config;
		102	__u32 max_inline_data;
		103	};
		104
		105	struct hns_roce_ib_alloc_ucontext {
		106	__u32 config;
		107	__u32 reserved;
93	};	108	};
94		109
95	struct hns_roce_ib_alloc_pd_resp {	110	struct hns_roce_ib_alloc_pd_resp {

include/uapi/rdma/ib_user_ioctl_verbs.h+3−0

 	IB_UVERBS_ACCESS_ZERO_BASED = 1 << 5,
 	IB_UVERBS_ACCESS_ON_DEMAND = 1 << 6,
 	IB_UVERBS_ACCESS_HUGETLB = 1 << 7,
+	IB_UVERBS_ACCESS_FLUSH_GLOBAL = 1 << 8,
+	IB_UVERBS_ACCESS_FLUSH_PERSISTENT = 1 << 9,
 	IB_UVERBS_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_OPTIONAL_FIRST,
 	IB_UVERBS_ACCESS_OPTIONAL_RANGE =
 	RDMA_DRIVER_EFA,
 	RDMA_DRIVER_SIW,
 	RDMA_DRIVER_ERDMA,
+	RDMA_DRIVER_MANA,
 };
 enum ib_uverbs_gid_type {

include/uapi/rdma/ib_user_verbs.h+21−0

 	IB_USER_VERBS_EX_CMD_MODIFY_CQ
 };
+/* see IBA A19.4.1.1 Placement Types */
+enum ib_placement_type {
+	IB_FLUSH_GLOBAL = 1U << 0,
+	IB_FLUSH_PERSISTENT = 1U << 1,
+};
+/* see IBA A19.4.1.2 Selectivity Level */
+enum ib_selectivity_level {
+	IB_FLUSH_RANGE = 0,
+	IB_FLUSH_MR,
+};
 /*
  * Make sure that all structs defined in this file remain laid out so
  * that they pack the same way on 32-bit and 64-bit architectures (to
 	IB_UVERBS_WC_BIND_MW = 5,
 	IB_UVERBS_WC_LOCAL_INV = 6,
 	IB_UVERBS_WC_TSO = 7,
+	IB_UVERBS_WC_FLUSH = 8,
+	IB_UVERBS_WC_ATOMIC_WRITE = 9,
 };
 struct ib_uverbs_wc {
 	IB_UVERBS_WR_RDMA_READ_WITH_INV = 11,
 	IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP = 12,
 	IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD = 13,
+	IB_UVERBS_WR_FLUSH = 14,
+	IB_UVERBS_WR_ATOMIC_WRITE = 15,
 	/* Review enum ib_wr_opcode before modifying this */
 };
 	/* Deprecated. Please use IB_UVERBS_RAW_PACKET_CAP_SCATTER_FCS. */
 	IB_UVERBS_DEVICE_RAW_SCATTER_FCS = 1ULL << 34,
 	IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING = 1ULL << 36,
+	/* Flush placement types */
+	IB_UVERBS_DEVICE_FLUSH_GLOBAL = 1ULL << 38,
+	IB_UVERBS_DEVICE_FLUSH_PERSISTENT = 1ULL << 39,
+	/* Atomic write attributes */
+	IB_UVERBS_DEVICE_ATOMIC_WRITE = 1ULL << 40,
 };
 enum ib_uverbs_raw_packet_caps {

include/uapi/rdma/mana-abi.h+66−0

@@ -0,0 +1,66 @@
	1	/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) */
	2	/*
	3	* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
	4	*/
	5
	6	#ifndef MANA_ABI_USER_H
	7	#define MANA_ABI_USER_H
	8
	9	#include <linux/types.h>
	10	#include <rdma/ib_user_ioctl_verbs.h>
	11
	12	/*
	13	* Increment this value if any changes that break userspace ABI
	14	* compatibility are made.
	15	*/
	16
	17	#define MANA_IB_UVERBS_ABI_VERSION 1
	18
	19	struct mana_ib_create_cq {
	20	__aligned_u64 buf_addr;
	21	};
	22
	23	struct mana_ib_create_qp {
	24	__aligned_u64 sq_buf_addr;
	25	__u32 sq_buf_size;
	26	__u32 port;
	27	};
	28
	29	struct mana_ib_create_qp_resp {
	30	__u32 sqid;
	31	__u32 cqid;
	32	__u32 tx_vp_offset;
	33	__u32 reserved;
	34	};
	35
	36	struct mana_ib_create_wq {
	37	__aligned_u64 wq_buf_addr;
	38	__u32 wq_buf_size;
	39	__u32 reserved;
	40	};
	41
	42	/* RX Hash function flags */
	43	enum mana_ib_rx_hash_function_flags {
	44	MANA_IB_RX_HASH_FUNC_TOEPLITZ = 1 << 0,
	45	};
	46
	47	struct mana_ib_create_qp_rss {
	48	__aligned_u64 rx_hash_fields_mask;
	49	__u8 rx_hash_function;
	50	__u8 reserved[7];
	51	__u32 rx_hash_key_len;
	52	__u8 rx_hash_key[40];
	53	__u32 port;
	54	};
	55
	56	struct rss_resp_entry {
	57	__u32 cqid;
	58	__u32 wqid;
	59	};
	60
	61	struct mana_ib_create_qp_rss_resp {
	62	__aligned_u64 num_entries;
	63	struct rss_resp_entry entries[64];
	64	};
	65
	66	#endif

include/uapi/rdma/rdma_user_rxe.h+8−0

 		__u32		invalidate_rkey;
 	} ex;
 	union {
+		struct {
+			__aligned_u64 remote_addr;
+			__u32	length;
+			__u32	rkey;
+			__u8	type;
+			__u8	level;
+		} flush;
 		struct {
 			__aligned_u64 remote_addr;
 			__u32	rkey;
 	__u32			reserved;
 	union {
 		__DECLARE_FLEX_ARRAY(__u8, inline_data);
+		__DECLARE_FLEX_ARRAY(__u8, atomic_wr);
 		__DECLARE_FLEX_ARRAY(struct rxe_sge, sge);
 	};
 };

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -78,6 +78,7 @@ config INFINIBAND_VIRT_DMA · 77 unmodified lines
78	def_bool !HIGHMEM	78	def_bool !HIGHMEM
79		79
80	if INFINIBAND_USER_ACCESS \|\| !INFINIBAND_USER_ACCESS	80	if INFINIBAND_USER_ACCESS \|\| !INFINIBAND_USER_ACCESS
		81	if !UML
81	source "drivers/infiniband/hw/bnxt_re/Kconfig"	82	source "drivers/infiniband/hw/bnxt_re/Kconfig"
82	source "drivers/infiniband/hw/cxgb4/Kconfig"	83	source "drivers/infiniband/hw/cxgb4/Kconfig"
83	source "drivers/infiniband/hw/efa/Kconfig"	84	source "drivers/infiniband/hw/efa/Kconfig"
@@ -85,6 +86,7 @@ source "drivers/infiniband/hw/erdma/Kconfig" · 1 unmodified line
85	source "drivers/infiniband/hw/hfi1/Kconfig"	86	source "drivers/infiniband/hw/hfi1/Kconfig"
86	source "drivers/infiniband/hw/hns/Kconfig"	87	source "drivers/infiniband/hw/hns/Kconfig"
87	source "drivers/infiniband/hw/irdma/Kconfig"	88	source "drivers/infiniband/hw/irdma/Kconfig"
		89	source "drivers/infiniband/hw/mana/Kconfig"
88	source "drivers/infiniband/hw/mlx4/Kconfig"	90	source "drivers/infiniband/hw/mlx4/Kconfig"
89	source "drivers/infiniband/hw/mlx5/Kconfig"	91	source "drivers/infiniband/hw/mlx5/Kconfig"
90	source "drivers/infiniband/hw/mthca/Kconfig"	92	source "drivers/infiniband/hw/mthca/Kconfig"
@@ -94,6 +96,7 @@ source "drivers/infiniband/hw/qib/Kconfig" · 3 unmodified lines
94	source "drivers/infiniband/hw/usnic/Kconfig"	96	source "drivers/infiniband/hw/usnic/Kconfig"
95	source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"	97	source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"
96	source "drivers/infiniband/sw/rdmavt/Kconfig"	98	source "drivers/infiniband/sw/rdmavt/Kconfig"
		99	endif # !UML
97	source "drivers/infiniband/sw/rxe/Kconfig"	100	source "drivers/infiniband/sw/rxe/Kconfig"
98	source "drivers/infiniband/sw/siw/Kconfig"	101	source "drivers/infiniband/sw/siw/Kconfig"
99	endif	102	endif