History

RDMA/bnxt_re: Add bnxt_re RoCE driver

This patch introduces the RoCE driver for the Broadcom NetXtreme-E 10/25/40/50G RoCE HCAs. The RoCE driver is a two part driver that relies on the parent bnxt_en NIC driver to operate. The changes needed in the bnxt_en driver have already been incorporated via Dave Miller's net tree into the mainline kernel. The vendor official git repository for this driver is available on github as: https://github.com/Broadcom/linux-rdma-nxt/

Selvin Xavier committed 9 years ago

Commit 1ac5a401 parent 24dc83114 files · +13347 −0Browse files

Unified · Split

14 files
  1. drivers/infiniband/hw/bnxt_re/bnxt_re.h +146 −0
  2. drivers/infiniband/hw/bnxt_re/ib_verbs.c +3202 −0
  3. drivers/infiniband/hw/bnxt_re/ib_verbs.h +197 −0
  4. drivers/infiniband/hw/bnxt_re/main.c +1315 −0
  5. drivers/infiniband/hw/bnxt_re/qplib_fp.c +2167 −0
  6. drivers/infiniband/hw/bnxt_re/qplib_fp.h +439 −0
  7. drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +694 −0
  8. drivers/infiniband/hw/bnxt_re/qplib_rcfw.h +231 −0
  9. drivers/infiniband/hw/bnxt_re/qplib_res.c +825 −0
  10. drivers/infiniband/hw/bnxt_re/qplib_res.h +223 −0
  11. drivers/infiniband/hw/bnxt_re/qplib_sp.c +838 −0
  12. drivers/infiniband/hw/bnxt_re/qplib_sp.h +160 −0
  13. drivers/infiniband/hw/bnxt_re/roce_hsi.h +2821 −0
  14. include/uapi/rdma/bnxt_re-abi.h +89 −0
drivers/infiniband/hw/bnxt_re/bnxt_re.h+146−0
@@ -0,0 +1,146 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: Slow Path Operators (header)
37+ *
38+ */
39+
40+#ifndef __BNXT_RE_H__
41+#define __BNXT_RE_H__
42+#define ROCE_DRV_MODULE_NAME "bnxt_re"
43+#define ROCE_DRV_MODULE_VERSION "1.0.0"
44+
45+#define BNXT_RE_DESC "Broadcom NetXtreme-C/E RoCE Driver"
46+
47+#define BNXT_RE_PAGE_SIZE_4K BIT(12)
48+#define BNXT_RE_PAGE_SIZE_8K BIT(13)
49+#define BNXT_RE_PAGE_SIZE_64K BIT(16)
50+#define BNXT_RE_PAGE_SIZE_2M BIT(21)
51+#define BNXT_RE_PAGE_SIZE_8M BIT(23)
52+#define BNXT_RE_PAGE_SIZE_1G BIT(30)
53+
54+#define BNXT_RE_MAX_QPC_COUNT (64 * 1024)
55+#define BNXT_RE_MAX_MRW_COUNT (64 * 1024)
56+#define BNXT_RE_MAX_SRQC_COUNT (64 * 1024)
57+#define BNXT_RE_MAX_CQ_COUNT (64 * 1024)
58+
59+struct bnxt_re_work {
60+ struct work_struct work;
61+ unsigned long event;
62+ struct bnxt_re_dev *rdev;
63+ struct net_device *vlan_dev;
64+};
65+
66+struct bnxt_re_sqp_entries {
67+ struct bnxt_qplib_sge sge;
68+ u64 wrid;
69+ /* For storing the actual qp1 cqe */
70+ struct bnxt_qplib_cqe cqe;
71+ struct bnxt_re_qp *qp1_qp;
72+};
73+
74+#define BNXT_RE_MIN_MSIX 2
75+#define BNXT_RE_MAX_MSIX 16
76+#define BNXT_RE_AEQ_IDX 0
77+#define BNXT_RE_NQ_IDX 1
78+
79+struct bnxt_re_dev {
80+ struct ib_device ibdev;
81+ struct list_head list;
82+ unsigned long flags;
83+#define BNXT_RE_FLAG_NETDEV_REGISTERED 0
84+#define BNXT_RE_FLAG_IBDEV_REGISTERED 1
85+#define BNXT_RE_FLAG_GOT_MSIX 2
86+#define BNXT_RE_FLAG_RCFW_CHANNEL_EN 8
87+#define BNXT_RE_FLAG_QOS_WORK_REG 16
88+ struct net_device *netdev;
89+ unsigned int version, major, minor;
90+ struct bnxt_en_dev *en_dev;
91+ struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX];
92+ int num_msix;
93+
94+ int id;
95+
96+ struct delayed_work worker;
97+ u8 cur_prio_map;
98+
99+ /* FP Notification Queue (CQ & SRQ) */
100+ struct tasklet_struct nq_task;
101+
102+ /* RCFW Channel */
103+ struct bnxt_qplib_rcfw rcfw;
104+
105+ /* NQ */
106+ struct bnxt_qplib_nq nq;
107+
108+ /* Device Resources */
109+ struct bnxt_qplib_dev_attr dev_attr;
110+ struct bnxt_qplib_ctx qplib_ctx;
111+ struct bnxt_qplib_res qplib_res;
112+ struct bnxt_qplib_dpi dpi_privileged;
113+
114+ atomic_t qp_count;
115+ struct mutex qp_lock; /* protect qp list */
116+ struct list_head qp_list;
117+
118+ atomic_t cq_count;
119+ atomic_t srq_count;
120+ atomic_t mr_count;
121+ atomic_t mw_count;
122+ /* Max of 2 lossless traffic class supported per port */
123+ u16 cosq[2];
124+
125+ /* QP for for handling QP1 packets */
126+ u32 sqp_id;
127+ struct bnxt_re_qp *qp1_sqp;
128+ struct bnxt_re_ah *sqp_ah;
129+ struct bnxt_re_sqp_entries sqp_tbl[1024];
130+};
131+
132+#define to_bnxt_re_dev(ptr, member) \
133+ container_of((ptr), struct bnxt_re_dev, member)
134+
135+#define BNXT_RE_ROCE_V1_PACKET 0
136+#define BNXT_RE_ROCEV2_IPV4_PACKET 2
137+#define BNXT_RE_ROCEV2_IPV6_PACKET 3
138+
139+static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev)
140+{
141+ if (rdev)
142+ return &rdev->ibdev.dev;
143+ return NULL;
144+}
145+
146+#endif
drivers/infiniband/hw/bnxt_re/ib_verbs.c+3202−0
@@ -0,0 +1,3202 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: IB Verbs interpreter
37+ */
38+
39+#include <linux/interrupt.h>
40+#include <linux/types.h>
41+#include <linux/pci.h>
42+#include <linux/netdevice.h>
43+#include <linux/if_ether.h>
44+
45+#include <rdma/ib_verbs.h>
46+#include <rdma/ib_user_verbs.h>
47+#include <rdma/ib_umem.h>
48+#include <rdma/ib_addr.h>
49+#include <rdma/ib_mad.h>
50+#include <rdma/ib_cache.h>
51+
52+#include "bnxt_ulp.h"
53+
54+#include "roce_hsi.h"
55+#include "qplib_res.h"
56+#include "qplib_sp.h"
57+#include "qplib_fp.h"
58+#include "qplib_rcfw.h"
59+
60+#include "bnxt_re.h"
61+#include "ib_verbs.h"
62+#include <rdma/bnxt_re-abi.h>
63+
64+static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
65+ struct bnxt_qplib_sge *sg_list, int num)
66+{
67+ int i, total = 0;
68+
69+ for (i = 0; i < num; i++) {
70+ sg_list[i].addr = ib_sg_list[i].addr;
71+ sg_list[i].lkey = ib_sg_list[i].lkey;
72+ sg_list[i].size = ib_sg_list[i].length;
73+ total += sg_list[i].size;
74+ }
75+ return total;
76+}
77+
78+/* Device */
79+struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num)
80+{
81+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
82+ struct net_device *netdev = NULL;
83+
84+ rcu_read_lock();
85+ if (rdev)
86+ netdev = rdev->netdev;
87+ if (netdev)
88+ dev_hold(netdev);
89+
90+ rcu_read_unlock();
91+ return netdev;
92+}
93+
94+int bnxt_re_query_device(struct ib_device *ibdev,
95+ struct ib_device_attr *ib_attr,
96+ struct ib_udata *udata)
97+{
98+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
99+ struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
100+
101+ memset(ib_attr, 0, sizeof(*ib_attr));
102+
103+ ib_attr->fw_ver = (u64)(unsigned long)(dev_attr->fw_ver);
104+ bnxt_qplib_get_guid(rdev->netdev->dev_addr,
105+ (u8 *)&ib_attr->sys_image_guid);
106+ ib_attr->max_mr_size = ~0ull;
107+ ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_8K |
108+ BNXT_RE_PAGE_SIZE_64K | BNXT_RE_PAGE_SIZE_2M |
109+ BNXT_RE_PAGE_SIZE_8M | BNXT_RE_PAGE_SIZE_1G;
110+
111+ ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
112+ ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
113+ ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
114+ ib_attr->max_qp = dev_attr->max_qp;
115+ ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
116+ ib_attr->device_cap_flags =
117+ IB_DEVICE_CURR_QP_STATE_MOD
118+ | IB_DEVICE_RC_RNR_NAK_GEN
119+ | IB_DEVICE_SHUTDOWN_PORT
120+ | IB_DEVICE_SYS_IMAGE_GUID
121+ | IB_DEVICE_LOCAL_DMA_LKEY
122+ | IB_DEVICE_RESIZE_MAX_WR
123+ | IB_DEVICE_PORT_ACTIVE_EVENT
124+ | IB_DEVICE_N_NOTIFY_CQ
125+ | IB_DEVICE_MEM_WINDOW
126+ | IB_DEVICE_MEM_WINDOW_TYPE_2B
127+ | IB_DEVICE_MEM_MGT_EXTENSIONS;
128+ ib_attr->max_sge = dev_attr->max_qp_sges;
129+ ib_attr->max_sge_rd = dev_attr->max_qp_sges;
130+ ib_attr->max_cq = dev_attr->max_cq;
131+ ib_attr->max_cqe = dev_attr->max_cq_wqes;
132+ ib_attr->max_mr = dev_attr->max_mr;
133+ ib_attr->max_pd = dev_attr->max_pd;
134+ ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
135+ ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_rd_atom;
136+ ib_attr->atomic_cap = IB_ATOMIC_HCA;
137+ ib_attr->masked_atomic_cap = IB_ATOMIC_HCA;
138+
139+ ib_attr->max_ee_rd_atom = 0;
140+ ib_attr->max_res_rd_atom = 0;
141+ ib_attr->max_ee_init_rd_atom = 0;
142+ ib_attr->max_ee = 0;
143+ ib_attr->max_rdd = 0;
144+ ib_attr->max_mw = dev_attr->max_mw;
145+ ib_attr->max_raw_ipv6_qp = 0;
146+ ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
147+ ib_attr->max_mcast_grp = 0;
148+ ib_attr->max_mcast_qp_attach = 0;
149+ ib_attr->max_total_mcast_qp_attach = 0;
150+ ib_attr->max_ah = dev_attr->max_ah;
151+
152+ ib_attr->max_fmr = dev_attr->max_fmr;
153+ ib_attr->max_map_per_fmr = 1; /* ? */
154+
155+ ib_attr->max_srq = dev_attr->max_srq;
156+ ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
157+ ib_attr->max_srq_sge = dev_attr->max_srq_sges;
158+
159+ ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
160+
161+ ib_attr->max_pkeys = 1;
162+ ib_attr->local_ca_ack_delay = 0;
163+ return 0;
164+}
165+
166+int bnxt_re_modify_device(struct ib_device *ibdev,
167+ int device_modify_mask,
168+ struct ib_device_modify *device_modify)
169+{
170+ switch (device_modify_mask) {
171+ case IB_DEVICE_MODIFY_SYS_IMAGE_GUID:
172+ /* Modify the GUID requires the modification of the GID table */
173+ /* GUID should be made as READ-ONLY */
174+ break;
175+ case IB_DEVICE_MODIFY_NODE_DESC:
176+ /* Node Desc should be made as READ-ONLY */
177+ break;
178+ default:
179+ break;
180+ }
181+ return 0;
182+}
183+
184+static void __to_ib_speed_width(struct net_device *netdev, u8 *speed, u8 *width)
185+{
186+ struct ethtool_link_ksettings lksettings;
187+ u32 espeed;
188+
189+ if (netdev->ethtool_ops && netdev->ethtool_ops->get_link_ksettings) {
190+ memset(&lksettings, 0, sizeof(lksettings));
191+ rtnl_lock();
192+ netdev->ethtool_ops->get_link_ksettings(netdev, &lksettings);
193+ rtnl_unlock();
194+ espeed = lksettings.base.speed;
195+ } else {
196+ espeed = SPEED_UNKNOWN;
197+ }
198+ switch (espeed) {
199+ case SPEED_1000:
200+ *speed = IB_SPEED_SDR;
201+ *width = IB_WIDTH_1X;
202+ break;
203+ case SPEED_10000:
204+ *speed = IB_SPEED_QDR;
205+ *width = IB_WIDTH_1X;
206+ break;
207+ case SPEED_20000:
208+ *speed = IB_SPEED_DDR;
209+ *width = IB_WIDTH_4X;
210+ break;
211+ case SPEED_25000:
212+ *speed = IB_SPEED_EDR;
213+ *width = IB_WIDTH_1X;
214+ break;
215+ case SPEED_40000:
216+ *speed = IB_SPEED_QDR;
217+ *width = IB_WIDTH_4X;
218+ break;
219+ case SPEED_50000:
220+ break;
221+ default:
222+ *speed = IB_SPEED_SDR;
223+ *width = IB_WIDTH_1X;
224+ break;
225+ }
226+}
227+
228+/* Port */
229+int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
230+ struct ib_port_attr *port_attr)
231+{
232+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
233+ struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
234+
235+ memset(port_attr, 0, sizeof(*port_attr));
236+
237+ if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
238+ port_attr->state = IB_PORT_ACTIVE;
239+ port_attr->phys_state = 5;
240+ } else {
241+ port_attr->state = IB_PORT_DOWN;
242+ port_attr->phys_state = 3;
243+ }
244+ port_attr->max_mtu = IB_MTU_4096;
245+ port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
246+ port_attr->gid_tbl_len = dev_attr->max_sgid;
247+ port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
248+ IB_PORT_DEVICE_MGMT_SUP |
249+ IB_PORT_VENDOR_CLASS_SUP |
250+ IB_PORT_IP_BASED_GIDS;
251+
252+ /* Max MSG size set to 2G for now */
253+ port_attr->max_msg_sz = 0x80000000;
254+ port_attr->bad_pkey_cntr = 0;
255+ port_attr->qkey_viol_cntr = 0;
256+ port_attr->pkey_tbl_len = dev_attr->max_pkey;
257+ port_attr->lid = 0;
258+ port_attr->sm_lid = 0;
259+ port_attr->lmc = 0;
260+ port_attr->max_vl_num = 4;
261+ port_attr->sm_sl = 0;
262+ port_attr->subnet_timeout = 0;
263+ port_attr->init_type_reply = 0;
264+ /* call the underlying netdev's ethtool hooks to query speed settings
265+ * for which we acquire rtnl_lock _only_ if it's registered with
266+ * IB stack to avoid race in the NETDEV_UNREG path
267+ */
268+ if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
269+ __to_ib_speed_width(rdev->netdev, &port_attr->active_speed,
270+ &port_attr->active_width);
271+ return 0;
272+}
273+
274+int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
275+ int port_modify_mask,
276+ struct ib_port_modify *port_modify)
277+{
278+ switch (port_modify_mask) {
279+ case IB_PORT_SHUTDOWN:
280+ break;
281+ case IB_PORT_INIT_TYPE:
282+ break;
283+ case IB_PORT_RESET_QKEY_CNTR:
284+ break;
285+ default:
286+ break;
287+ }
288+ return 0;
289+}
290+
291+int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
292+ struct ib_port_immutable *immutable)
293+{
294+ struct ib_port_attr port_attr;
295+
296+ if (bnxt_re_query_port(ibdev, port_num, &port_attr))
297+ return -EINVAL;
298+
299+ immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
300+ immutable->gid_tbl_len = port_attr.gid_tbl_len;
301+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
302+ immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
303+ immutable->max_mad_size = IB_MGMT_MAD_SIZE;
304+ return 0;
305+}
306+
307+int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
308+ u16 index, u16 *pkey)
309+{
310+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
311+
312+ /* Ignore port_num */
313+
314+ memset(pkey, 0, sizeof(*pkey));
315+ return bnxt_qplib_get_pkey(&rdev->qplib_res,
316+ &rdev->qplib_res.pkey_tbl, index, pkey);
317+}
318+
319+int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
320+ int index, union ib_gid *gid)
321+{
322+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
323+ int rc = 0;
324+
325+ /* Ignore port_num */
326+ memset(gid, 0, sizeof(*gid));
327+ rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
328+ &rdev->qplib_res.sgid_tbl, index,
329+ (struct bnxt_qplib_gid *)gid);
330+ return rc;
331+}
332+
333+int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
334+ unsigned int index, void **context)
335+{
336+ int rc = 0;
337+ struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
338+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
339+ struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
340+
341+ /* Delete the entry from the hardware */
342+ ctx = *context;
343+ if (!ctx)
344+ return -EINVAL;
345+
346+ if (sgid_tbl && sgid_tbl->active) {
347+ if (ctx->idx >= sgid_tbl->max)
348+ return -EINVAL;
349+ ctx->refcnt--;
350+ if (!ctx->refcnt) {
351+ rc = bnxt_qplib_del_sgid
352+ (sgid_tbl,
353+ &sgid_tbl->tbl[ctx->idx], true);
354+ if (rc)
355+ dev_err(rdev_to_dev(rdev),
356+ "Failed to remove GID: %#x", rc);
357+ ctx_tbl = sgid_tbl->ctx;
358+ ctx_tbl[ctx->idx] = NULL;
359+ kfree(ctx);
360+ }
361+ } else {
362+ return -EINVAL;
363+ }
364+ return rc;
365+}
366+
367+int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
368+ unsigned int index, const union ib_gid *gid,
369+ const struct ib_gid_attr *attr, void **context)
370+{
371+ int rc;
372+ u32 tbl_idx = 0;
373+ u16 vlan_id = 0xFFFF;
374+ struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
375+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
376+ struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
377+
378+ if ((attr->ndev) && is_vlan_dev(attr->ndev))
379+ vlan_id = vlan_dev_vlan_id(attr->ndev);
380+
381+ rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)gid,
382+ rdev->qplib_res.netdev->dev_addr,
383+ vlan_id, true, &tbl_idx);
384+ if (rc == -EALREADY) {
385+ ctx_tbl = sgid_tbl->ctx;
386+ ctx_tbl[tbl_idx]->refcnt++;
387+ *context = ctx_tbl[tbl_idx];
388+ return 0;
389+ }
390+
391+ if (rc < 0) {
392+ dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc);
393+ return rc;
394+ }
395+
396+ ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
397+ if (!ctx)
398+ return -ENOMEM;
399+ ctx_tbl = sgid_tbl->ctx;
400+ ctx->idx = tbl_idx;
401+ ctx->refcnt = 1;
402+ ctx_tbl[tbl_idx] = ctx;
403+
404+ return rc;
405+}
406+
407+enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
408+ u8 port_num)
409+{
410+ return IB_LINK_LAYER_ETHERNET;
411+}
412+
413+/* Protection Domains */
414+int bnxt_re_dealloc_pd(struct ib_pd *ib_pd)
415+{
416+ struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
417+ struct bnxt_re_dev *rdev = pd->rdev;
418+ int rc;
419+
420+ if (ib_pd->uobject && pd->dpi.dbr) {
421+ struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
422+ struct bnxt_re_ucontext *ucntx;
423+
424+ /* Free DPI only if this is the first PD allocated by the
425+ * application and mark the context dpi as NULL
426+ */
427+ ucntx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
428+
429+ rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
430+ &rdev->qplib_res.dpi_tbl,
431+ &pd->dpi);
432+ if (rc)
433+ dev_err(rdev_to_dev(rdev), "Failed to deallocate HW DPI");
434+ /* Don't fail, continue*/
435+ ucntx->dpi = NULL;
436+ }
437+
438+ rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res,
439+ &rdev->qplib_res.pd_tbl,
440+ &pd->qplib_pd);
441+ if (rc) {
442+ dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD");
443+ return rc;
444+ }
445+
446+ kfree(pd);
447+ return 0;
448+}
449+
450+struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev,
451+ struct ib_ucontext *ucontext,
452+ struct ib_udata *udata)
453+{
454+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
455+ struct bnxt_re_ucontext *ucntx = container_of(ucontext,
456+ struct bnxt_re_ucontext,
457+ ib_uctx);
458+ struct bnxt_re_pd *pd;
459+ int rc;
460+
461+ pd = kzalloc(sizeof(*pd), GFP_KERNEL);
462+ if (!pd)
463+ return ERR_PTR(-ENOMEM);
464+
465+ pd->rdev = rdev;
466+ if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) {
467+ dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD");
468+ rc = -ENOMEM;
469+ goto fail;
470+ }
471+
472+ if (udata) {
473+ struct bnxt_re_pd_resp resp;
474+
475+ if (!ucntx->dpi) {
476+ /* Allocate DPI in alloc_pd to avoid failing of
477+ * ibv_devinfo and family of application when DPIs
478+ * are depleted.
479+ */
480+ if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
481+ &pd->dpi, ucntx)) {
482+ rc = -ENOMEM;
483+ goto dbfail;
484+ }
485+ ucntx->dpi = &pd->dpi;
486+ }
487+
488+ resp.pdid = pd->qplib_pd.id;
489+ /* Still allow mapping this DBR to the new user PD. */
490+ resp.dpi = ucntx->dpi->dpi;
491+ resp.dbr = (u64)ucntx->dpi->umdbr;
492+
493+ rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
494+ if (rc) {
495+ dev_err(rdev_to_dev(rdev),
496+ "Failed to copy user response\n");
497+ goto dbfail;
498+ }
499+ }
500+
501+ return &pd->ib_pd;
502+dbfail:
503+ (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
504+ &pd->qplib_pd);
505+fail:
506+ kfree(pd);
507+ return ERR_PTR(rc);
508+}
509+
510+/* Address Handles */
511+int bnxt_re_destroy_ah(struct ib_ah *ib_ah)
512+{
513+ struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
514+ struct bnxt_re_dev *rdev = ah->rdev;
515+ int rc;
516+
517+ rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah);
518+ if (rc) {
519+ dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH");
520+ return rc;
521+ }
522+ kfree(ah);
523+ return 0;
524+}
525+
526+struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
527+ struct ib_ah_attr *ah_attr,
528+ struct ib_udata *udata)
529+{
530+ struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
531+ struct bnxt_re_dev *rdev = pd->rdev;
532+ struct bnxt_re_ah *ah;
533+ int rc;
534+ u16 vlan_tag;
535+ u8 nw_type;
536+
537+ struct ib_gid_attr sgid_attr;
538+
539+ if (!(ah_attr->ah_flags & IB_AH_GRH)) {
540+ dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set");
541+ return ERR_PTR(-EINVAL);
542+ }
543+ ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
544+ if (!ah)
545+ return ERR_PTR(-ENOMEM);
546+
547+ ah->rdev = rdev;
548+ ah->qplib_ah.pd = &pd->qplib_pd;
549+
550+ /* Supply the configuration for the HW */
551+ memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw,
552+ sizeof(union ib_gid));
553+ /*
554+ * If RoCE V2 is enabled, stack will have two entries for
555+ * each GID entry. Avoiding this duplicte entry in HW. Dividing
556+ * the GID index by 2 for RoCE V2
557+ */
558+ ah->qplib_ah.sgid_index = ah_attr->grh.sgid_index / 2;
559+ ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index;
560+ ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class;
561+ ah->qplib_ah.flow_label = ah_attr->grh.flow_label;
562+ ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit;
563+ ah->qplib_ah.sl = ah_attr->sl;
564+ if (ib_pd->uobject &&
565+ !rdma_is_multicast_addr((struct in6_addr *)
566+ ah_attr->grh.dgid.raw) &&
567+ !rdma_link_local_addr((struct in6_addr *)
568+ ah_attr->grh.dgid.raw)) {
569+ union ib_gid sgid;
570+
571+ rc = ib_get_cached_gid(&rdev->ibdev, 1,
572+ ah_attr->grh.sgid_index, &sgid,
573+ &sgid_attr);
574+ if (rc) {
575+ dev_err(rdev_to_dev(rdev),
576+ "Failed to query gid at index %d",
577+ ah_attr->grh.sgid_index);
578+ goto fail;
579+ }
580+ if (sgid_attr.ndev) {
581+ if (is_vlan_dev(sgid_attr.ndev))
582+ vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev);
583+ dev_put(sgid_attr.ndev);
584+ }
585+ /* Get network header type for this GID */
586+ nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
587+ switch (nw_type) {
588+ case RDMA_NETWORK_IPV4:
589+ ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
590+ break;
591+ case RDMA_NETWORK_IPV6:
592+ ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
593+ break;
594+ default:
595+ ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1;
596+ break;
597+ }
598+ rc = rdma_addr_find_l2_eth_by_grh(&sgid, &ah_attr->grh.dgid,
599+ ah_attr->dmac, &vlan_tag,
600+ &sgid_attr.ndev->ifindex,
601+ NULL);
602+ if (rc) {
603+ dev_err(rdev_to_dev(rdev), "Failed to get dmac\n");
604+ goto fail;
605+ }
606+ }
607+
608+ memcpy(ah->qplib_ah.dmac, ah_attr->dmac, ETH_ALEN);
609+ rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
610+ if (rc) {
611+ dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH");
612+ goto fail;
613+ }
614+
615+ /* Write AVID to shared page. */
616+ if (ib_pd->uobject) {
617+ struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
618+ struct bnxt_re_ucontext *uctx;
619+ unsigned long flag;
620+ u32 *wrptr;
621+
622+ uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
623+ spin_lock_irqsave(&uctx->sh_lock, flag);
624+ wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
625+ *wrptr = ah->qplib_ah.id;
626+ wmb(); /* make sure cache is updated. */
627+ spin_unlock_irqrestore(&uctx->sh_lock, flag);
628+ }
629+
630+ return &ah->ib_ah;
631+
632+fail:
633+ kfree(ah);
634+ return ERR_PTR(rc);
635+}
636+
637+int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
638+{
639+ return 0;
640+}
641+
642+int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
643+{
644+ struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
645+
646+ memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data,
647+ sizeof(union ib_gid));
648+ ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index;
649+ ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class;
650+ ah_attr->sl = ah->qplib_ah.sl;
651+ memcpy(ah_attr->dmac, ah->qplib_ah.dmac, ETH_ALEN);
652+ ah_attr->ah_flags = IB_AH_GRH;
653+ ah_attr->port_num = 1;
654+ ah_attr->static_rate = 0;
655+ return 0;
656+}
657+
658+/* Queue Pairs */
659+int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
660+{
661+ struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
662+ struct bnxt_re_dev *rdev = qp->rdev;
663+ int rc;
664+
665+ rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
666+ if (rc) {
667+ dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP");
668+ return rc;
669+ }
670+ if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) {
671+ rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
672+ &rdev->sqp_ah->qplib_ah);
673+ if (rc) {
674+ dev_err(rdev_to_dev(rdev),
675+ "Failed to destroy HW AH for shadow QP");
676+ return rc;
677+ }
678+
679+ rc = bnxt_qplib_destroy_qp(&rdev->qplib_res,
680+ &rdev->qp1_sqp->qplib_qp);
681+ if (rc) {
682+ dev_err(rdev_to_dev(rdev),
683+ "Failed to destroy Shadow QP");
684+ return rc;
685+ }
686+ mutex_lock(&rdev->qp_lock);
687+ list_del(&rdev->qp1_sqp->list);
688+ atomic_dec(&rdev->qp_count);
689+ mutex_unlock(&rdev->qp_lock);
690+
691+ kfree(rdev->sqp_ah);
692+ kfree(rdev->qp1_sqp);
693+ }
694+
695+ if (qp->rumem && !IS_ERR(qp->rumem))
696+ ib_umem_release(qp->rumem);
697+ if (qp->sumem && !IS_ERR(qp->sumem))
698+ ib_umem_release(qp->sumem);
699+
700+ mutex_lock(&rdev->qp_lock);
701+ list_del(&qp->list);
702+ atomic_dec(&rdev->qp_count);
703+ mutex_unlock(&rdev->qp_lock);
704+ kfree(qp);
705+ return 0;
706+}
707+
708+static u8 __from_ib_qp_type(enum ib_qp_type type)
709+{
710+ switch (type) {
711+ case IB_QPT_GSI:
712+ return CMDQ_CREATE_QP1_TYPE_GSI;
713+ case IB_QPT_RC:
714+ return CMDQ_CREATE_QP_TYPE_RC;
715+ case IB_QPT_UD:
716+ return CMDQ_CREATE_QP_TYPE_UD;
717+ default:
718+ return IB_QPT_MAX;
719+ }
720+}
721+
722+static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
723+ struct bnxt_re_qp *qp, struct ib_udata *udata)
724+{
725+ struct bnxt_re_qp_req ureq;
726+ struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
727+ struct ib_umem *umem;
728+ int bytes = 0;
729+ struct ib_ucontext *context = pd->ib_pd.uobject->context;
730+ struct bnxt_re_ucontext *cntx = container_of(context,
731+ struct bnxt_re_ucontext,
732+ ib_uctx);
733+ if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
734+ return -EFAULT;
735+
736+ bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
737+ /* Consider mapping PSN search memory only for RC QPs. */
738+ if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC)
739+ bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search));
740+ bytes = PAGE_ALIGN(bytes);
741+ umem = ib_umem_get(context, ureq.qpsva, bytes,
742+ IB_ACCESS_LOCAL_WRITE, 1);
743+ if (IS_ERR(umem))
744+ return PTR_ERR(umem);
745+
746+ qp->sumem = umem;
747+ qplib_qp->sq.sglist = umem->sg_head.sgl;
748+ qplib_qp->sq.nmap = umem->nmap;
749+ qplib_qp->qp_handle = ureq.qp_handle;
750+
751+ if (!qp->qplib_qp.srq) {
752+ bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
753+ bytes = PAGE_ALIGN(bytes);
754+ umem = ib_umem_get(context, ureq.qprva, bytes,
755+ IB_ACCESS_LOCAL_WRITE, 1);
756+ if (IS_ERR(umem))
757+ goto rqfail;
758+ qp->rumem = umem;
759+ qplib_qp->rq.sglist = umem->sg_head.sgl;
760+ qplib_qp->rq.nmap = umem->nmap;
761+ }
762+
763+ qplib_qp->dpi = cntx->dpi;
764+ return 0;
765+rqfail:
766+ ib_umem_release(qp->sumem);
767+ qp->sumem = NULL;
768+ qplib_qp->sq.sglist = NULL;
769+ qplib_qp->sq.nmap = 0;
770+
771+ return PTR_ERR(umem);
772+}
773+
774+static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
775+ (struct bnxt_re_pd *pd,
776+ struct bnxt_qplib_res *qp1_res,
777+ struct bnxt_qplib_qp *qp1_qp)
778+{
779+ struct bnxt_re_dev *rdev = pd->rdev;
780+ struct bnxt_re_ah *ah;
781+ union ib_gid sgid;
782+ int rc;
783+
784+ ah = kzalloc(sizeof(*ah), GFP_KERNEL);
785+ if (!ah)
786+ return NULL;
787+
788+ memset(ah, 0, sizeof(*ah));
789+ ah->rdev = rdev;
790+ ah->qplib_ah.pd = &pd->qplib_pd;
791+
792+ rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
793+ if (rc)
794+ goto fail;
795+
796+ /* supply the dgid data same as sgid */
797+ memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
798+ sizeof(union ib_gid));
799+ ah->qplib_ah.sgid_index = 0;
800+
801+ ah->qplib_ah.traffic_class = 0;
802+ ah->qplib_ah.flow_label = 0;
803+ ah->qplib_ah.hop_limit = 1;
804+ ah->qplib_ah.sl = 0;
805+ /* Have DMAC same as SMAC */
806+ ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
807+
808+ rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
809+ if (rc) {
810+ dev_err(rdev_to_dev(rdev),
811+ "Failed to allocate HW AH for Shadow QP");
812+ goto fail;
813+ }
814+
815+ return ah;
816+
817+fail:
818+ kfree(ah);
819+ return NULL;
820+}
821+
822+static struct bnxt_re_qp *bnxt_re_create_shadow_qp
823+ (struct bnxt_re_pd *pd,
824+ struct bnxt_qplib_res *qp1_res,
825+ struct bnxt_qplib_qp *qp1_qp)
826+{
827+ struct bnxt_re_dev *rdev = pd->rdev;
828+ struct bnxt_re_qp *qp;
829+ int rc;
830+
831+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
832+ if (!qp)
833+ return NULL;
834+
835+ memset(qp, 0, sizeof(*qp));
836+ qp->rdev = rdev;
837+
838+ /* Initialize the shadow QP structure from the QP1 values */
839+ ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
840+
841+ qp->qplib_qp.pd = &pd->qplib_pd;
842+ qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
843+ qp->qplib_qp.type = IB_QPT_UD;
844+
845+ qp->qplib_qp.max_inline_data = 0;
846+ qp->qplib_qp.sig_type = true;
847+
848+ /* Shadow QP SQ depth should be same as QP1 RQ depth */
849+ qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
850+ qp->qplib_qp.sq.max_sge = 2;
851+
852+ qp->qplib_qp.scq = qp1_qp->scq;
853+ qp->qplib_qp.rcq = qp1_qp->rcq;
854+
855+ qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
856+ qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
857+
858+ qp->qplib_qp.mtu = qp1_qp->mtu;
859+
860+ qp->qplib_qp.sq_hdr_buf_size = 0;
861+ qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
862+ qp->qplib_qp.dpi = &rdev->dpi_privileged;
863+
864+ rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
865+ if (rc)
866+ goto fail;
867+
868+ rdev->sqp_id = qp->qplib_qp.id;
869+
870+ spin_lock_init(&qp->sq_lock);
871+ INIT_LIST_HEAD(&qp->list);
872+ mutex_lock(&rdev->qp_lock);
873+ list_add_tail(&qp->list, &rdev->qp_list);
874+ atomic_inc(&rdev->qp_count);
875+ mutex_unlock(&rdev->qp_lock);
876+ return qp;
877+fail:
878+ kfree(qp);
879+ return NULL;
880+}
881+
882+struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
883+ struct ib_qp_init_attr *qp_init_attr,
884+ struct ib_udata *udata)
885+{
886+ struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
887+ struct bnxt_re_dev *rdev = pd->rdev;
888+ struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
889+ struct bnxt_re_qp *qp;
890+ struct bnxt_re_cq *cq;
891+ int rc, entries;
892+
893+ if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
894+ (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) ||
895+ (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) ||
896+ (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) ||
897+ (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data))
898+ return ERR_PTR(-EINVAL);
899+
900+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
901+ if (!qp)
902+ return ERR_PTR(-ENOMEM);
903+
904+ qp->rdev = rdev;
905+ ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
906+ qp->qplib_qp.pd = &pd->qplib_pd;
907+ qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
908+ qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type);
909+ if (qp->qplib_qp.type == IB_QPT_MAX) {
910+ dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported",
911+ qp->qplib_qp.type);
912+ rc = -EINVAL;
913+ goto fail;
914+ }
915+ qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data;
916+ qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type ==
917+ IB_SIGNAL_ALL_WR) ? true : false);
918+
919+ entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1);
920+ qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
921+ dev_attr->max_qp_wqes + 1);
922+
923+ qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge;
924+ if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges)
925+ qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges;
926+
927+ if (qp_init_attr->send_cq) {
928+ cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq,
929+ ib_cq);
930+ if (!cq) {
931+ dev_err(rdev_to_dev(rdev), "Send CQ not found");
932+ rc = -EINVAL;
933+ goto fail;
934+ }
935+ qp->qplib_qp.scq = &cq->qplib_cq;
936+ }
937+
938+ if (qp_init_attr->recv_cq) {
939+ cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq,
940+ ib_cq);
941+ if (!cq) {
942+ dev_err(rdev_to_dev(rdev), "Receive CQ not found");
943+ rc = -EINVAL;
944+ goto fail;
945+ }
946+ qp->qplib_qp.rcq = &cq->qplib_cq;
947+ }
948+
949+ if (qp_init_attr->srq) {
950+ dev_err(rdev_to_dev(rdev), "SRQ not supported");
951+ rc = -ENOTSUPP;
952+ goto fail;
953+ } else {
954+ /* Allocate 1 more than what's provided so posting max doesn't
955+ * mean empty
956+ */
957+ entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1);
958+ qp->qplib_qp.rq.max_wqe = min_t(u32, entries,
959+ dev_attr->max_qp_wqes + 1);
960+
961+ qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge;
962+ if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges)
963+ qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
964+ }
965+
966+ qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
967+
968+ if (qp_init_attr->qp_type == IB_QPT_GSI) {
969+ qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
970+ if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges)
971+ qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
972+ qp->qplib_qp.sq.max_sge++;
973+ if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges)
974+ qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges;
975+
976+ qp->qplib_qp.rq_hdr_buf_size =
977+ BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
978+
979+ qp->qplib_qp.sq_hdr_buf_size =
980+ BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
981+ qp->qplib_qp.dpi = &rdev->dpi_privileged;
982+ rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp);
983+ if (rc) {
984+ dev_err(rdev_to_dev(rdev), "Failed to create HW QP1");
985+ goto fail;
986+ }
987+ /* Create a shadow QP to handle the QP1 traffic */
988+ rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res,
989+ &qp->qplib_qp);
990+ if (!rdev->qp1_sqp) {
991+ rc = -EINVAL;
992+ dev_err(rdev_to_dev(rdev),
993+ "Failed to create Shadow QP for QP1");
994+ goto qp_destroy;
995+ }
996+ rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
997+ &qp->qplib_qp);
998+ if (!rdev->sqp_ah) {
999+ bnxt_qplib_destroy_qp(&rdev->qplib_res,
1000+ &rdev->qp1_sqp->qplib_qp);
1001+ rc = -EINVAL;
1002+ dev_err(rdev_to_dev(rdev),
1003+ "Failed to create AH entry for ShadowQP");
1004+ goto qp_destroy;
1005+ }
1006+
1007+ } else {
1008+ qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom;
1009+ qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1010+ if (udata) {
1011+ rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1012+ if (rc)
1013+ goto fail;
1014+ } else {
1015+ qp->qplib_qp.dpi = &rdev->dpi_privileged;
1016+ }
1017+
1018+ rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1019+ if (rc) {
1020+ dev_err(rdev_to_dev(rdev), "Failed to create HW QP");
1021+ goto fail;
1022+ }
1023+ }
1024+
1025+ qp->ib_qp.qp_num = qp->qplib_qp.id;
1026+ spin_lock_init(&qp->sq_lock);
1027+
1028+ if (udata) {
1029+ struct bnxt_re_qp_resp resp;
1030+
1031+ resp.qpid = qp->ib_qp.qp_num;
1032+ resp.rsvd = 0;
1033+ rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1034+ if (rc) {
1035+ dev_err(rdev_to_dev(rdev), "Failed to copy QP udata");
1036+ goto qp_destroy;
1037+ }
1038+ }
1039+ INIT_LIST_HEAD(&qp->list);
1040+ mutex_lock(&rdev->qp_lock);
1041+ list_add_tail(&qp->list, &rdev->qp_list);
1042+ atomic_inc(&rdev->qp_count);
1043+ mutex_unlock(&rdev->qp_lock);
1044+
1045+ return &qp->ib_qp;
1046+qp_destroy:
1047+ bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1048+fail:
1049+ kfree(qp);
1050+ return ERR_PTR(rc);
1051+}
1052+
1053+static u8 __from_ib_qp_state(enum ib_qp_state state)
1054+{
1055+ switch (state) {
1056+ case IB_QPS_RESET:
1057+ return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1058+ case IB_QPS_INIT:
1059+ return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1060+ case IB_QPS_RTR:
1061+ return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1062+ case IB_QPS_RTS:
1063+ return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1064+ case IB_QPS_SQD:
1065+ return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1066+ case IB_QPS_SQE:
1067+ return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1068+ case IB_QPS_ERR:
1069+ default:
1070+ return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1071+ }
1072+}
1073+
1074+static enum ib_qp_state __to_ib_qp_state(u8 state)
1075+{
1076+ switch (state) {
1077+ case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1078+ return IB_QPS_RESET;
1079+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1080+ return IB_QPS_INIT;
1081+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1082+ return IB_QPS_RTR;
1083+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1084+ return IB_QPS_RTS;
1085+ case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1086+ return IB_QPS_SQD;
1087+ case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1088+ return IB_QPS_SQE;
1089+ case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1090+ default:
1091+ return IB_QPS_ERR;
1092+ }
1093+}
1094+
1095+static u32 __from_ib_mtu(enum ib_mtu mtu)
1096+{
1097+ switch (mtu) {
1098+ case IB_MTU_256:
1099+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1100+ case IB_MTU_512:
1101+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1102+ case IB_MTU_1024:
1103+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1104+ case IB_MTU_2048:
1105+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1106+ case IB_MTU_4096:
1107+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1108+ default:
1109+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1110+ }
1111+}
1112+
1113+static enum ib_mtu __to_ib_mtu(u32 mtu)
1114+{
1115+ switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1116+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1117+ return IB_MTU_256;
1118+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1119+ return IB_MTU_512;
1120+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1121+ return IB_MTU_1024;
1122+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1123+ return IB_MTU_2048;
1124+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1125+ return IB_MTU_4096;
1126+ default:
1127+ return IB_MTU_2048;
1128+ }
1129+}
1130+
1131+static int __from_ib_access_flags(int iflags)
1132+{
1133+ int qflags = 0;
1134+
1135+ if (iflags & IB_ACCESS_LOCAL_WRITE)
1136+ qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1137+ if (iflags & IB_ACCESS_REMOTE_READ)
1138+ qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
1139+ if (iflags & IB_ACCESS_REMOTE_WRITE)
1140+ qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
1141+ if (iflags & IB_ACCESS_REMOTE_ATOMIC)
1142+ qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
1143+ if (iflags & IB_ACCESS_MW_BIND)
1144+ qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
1145+ if (iflags & IB_ZERO_BASED)
1146+ qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
1147+ if (iflags & IB_ACCESS_ON_DEMAND)
1148+ qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
1149+ return qflags;
1150+};
1151+
1152+static enum ib_access_flags __to_ib_access_flags(int qflags)
1153+{
1154+ enum ib_access_flags iflags = 0;
1155+
1156+ if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
1157+ iflags |= IB_ACCESS_LOCAL_WRITE;
1158+ if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
1159+ iflags |= IB_ACCESS_REMOTE_WRITE;
1160+ if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
1161+ iflags |= IB_ACCESS_REMOTE_READ;
1162+ if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
1163+ iflags |= IB_ACCESS_REMOTE_ATOMIC;
1164+ if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
1165+ iflags |= IB_ACCESS_MW_BIND;
1166+ if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
1167+ iflags |= IB_ZERO_BASED;
1168+ if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
1169+ iflags |= IB_ACCESS_ON_DEMAND;
1170+ return iflags;
1171+};
1172+
1173+static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1174+ struct bnxt_re_qp *qp1_qp,
1175+ int qp_attr_mask)
1176+{
1177+ struct bnxt_re_qp *qp = rdev->qp1_sqp;
1178+ int rc = 0;
1179+
1180+ if (qp_attr_mask & IB_QP_STATE) {
1181+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1182+ qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1183+ }
1184+ if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1185+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1186+ qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1187+ }
1188+
1189+ if (qp_attr_mask & IB_QP_QKEY) {
1190+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1191+ /* Using a Random QKEY */
1192+ qp->qplib_qp.qkey = 0x81818181;
1193+ }
1194+ if (qp_attr_mask & IB_QP_SQ_PSN) {
1195+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1196+ qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1197+ }
1198+
1199+ rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1200+ if (rc)
1201+ dev_err(rdev_to_dev(rdev),
1202+ "Failed to modify Shadow QP for QP1");
1203+ return rc;
1204+}
1205+
1206+int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1207+ int qp_attr_mask, struct ib_udata *udata)
1208+{
1209+ struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1210+ struct bnxt_re_dev *rdev = qp->rdev;
1211+ struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1212+ enum ib_qp_state curr_qp_state, new_qp_state;
1213+ int rc, entries;
1214+ int status;
1215+ union ib_gid sgid;
1216+ struct ib_gid_attr sgid_attr;
1217+ u8 nw_type;
1218+
1219+ qp->qplib_qp.modify_flags = 0;
1220+ if (qp_attr_mask & IB_QP_STATE) {
1221+ curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1222+ new_qp_state = qp_attr->qp_state;
1223+ if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
1224+ ib_qp->qp_type, qp_attr_mask,
1225+ IB_LINK_LAYER_ETHERNET)) {
1226+ dev_err(rdev_to_dev(rdev),
1227+ "Invalid attribute mask: %#x specified ",
1228+ qp_attr_mask);
1229+ dev_err(rdev_to_dev(rdev),
1230+ "for qpn: %#x type: %#x",
1231+ ib_qp->qp_num, ib_qp->qp_type);
1232+ dev_err(rdev_to_dev(rdev),
1233+ "curr_qp_state=0x%x, new_qp_state=0x%x\n",
1234+ curr_qp_state, new_qp_state);
1235+ return -EINVAL;
1236+ }
1237+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1238+ qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
1239+ }
1240+ if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1241+ qp->qplib_qp.modify_flags |=
1242+ CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1243+ qp->qplib_qp.en_sqd_async_notify = true;
1244+ }
1245+ if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1246+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1247+ qp->qplib_qp.access =
1248+ __from_ib_access_flags(qp_attr->qp_access_flags);
1249+ /* LOCAL_WRITE access must be set to allow RC receive */
1250+ qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1251+ }
1252+ if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1253+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1254+ qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1255+ }
1256+ if (qp_attr_mask & IB_QP_QKEY) {
1257+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1258+ qp->qplib_qp.qkey = qp_attr->qkey;
1259+ }
1260+ if (qp_attr_mask & IB_QP_AV) {
1261+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1262+ CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1263+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1264+ CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1265+ CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1266+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1267+ CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1268+ memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw,
1269+ sizeof(qp->qplib_qp.ah.dgid.data));
1270+ qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label;
1271+ /* If RoCE V2 is enabled, stack will have two entries for
1272+ * each GID entry. Avoiding this duplicte entry in HW. Dividing
1273+ * the GID index by 2 for RoCE V2
1274+ */
1275+ qp->qplib_qp.ah.sgid_index =
1276+ qp_attr->ah_attr.grh.sgid_index / 2;
1277+ qp->qplib_qp.ah.host_sgid_index =
1278+ qp_attr->ah_attr.grh.sgid_index;
1279+ qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit;
1280+ qp->qplib_qp.ah.traffic_class =
1281+ qp_attr->ah_attr.grh.traffic_class;
1282+ qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl;
1283+ ether_addr_copy(qp->qplib_qp.ah.dmac, qp_attr->ah_attr.dmac);
1284+
1285+ status = ib_get_cached_gid(&rdev->ibdev, 1,
1286+ qp_attr->ah_attr.grh.sgid_index,
1287+ &sgid, &sgid_attr);
1288+ if (!status && sgid_attr.ndev) {
1289+ memcpy(qp->qplib_qp.smac, sgid_attr.ndev->dev_addr,
1290+ ETH_ALEN);
1291+ dev_put(sgid_attr.ndev);
1292+ nw_type = ib_gid_to_network_type(sgid_attr.gid_type,
1293+ &sgid);
1294+ switch (nw_type) {
1295+ case RDMA_NETWORK_IPV4:
1296+ qp->qplib_qp.nw_type =
1297+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
1298+ break;
1299+ case RDMA_NETWORK_IPV6:
1300+ qp->qplib_qp.nw_type =
1301+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
1302+ break;
1303+ default:
1304+ qp->qplib_qp.nw_type =
1305+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
1306+ break;
1307+ }
1308+ }
1309+ }
1310+
1311+ if (qp_attr_mask & IB_QP_PATH_MTU) {
1312+ qp->qplib_qp.modify_flags |=
1313+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1314+ qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
1315+ } else if (qp_attr->qp_state == IB_QPS_RTR) {
1316+ qp->qplib_qp.modify_flags |=
1317+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1318+ qp->qplib_qp.path_mtu =
1319+ __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
1320+ }
1321+
1322+ if (qp_attr_mask & IB_QP_TIMEOUT) {
1323+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
1324+ qp->qplib_qp.timeout = qp_attr->timeout;
1325+ }
1326+ if (qp_attr_mask & IB_QP_RETRY_CNT) {
1327+ qp->qplib_qp.modify_flags |=
1328+ CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
1329+ qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
1330+ }
1331+ if (qp_attr_mask & IB_QP_RNR_RETRY) {
1332+ qp->qplib_qp.modify_flags |=
1333+ CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
1334+ qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
1335+ }
1336+ if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
1337+ qp->qplib_qp.modify_flags |=
1338+ CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
1339+ qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
1340+ }
1341+ if (qp_attr_mask & IB_QP_RQ_PSN) {
1342+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
1343+ qp->qplib_qp.rq.psn = qp_attr->rq_psn;
1344+ }
1345+ if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1346+ qp->qplib_qp.modify_flags |=
1347+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
1348+ qp->qplib_qp.max_rd_atomic = qp_attr->max_rd_atomic;
1349+ }
1350+ if (qp_attr_mask & IB_QP_SQ_PSN) {
1351+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1352+ qp->qplib_qp.sq.psn = qp_attr->sq_psn;
1353+ }
1354+ if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1355+ qp->qplib_qp.modify_flags |=
1356+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
1357+ qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
1358+ }
1359+ if (qp_attr_mask & IB_QP_CAP) {
1360+ qp->qplib_qp.modify_flags |=
1361+ CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
1362+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
1363+ CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
1364+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
1365+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
1366+ if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
1367+ (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
1368+ (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
1369+ (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
1370+ (qp_attr->cap.max_inline_data >=
1371+ dev_attr->max_inline_data)) {
1372+ dev_err(rdev_to_dev(rdev),
1373+ "Create QP failed - max exceeded");
1374+ return -EINVAL;
1375+ }
1376+ entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
1377+ qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
1378+ dev_attr->max_qp_wqes + 1);
1379+ qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
1380+ if (qp->qplib_qp.rq.max_wqe) {
1381+ entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
1382+ qp->qplib_qp.rq.max_wqe =
1383+ min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1384+ qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
1385+ } else {
1386+ /* SRQ was used prior, just ignore the RQ caps */
1387+ }
1388+ }
1389+ if (qp_attr_mask & IB_QP_DEST_QPN) {
1390+ qp->qplib_qp.modify_flags |=
1391+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
1392+ qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
1393+ }
1394+ rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1395+ if (rc) {
1396+ dev_err(rdev_to_dev(rdev), "Failed to modify HW QP");
1397+ return rc;
1398+ }
1399+ if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp)
1400+ rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
1401+ return rc;
1402+}
1403+
1404+int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1405+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1406+{
1407+ struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1408+ struct bnxt_re_dev *rdev = qp->rdev;
1409+ struct bnxt_qplib_qp qplib_qp;
1410+ int rc;
1411+
1412+ memset(&qplib_qp, 0, sizeof(struct bnxt_qplib_qp));
1413+ qplib_qp.id = qp->qplib_qp.id;
1414+ qplib_qp.ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
1415+
1416+ rc = bnxt_qplib_query_qp(&rdev->qplib_res, &qplib_qp);
1417+ if (rc) {
1418+ dev_err(rdev_to_dev(rdev), "Failed to query HW QP");
1419+ return rc;
1420+ }
1421+ qp_attr->qp_state = __to_ib_qp_state(qplib_qp.state);
1422+ qp_attr->en_sqd_async_notify = qplib_qp.en_sqd_async_notify ? 1 : 0;
1423+ qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp.access);
1424+ qp_attr->pkey_index = qplib_qp.pkey_index;
1425+ qp_attr->qkey = qplib_qp.qkey;
1426+ memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp.ah.dgid.data,
1427+ sizeof(qplib_qp.ah.dgid.data));
1428+ qp_attr->ah_attr.grh.flow_label = qplib_qp.ah.flow_label;
1429+ qp_attr->ah_attr.grh.sgid_index = qplib_qp.ah.host_sgid_index;
1430+ qp_attr->ah_attr.grh.hop_limit = qplib_qp.ah.hop_limit;
1431+ qp_attr->ah_attr.grh.traffic_class = qplib_qp.ah.traffic_class;
1432+ qp_attr->ah_attr.sl = qplib_qp.ah.sl;
1433+ ether_addr_copy(qp_attr->ah_attr.dmac, qplib_qp.ah.dmac);
1434+ qp_attr->path_mtu = __to_ib_mtu(qplib_qp.path_mtu);
1435+ qp_attr->timeout = qplib_qp.timeout;
1436+ qp_attr->retry_cnt = qplib_qp.retry_cnt;
1437+ qp_attr->rnr_retry = qplib_qp.rnr_retry;
1438+ qp_attr->min_rnr_timer = qplib_qp.min_rnr_timer;
1439+ qp_attr->rq_psn = qplib_qp.rq.psn;
1440+ qp_attr->max_rd_atomic = qplib_qp.max_rd_atomic;
1441+ qp_attr->sq_psn = qplib_qp.sq.psn;
1442+ qp_attr->max_dest_rd_atomic = qplib_qp.max_dest_rd_atomic;
1443+ qp_init_attr->sq_sig_type = qplib_qp.sig_type ? IB_SIGNAL_ALL_WR :
1444+ IB_SIGNAL_REQ_WR;
1445+ qp_attr->dest_qp_num = qplib_qp.dest_qpn;
1446+
1447+ qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
1448+ qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
1449+ qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
1450+ qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
1451+ qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
1452+ qp_init_attr->cap = qp_attr->cap;
1453+
1454+ return 0;
1455+}
1456+
1457+/* Routine for sending QP1 packets for RoCE V1 an V2
1458+ */
1459+static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
1460+ struct ib_send_wr *wr,
1461+ struct bnxt_qplib_swqe *wqe,
1462+ int payload_size)
1463+{
1464+ struct ib_device *ibdev = &qp->rdev->ibdev;
1465+ struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
1466+ ib_ah);
1467+ struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
1468+ struct bnxt_qplib_sge sge;
1469+ union ib_gid sgid;
1470+ u8 nw_type;
1471+ u16 ether_type;
1472+ struct ib_gid_attr sgid_attr;
1473+ union ib_gid dgid;
1474+ bool is_eth = false;
1475+ bool is_vlan = false;
1476+ bool is_grh = false;
1477+ bool is_udp = false;
1478+ u8 ip_version = 0;
1479+ u16 vlan_id = 0xFFFF;
1480+ void *buf;
1481+ int i, rc = 0, size;
1482+
1483+ memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
1484+
1485+ rc = ib_get_cached_gid(ibdev, 1,
1486+ qplib_ah->host_sgid_index, &sgid,
1487+ &sgid_attr);
1488+ if (rc) {
1489+ dev_err(rdev_to_dev(qp->rdev),
1490+ "Failed to query gid at index %d",
1491+ qplib_ah->host_sgid_index);
1492+ return rc;
1493+ }
1494+ if (sgid_attr.ndev) {
1495+ if (is_vlan_dev(sgid_attr.ndev))
1496+ vlan_id = vlan_dev_vlan_id(sgid_attr.ndev);
1497+ dev_put(sgid_attr.ndev);
1498+ }
1499+ /* Get network header type for this GID */
1500+ nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
1501+ switch (nw_type) {
1502+ case RDMA_NETWORK_IPV4:
1503+ nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
1504+ break;
1505+ case RDMA_NETWORK_IPV6:
1506+ nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
1507+ break;
1508+ default:
1509+ nw_type = BNXT_RE_ROCE_V1_PACKET;
1510+ break;
1511+ }
1512+ memcpy(&dgid.raw, &qplib_ah->dgid, 16);
1513+ is_udp = sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
1514+ if (is_udp) {
1515+ if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) {
1516+ ip_version = 4;
1517+ ether_type = ETH_P_IP;
1518+ } else {
1519+ ip_version = 6;
1520+ ether_type = ETH_P_IPV6;
1521+ }
1522+ is_grh = false;
1523+ } else {
1524+ ether_type = ETH_P_IBOE;
1525+ is_grh = true;
1526+ }
1527+
1528+ is_eth = true;
1529+ is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false;
1530+
1531+ ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
1532+ ip_version, is_udp, 0, &qp->qp1_hdr);
1533+
1534+ /* ETH */
1535+ ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
1536+ ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
1537+
1538+ /* For vlan, check the sgid for vlan existence */
1539+
1540+ if (!is_vlan) {
1541+ qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
1542+ } else {
1543+ qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
1544+ qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
1545+ }
1546+
1547+ if (is_grh || (ip_version == 6)) {
1548+ memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid));
1549+ memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
1550+ sizeof(sgid));
1551+ qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit;
1552+ }
1553+
1554+ if (ip_version == 4) {
1555+ qp->qp1_hdr.ip4.tos = 0;
1556+ qp->qp1_hdr.ip4.id = 0;
1557+ qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
1558+ qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
1559+
1560+ memcpy(&qp->qp1_hdr.ip4.saddr, sgid.raw + 12, 4);
1561+ memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
1562+ qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
1563+ }
1564+
1565+ if (is_udp) {
1566+ qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
1567+ qp->qp1_hdr.udp.sport = htons(0x8CD1);
1568+ qp->qp1_hdr.udp.csum = 0;
1569+ }
1570+
1571+ /* BTH */
1572+ if (wr->opcode == IB_WR_SEND_WITH_IMM) {
1573+ qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
1574+ qp->qp1_hdr.immediate_present = 1;
1575+ } else {
1576+ qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
1577+ }
1578+ if (wr->send_flags & IB_SEND_SOLICITED)
1579+ qp->qp1_hdr.bth.solicited_event = 1;
1580+ /* pad_count */
1581+ qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
1582+
1583+ /* P_key for QP1 is for all members */
1584+ qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
1585+ qp->qp1_hdr.bth.destination_qpn = IB_QP1;
1586+ qp->qp1_hdr.bth.ack_req = 0;
1587+ qp->send_psn++;
1588+ qp->send_psn &= BTH_PSN_MASK;
1589+ qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
1590+ /* DETH */
1591+ /* Use the priviledged Q_Key for QP1 */
1592+ qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
1593+ qp->qp1_hdr.deth.source_qpn = IB_QP1;
1594+
1595+ /* Pack the QP1 to the transmit buffer */
1596+ buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
1597+ if (buf) {
1598+ size = ib_ud_header_pack(&qp->qp1_hdr, buf);
1599+ for (i = wqe->num_sge; i; i--) {
1600+ wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
1601+ wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
1602+ wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
1603+ }
1604+
1605+ /*
1606+ * Max Header buf size for IPV6 RoCE V2 is 86,
1607+ * which is same as the QP1 SQ header buffer.
1608+ * Header buf size for IPV4 RoCE V2 can be 66.
1609+ * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
1610+ * Subtract 20 bytes from QP1 SQ header buf size
1611+ */
1612+ if (is_udp && ip_version == 4)
1613+ sge.size -= 20;
1614+ /*
1615+ * Max Header buf size for RoCE V1 is 78.
1616+ * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
1617+ * Subtract 8 bytes from QP1 SQ header buf size
1618+ */
1619+ if (!is_udp)
1620+ sge.size -= 8;
1621+
1622+ /* Subtract 4 bytes for non vlan packets */
1623+ if (!is_vlan)
1624+ sge.size -= 4;
1625+
1626+ wqe->sg_list[0].addr = sge.addr;
1627+ wqe->sg_list[0].lkey = sge.lkey;
1628+ wqe->sg_list[0].size = sge.size;
1629+ wqe->num_sge++;
1630+
1631+ } else {
1632+ dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!");
1633+ rc = -ENOMEM;
1634+ }
1635+ return rc;
1636+}
1637+
1638+/* For the MAD layer, it only provides the recv SGE the size of
1639+ * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH,
1640+ * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire
1641+ * receive packet (334 bytes) with no VLAN and then copy the GRH
1642+ * and the MAD datagram out to the provided SGE.
1643+ */
1644+static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
1645+ struct ib_recv_wr *wr,
1646+ struct bnxt_qplib_swqe *wqe,
1647+ int payload_size)
1648+{
1649+ struct bnxt_qplib_sge ref, sge;
1650+ u32 rq_prod_index;
1651+ struct bnxt_re_sqp_entries *sqp_entry;
1652+
1653+ rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
1654+
1655+ if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
1656+ return -ENOMEM;
1657+
1658+ /* Create 1 SGE to receive the entire
1659+ * ethernet packet
1660+ */
1661+ /* Save the reference from ULP */
1662+ ref.addr = wqe->sg_list[0].addr;
1663+ ref.lkey = wqe->sg_list[0].lkey;
1664+ ref.size = wqe->sg_list[0].size;
1665+
1666+ sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index];
1667+
1668+ /* SGE 1 */
1669+ wqe->sg_list[0].addr = sge.addr;
1670+ wqe->sg_list[0].lkey = sge.lkey;
1671+ wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1672+ sge.size -= wqe->sg_list[0].size;
1673+
1674+ sqp_entry->sge.addr = ref.addr;
1675+ sqp_entry->sge.lkey = ref.lkey;
1676+ sqp_entry->sge.size = ref.size;
1677+ /* Store the wrid for reporting completion */
1678+ sqp_entry->wrid = wqe->wr_id;
1679+ /* change the wqe->wrid to table index */
1680+ wqe->wr_id = rq_prod_index;
1681+ return 0;
1682+}
1683+
1684+static int is_ud_qp(struct bnxt_re_qp *qp)
1685+{
1686+ return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD;
1687+}
1688+
1689+static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
1690+ struct ib_send_wr *wr,
1691+ struct bnxt_qplib_swqe *wqe)
1692+{
1693+ struct bnxt_re_ah *ah = NULL;
1694+
1695+ if (is_ud_qp(qp)) {
1696+ ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
1697+ wqe->send.q_key = ud_wr(wr)->remote_qkey;
1698+ wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
1699+ wqe->send.avid = ah->qplib_ah.id;
1700+ }
1701+ switch (wr->opcode) {
1702+ case IB_WR_SEND:
1703+ wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
1704+ break;
1705+ case IB_WR_SEND_WITH_IMM:
1706+ wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
1707+ wqe->send.imm_data = wr->ex.imm_data;
1708+ break;
1709+ case IB_WR_SEND_WITH_INV:
1710+ wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
1711+ wqe->send.inv_key = wr->ex.invalidate_rkey;
1712+ break;
1713+ default:
1714+ return -EINVAL;
1715+ }
1716+ if (wr->send_flags & IB_SEND_SIGNALED)
1717+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1718+ if (wr->send_flags & IB_SEND_FENCE)
1719+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1720+ if (wr->send_flags & IB_SEND_SOLICITED)
1721+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1722+ if (wr->send_flags & IB_SEND_INLINE)
1723+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
1724+
1725+ return 0;
1726+}
1727+
1728+static int bnxt_re_build_rdma_wqe(struct ib_send_wr *wr,
1729+ struct bnxt_qplib_swqe *wqe)
1730+{
1731+ switch (wr->opcode) {
1732+ case IB_WR_RDMA_WRITE:
1733+ wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
1734+ break;
1735+ case IB_WR_RDMA_WRITE_WITH_IMM:
1736+ wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
1737+ wqe->rdma.imm_data = wr->ex.imm_data;
1738+ break;
1739+ case IB_WR_RDMA_READ:
1740+ wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
1741+ wqe->rdma.inv_key = wr->ex.invalidate_rkey;
1742+ break;
1743+ default:
1744+ return -EINVAL;
1745+ }
1746+ wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
1747+ wqe->rdma.r_key = rdma_wr(wr)->rkey;
1748+ if (wr->send_flags & IB_SEND_SIGNALED)
1749+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1750+ if (wr->send_flags & IB_SEND_FENCE)
1751+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1752+ if (wr->send_flags & IB_SEND_SOLICITED)
1753+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1754+ if (wr->send_flags & IB_SEND_INLINE)
1755+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
1756+
1757+ return 0;
1758+}
1759+
1760+static int bnxt_re_build_atomic_wqe(struct ib_send_wr *wr,
1761+ struct bnxt_qplib_swqe *wqe)
1762+{
1763+ switch (wr->opcode) {
1764+ case IB_WR_ATOMIC_CMP_AND_SWP:
1765+ wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
1766+ wqe->atomic.swap_data = atomic_wr(wr)->swap;
1767+ break;
1768+ case IB_WR_ATOMIC_FETCH_AND_ADD:
1769+ wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
1770+ wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
1771+ break;
1772+ default:
1773+ return -EINVAL;
1774+ }
1775+ wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
1776+ wqe->atomic.r_key = atomic_wr(wr)->rkey;
1777+ if (wr->send_flags & IB_SEND_SIGNALED)
1778+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1779+ if (wr->send_flags & IB_SEND_FENCE)
1780+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1781+ if (wr->send_flags & IB_SEND_SOLICITED)
1782+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1783+ return 0;
1784+}
1785+
1786+static int bnxt_re_build_inv_wqe(struct ib_send_wr *wr,
1787+ struct bnxt_qplib_swqe *wqe)
1788+{
1789+ wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
1790+ wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
1791+
1792+ if (wr->send_flags & IB_SEND_SIGNALED)
1793+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1794+ if (wr->send_flags & IB_SEND_FENCE)
1795+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1796+ if (wr->send_flags & IB_SEND_SOLICITED)
1797+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1798+
1799+ return 0;
1800+}
1801+
1802+static int bnxt_re_build_reg_wqe(struct ib_reg_wr *wr,
1803+ struct bnxt_qplib_swqe *wqe)
1804+{
1805+ struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
1806+ struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
1807+ int access = wr->access;
1808+
1809+ wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
1810+ wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
1811+ wqe->frmr.page_list = mr->pages;
1812+ wqe->frmr.page_list_len = mr->npages;
1813+ wqe->frmr.levels = qplib_frpl->hwq.level + 1;
1814+ wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
1815+
1816+ if (wr->wr.send_flags & IB_SEND_FENCE)
1817+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1818+ if (wr->wr.send_flags & IB_SEND_SIGNALED)
1819+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1820+
1821+ if (access & IB_ACCESS_LOCAL_WRITE)
1822+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1823+ if (access & IB_ACCESS_REMOTE_READ)
1824+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
1825+ if (access & IB_ACCESS_REMOTE_WRITE)
1826+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
1827+ if (access & IB_ACCESS_REMOTE_ATOMIC)
1828+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
1829+ if (access & IB_ACCESS_MW_BIND)
1830+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
1831+
1832+ wqe->frmr.l_key = wr->key;
1833+ wqe->frmr.length = wr->mr->length;
1834+ wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1;
1835+ wqe->frmr.va = wr->mr->iova;
1836+ return 0;
1837+}
1838+
1839+static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
1840+ struct ib_send_wr *wr,
1841+ struct bnxt_qplib_swqe *wqe)
1842+{
1843+ /* Copy the inline data to the data field */
1844+ u8 *in_data;
1845+ u32 i, sge_len;
1846+ void *sge_addr;
1847+
1848+ in_data = wqe->inline_data;
1849+ for (i = 0; i < wr->num_sge; i++) {
1850+ sge_addr = (void *)(unsigned long)
1851+ wr->sg_list[i].addr;
1852+ sge_len = wr->sg_list[i].length;
1853+
1854+ if ((sge_len + wqe->inline_len) >
1855+ BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1856+ dev_err(rdev_to_dev(rdev),
1857+ "Inline data size requested > supported value");
1858+ return -EINVAL;
1859+ }
1860+ sge_len = wr->sg_list[i].length;
1861+
1862+ memcpy(in_data, sge_addr, sge_len);
1863+ in_data += wr->sg_list[i].length;
1864+ wqe->inline_len += wr->sg_list[i].length;
1865+ }
1866+ return wqe->inline_len;
1867+}
1868+
1869+static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
1870+ struct ib_send_wr *wr,
1871+ struct bnxt_qplib_swqe *wqe)
1872+{
1873+ int payload_sz = 0;
1874+
1875+ if (wr->send_flags & IB_SEND_INLINE)
1876+ payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
1877+ else
1878+ payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
1879+ wqe->num_sge);
1880+
1881+ return payload_sz;
1882+}
1883+
1884+static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
1885+ struct bnxt_re_qp *qp,
1886+ struct ib_send_wr *wr)
1887+{
1888+ struct bnxt_qplib_swqe wqe;
1889+ int rc = 0, payload_sz = 0;
1890+ unsigned long flags;
1891+
1892+ spin_lock_irqsave(&qp->sq_lock, flags);
1893+ memset(&wqe, 0, sizeof(wqe));
1894+ while (wr) {
1895+ /* House keeping */
1896+ memset(&wqe, 0, sizeof(wqe));
1897+
1898+ /* Common */
1899+ wqe.num_sge = wr->num_sge;
1900+ if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
1901+ dev_err(rdev_to_dev(rdev),
1902+ "Limit exceeded for Send SGEs");
1903+ rc = -EINVAL;
1904+ goto bad;
1905+ }
1906+
1907+ payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
1908+ if (payload_sz < 0) {
1909+ rc = -EINVAL;
1910+ goto bad;
1911+ }
1912+ wqe.wr_id = wr->wr_id;
1913+
1914+ wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
1915+
1916+ rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
1917+ if (!rc)
1918+ rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
1919+bad:
1920+ if (rc) {
1921+ dev_err(rdev_to_dev(rdev),
1922+ "Post send failed opcode = %#x rc = %d",
1923+ wr->opcode, rc);
1924+ break;
1925+ }
1926+ wr = wr->next;
1927+ }
1928+ bnxt_qplib_post_send_db(&qp->qplib_qp);
1929+ spin_unlock_irqrestore(&qp->sq_lock, flags);
1930+ return rc;
1931+}
1932+
1933+int bnxt_re_post_send(struct ib_qp *ib_qp, struct ib_send_wr *wr,
1934+ struct ib_send_wr **bad_wr)
1935+{
1936+ struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1937+ struct bnxt_qplib_swqe wqe;
1938+ int rc = 0, payload_sz = 0;
1939+ unsigned long flags;
1940+
1941+ spin_lock_irqsave(&qp->sq_lock, flags);
1942+ while (wr) {
1943+ /* House keeping */
1944+ memset(&wqe, 0, sizeof(wqe));
1945+
1946+ /* Common */
1947+ wqe.num_sge = wr->num_sge;
1948+ if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
1949+ dev_err(rdev_to_dev(qp->rdev),
1950+ "Limit exceeded for Send SGEs");
1951+ rc = -EINVAL;
1952+ goto bad;
1953+ }
1954+
1955+ payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
1956+ if (payload_sz < 0) {
1957+ rc = -EINVAL;
1958+ goto bad;
1959+ }
1960+ wqe.wr_id = wr->wr_id;
1961+
1962+ switch (wr->opcode) {
1963+ case IB_WR_SEND:
1964+ case IB_WR_SEND_WITH_IMM:
1965+ if (ib_qp->qp_type == IB_QPT_GSI) {
1966+ rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
1967+ payload_sz);
1968+ if (rc)
1969+ goto bad;
1970+ wqe.rawqp1.lflags |=
1971+ SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
1972+ }
1973+ switch (wr->send_flags) {
1974+ case IB_SEND_IP_CSUM:
1975+ wqe.rawqp1.lflags |=
1976+ SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
1977+ break;
1978+ default:
1979+ break;
1980+ }
1981+ /* Fall thru to build the wqe */
1982+ case IB_WR_SEND_WITH_INV:
1983+ rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
1984+ break;
1985+ case IB_WR_RDMA_WRITE:
1986+ case IB_WR_RDMA_WRITE_WITH_IMM:
1987+ case IB_WR_RDMA_READ:
1988+ rc = bnxt_re_build_rdma_wqe(wr, &wqe);
1989+ break;
1990+ case IB_WR_ATOMIC_CMP_AND_SWP:
1991+ case IB_WR_ATOMIC_FETCH_AND_ADD:
1992+ rc = bnxt_re_build_atomic_wqe(wr, &wqe);
1993+ break;
1994+ case IB_WR_RDMA_READ_WITH_INV:
1995+ dev_err(rdev_to_dev(qp->rdev),
1996+ "RDMA Read with Invalidate is not supported");
1997+ rc = -EINVAL;
1998+ goto bad;
1999+ case IB_WR_LOCAL_INV:
2000+ rc = bnxt_re_build_inv_wqe(wr, &wqe);
2001+ break;
2002+ case IB_WR_REG_MR:
2003+ rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2004+ break;
2005+ default:
2006+ /* Unsupported WRs */
2007+ dev_err(rdev_to_dev(qp->rdev),
2008+ "WR (%#x) is not supported", wr->opcode);
2009+ rc = -EINVAL;
2010+ goto bad;
2011+ }
2012+ if (!rc)
2013+ rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2014+bad:
2015+ if (rc) {
2016+ dev_err(rdev_to_dev(qp->rdev),
2017+ "post_send failed op:%#x qps = %#x rc = %d\n",
2018+ wr->opcode, qp->qplib_qp.state, rc);
2019+ *bad_wr = wr;
2020+ break;
2021+ }
2022+ wr = wr->next;
2023+ }
2024+ bnxt_qplib_post_send_db(&qp->qplib_qp);
2025+ spin_unlock_irqrestore(&qp->sq_lock, flags);
2026+
2027+ return rc;
2028+}
2029+
2030+static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2031+ struct bnxt_re_qp *qp,
2032+ struct ib_recv_wr *wr)
2033+{
2034+ struct bnxt_qplib_swqe wqe;
2035+ int rc = 0, payload_sz = 0;
2036+
2037+ memset(&wqe, 0, sizeof(wqe));
2038+ while (wr) {
2039+ /* House keeping */
2040+ memset(&wqe, 0, sizeof(wqe));
2041+
2042+ /* Common */
2043+ wqe.num_sge = wr->num_sge;
2044+ if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2045+ dev_err(rdev_to_dev(rdev),
2046+ "Limit exceeded for Receive SGEs");
2047+ rc = -EINVAL;
2048+ break;
2049+ }
2050+ payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2051+ wr->num_sge);
2052+ wqe.wr_id = wr->wr_id;
2053+ wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2054+
2055+ rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2056+ if (rc)
2057+ break;
2058+
2059+ wr = wr->next;
2060+ }
2061+ if (!rc)
2062+ bnxt_qplib_post_recv_db(&qp->qplib_qp);
2063+ return rc;
2064+}
2065+
2066+int bnxt_re_post_recv(struct ib_qp *ib_qp, struct ib_recv_wr *wr,
2067+ struct ib_recv_wr **bad_wr)
2068+{
2069+ struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2070+ struct bnxt_qplib_swqe wqe;
2071+ int rc = 0, payload_sz = 0;
2072+
2073+ while (wr) {
2074+ /* House keeping */
2075+ memset(&wqe, 0, sizeof(wqe));
2076+
2077+ /* Common */
2078+ wqe.num_sge = wr->num_sge;
2079+ if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2080+ dev_err(rdev_to_dev(qp->rdev),
2081+ "Limit exceeded for Receive SGEs");
2082+ rc = -EINVAL;
2083+ *bad_wr = wr;
2084+ break;
2085+ }
2086+
2087+ payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2088+ wr->num_sge);
2089+ wqe.wr_id = wr->wr_id;
2090+ wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2091+
2092+ if (ib_qp->qp_type == IB_QPT_GSI)
2093+ rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2094+ payload_sz);
2095+ if (!rc)
2096+ rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2097+ if (rc) {
2098+ *bad_wr = wr;
2099+ break;
2100+ }
2101+ wr = wr->next;
2102+ }
2103+ bnxt_qplib_post_recv_db(&qp->qplib_qp);
2104+ return rc;
2105+}
2106+
2107+/* Completion Queues */
2108+int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
2109+{
2110+ struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2111+ struct bnxt_re_dev *rdev = cq->rdev;
2112+ int rc;
2113+
2114+ rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2115+ if (rc) {
2116+ dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ");
2117+ return rc;
2118+ }
2119+ if (cq->umem && !IS_ERR(cq->umem))
2120+ ib_umem_release(cq->umem);
2121+
2122+ if (cq) {
2123+ kfree(cq->cql);
2124+ kfree(cq);
2125+ }
2126+ atomic_dec(&rdev->cq_count);
2127+ rdev->nq.budget--;
2128+ return 0;
2129+}
2130+
2131+struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
2132+ const struct ib_cq_init_attr *attr,
2133+ struct ib_ucontext *context,
2134+ struct ib_udata *udata)
2135+{
2136+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
2137+ struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
2138+ struct bnxt_re_cq *cq = NULL;
2139+ int rc, entries;
2140+ int cqe = attr->cqe;
2141+
2142+ /* Validate CQ fields */
2143+ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
2144+ dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded");
2145+ return ERR_PTR(-EINVAL);
2146+ }
2147+ cq = kzalloc(sizeof(*cq), GFP_KERNEL);
2148+ if (!cq)
2149+ return ERR_PTR(-ENOMEM);
2150+
2151+ cq->rdev = rdev;
2152+ cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2153+
2154+ entries = roundup_pow_of_two(cqe + 1);
2155+ if (entries > dev_attr->max_cq_wqes + 1)
2156+ entries = dev_attr->max_cq_wqes + 1;
2157+
2158+ if (context) {
2159+ struct bnxt_re_cq_req req;
2160+ struct bnxt_re_ucontext *uctx = container_of
2161+ (context,
2162+ struct bnxt_re_ucontext,
2163+ ib_uctx);
2164+ if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2165+ rc = -EFAULT;
2166+ goto fail;
2167+ }
2168+
2169+ cq->umem = ib_umem_get(context, req.cq_va,
2170+ entries * sizeof(struct cq_base),
2171+ IB_ACCESS_LOCAL_WRITE, 1);
2172+ if (IS_ERR(cq->umem)) {
2173+ rc = PTR_ERR(cq->umem);
2174+ goto fail;
2175+ }
2176+ cq->qplib_cq.sghead = cq->umem->sg_head.sgl;
2177+ cq->qplib_cq.nmap = cq->umem->nmap;
2178+ cq->qplib_cq.dpi = uctx->dpi;
2179+ } else {
2180+ cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2181+ cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2182+ GFP_KERNEL);
2183+ if (!cq->cql) {
2184+ rc = -ENOMEM;
2185+ goto fail;
2186+ }
2187+
2188+ cq->qplib_cq.dpi = &rdev->dpi_privileged;
2189+ cq->qplib_cq.sghead = NULL;
2190+ cq->qplib_cq.nmap = 0;
2191+ }
2192+ cq->qplib_cq.max_wqe = entries;
2193+ cq->qplib_cq.cnq_hw_ring_id = rdev->nq.ring_id;
2194+
2195+ rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2196+ if (rc) {
2197+ dev_err(rdev_to_dev(rdev), "Failed to create HW CQ");
2198+ goto fail;
2199+ }
2200+
2201+ cq->ib_cq.cqe = entries;
2202+ cq->cq_period = cq->qplib_cq.period;
2203+ rdev->nq.budget++;
2204+
2205+ atomic_inc(&rdev->cq_count);
2206+
2207+ if (context) {
2208+ struct bnxt_re_cq_resp resp;
2209+
2210+ resp.cqid = cq->qplib_cq.id;
2211+ resp.tail = cq->qplib_cq.hwq.cons;
2212+ resp.phase = cq->qplib_cq.period;
2213+ resp.rsvd = 0;
2214+ rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
2215+ if (rc) {
2216+ dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata");
2217+ bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2218+ goto c2fail;
2219+ }
2220+ }
2221+
2222+ return &cq->ib_cq;
2223+
2224+c2fail:
2225+ if (context)
2226+ ib_umem_release(cq->umem);
2227+fail:
2228+ kfree(cq->cql);
2229+ kfree(cq);
2230+ return ERR_PTR(rc);
2231+}
2232+
2233+static u8 __req_to_ib_wc_status(u8 qstatus)
2234+{
2235+ switch (qstatus) {
2236+ case CQ_REQ_STATUS_OK:
2237+ return IB_WC_SUCCESS;
2238+ case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
2239+ return IB_WC_BAD_RESP_ERR;
2240+ case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
2241+ return IB_WC_LOC_LEN_ERR;
2242+ case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
2243+ return IB_WC_LOC_QP_OP_ERR;
2244+ case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
2245+ return IB_WC_LOC_PROT_ERR;
2246+ case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
2247+ return IB_WC_GENERAL_ERR;
2248+ case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
2249+ return IB_WC_REM_INV_REQ_ERR;
2250+ case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
2251+ return IB_WC_REM_ACCESS_ERR;
2252+ case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
2253+ return IB_WC_REM_OP_ERR;
2254+ case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
2255+ return IB_WC_RNR_RETRY_EXC_ERR;
2256+ case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
2257+ return IB_WC_RETRY_EXC_ERR;
2258+ case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
2259+ return IB_WC_WR_FLUSH_ERR;
2260+ default:
2261+ return IB_WC_GENERAL_ERR;
2262+ }
2263+ return 0;
2264+}
2265+
2266+static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
2267+{
2268+ switch (qstatus) {
2269+ case CQ_RES_RAWETH_QP1_STATUS_OK:
2270+ return IB_WC_SUCCESS;
2271+ case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
2272+ return IB_WC_LOC_ACCESS_ERR;
2273+ case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
2274+ return IB_WC_LOC_LEN_ERR;
2275+ case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
2276+ return IB_WC_LOC_PROT_ERR;
2277+ case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
2278+ return IB_WC_LOC_QP_OP_ERR;
2279+ case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
2280+ return IB_WC_GENERAL_ERR;
2281+ case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
2282+ return IB_WC_WR_FLUSH_ERR;
2283+ case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
2284+ return IB_WC_WR_FLUSH_ERR;
2285+ default:
2286+ return IB_WC_GENERAL_ERR;
2287+ }
2288+}
2289+
2290+static u8 __rc_to_ib_wc_status(u8 qstatus)
2291+{
2292+ switch (qstatus) {
2293+ case CQ_RES_RC_STATUS_OK:
2294+ return IB_WC_SUCCESS;
2295+ case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
2296+ return IB_WC_LOC_ACCESS_ERR;
2297+ case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
2298+ return IB_WC_LOC_LEN_ERR;
2299+ case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
2300+ return IB_WC_LOC_PROT_ERR;
2301+ case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
2302+ return IB_WC_LOC_QP_OP_ERR;
2303+ case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
2304+ return IB_WC_GENERAL_ERR;
2305+ case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
2306+ return IB_WC_REM_INV_REQ_ERR;
2307+ case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
2308+ return IB_WC_WR_FLUSH_ERR;
2309+ case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
2310+ return IB_WC_WR_FLUSH_ERR;
2311+ default:
2312+ return IB_WC_GENERAL_ERR;
2313+ }
2314+}
2315+
2316+static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
2317+{
2318+ switch (cqe->type) {
2319+ case BNXT_QPLIB_SWQE_TYPE_SEND:
2320+ wc->opcode = IB_WC_SEND;
2321+ break;
2322+ case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2323+ wc->opcode = IB_WC_SEND;
2324+ wc->wc_flags |= IB_WC_WITH_IMM;
2325+ break;
2326+ case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
2327+ wc->opcode = IB_WC_SEND;
2328+ wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2329+ break;
2330+ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
2331+ wc->opcode = IB_WC_RDMA_WRITE;
2332+ break;
2333+ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
2334+ wc->opcode = IB_WC_RDMA_WRITE;
2335+ wc->wc_flags |= IB_WC_WITH_IMM;
2336+ break;
2337+ case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
2338+ wc->opcode = IB_WC_RDMA_READ;
2339+ break;
2340+ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
2341+ wc->opcode = IB_WC_COMP_SWAP;
2342+ break;
2343+ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
2344+ wc->opcode = IB_WC_FETCH_ADD;
2345+ break;
2346+ case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
2347+ wc->opcode = IB_WC_LOCAL_INV;
2348+ break;
2349+ case BNXT_QPLIB_SWQE_TYPE_REG_MR:
2350+ wc->opcode = IB_WC_REG_MR;
2351+ break;
2352+ default:
2353+ wc->opcode = IB_WC_SEND;
2354+ break;
2355+ }
2356+
2357+ wc->status = __req_to_ib_wc_status(cqe->status);
2358+}
2359+
2360+static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
2361+ u16 raweth_qp1_flags2)
2362+{
2363+ bool is_udp = false, is_ipv6 = false, is_ipv4 = false;
2364+
2365+ /* raweth_qp1_flags Bit 9-6 indicates itype */
2366+ if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
2367+ != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
2368+ return -1;
2369+
2370+ if (raweth_qp1_flags2 &
2371+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
2372+ raweth_qp1_flags2 &
2373+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
2374+ is_udp = true;
2375+ /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
2376+ (raweth_qp1_flags2 &
2377+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
2378+ (is_ipv6 = true) : (is_ipv4 = true);
2379+ return ((is_ipv6) ?
2380+ BNXT_RE_ROCEV2_IPV6_PACKET :
2381+ BNXT_RE_ROCEV2_IPV4_PACKET);
2382+ } else {
2383+ return BNXT_RE_ROCE_V1_PACKET;
2384+ }
2385+}
2386+
2387+static int bnxt_re_to_ib_nw_type(int nw_type)
2388+{
2389+ u8 nw_hdr_type = 0xFF;
2390+
2391+ switch (nw_type) {
2392+ case BNXT_RE_ROCE_V1_PACKET:
2393+ nw_hdr_type = RDMA_NETWORK_ROCE_V1;
2394+ break;
2395+ case BNXT_RE_ROCEV2_IPV4_PACKET:
2396+ nw_hdr_type = RDMA_NETWORK_IPV4;
2397+ break;
2398+ case BNXT_RE_ROCEV2_IPV6_PACKET:
2399+ nw_hdr_type = RDMA_NETWORK_IPV6;
2400+ break;
2401+ }
2402+ return nw_hdr_type;
2403+}
2404+
2405+static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
2406+ void *rq_hdr_buf)
2407+{
2408+ u8 *tmp_buf = NULL;
2409+ struct ethhdr *eth_hdr;
2410+ u16 eth_type;
2411+ bool rc = false;
2412+
2413+ tmp_buf = (u8 *)rq_hdr_buf;
2414+ /*
2415+ * If dest mac is not same as I/F mac, this could be a
2416+ * loopback address or multicast address, check whether
2417+ * it is a loopback packet
2418+ */
2419+ if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
2420+ tmp_buf += 4;
2421+ /* Check the ether type */
2422+ eth_hdr = (struct ethhdr *)tmp_buf;
2423+ eth_type = ntohs(eth_hdr->h_proto);
2424+ switch (eth_type) {
2425+ case ETH_P_IBOE:
2426+ rc = true;
2427+ break;
2428+ case ETH_P_IP:
2429+ case ETH_P_IPV6: {
2430+ u32 len;
2431+ struct udphdr *udp_hdr;
2432+
2433+ len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
2434+ sizeof(struct ipv6hdr));
2435+ tmp_buf += sizeof(struct ethhdr) + len;
2436+ udp_hdr = (struct udphdr *)tmp_buf;
2437+ if (ntohs(udp_hdr->dest) ==
2438+ ROCE_V2_UDP_DPORT)
2439+ rc = true;
2440+ break;
2441+ }
2442+ default:
2443+ break;
2444+ }
2445+ }
2446+
2447+ return rc;
2448+}
2449+
2450+static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp,
2451+ struct bnxt_qplib_cqe *cqe)
2452+{
2453+ struct bnxt_re_dev *rdev = qp1_qp->rdev;
2454+ struct bnxt_re_sqp_entries *sqp_entry = NULL;
2455+ struct bnxt_re_qp *qp = rdev->qp1_sqp;
2456+ struct ib_send_wr *swr;
2457+ struct ib_ud_wr udwr;
2458+ struct ib_recv_wr rwr;
2459+ int pkt_type = 0;
2460+ u32 tbl_idx;
2461+ void *rq_hdr_buf;
2462+ dma_addr_t rq_hdr_buf_map;
2463+ dma_addr_t shrq_hdr_buf_map;
2464+ u32 offset = 0;
2465+ u32 skip_bytes = 0;
2466+ struct ib_sge s_sge[2];
2467+ struct ib_sge r_sge[2];
2468+ int rc;
2469+
2470+ memset(&udwr, 0, sizeof(udwr));
2471+ memset(&rwr, 0, sizeof(rwr));
2472+ memset(&s_sge, 0, sizeof(s_sge));
2473+ memset(&r_sge, 0, sizeof(r_sge));
2474+
2475+ swr = &udwr.wr;
2476+ tbl_idx = cqe->wr_id;
2477+
2478+ rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf +
2479+ (tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size);
2480+ rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp,
2481+ tbl_idx);
2482+
2483+ /* Shadow QP header buffer */
2484+ shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp,
2485+ tbl_idx);
2486+ sqp_entry = &rdev->sqp_tbl[tbl_idx];
2487+
2488+ /* Store this cqe */
2489+ memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
2490+ sqp_entry->qp1_qp = qp1_qp;
2491+
2492+ /* Find packet type from the cqe */
2493+
2494+ pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
2495+ cqe->raweth_qp1_flags2);
2496+ if (pkt_type < 0) {
2497+ dev_err(rdev_to_dev(rdev), "Invalid packet\n");
2498+ return -EINVAL;
2499+ }
2500+
2501+ /* Adjust the offset for the user buffer and post in the rq */
2502+
2503+ if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
2504+ offset = 20;
2505+
2506+ /*
2507+ * QP1 loopback packet has 4 bytes of internal header before
2508+ * ether header. Skip these four bytes.
2509+ */
2510+ if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
2511+ skip_bytes = 4;
2512+
2513+ /* First send SGE . Skip the ether header*/
2514+ s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
2515+ + skip_bytes;
2516+ s_sge[0].lkey = 0xFFFFFFFF;
2517+ s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
2518+ BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
2519+
2520+ /* Second Send SGE */
2521+ s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
2522+ BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
2523+ if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
2524+ s_sge[1].addr += 8;
2525+ s_sge[1].lkey = 0xFFFFFFFF;
2526+ s_sge[1].length = 256;
2527+
2528+ /* First recv SGE */
2529+
2530+ r_sge[0].addr = shrq_hdr_buf_map;
2531+ r_sge[0].lkey = 0xFFFFFFFF;
2532+ r_sge[0].length = 40;
2533+
2534+ r_sge[1].addr = sqp_entry->sge.addr + offset;
2535+ r_sge[1].lkey = sqp_entry->sge.lkey;
2536+ r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
2537+
2538+ /* Create receive work request */
2539+ rwr.num_sge = 2;
2540+ rwr.sg_list = r_sge;
2541+ rwr.wr_id = tbl_idx;
2542+ rwr.next = NULL;
2543+
2544+ rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr);
2545+ if (rc) {
2546+ dev_err(rdev_to_dev(rdev),
2547+ "Failed to post Rx buffers to shadow QP");
2548+ return -ENOMEM;
2549+ }
2550+
2551+ swr->num_sge = 2;
2552+ swr->sg_list = s_sge;
2553+ swr->wr_id = tbl_idx;
2554+ swr->opcode = IB_WR_SEND;
2555+ swr->next = NULL;
2556+
2557+ udwr.ah = &rdev->sqp_ah->ib_ah;
2558+ udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id;
2559+ udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey;
2560+
2561+ /* post data received in the send queue */
2562+ rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr);
2563+
2564+ return 0;
2565+}
2566+
2567+static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
2568+ struct bnxt_qplib_cqe *cqe)
2569+{
2570+ wc->opcode = IB_WC_RECV;
2571+ wc->status = __rawqp1_to_ib_wc_status(cqe->status);
2572+ wc->wc_flags |= IB_WC_GRH;
2573+}
2574+
2575+static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
2576+ struct bnxt_qplib_cqe *cqe)
2577+{
2578+ wc->opcode = IB_WC_RECV;
2579+ wc->status = __rc_to_ib_wc_status(cqe->status);
2580+
2581+ if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
2582+ wc->wc_flags |= IB_WC_WITH_IMM;
2583+ if (cqe->flags & CQ_RES_RC_FLAGS_INV)
2584+ wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2585+ if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
2586+ (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
2587+ wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2588+}
2589+
2590+static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp,
2591+ struct ib_wc *wc,
2592+ struct bnxt_qplib_cqe *cqe)
2593+{
2594+ u32 tbl_idx;
2595+ struct bnxt_re_dev *rdev = qp->rdev;
2596+ struct bnxt_re_qp *qp1_qp = NULL;
2597+ struct bnxt_qplib_cqe *orig_cqe = NULL;
2598+ struct bnxt_re_sqp_entries *sqp_entry = NULL;
2599+ int nw_type;
2600+
2601+ tbl_idx = cqe->wr_id;
2602+
2603+ sqp_entry = &rdev->sqp_tbl[tbl_idx];
2604+ qp1_qp = sqp_entry->qp1_qp;
2605+ orig_cqe = &sqp_entry->cqe;
2606+
2607+ wc->wr_id = sqp_entry->wrid;
2608+ wc->byte_len = orig_cqe->length;
2609+ wc->qp = &qp1_qp->ib_qp;
2610+
2611+ wc->ex.imm_data = orig_cqe->immdata;
2612+ wc->src_qp = orig_cqe->src_qp;
2613+ memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
2614+ wc->port_num = 1;
2615+ wc->vendor_err = orig_cqe->status;
2616+
2617+ wc->opcode = IB_WC_RECV;
2618+ wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
2619+ wc->wc_flags |= IB_WC_GRH;
2620+
2621+ nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
2622+ orig_cqe->raweth_qp1_flags2);
2623+ if (nw_type >= 0) {
2624+ wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
2625+ wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
2626+ }
2627+}
2628+
2629+static void bnxt_re_process_res_ud_wc(struct ib_wc *wc,
2630+ struct bnxt_qplib_cqe *cqe)
2631+{
2632+ wc->opcode = IB_WC_RECV;
2633+ wc->status = __rc_to_ib_wc_status(cqe->status);
2634+
2635+ if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
2636+ wc->wc_flags |= IB_WC_WITH_IMM;
2637+ if (cqe->flags & CQ_RES_RC_FLAGS_INV)
2638+ wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2639+ if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
2640+ (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
2641+ wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2642+}
2643+
2644+int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
2645+{
2646+ struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2647+ struct bnxt_re_qp *qp;
2648+ struct bnxt_qplib_cqe *cqe;
2649+ int i, ncqe, budget;
2650+ u32 tbl_idx;
2651+ struct bnxt_re_sqp_entries *sqp_entry = NULL;
2652+ unsigned long flags;
2653+
2654+ spin_lock_irqsave(&cq->cq_lock, flags);
2655+ budget = min_t(u32, num_entries, cq->max_cql);
2656+ if (!cq->cql) {
2657+ dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use");
2658+ goto exit;
2659+ }
2660+ cqe = &cq->cql[0];
2661+ while (budget) {
2662+ ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget);
2663+ if (!ncqe)
2664+ break;
2665+
2666+ for (i = 0; i < ncqe; i++, cqe++) {
2667+ /* Transcribe each qplib_wqe back to ib_wc */
2668+ memset(wc, 0, sizeof(*wc));
2669+
2670+ wc->wr_id = cqe->wr_id;
2671+ wc->byte_len = cqe->length;
2672+ qp = container_of
2673+ ((struct bnxt_qplib_qp *)
2674+ (unsigned long)(cqe->qp_handle),
2675+ struct bnxt_re_qp, qplib_qp);
2676+ if (!qp) {
2677+ dev_err(rdev_to_dev(cq->rdev),
2678+ "POLL CQ : bad QP handle");
2679+ continue;
2680+ }
2681+ wc->qp = &qp->ib_qp;
2682+ wc->ex.imm_data = cqe->immdata;
2683+ wc->src_qp = cqe->src_qp;
2684+ memcpy(wc->smac, cqe->smac, ETH_ALEN);
2685+ wc->port_num = 1;
2686+ wc->vendor_err = cqe->status;
2687+
2688+ switch (cqe->opcode) {
2689+ case CQ_BASE_CQE_TYPE_REQ:
2690+ if (qp->qplib_qp.id ==
2691+ qp->rdev->qp1_sqp->qplib_qp.id) {
2692+ /* Handle this completion with
2693+ * the stored completion
2694+ */
2695+ memset(wc, 0, sizeof(*wc));
2696+ continue;
2697+ }
2698+ bnxt_re_process_req_wc(wc, cqe);
2699+ break;
2700+ case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2701+ if (!cqe->status) {
2702+ int rc = 0;
2703+
2704+ rc = bnxt_re_process_raw_qp_pkt_rx
2705+ (qp, cqe);
2706+ if (!rc) {
2707+ memset(wc, 0, sizeof(*wc));
2708+ continue;
2709+ }
2710+ cqe->status = -1;
2711+ }
2712+ /* Errors need not be looped back.
2713+ * But change the wr_id to the one
2714+ * stored in the table
2715+ */
2716+ tbl_idx = cqe->wr_id;
2717+ sqp_entry = &cq->rdev->sqp_tbl[tbl_idx];
2718+ wc->wr_id = sqp_entry->wrid;
2719+ bnxt_re_process_res_rawqp1_wc(wc, cqe);
2720+ break;
2721+ case CQ_BASE_CQE_TYPE_RES_RC:
2722+ bnxt_re_process_res_rc_wc(wc, cqe);
2723+ break;
2724+ case CQ_BASE_CQE_TYPE_RES_UD:
2725+ if (qp->qplib_qp.id ==
2726+ qp->rdev->qp1_sqp->qplib_qp.id) {
2727+ /* Handle this completion with
2728+ * the stored completion
2729+ */
2730+ if (cqe->status) {
2731+ continue;
2732+ } else {
2733+ bnxt_re_process_res_shadow_qp_wc
2734+ (qp, wc, cqe);
2735+ break;
2736+ }
2737+ }
2738+ bnxt_re_process_res_ud_wc(wc, cqe);
2739+ break;
2740+ default:
2741+ dev_err(rdev_to_dev(cq->rdev),
2742+ "POLL CQ : type 0x%x not handled",
2743+ cqe->opcode);
2744+ continue;
2745+ }
2746+ wc++;
2747+ budget--;
2748+ }
2749+ }
2750+exit:
2751+ spin_unlock_irqrestore(&cq->cq_lock, flags);
2752+ return num_entries - budget;
2753+}
2754+
2755+int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
2756+ enum ib_cq_notify_flags ib_cqn_flags)
2757+{
2758+ struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2759+ int type = 0;
2760+
2761+ /* Trigger on the very next completion */
2762+ if (ib_cqn_flags & IB_CQ_NEXT_COMP)
2763+ type = DBR_DBR_TYPE_CQ_ARMALL;
2764+ /* Trigger on the next solicited completion */
2765+ else if (ib_cqn_flags & IB_CQ_SOLICITED)
2766+ type = DBR_DBR_TYPE_CQ_ARMSE;
2767+
2768+ bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
2769+
2770+ return 0;
2771+}
2772+
2773+/* Memory Regions */
2774+struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
2775+{
2776+ struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2777+ struct bnxt_re_dev *rdev = pd->rdev;
2778+ struct bnxt_re_mr *mr;
2779+ u64 pbl = 0;
2780+ int rc;
2781+
2782+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2783+ if (!mr)
2784+ return ERR_PTR(-ENOMEM);
2785+
2786+ mr->rdev = rdev;
2787+ mr->qplib_mr.pd = &pd->qplib_pd;
2788+ mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
2789+ mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2790+
2791+ /* Allocate and register 0 as the address */
2792+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
2793+ if (rc)
2794+ goto fail;
2795+
2796+ mr->qplib_mr.hwq.level = PBL_LVL_MAX;
2797+ mr->qplib_mr.total_size = -1; /* Infinte length */
2798+ rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false);
2799+ if (rc)
2800+ goto fail_mr;
2801+
2802+ mr->ib_mr.lkey = mr->qplib_mr.lkey;
2803+ if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
2804+ IB_ACCESS_REMOTE_ATOMIC))
2805+ mr->ib_mr.rkey = mr->ib_mr.lkey;
2806+ atomic_inc(&rdev->mr_count);
2807+
2808+ return &mr->ib_mr;
2809+
2810+fail_mr:
2811+ bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2812+fail:
2813+ kfree(mr);
2814+ return ERR_PTR(rc);
2815+}
2816+
2817+int bnxt_re_dereg_mr(struct ib_mr *ib_mr)
2818+{
2819+ struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2820+ struct bnxt_re_dev *rdev = mr->rdev;
2821+ int rc = 0;
2822+
2823+ if (mr->npages && mr->pages) {
2824+ rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
2825+ &mr->qplib_frpl);
2826+ kfree(mr->pages);
2827+ mr->npages = 0;
2828+ mr->pages = NULL;
2829+ }
2830+ rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2831+
2832+ if (!IS_ERR(mr->ib_umem) && mr->ib_umem)
2833+ ib_umem_release(mr->ib_umem);
2834+
2835+ kfree(mr);
2836+ atomic_dec(&rdev->mr_count);
2837+ return rc;
2838+}
2839+
2840+static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
2841+{
2842+ struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2843+
2844+ if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
2845+ return -ENOMEM;
2846+
2847+ mr->pages[mr->npages++] = addr;
2848+ return 0;
2849+}
2850+
2851+int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
2852+ unsigned int *sg_offset)
2853+{
2854+ struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2855+
2856+ mr->npages = 0;
2857+ return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
2858+}
2859+
2860+struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
2861+ u32 max_num_sg)
2862+{
2863+ struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2864+ struct bnxt_re_dev *rdev = pd->rdev;
2865+ struct bnxt_re_mr *mr = NULL;
2866+ int rc;
2867+
2868+ if (type != IB_MR_TYPE_MEM_REG) {
2869+ dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type);
2870+ return ERR_PTR(-EINVAL);
2871+ }
2872+ if (max_num_sg > MAX_PBL_LVL_1_PGS)
2873+ return ERR_PTR(-EINVAL);
2874+
2875+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2876+ if (!mr)
2877+ return ERR_PTR(-ENOMEM);
2878+
2879+ mr->rdev = rdev;
2880+ mr->qplib_mr.pd = &pd->qplib_pd;
2881+ mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
2882+ mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2883+
2884+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
2885+ if (rc)
2886+ goto fail;
2887+
2888+ mr->ib_mr.lkey = mr->qplib_mr.lkey;
2889+ mr->ib_mr.rkey = mr->ib_mr.lkey;
2890+
2891+ mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
2892+ if (!mr->pages) {
2893+ rc = -ENOMEM;
2894+ goto fail;
2895+ }
2896+ rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
2897+ &mr->qplib_frpl, max_num_sg);
2898+ if (rc) {
2899+ dev_err(rdev_to_dev(rdev),
2900+ "Failed to allocate HW FR page list");
2901+ goto fail_mr;
2902+ }
2903+
2904+ atomic_inc(&rdev->mr_count);
2905+ return &mr->ib_mr;
2906+
2907+fail_mr:
2908+ bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2909+fail:
2910+ kfree(mr->pages);
2911+ kfree(mr);
2912+ return ERR_PTR(rc);
2913+}
2914+
2915+/* Fast Memory Regions */
2916+struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *ib_pd, int mr_access_flags,
2917+ struct ib_fmr_attr *fmr_attr)
2918+{
2919+ struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2920+ struct bnxt_re_dev *rdev = pd->rdev;
2921+ struct bnxt_re_fmr *fmr;
2922+ int rc;
2923+
2924+ if (fmr_attr->max_pages > MAX_PBL_LVL_2_PGS ||
2925+ fmr_attr->max_maps > rdev->dev_attr.max_map_per_fmr) {
2926+ dev_err(rdev_to_dev(rdev), "Allocate FMR exceeded Max limit");
2927+ return ERR_PTR(-ENOMEM);
2928+ }
2929+ fmr = kzalloc(sizeof(*fmr), GFP_KERNEL);
2930+ if (!fmr)
2931+ return ERR_PTR(-ENOMEM);
2932+
2933+ fmr->rdev = rdev;
2934+ fmr->qplib_fmr.pd = &pd->qplib_pd;
2935+ fmr->qplib_fmr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2936+
2937+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &fmr->qplib_fmr);
2938+ if (rc)
2939+ goto fail;
2940+
2941+ fmr->qplib_fmr.flags = __from_ib_access_flags(mr_access_flags);
2942+ fmr->ib_fmr.lkey = fmr->qplib_fmr.lkey;
2943+ fmr->ib_fmr.rkey = fmr->ib_fmr.lkey;
2944+
2945+ atomic_inc(&rdev->mr_count);
2946+ return &fmr->ib_fmr;
2947+fail:
2948+ kfree(fmr);
2949+ return ERR_PTR(rc);
2950+}
2951+
2952+int bnxt_re_map_phys_fmr(struct ib_fmr *ib_fmr, u64 *page_list, int list_len,
2953+ u64 iova)
2954+{
2955+ struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr,
2956+ ib_fmr);
2957+ struct bnxt_re_dev *rdev = fmr->rdev;
2958+ int rc;
2959+
2960+ fmr->qplib_fmr.va = iova;
2961+ fmr->qplib_fmr.total_size = list_len * PAGE_SIZE;
2962+
2963+ rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &fmr->qplib_fmr, page_list,
2964+ list_len, true);
2965+ if (rc)
2966+ dev_err(rdev_to_dev(rdev), "Failed to map FMR for lkey = 0x%x!",
2967+ fmr->ib_fmr.lkey);
2968+ return rc;
2969+}
2970+
2971+int bnxt_re_unmap_fmr(struct list_head *fmr_list)
2972+{
2973+ struct bnxt_re_dev *rdev;
2974+ struct bnxt_re_fmr *fmr;
2975+ struct ib_fmr *ib_fmr;
2976+ int rc = 0;
2977+
2978+ /* Validate each FMRs inside the fmr_list */
2979+ list_for_each_entry(ib_fmr, fmr_list, list) {
2980+ fmr = container_of(ib_fmr, struct bnxt_re_fmr, ib_fmr);
2981+ rdev = fmr->rdev;
2982+
2983+ if (rdev) {
2984+ rc = bnxt_qplib_dereg_mrw(&rdev->qplib_res,
2985+ &fmr->qplib_fmr, true);
2986+ if (rc)
2987+ break;
2988+ }
2989+ }
2990+ return rc;
2991+}
2992+
2993+int bnxt_re_dealloc_fmr(struct ib_fmr *ib_fmr)
2994+{
2995+ struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr,
2996+ ib_fmr);
2997+ struct bnxt_re_dev *rdev = fmr->rdev;
2998+ int rc;
2999+
3000+ rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &fmr->qplib_fmr);
3001+ if (rc)
3002+ dev_err(rdev_to_dev(rdev), "Failed to free FMR");
3003+
3004+ kfree(fmr);
3005+ atomic_dec(&rdev->mr_count);
3006+ return rc;
3007+}
3008+
3009+/* uverbs */
3010+struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
3011+ u64 virt_addr, int mr_access_flags,
3012+ struct ib_udata *udata)
3013+{
3014+ struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3015+ struct bnxt_re_dev *rdev = pd->rdev;
3016+ struct bnxt_re_mr *mr;
3017+ struct ib_umem *umem;
3018+ u64 *pbl_tbl, *pbl_tbl_orig;
3019+ int i, umem_pgs, pages, page_shift, rc;
3020+ struct scatterlist *sg;
3021+ int entry;
3022+
3023+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3024+ if (!mr)
3025+ return ERR_PTR(-ENOMEM);
3026+
3027+ mr->rdev = rdev;
3028+ mr->qplib_mr.pd = &pd->qplib_pd;
3029+ mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3030+ mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
3031+
3032+ umem = ib_umem_get(ib_pd->uobject->context, start, length,
3033+ mr_access_flags, 0);
3034+ if (IS_ERR(umem)) {
3035+ dev_err(rdev_to_dev(rdev), "Failed to get umem");
3036+ rc = -EFAULT;
3037+ goto free_mr;
3038+ }
3039+ mr->ib_umem = umem;
3040+
3041+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3042+ if (rc) {
3043+ dev_err(rdev_to_dev(rdev), "Failed to allocate MR");
3044+ goto release_umem;
3045+ }
3046+ /* The fixed portion of the rkey is the same as the lkey */
3047+ mr->ib_mr.rkey = mr->qplib_mr.rkey;
3048+
3049+ mr->qplib_mr.va = virt_addr;
3050+ umem_pgs = ib_umem_page_count(umem);
3051+ if (!umem_pgs) {
3052+ dev_err(rdev_to_dev(rdev), "umem is invalid!");
3053+ rc = -EINVAL;
3054+ goto free_mrw;
3055+ }
3056+ mr->qplib_mr.total_size = length;
3057+
3058+ pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL);
3059+ if (!pbl_tbl) {
3060+ rc = -EINVAL;
3061+ goto free_mrw;
3062+ }
3063+ pbl_tbl_orig = pbl_tbl;
3064+
3065+ page_shift = ilog2(umem->page_size);
3066+ if (umem->hugetlb) {
3067+ dev_err(rdev_to_dev(rdev), "umem hugetlb not supported!");
3068+ rc = -EFAULT;
3069+ goto fail;
3070+ }
3071+ if (umem->page_size != PAGE_SIZE) {
3072+ dev_err(rdev_to_dev(rdev), "umem page size unsupported!");
3073+ rc = -EFAULT;
3074+ goto fail;
3075+ }
3076+ /* Map umem buf ptrs to the PBL */
3077+ for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
3078+ pages = sg_dma_len(sg) >> page_shift;
3079+ for (i = 0; i < pages; i++, pbl_tbl++)
3080+ *pbl_tbl = sg_dma_address(sg) + (i << page_shift);
3081+ }
3082+ rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl_orig,
3083+ umem_pgs, false);
3084+ if (rc) {
3085+ dev_err(rdev_to_dev(rdev), "Failed to register user MR");
3086+ goto fail;
3087+ }
3088+
3089+ kfree(pbl_tbl_orig);
3090+
3091+ mr->ib_mr.lkey = mr->qplib_mr.lkey;
3092+ mr->ib_mr.rkey = mr->qplib_mr.lkey;
3093+ atomic_inc(&rdev->mr_count);
3094+
3095+ return &mr->ib_mr;
3096+fail:
3097+ kfree(pbl_tbl_orig);
3098+free_mrw:
3099+ bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3100+release_umem:
3101+ ib_umem_release(umem);
3102+free_mr:
3103+ kfree(mr);
3104+ return ERR_PTR(rc);
3105+}
3106+
3107+struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev,
3108+ struct ib_udata *udata)
3109+{
3110+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
3111+ struct bnxt_re_uctx_resp resp;
3112+ struct bnxt_re_ucontext *uctx;
3113+ struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
3114+ int rc;
3115+
3116+ dev_dbg(rdev_to_dev(rdev), "ABI version requested %d",
3117+ ibdev->uverbs_abi_ver);
3118+
3119+ if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
3120+ dev_dbg(rdev_to_dev(rdev), " is different from the device %d ",
3121+ BNXT_RE_ABI_VERSION);
3122+ return ERR_PTR(-EPERM);
3123+ }
3124+
3125+ uctx = kzalloc(sizeof(*uctx), GFP_KERNEL);
3126+ if (!uctx)
3127+ return ERR_PTR(-ENOMEM);
3128+
3129+ uctx->rdev = rdev;
3130+
3131+ uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
3132+ if (!uctx->shpg) {
3133+ rc = -ENOMEM;
3134+ goto fail;
3135+ }
3136+ spin_lock_init(&uctx->sh_lock);
3137+
3138+ resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/
3139+ resp.max_qp = rdev->qplib_ctx.qpc_count;
3140+ resp.pg_size = PAGE_SIZE;
3141+ resp.cqe_sz = sizeof(struct cq_base);
3142+ resp.max_cqd = dev_attr->max_cq_wqes;
3143+ resp.rsvd = 0;
3144+
3145+ rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
3146+ if (rc) {
3147+ dev_err(rdev_to_dev(rdev), "Failed to copy user context");
3148+ rc = -EFAULT;
3149+ goto cfail;
3150+ }
3151+
3152+ return &uctx->ib_uctx;
3153+cfail:
3154+ free_page((unsigned long)uctx->shpg);
3155+ uctx->shpg = NULL;
3156+fail:
3157+ kfree(uctx);
3158+ return ERR_PTR(rc);
3159+}
3160+
3161+int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
3162+{
3163+ struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3164+ struct bnxt_re_ucontext,
3165+ ib_uctx);
3166+ if (uctx->shpg)
3167+ free_page((unsigned long)uctx->shpg);
3168+ kfree(uctx);
3169+ return 0;
3170+}
3171+
3172+/* Helper function to mmap the virtual memory from user app */
3173+int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
3174+{
3175+ struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3176+ struct bnxt_re_ucontext,
3177+ ib_uctx);
3178+ struct bnxt_re_dev *rdev = uctx->rdev;
3179+ u64 pfn;
3180+
3181+ if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3182+ return -EINVAL;
3183+
3184+ if (vma->vm_pgoff) {
3185+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3186+ if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
3187+ PAGE_SIZE, vma->vm_page_prot)) {
3188+ dev_err(rdev_to_dev(rdev), "Failed to map DPI");
3189+ return -EAGAIN;
3190+ }
3191+ } else {
3192+ pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT;
3193+ if (remap_pfn_range(vma, vma->vm_start,
3194+ pfn, PAGE_SIZE, vma->vm_page_prot)) {
3195+ dev_err(rdev_to_dev(rdev),
3196+ "Failed to map shared page");
3197+ return -EAGAIN;
3198+ }
3199+ }
3200+
3201+ return 0;
3202+}
drivers/infiniband/hw/bnxt_re/ib_verbs.h+197−0
@@ -0,0 +1,197 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: IB Verbs interpreter (header)
37+ */
38+
39+#ifndef __BNXT_RE_IB_VERBS_H__
40+#define __BNXT_RE_IB_VERBS_H__
41+
42+struct bnxt_re_gid_ctx {
43+ u32 idx;
44+ u32 refcnt;
45+};
46+
47+struct bnxt_re_pd {
48+ struct bnxt_re_dev *rdev;
49+ struct ib_pd ib_pd;
50+ struct bnxt_qplib_pd qplib_pd;
51+ struct bnxt_qplib_dpi dpi;
52+};
53+
54+struct bnxt_re_ah {
55+ struct bnxt_re_dev *rdev;
56+ struct ib_ah ib_ah;
57+ struct bnxt_qplib_ah qplib_ah;
58+};
59+
60+struct bnxt_re_qp {
61+ struct list_head list;
62+ struct bnxt_re_dev *rdev;
63+ struct ib_qp ib_qp;
64+ spinlock_t sq_lock; /* protect sq */
65+ struct bnxt_qplib_qp qplib_qp;
66+ struct ib_umem *sumem;
67+ struct ib_umem *rumem;
68+ /* QP1 */
69+ u32 send_psn;
70+ struct ib_ud_header qp1_hdr;
71+};
72+
73+struct bnxt_re_cq {
74+ struct bnxt_re_dev *rdev;
75+ spinlock_t cq_lock; /* protect cq */
76+ u16 cq_count;
77+ u16 cq_period;
78+ struct ib_cq ib_cq;
79+ struct bnxt_qplib_cq qplib_cq;
80+ struct bnxt_qplib_cqe *cql;
81+#define MAX_CQL_PER_POLL 1024
82+ u32 max_cql;
83+ struct ib_umem *umem;
84+};
85+
86+struct bnxt_re_mr {
87+ struct bnxt_re_dev *rdev;
88+ struct ib_mr ib_mr;
89+ struct ib_umem *ib_umem;
90+ struct bnxt_qplib_mrw qplib_mr;
91+ u32 npages;
92+ u64 *pages;
93+ struct bnxt_qplib_frpl qplib_frpl;
94+};
95+
96+struct bnxt_re_frpl {
97+ struct bnxt_re_dev *rdev;
98+ struct bnxt_qplib_frpl qplib_frpl;
99+ u64 *page_list;
100+};
101+
102+struct bnxt_re_fmr {
103+ struct bnxt_re_dev *rdev;
104+ struct ib_fmr ib_fmr;
105+ struct bnxt_qplib_mrw qplib_fmr;
106+};
107+
108+struct bnxt_re_mw {
109+ struct bnxt_re_dev *rdev;
110+ struct ib_mw ib_mw;
111+ struct bnxt_qplib_mrw qplib_mw;
112+};
113+
114+struct bnxt_re_ucontext {
115+ struct bnxt_re_dev *rdev;
116+ struct ib_ucontext ib_uctx;
117+ struct bnxt_qplib_dpi *dpi;
118+ void *shpg;
119+ spinlock_t sh_lock; /* protect shpg */
120+};
121+
122+struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num);
123+
124+int bnxt_re_query_device(struct ib_device *ibdev,
125+ struct ib_device_attr *ib_attr,
126+ struct ib_udata *udata);
127+int bnxt_re_modify_device(struct ib_device *ibdev,
128+ int device_modify_mask,
129+ struct ib_device_modify *device_modify);
130+int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
131+ struct ib_port_attr *port_attr);
132+int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
133+ int port_modify_mask,
134+ struct ib_port_modify *port_modify);
135+int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
136+ struct ib_port_immutable *immutable);
137+int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
138+ u16 index, u16 *pkey);
139+int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
140+ unsigned int index, void **context);
141+int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
142+ unsigned int index, const union ib_gid *gid,
143+ const struct ib_gid_attr *attr, void **context);
144+int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
145+ int index, union ib_gid *gid);
146+enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
147+ u8 port_num);
148+struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev,
149+ struct ib_ucontext *context,
150+ struct ib_udata *udata);
151+int bnxt_re_dealloc_pd(struct ib_pd *pd);
152+struct ib_ah *bnxt_re_create_ah(struct ib_pd *pd,
153+ struct ib_ah_attr *ah_attr,
154+ struct ib_udata *udata);
155+int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
156+int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
157+int bnxt_re_destroy_ah(struct ib_ah *ah);
158+struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd,
159+ struct ib_qp_init_attr *qp_init_attr,
160+ struct ib_udata *udata);
161+int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
162+ int qp_attr_mask, struct ib_udata *udata);
163+int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
164+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
165+int bnxt_re_destroy_qp(struct ib_qp *qp);
166+int bnxt_re_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr,
167+ struct ib_send_wr **bad_send_wr);
168+int bnxt_re_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
169+ struct ib_recv_wr **bad_recv_wr);
170+struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
171+ const struct ib_cq_init_attr *attr,
172+ struct ib_ucontext *context,
173+ struct ib_udata *udata);
174+int bnxt_re_destroy_cq(struct ib_cq *cq);
175+int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
176+int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
177+struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
178+
179+int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
180+ unsigned int *sg_offset);
181+struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type,
182+ u32 max_num_sg);
183+int bnxt_re_dereg_mr(struct ib_mr *mr);
184+struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
185+ struct ib_fmr_attr *fmr_attr);
186+int bnxt_re_map_phys_fmr(struct ib_fmr *fmr, u64 *page_list, int list_len,
187+ u64 iova);
188+int bnxt_re_unmap_fmr(struct list_head *fmr_list);
189+int bnxt_re_dealloc_fmr(struct ib_fmr *fmr);
190+struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
191+ u64 virt_addr, int mr_access_flags,
192+ struct ib_udata *udata);
193+struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev,
194+ struct ib_udata *udata);
195+int bnxt_re_dealloc_ucontext(struct ib_ucontext *context);
196+int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
197+#endif /* __BNXT_RE_IB_VERBS_H__ */
drivers/infiniband/hw/bnxt_re/main.c+1315−0
@@ -0,0 +1,1315 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: Main component of the bnxt_re driver
37+ */
38+
39+#include <linux/module.h>
40+#include <linux/netdevice.h>
41+#include <linux/ethtool.h>
42+#include <linux/mutex.h>
43+#include <linux/list.h>
44+#include <linux/rculist.h>
45+#include <linux/spinlock.h>
46+#include <linux/pci.h>
47+#include <net/dcbnl.h>
48+#include <net/ipv6.h>
49+#include <net/addrconf.h>
50+#include <linux/if_ether.h>
51+
52+#include <rdma/ib_verbs.h>
53+#include <rdma/ib_user_verbs.h>
54+#include <rdma/ib_umem.h>
55+#include <rdma/ib_addr.h>
56+
57+#include "bnxt_ulp.h"
58+#include "roce_hsi.h"
59+#include "qplib_res.h"
60+#include "qplib_sp.h"
61+#include "qplib_fp.h"
62+#include "qplib_rcfw.h"
63+#include "bnxt_re.h"
64+#include "ib_verbs.h"
65+#include <rdma/bnxt_re-abi.h>
66+#include "bnxt.h"
67+static char version[] =
68+ BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n";
69+
70+MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>");
71+MODULE_DESCRIPTION(BNXT_RE_DESC " Driver");
72+MODULE_LICENSE("Dual BSD/GPL");
73+MODULE_VERSION(ROCE_DRV_MODULE_VERSION);
74+
75+/* globals */
76+static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);
77+/* Mutex to protect the list of bnxt_re devices added */
78+static DEFINE_MUTEX(bnxt_re_dev_lock);
79+static struct workqueue_struct *bnxt_re_wq;
80+
81+/* for handling bnxt_en callbacks later */
82+static void bnxt_re_stop(void *p)
83+{
84+}
85+
86+static void bnxt_re_start(void *p)
87+{
88+}
89+
90+static void bnxt_re_sriov_config(void *p, int num_vfs)
91+{
92+}
93+
94+static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
95+ .ulp_async_notifier = NULL,
96+ .ulp_stop = bnxt_re_stop,
97+ .ulp_start = bnxt_re_start,
98+ .ulp_sriov_config = bnxt_re_sriov_config
99+};
100+
101+/* RoCE -> Net driver */
102+
103+/* Driver registration routines used to let the networking driver (bnxt_en)
104+ * to know that the RoCE driver is now installed
105+ */
106+static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev, bool lock_wait)
107+{
108+ struct bnxt_en_dev *en_dev;
109+ int rc;
110+
111+ if (!rdev)
112+ return -EINVAL;
113+
114+ en_dev = rdev->en_dev;
115+ /* Acquire rtnl lock if it is not invokded from netdev event */
116+ if (lock_wait)
117+ rtnl_lock();
118+
119+ rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev,
120+ BNXT_ROCE_ULP);
121+ if (lock_wait)
122+ rtnl_unlock();
123+ return rc;
124+}
125+
126+static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
127+{
128+ struct bnxt_en_dev *en_dev;
129+ int rc = 0;
130+
131+ if (!rdev)
132+ return -EINVAL;
133+
134+ en_dev = rdev->en_dev;
135+
136+ rtnl_lock();
137+ rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP,
138+ &bnxt_re_ulp_ops, rdev);
139+ rtnl_unlock();
140+ return rc;
141+}
142+
143+static int bnxt_re_free_msix(struct bnxt_re_dev *rdev, bool lock_wait)
144+{
145+ struct bnxt_en_dev *en_dev;
146+ int rc;
147+
148+ if (!rdev)
149+ return -EINVAL;
150+
151+ en_dev = rdev->en_dev;
152+
153+ if (lock_wait)
154+ rtnl_lock();
155+
156+ rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP);
157+
158+ if (lock_wait)
159+ rtnl_unlock();
160+ return rc;
161+}
162+
163+static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
164+{
165+ int rc = 0, num_msix_want = BNXT_RE_MIN_MSIX, num_msix_got;
166+ struct bnxt_en_dev *en_dev;
167+
168+ if (!rdev)
169+ return -EINVAL;
170+
171+ en_dev = rdev->en_dev;
172+
173+ rtnl_lock();
174+ num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP,
175+ rdev->msix_entries,
176+ num_msix_want);
177+ if (num_msix_got < BNXT_RE_MIN_MSIX) {
178+ rc = -EINVAL;
179+ goto done;
180+ }
181+ if (num_msix_got != num_msix_want) {
182+ dev_warn(rdev_to_dev(rdev),
183+ "Requested %d MSI-X vectors, got %d\n",
184+ num_msix_want, num_msix_got);
185+ }
186+ rdev->num_msix = num_msix_got;
187+done:
188+ rtnl_unlock();
189+ return rc;
190+}
191+
192+static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr,
193+ u16 opcd, u16 crid, u16 trid)
194+{
195+ hdr->req_type = cpu_to_le16(opcd);
196+ hdr->cmpl_ring = cpu_to_le16(crid);
197+ hdr->target_id = cpu_to_le16(trid);
198+}
199+
200+static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg,
201+ int msg_len, void *resp, int resp_max_len,
202+ int timeout)
203+{
204+ fw_msg->msg = msg;
205+ fw_msg->msg_len = msg_len;
206+ fw_msg->resp = resp;
207+ fw_msg->resp_max_len = resp_max_len;
208+ fw_msg->timeout = timeout;
209+}
210+
211+static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id,
212+ bool lock_wait)
213+{
214+ struct bnxt_en_dev *en_dev = rdev->en_dev;
215+ struct hwrm_ring_free_input req = {0};
216+ struct hwrm_ring_free_output resp;
217+ struct bnxt_fw_msg fw_msg;
218+ bool do_unlock = false;
219+ int rc = -EINVAL;
220+
221+ if (!en_dev)
222+ return rc;
223+
224+ memset(&fw_msg, 0, sizeof(fw_msg));
225+ if (lock_wait) {
226+ rtnl_lock();
227+ do_unlock = true;
228+ }
229+
230+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1);
231+ req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
232+ req.ring_id = cpu_to_le16(fw_ring_id);
233+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
234+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
235+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
236+ if (rc)
237+ dev_err(rdev_to_dev(rdev),
238+ "Failed to free HW ring:%d :%#x", req.ring_id, rc);
239+ if (do_unlock)
240+ rtnl_unlock();
241+ return rc;
242+}
243+
244+static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr,
245+ int pages, int type, u32 ring_mask,
246+ u32 map_index, u16 *fw_ring_id)
247+{
248+ struct bnxt_en_dev *en_dev = rdev->en_dev;
249+ struct hwrm_ring_alloc_input req = {0};
250+ struct hwrm_ring_alloc_output resp;
251+ struct bnxt_fw_msg fw_msg;
252+ int rc = -EINVAL;
253+
254+ if (!en_dev)
255+ return rc;
256+
257+ memset(&fw_msg, 0, sizeof(fw_msg));
258+ rtnl_lock();
259+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1);
260+ req.enables = 0;
261+ req.page_tbl_addr = cpu_to_le64(dma_arr[0]);
262+ if (pages > 1) {
263+ /* Page size is in log2 units */
264+ req.page_size = BNXT_PAGE_SHIFT;
265+ req.page_tbl_depth = 1;
266+ }
267+ req.fbo = 0;
268+ /* Association of ring index with doorbell index and MSIX number */
269+ req.logical_id = cpu_to_le16(map_index);
270+ req.length = cpu_to_le32(ring_mask + 1);
271+ req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
272+ req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
273+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
274+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
275+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
276+ if (!rc)
277+ *fw_ring_id = le16_to_cpu(resp.ring_id);
278+
279+ rtnl_unlock();
280+ return rc;
281+}
282+
283+static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
284+ u32 fw_stats_ctx_id, bool lock_wait)
285+{
286+ struct bnxt_en_dev *en_dev = rdev->en_dev;
287+ struct hwrm_stat_ctx_free_input req = {0};
288+ struct bnxt_fw_msg fw_msg;
289+ bool do_unlock = false;
290+ int rc = -EINVAL;
291+
292+ if (!en_dev)
293+ return rc;
294+
295+ memset(&fw_msg, 0, sizeof(fw_msg));
296+ if (lock_wait) {
297+ rtnl_lock();
298+ do_unlock = true;
299+ }
300+
301+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1);
302+ req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);
303+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&req,
304+ sizeof(req), DFLT_HWRM_CMD_TIMEOUT);
305+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
306+ if (rc)
307+ dev_err(rdev_to_dev(rdev),
308+ "Failed to free HW stats context %#x", rc);
309+
310+ if (do_unlock)
311+ rtnl_unlock();
312+ return rc;
313+}
314+
315+static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev,
316+ dma_addr_t dma_map,
317+ u32 *fw_stats_ctx_id)
318+{
319+ struct hwrm_stat_ctx_alloc_output resp = {0};
320+ struct hwrm_stat_ctx_alloc_input req = {0};
321+ struct bnxt_en_dev *en_dev = rdev->en_dev;
322+ struct bnxt_fw_msg fw_msg;
323+ int rc = -EINVAL;
324+
325+ *fw_stats_ctx_id = INVALID_STATS_CTX_ID;
326+
327+ if (!en_dev)
328+ return rc;
329+
330+ memset(&fw_msg, 0, sizeof(fw_msg));
331+ rtnl_lock();
332+
333+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1);
334+ req.update_period_ms = cpu_to_le32(1000);
335+ req.stats_dma_addr = cpu_to_le64(dma_map);
336+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
337+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
338+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
339+ if (!rc)
340+ *fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id);
341+
342+ rtnl_unlock();
343+ return rc;
344+}
345+
346+/* Device */
347+
348+static bool is_bnxt_re_dev(struct net_device *netdev)
349+{
350+ struct ethtool_drvinfo drvinfo;
351+
352+ if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) {
353+ memset(&drvinfo, 0, sizeof(drvinfo));
354+ netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo);
355+
356+ if (strcmp(drvinfo.driver, "bnxt_en"))
357+ return false;
358+ return true;
359+ }
360+ return false;
361+}
362+
363+static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
364+{
365+ struct bnxt_re_dev *rdev;
366+
367+ rcu_read_lock();
368+ list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) {
369+ if (rdev->netdev == netdev) {
370+ rcu_read_unlock();
371+ return rdev;
372+ }
373+ }
374+ rcu_read_unlock();
375+ return NULL;
376+}
377+
378+static void bnxt_re_dev_unprobe(struct net_device *netdev,
379+ struct bnxt_en_dev *en_dev)
380+{
381+ dev_put(netdev);
382+ module_put(en_dev->pdev->driver->driver.owner);
383+}
384+
385+static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev)
386+{
387+ struct bnxt *bp = netdev_priv(netdev);
388+ struct bnxt_en_dev *en_dev;
389+ struct pci_dev *pdev;
390+
391+ /* Call bnxt_en's RoCE probe via indirect API */
392+ if (!bp->ulp_probe)
393+ return ERR_PTR(-EINVAL);
394+
395+ en_dev = bp->ulp_probe(netdev);
396+ if (IS_ERR(en_dev))
397+ return en_dev;
398+
399+ pdev = en_dev->pdev;
400+ if (!pdev)
401+ return ERR_PTR(-EINVAL);
402+
403+ if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) {
404+ dev_dbg(&pdev->dev,
405+ "%s: probe error: RoCE is not supported on this device",
406+ ROCE_DRV_MODULE_NAME);
407+ return ERR_PTR(-ENODEV);
408+ }
409+
410+ /* Bump net device reference count */
411+ if (!try_module_get(pdev->driver->driver.owner))
412+ return ERR_PTR(-ENODEV);
413+
414+ dev_hold(netdev);
415+
416+ return en_dev;
417+}
418+
419+static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev)
420+{
421+ ib_unregister_device(&rdev->ibdev);
422+}
423+
424+static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
425+{
426+ struct ib_device *ibdev = &rdev->ibdev;
427+
428+ /* ib device init */
429+ ibdev->owner = THIS_MODULE;
430+ ibdev->node_type = RDMA_NODE_IB_CA;
431+ strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX);
432+ strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA",
433+ strlen(BNXT_RE_DESC) + 5);
434+ ibdev->phys_port_cnt = 1;
435+
436+ bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid);
437+
438+ ibdev->num_comp_vectors = 1;
439+ ibdev->dma_device = &rdev->en_dev->pdev->dev;
440+ ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
441+
442+ /* User space */
443+ ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION;
444+ ibdev->uverbs_cmd_mask =
445+ (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
446+ (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
447+ (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
448+ (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
449+ (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
450+ (1ull << IB_USER_VERBS_CMD_REG_MR) |
451+ (1ull << IB_USER_VERBS_CMD_REREG_MR) |
452+ (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
453+ (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
454+ (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
455+ (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
456+ (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
457+ (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
458+ (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
459+ (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
460+ (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
461+ (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
462+ (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
463+ (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
464+ (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
465+ (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
466+ (1ull << IB_USER_VERBS_CMD_MODIFY_AH) |
467+ (1ull << IB_USER_VERBS_CMD_QUERY_AH) |
468+ (1ull << IB_USER_VERBS_CMD_DESTROY_AH);
469+ /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */
470+
471+ /* Kernel verbs */
472+ ibdev->query_device = bnxt_re_query_device;
473+ ibdev->modify_device = bnxt_re_modify_device;
474+
475+ ibdev->query_port = bnxt_re_query_port;
476+ ibdev->modify_port = bnxt_re_modify_port;
477+ ibdev->get_port_immutable = bnxt_re_get_port_immutable;
478+ ibdev->query_pkey = bnxt_re_query_pkey;
479+ ibdev->query_gid = bnxt_re_query_gid;
480+ ibdev->get_netdev = bnxt_re_get_netdev;
481+ ibdev->add_gid = bnxt_re_add_gid;
482+ ibdev->del_gid = bnxt_re_del_gid;
483+ ibdev->get_link_layer = bnxt_re_get_link_layer;
484+
485+ ibdev->alloc_pd = bnxt_re_alloc_pd;
486+ ibdev->dealloc_pd = bnxt_re_dealloc_pd;
487+
488+ ibdev->create_ah = bnxt_re_create_ah;
489+ ibdev->modify_ah = bnxt_re_modify_ah;
490+ ibdev->query_ah = bnxt_re_query_ah;
491+ ibdev->destroy_ah = bnxt_re_destroy_ah;
492+
493+ ibdev->create_qp = bnxt_re_create_qp;
494+ ibdev->modify_qp = bnxt_re_modify_qp;
495+ ibdev->query_qp = bnxt_re_query_qp;
496+ ibdev->destroy_qp = bnxt_re_destroy_qp;
497+
498+ ibdev->post_send = bnxt_re_post_send;
499+ ibdev->post_recv = bnxt_re_post_recv;
500+
501+ ibdev->create_cq = bnxt_re_create_cq;
502+ ibdev->destroy_cq = bnxt_re_destroy_cq;
503+ ibdev->poll_cq = bnxt_re_poll_cq;
504+ ibdev->req_notify_cq = bnxt_re_req_notify_cq;
505+
506+ ibdev->get_dma_mr = bnxt_re_get_dma_mr;
507+ ibdev->dereg_mr = bnxt_re_dereg_mr;
508+ ibdev->alloc_mr = bnxt_re_alloc_mr;
509+ ibdev->map_mr_sg = bnxt_re_map_mr_sg;
510+ ibdev->alloc_fmr = bnxt_re_alloc_fmr;
511+ ibdev->map_phys_fmr = bnxt_re_map_phys_fmr;
512+ ibdev->unmap_fmr = bnxt_re_unmap_fmr;
513+ ibdev->dealloc_fmr = bnxt_re_dealloc_fmr;
514+
515+ ibdev->reg_user_mr = bnxt_re_reg_user_mr;
516+ ibdev->alloc_ucontext = bnxt_re_alloc_ucontext;
517+ ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext;
518+ ibdev->mmap = bnxt_re_mmap;
519+
520+ return ib_register_device(ibdev, NULL);
521+}
522+
523+static ssize_t show_rev(struct device *device, struct device_attribute *attr,
524+ char *buf)
525+{
526+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
527+
528+ return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
529+}
530+
531+static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
532+ char *buf)
533+{
534+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
535+
536+ return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->dev_attr.fw_ver);
537+}
538+
539+static ssize_t show_hca(struct device *device, struct device_attribute *attr,
540+ char *buf)
541+{
542+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
543+
544+ return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc);
545+}
546+
547+static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL);
548+static DEVICE_ATTR(fw_rev, 0444, show_fw_ver, NULL);
549+static DEVICE_ATTR(hca_type, 0444, show_hca, NULL);
550+
551+static struct device_attribute *bnxt_re_attributes[] = {
552+ &dev_attr_hw_rev,
553+ &dev_attr_fw_rev,
554+ &dev_attr_hca_type
555+};
556+
557+static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
558+{
559+ dev_put(rdev->netdev);
560+ rdev->netdev = NULL;
561+
562+ mutex_lock(&bnxt_re_dev_lock);
563+ list_del_rcu(&rdev->list);
564+ mutex_unlock(&bnxt_re_dev_lock);
565+
566+ synchronize_rcu();
567+ flush_workqueue(bnxt_re_wq);
568+
569+ ib_dealloc_device(&rdev->ibdev);
570+ /* rdev is gone */
571+}
572+
573+static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
574+ struct bnxt_en_dev *en_dev)
575+{
576+ struct bnxt_re_dev *rdev;
577+
578+ /* Allocate bnxt_re_dev instance here */
579+ rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev));
580+ if (!rdev) {
581+ dev_err(NULL, "%s: bnxt_re_dev allocation failure!",
582+ ROCE_DRV_MODULE_NAME);
583+ return NULL;
584+ }
585+ /* Default values */
586+ rdev->netdev = netdev;
587+ dev_hold(rdev->netdev);
588+ rdev->en_dev = en_dev;
589+ rdev->id = rdev->en_dev->pdev->devfn;
590+ INIT_LIST_HEAD(&rdev->qp_list);
591+ mutex_init(&rdev->qp_lock);
592+ atomic_set(&rdev->qp_count, 0);
593+ atomic_set(&rdev->cq_count, 0);
594+ atomic_set(&rdev->srq_count, 0);
595+ atomic_set(&rdev->mr_count, 0);
596+ atomic_set(&rdev->mw_count, 0);
597+ rdev->cosq[0] = 0xFFFF;
598+ rdev->cosq[1] = 0xFFFF;
599+
600+ mutex_lock(&bnxt_re_dev_lock);
601+ list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list);
602+ mutex_unlock(&bnxt_re_dev_lock);
603+ return rdev;
604+}
605+
606+static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
607+ struct creq_func_event *aeqe)
608+{
609+ switch (aeqe->event) {
610+ case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
611+ break;
612+ case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
613+ break;
614+ case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
615+ break;
616+ case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
617+ break;
618+ case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
619+ break;
620+ case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
621+ break;
622+ case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
623+ break;
624+ case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
625+ break;
626+ case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
627+ break;
628+ case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
629+ break;
630+ case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
631+ break;
632+ default:
633+ return -EINVAL;
634+ }
635+ return 0;
636+}
637+
638+static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
639+ struct bnxt_qplib_cq *handle)
640+{
641+ struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq,
642+ qplib_cq);
643+
644+ if (!cq) {
645+ dev_err(NULL, "%s: CQ is NULL, CQN not handled",
646+ ROCE_DRV_MODULE_NAME);
647+ return -EINVAL;
648+ }
649+ if (cq->ib_cq.comp_handler) {
650+ /* Lock comp_handler? */
651+ (*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context);
652+ }
653+
654+ return 0;
655+}
656+
657+static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
658+{
659+ if (rdev->nq.hwq.max_elements)
660+ bnxt_qplib_disable_nq(&rdev->nq);
661+
662+ if (rdev->qplib_res.rcfw)
663+ bnxt_qplib_cleanup_res(&rdev->qplib_res);
664+}
665+
666+static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
667+{
668+ int rc = 0;
669+
670+ bnxt_qplib_init_res(&rdev->qplib_res);
671+
672+ if (rdev->msix_entries[BNXT_RE_NQ_IDX].vector <= 0)
673+ return -EINVAL;
674+
675+ rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq,
676+ rdev->msix_entries[BNXT_RE_NQ_IDX].vector,
677+ rdev->msix_entries[BNXT_RE_NQ_IDX].db_offset,
678+ &bnxt_re_cqn_handler,
679+ NULL);
680+
681+ if (rc)
682+ dev_err(rdev_to_dev(rdev), "Failed to enable NQ: %#x", rc);
683+
684+ return rc;
685+}
686+
687+static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait)
688+{
689+ if (rdev->nq.hwq.max_elements) {
690+ bnxt_re_net_ring_free(rdev, rdev->nq.ring_id, lock_wait);
691+ bnxt_qplib_free_nq(&rdev->nq);
692+ }
693+ if (rdev->qplib_res.dpi_tbl.max) {
694+ bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
695+ &rdev->qplib_res.dpi_tbl,
696+ &rdev->dpi_privileged);
697+ }
698+ if (rdev->qplib_res.rcfw) {
699+ bnxt_qplib_free_res(&rdev->qplib_res);
700+ rdev->qplib_res.rcfw = NULL;
701+ }
702+}
703+
704+static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
705+{
706+ int rc = 0;
707+
708+ /* Configure and allocate resources for qplib */
709+ rdev->qplib_res.rcfw = &rdev->rcfw;
710+ rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr);
711+ if (rc)
712+ goto fail;
713+
714+ rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev,
715+ rdev->netdev, &rdev->dev_attr);
716+ if (rc)
717+ goto fail;
718+
719+ rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
720+ &rdev->dpi_privileged,
721+ rdev);
722+ if (rc)
723+ goto fail;
724+
725+ rdev->nq.hwq.max_elements = BNXT_RE_MAX_CQ_COUNT +
726+ BNXT_RE_MAX_SRQC_COUNT + 2;
727+ rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq);
728+ if (rc) {
729+ dev_err(rdev_to_dev(rdev),
730+ "Failed to allocate NQ memory: %#x", rc);
731+ goto fail;
732+ }
733+ rc = bnxt_re_net_ring_alloc
734+ (rdev, rdev->nq.hwq.pbl[PBL_LVL_0].pg_map_arr,
735+ rdev->nq.hwq.pbl[rdev->nq.hwq.level].pg_count,
736+ HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_NQE_MAX_CNT - 1,
737+ rdev->msix_entries[BNXT_RE_NQ_IDX].ring_idx,
738+ &rdev->nq.ring_id);
739+ if (rc) {
740+ dev_err(rdev_to_dev(rdev),
741+ "Failed to allocate NQ ring: %#x", rc);
742+ goto free_nq;
743+ }
744+ return 0;
745+free_nq:
746+ bnxt_qplib_free_nq(&rdev->nq);
747+fail:
748+ rdev->qplib_res.rcfw = NULL;
749+ return rc;
750+}
751+
752+static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
753+ u8 port_num, enum ib_event_type event)
754+{
755+ struct ib_event ib_event;
756+
757+ ib_event.device = ibdev;
758+ if (qp)
759+ ib_event.element.qp = qp;
760+ else
761+ ib_event.element.port_num = port_num;
762+ ib_event.event = event;
763+ ib_dispatch_event(&ib_event);
764+}
765+
766+#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02
767+static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir,
768+ u64 *cid_map)
769+{
770+ struct hwrm_queue_pri2cos_qcfg_input req = {0};
771+ struct bnxt *bp = netdev_priv(rdev->netdev);
772+ struct hwrm_queue_pri2cos_qcfg_output resp;
773+ struct bnxt_en_dev *en_dev = rdev->en_dev;
774+ struct bnxt_fw_msg fw_msg;
775+ u32 flags = 0;
776+ u8 *qcfgmap, *tmp_map;
777+ int rc = 0, i;
778+
779+ if (!cid_map)
780+ return -EINVAL;
781+
782+ memset(&fw_msg, 0, sizeof(fw_msg));
783+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
784+ HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
785+ flags |= (dir & 0x01);
786+ flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN;
787+ req.flags = cpu_to_le32(flags);
788+ req.port_id = bp->pf.port_id;
789+
790+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
791+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
792+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
793+ if (rc)
794+ return rc;
795+
796+ if (resp.queue_cfg_info) {
797+ dev_warn(rdev_to_dev(rdev),
798+ "Asymmetric cos queue configuration detected");
799+ dev_warn(rdev_to_dev(rdev),
800+ " on device, QoS may not be fully functional\n");
801+ }
802+ qcfgmap = &resp.pri0_cos_queue_id;
803+ tmp_map = (u8 *)cid_map;
804+ for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++)
805+ tmp_map[i] = qcfgmap[i];
806+
807+ return rc;
808+}
809+
810+static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
811+ struct bnxt_re_qp *qp)
812+{
813+ return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp);
814+}
815+
816+static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)
817+{
818+ int mask = IB_QP_STATE;
819+ struct ib_qp_attr qp_attr;
820+ struct bnxt_re_qp *qp;
821+
822+ qp_attr.qp_state = IB_QPS_ERR;
823+ mutex_lock(&rdev->qp_lock);
824+ list_for_each_entry(qp, &rdev->qp_list, list) {
825+ /* Modify the state of all QPs except QP1/Shadow QP */
826+ if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) {
827+ if (qp->qplib_qp.state !=
828+ CMDQ_MODIFY_QP_NEW_STATE_RESET &&
829+ qp->qplib_qp.state !=
830+ CMDQ_MODIFY_QP_NEW_STATE_ERR) {
831+ bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp,
832+ 1, IB_EVENT_QP_FATAL);
833+ bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask,
834+ NULL);
835+ }
836+ }
837+ }
838+ mutex_unlock(&rdev->qp_lock);
839+}
840+
841+static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev)
842+{
843+ u32 prio_map = 0, tmp_map = 0;
844+ struct net_device *netdev;
845+ struct dcb_app app;
846+
847+ netdev = rdev->netdev;
848+
849+ memset(&app, 0, sizeof(app));
850+ app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE;
851+ app.protocol = ETH_P_IBOE;
852+ tmp_map = dcb_ieee_getapp_mask(netdev, &app);
853+ prio_map = tmp_map;
854+
855+ app.selector = IEEE_8021QAZ_APP_SEL_DGRAM;
856+ app.protocol = ROCE_V2_UDP_DPORT;
857+ tmp_map = dcb_ieee_getapp_mask(netdev, &app);
858+ prio_map |= tmp_map;
859+
860+ if (!prio_map)
861+ prio_map = -EFAULT;
862+ return prio_map;
863+}
864+
865+static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq)
866+{
867+ u16 prio;
868+ u8 id;
869+
870+ for (prio = 0, id = 0; prio < 8; prio++) {
871+ if (prio_map & (1 << prio)) {
872+ cosq[id] = cid_map[prio];
873+ id++;
874+ if (id == 2) /* Max 2 tcs supported */
875+ break;
876+ }
877+ }
878+}
879+
880+static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
881+{
882+ u8 prio_map = 0;
883+ u64 cid_map;
884+ int rc;
885+
886+ /* Get priority for roce */
887+ rc = bnxt_re_get_priority_mask(rdev);
888+ if (rc < 0)
889+ return rc;
890+ prio_map = (u8)rc;
891+
892+ if (prio_map == rdev->cur_prio_map)
893+ return 0;
894+ rdev->cur_prio_map = prio_map;
895+ /* Get cosq id for this priority */
896+ rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map);
897+ if (rc) {
898+ dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map);
899+ return rc;
900+ }
901+ /* Parse CoS IDs for app priority */
902+ bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq);
903+
904+ /* Config BONO. */
905+ rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq);
906+ if (rc) {
907+ dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n",
908+ rdev->cosq[0], rdev->cosq[1]);
909+ return rc;
910+ }
911+
912+ return 0;
913+}
914+
915+static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait)
916+{
917+ int i, rc;
918+
919+ if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
920+ for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++)
921+ device_remove_file(&rdev->ibdev.dev,
922+ bnxt_re_attributes[i]);
923+ /* Cleanup ib dev */
924+ bnxt_re_unregister_ib(rdev);
925+ }
926+ if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags))
927+ cancel_delayed_work(&rdev->worker);
928+
929+ bnxt_re_cleanup_res(rdev);
930+ bnxt_re_free_res(rdev, lock_wait);
931+
932+ if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) {
933+ rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw);
934+ if (rc)
935+ dev_warn(rdev_to_dev(rdev),
936+ "Failed to deinitialize RCFW: %#x", rc);
937+ bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id,
938+ lock_wait);
939+ bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx);
940+ bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
941+ bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, lock_wait);
942+ bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
943+ }
944+ if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) {
945+ rc = bnxt_re_free_msix(rdev, lock_wait);
946+ if (rc)
947+ dev_warn(rdev_to_dev(rdev),
948+ "Failed to free MSI-X vectors: %#x", rc);
949+ }
950+ if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) {
951+ rc = bnxt_re_unregister_netdev(rdev, lock_wait);
952+ if (rc)
953+ dev_warn(rdev_to_dev(rdev),
954+ "Failed to unregister with netdev: %#x", rc);
955+ }
956+}
957+
958+static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
959+{
960+ u32 i;
961+
962+ rdev->qplib_ctx.qpc_count = BNXT_RE_MAX_QPC_COUNT;
963+ rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT;
964+ rdev->qplib_ctx.srqc_count = BNXT_RE_MAX_SRQC_COUNT;
965+ rdev->qplib_ctx.cq_count = BNXT_RE_MAX_CQ_COUNT;
966+ for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
967+ rdev->qplib_ctx.tqm_count[i] =
968+ rdev->dev_attr.tqm_alloc_reqs[i];
969+}
970+
971+/* worker thread for polling periodic events. Now used for QoS programming*/
972+static void bnxt_re_worker(struct work_struct *work)
973+{
974+ struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
975+ worker.work);
976+
977+ bnxt_re_setup_qos(rdev);
978+ schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
979+}
980+
981+static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
982+{
983+ int i, j, rc;
984+
985+ /* Registered a new RoCE device instance to netdev */
986+ rc = bnxt_re_register_netdev(rdev);
987+ if (rc) {
988+ pr_err("Failed to register with netedev: %#x\n", rc);
989+ return -EINVAL;
990+ }
991+ set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
992+
993+ rc = bnxt_re_request_msix(rdev);
994+ if (rc) {
995+ pr_err("Failed to get MSI-X vectors: %#x\n", rc);
996+ rc = -EINVAL;
997+ goto fail;
998+ }
999+ set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags);
1000+
1001+ /* Establish RCFW Communication Channel to initialize the context
1002+ * memory for the function and all child VFs
1003+ */
1004+ rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw);
1005+ if (rc)
1006+ goto fail;
1007+
1008+ rc = bnxt_re_net_ring_alloc
1009+ (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr,
1010+ rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count,
1011+ HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1,
1012+ rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx,
1013+ &rdev->rcfw.creq_ring_id);
1014+ if (rc) {
1015+ pr_err("Failed to allocate CREQ: %#x\n", rc);
1016+ goto free_rcfw;
1017+ }
1018+ rc = bnxt_qplib_enable_rcfw_channel
1019+ (rdev->en_dev->pdev, &rdev->rcfw,
1020+ rdev->msix_entries[BNXT_RE_AEQ_IDX].vector,
1021+ rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset,
1022+ 0, &bnxt_re_aeq_handler);
1023+ if (rc) {
1024+ pr_err("Failed to enable RCFW channel: %#x\n", rc);
1025+ goto free_ring;
1026+ }
1027+
1028+ rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr);
1029+ if (rc)
1030+ goto disable_rcfw;
1031+ bnxt_re_set_resource_limits(rdev);
1032+
1033+ rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0);
1034+ if (rc) {
1035+ pr_err("Failed to allocate QPLIB context: %#x\n", rc);
1036+ goto disable_rcfw;
1037+ }
1038+ rc = bnxt_re_net_stats_ctx_alloc(rdev,
1039+ rdev->qplib_ctx.stats.dma_map,
1040+ &rdev->qplib_ctx.stats.fw_id);
1041+ if (rc) {
1042+ pr_err("Failed to allocate stats context: %#x\n", rc);
1043+ goto free_ctx;
1044+ }
1045+
1046+ rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, 0);
1047+ if (rc) {
1048+ pr_err("Failed to initialize RCFW: %#x\n", rc);
1049+ goto free_sctx;
1050+ }
1051+ set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
1052+
1053+ /* Resources based on the 'new' device caps */
1054+ rc = bnxt_re_alloc_res(rdev);
1055+ if (rc) {
1056+ pr_err("Failed to allocate resources: %#x\n", rc);
1057+ goto fail;
1058+ }
1059+ rc = bnxt_re_init_res(rdev);
1060+ if (rc) {
1061+ pr_err("Failed to initialize resources: %#x\n", rc);
1062+ goto fail;
1063+ }
1064+
1065+ rc = bnxt_re_setup_qos(rdev);
1066+ if (rc)
1067+ pr_info("RoCE priority not yet configured\n");
1068+
1069+ INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
1070+ set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags);
1071+ schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
1072+
1073+ /* Register ib dev */
1074+ rc = bnxt_re_register_ib(rdev);
1075+ if (rc) {
1076+ pr_err("Failed to register with IB: %#x\n", rc);
1077+ goto fail;
1078+ }
1079+ dev_info(rdev_to_dev(rdev), "Device registered successfully");
1080+ for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) {
1081+ rc = device_create_file(&rdev->ibdev.dev,
1082+ bnxt_re_attributes[i]);
1083+ if (rc) {
1084+ dev_err(rdev_to_dev(rdev),
1085+ "Failed to create IB sysfs: %#x", rc);
1086+ /* Must clean up all created device files */
1087+ for (j = 0; j < i; j++)
1088+ device_remove_file(&rdev->ibdev.dev,
1089+ bnxt_re_attributes[j]);
1090+ bnxt_re_unregister_ib(rdev);
1091+ goto fail;
1092+ }
1093+ }
1094+ set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
1095+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);
1096+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE);
1097+
1098+ return 0;
1099+free_sctx:
1100+ bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, true);
1101+free_ctx:
1102+ bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx);
1103+disable_rcfw:
1104+ bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
1105+free_ring:
1106+ bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, true);
1107+free_rcfw:
1108+ bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
1109+fail:
1110+ bnxt_re_ib_unreg(rdev, true);
1111+ return rc;
1112+}
1113+
1114+static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev)
1115+{
1116+ struct bnxt_en_dev *en_dev = rdev->en_dev;
1117+ struct net_device *netdev = rdev->netdev;
1118+
1119+ bnxt_re_dev_remove(rdev);
1120+
1121+ if (netdev)
1122+ bnxt_re_dev_unprobe(netdev, en_dev);
1123+}
1124+
1125+static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev)
1126+{
1127+ struct bnxt_en_dev *en_dev;
1128+ int rc = 0;
1129+
1130+ if (!is_bnxt_re_dev(netdev))
1131+ return -ENODEV;
1132+
1133+ en_dev = bnxt_re_dev_probe(netdev);
1134+ if (IS_ERR(en_dev)) {
1135+ if (en_dev != ERR_PTR(-ENODEV))
1136+ pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME);
1137+ rc = PTR_ERR(en_dev);
1138+ goto exit;
1139+ }
1140+ *rdev = bnxt_re_dev_add(netdev, en_dev);
1141+ if (!*rdev) {
1142+ rc = -ENOMEM;
1143+ bnxt_re_dev_unprobe(netdev, en_dev);
1144+ goto exit;
1145+ }
1146+exit:
1147+ return rc;
1148+}
1149+
1150+static void bnxt_re_remove_one(struct bnxt_re_dev *rdev)
1151+{
1152+ pci_dev_put(rdev->en_dev->pdev);
1153+}
1154+
1155+/* Handle all deferred netevents tasks */
1156+static void bnxt_re_task(struct work_struct *work)
1157+{
1158+ struct bnxt_re_work *re_work;
1159+ struct bnxt_re_dev *rdev;
1160+ int rc = 0;
1161+
1162+ re_work = container_of(work, struct bnxt_re_work, work);
1163+ rdev = re_work->rdev;
1164+
1165+ if (re_work->event != NETDEV_REGISTER &&
1166+ !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
1167+ return;
1168+
1169+ switch (re_work->event) {
1170+ case NETDEV_REGISTER:
1171+ rc = bnxt_re_ib_reg(rdev);
1172+ if (rc)
1173+ dev_err(rdev_to_dev(rdev),
1174+ "Failed to register with IB: %#x", rc);
1175+ break;
1176+ case NETDEV_UP:
1177+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
1178+ IB_EVENT_PORT_ACTIVE);
1179+ break;
1180+ case NETDEV_DOWN:
1181+ bnxt_re_dev_stop(rdev);
1182+ break;
1183+ case NETDEV_CHANGE:
1184+ if (!netif_carrier_ok(rdev->netdev))
1185+ bnxt_re_dev_stop(rdev);
1186+ else if (netif_carrier_ok(rdev->netdev))
1187+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
1188+ IB_EVENT_PORT_ACTIVE);
1189+ break;
1190+ default:
1191+ break;
1192+ }
1193+ kfree(re_work);
1194+}
1195+
1196+static void bnxt_re_init_one(struct bnxt_re_dev *rdev)
1197+{
1198+ pci_dev_get(rdev->en_dev->pdev);
1199+}
1200+
1201+/*
1202+ * "Notifier chain callback can be invoked for the same chain from
1203+ * different CPUs at the same time".
1204+ *
1205+ * For cases when the netdev is already present, our call to the
1206+ * register_netdevice_notifier() will actually get the rtnl_lock()
1207+ * before sending NETDEV_REGISTER and (if up) NETDEV_UP
1208+ * events.
1209+ *
1210+ * But for cases when the netdev is not already present, the notifier
1211+ * chain is subjected to be invoked from different CPUs simultaneously.
1212+ *
1213+ * This is protected by the netdev_mutex.
1214+ */
1215+static int bnxt_re_netdev_event(struct notifier_block *notifier,
1216+ unsigned long event, void *ptr)
1217+{
1218+ struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr);
1219+ struct bnxt_re_work *re_work;
1220+ struct bnxt_re_dev *rdev;
1221+ int rc = 0;
1222+ bool sch_work = false;
1223+
1224+ real_dev = rdma_vlan_dev_real_dev(netdev);
1225+ if (!real_dev)
1226+ real_dev = netdev;
1227+
1228+ rdev = bnxt_re_from_netdev(real_dev);
1229+ if (!rdev && event != NETDEV_REGISTER)
1230+ goto exit;
1231+ if (real_dev != netdev)
1232+ goto exit;
1233+
1234+ switch (event) {
1235+ case NETDEV_REGISTER:
1236+ if (rdev)
1237+ break;
1238+ rc = bnxt_re_dev_reg(&rdev, real_dev);
1239+ if (rc == -ENODEV)
1240+ break;
1241+ if (rc) {
1242+ pr_err("Failed to register with the device %s: %#x\n",
1243+ real_dev->name, rc);
1244+ break;
1245+ }
1246+ bnxt_re_init_one(rdev);
1247+ sch_work = true;
1248+ break;
1249+
1250+ case NETDEV_UNREGISTER:
1251+ bnxt_re_ib_unreg(rdev, false);
1252+ bnxt_re_remove_one(rdev);
1253+ bnxt_re_dev_unreg(rdev);
1254+ break;
1255+
1256+ default:
1257+ sch_work = true;
1258+ break;
1259+ }
1260+ if (sch_work) {
1261+ /* Allocate for the deferred task */
1262+ re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC);
1263+ if (re_work) {
1264+ re_work->rdev = rdev;
1265+ re_work->event = event;
1266+ re_work->vlan_dev = (real_dev == netdev ?
1267+ NULL : netdev);
1268+ INIT_WORK(&re_work->work, bnxt_re_task);
1269+ queue_work(bnxt_re_wq, &re_work->work);
1270+ }
1271+ }
1272+
1273+exit:
1274+ return NOTIFY_DONE;
1275+}
1276+
1277+static struct notifier_block bnxt_re_netdev_notifier = {
1278+ .notifier_call = bnxt_re_netdev_event
1279+};
1280+
1281+static int __init bnxt_re_mod_init(void)
1282+{
1283+ int rc = 0;
1284+
1285+ pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);
1286+
1287+ bnxt_re_wq = create_singlethread_workqueue("bnxt_re");
1288+ if (!bnxt_re_wq)
1289+ return -ENOMEM;
1290+
1291+ INIT_LIST_HEAD(&bnxt_re_dev_list);
1292+
1293+ rc = register_netdevice_notifier(&bnxt_re_netdev_notifier);
1294+ if (rc) {
1295+ pr_err("%s: Cannot register to netdevice_notifier",
1296+ ROCE_DRV_MODULE_NAME);
1297+ goto err_netdev;
1298+ }
1299+ return 0;
1300+
1301+err_netdev:
1302+ destroy_workqueue(bnxt_re_wq);
1303+
1304+ return rc;
1305+}
1306+
1307+static void __exit bnxt_re_mod_exit(void)
1308+{
1309+ unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
1310+ if (bnxt_re_wq)
1311+ destroy_workqueue(bnxt_re_wq);
1312+}
1313+
1314+module_init(bnxt_re_mod_init);
1315+module_exit(bnxt_re_mod_exit);
drivers/infiniband/hw/bnxt_re/qplib_fp.c+2167−0
@@ -0,0 +1,2167 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: Fast Path Operators
37+ */
38+
39+#include <linux/interrupt.h>
40+#include <linux/spinlock.h>
41+#include <linux/sched.h>
42+#include <linux/slab.h>
43+#include <linux/pci.h>
44+#include <linux/prefetch.h>
45+
46+#include "roce_hsi.h"
47+
48+#include "qplib_res.h"
49+#include "qplib_rcfw.h"
50+#include "qplib_sp.h"
51+#include "qplib_fp.h"
52+
53+static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
54+
55+static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
56+ struct bnxt_qplib_qp *qp)
57+{
58+ struct bnxt_qplib_q *rq = &qp->rq;
59+ struct bnxt_qplib_q *sq = &qp->sq;
60+
61+ if (qp->rq_hdr_buf)
62+ dma_free_coherent(&res->pdev->dev,
63+ rq->hwq.max_elements * qp->rq_hdr_buf_size,
64+ qp->rq_hdr_buf, qp->rq_hdr_buf_map);
65+ if (qp->sq_hdr_buf)
66+ dma_free_coherent(&res->pdev->dev,
67+ sq->hwq.max_elements * qp->sq_hdr_buf_size,
68+ qp->sq_hdr_buf, qp->sq_hdr_buf_map);
69+ qp->rq_hdr_buf = NULL;
70+ qp->sq_hdr_buf = NULL;
71+ qp->rq_hdr_buf_map = 0;
72+ qp->sq_hdr_buf_map = 0;
73+ qp->sq_hdr_buf_size = 0;
74+ qp->rq_hdr_buf_size = 0;
75+}
76+
77+static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
78+ struct bnxt_qplib_qp *qp)
79+{
80+ struct bnxt_qplib_q *rq = &qp->rq;
81+ struct bnxt_qplib_q *sq = &qp->rq;
82+ int rc = 0;
83+
84+ if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
85+ qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
86+ sq->hwq.max_elements *
87+ qp->sq_hdr_buf_size,
88+ &qp->sq_hdr_buf_map, GFP_KERNEL);
89+ if (!qp->sq_hdr_buf) {
90+ rc = -ENOMEM;
91+ dev_err(&res->pdev->dev,
92+ "QPLIB: Failed to create sq_hdr_buf");
93+ goto fail;
94+ }
95+ }
96+
97+ if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
98+ qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
99+ rq->hwq.max_elements *
100+ qp->rq_hdr_buf_size,
101+ &qp->rq_hdr_buf_map,
102+ GFP_KERNEL);
103+ if (!qp->rq_hdr_buf) {
104+ rc = -ENOMEM;
105+ dev_err(&res->pdev->dev,
106+ "QPLIB: Failed to create rq_hdr_buf");
107+ goto fail;
108+ }
109+ }
110+ return 0;
111+
112+fail:
113+ bnxt_qplib_free_qp_hdr_buf(res, qp);
114+ return rc;
115+}
116+
117+static void bnxt_qplib_service_nq(unsigned long data)
118+{
119+ struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
120+ struct bnxt_qplib_hwq *hwq = &nq->hwq;
121+ struct nq_base *nqe, **nq_ptr;
122+ int num_cqne_processed = 0;
123+ u32 sw_cons, raw_cons;
124+ u16 type;
125+ int budget = nq->budget;
126+ u64 q_handle;
127+
128+ /* Service the NQ until empty */
129+ raw_cons = hwq->cons;
130+ while (budget--) {
131+ sw_cons = HWQ_CMP(raw_cons, hwq);
132+ nq_ptr = (struct nq_base **)hwq->pbl_ptr;
133+ nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
134+ if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
135+ break;
136+
137+ type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
138+ switch (type) {
139+ case NQ_BASE_TYPE_CQ_NOTIFICATION:
140+ {
141+ struct nq_cn *nqcne = (struct nq_cn *)nqe;
142+
143+ q_handle = le32_to_cpu(nqcne->cq_handle_low);
144+ q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
145+ << 32;
146+ bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *)
147+ ((unsigned long)q_handle));
148+ if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *)
149+ ((unsigned long)q_handle)))
150+ num_cqne_processed++;
151+ else
152+ dev_warn(&nq->pdev->dev,
153+ "QPLIB: cqn - type 0x%x not handled",
154+ type);
155+ break;
156+ }
157+ case NQ_BASE_TYPE_DBQ_EVENT:
158+ break;
159+ default:
160+ dev_warn(&nq->pdev->dev,
161+ "QPLIB: nqe with type = 0x%x not handled",
162+ type);
163+ break;
164+ }
165+ raw_cons++;
166+ }
167+ if (hwq->cons != raw_cons) {
168+ hwq->cons = raw_cons;
169+ NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
170+ }
171+}
172+
173+static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
174+{
175+ struct bnxt_qplib_nq *nq = dev_instance;
176+ struct bnxt_qplib_hwq *hwq = &nq->hwq;
177+ struct nq_base **nq_ptr;
178+ u32 sw_cons;
179+
180+ /* Prefetch the NQ element */
181+ sw_cons = HWQ_CMP(hwq->cons, hwq);
182+ nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
183+ prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
184+
185+ /* Fan out to CPU affinitized kthreads? */
186+ tasklet_schedule(&nq->worker);
187+
188+ return IRQ_HANDLED;
189+}
190+
191+void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
192+{
193+ /* Make sure the HW is stopped! */
194+ synchronize_irq(nq->vector);
195+ tasklet_disable(&nq->worker);
196+ tasklet_kill(&nq->worker);
197+
198+ if (nq->requested) {
199+ free_irq(nq->vector, nq);
200+ nq->requested = false;
201+ }
202+ if (nq->bar_reg_iomem)
203+ iounmap(nq->bar_reg_iomem);
204+ nq->bar_reg_iomem = NULL;
205+
206+ nq->cqn_handler = NULL;
207+ nq->srqn_handler = NULL;
208+ nq->vector = 0;
209+}
210+
211+int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
212+ int msix_vector, int bar_reg_offset,
213+ int (*cqn_handler)(struct bnxt_qplib_nq *nq,
214+ struct bnxt_qplib_cq *),
215+ int (*srqn_handler)(struct bnxt_qplib_nq *nq,
216+ void *, u8 event))
217+{
218+ resource_size_t nq_base;
219+ int rc;
220+
221+ nq->pdev = pdev;
222+ nq->vector = msix_vector;
223+
224+ nq->cqn_handler = cqn_handler;
225+
226+ nq->srqn_handler = srqn_handler;
227+
228+ tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
229+
230+ nq->requested = false;
231+ rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq);
232+ if (rc) {
233+ dev_err(&nq->pdev->dev,
234+ "Failed to request IRQ for NQ: %#x", rc);
235+ bnxt_qplib_disable_nq(nq);
236+ goto fail;
237+ }
238+ nq->requested = true;
239+ nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
240+ nq->bar_reg_off = bar_reg_offset;
241+ nq_base = pci_resource_start(pdev, nq->bar_reg);
242+ if (!nq_base) {
243+ rc = -ENOMEM;
244+ goto fail;
245+ }
246+ nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
247+ if (!nq->bar_reg_iomem) {
248+ rc = -ENOMEM;
249+ goto fail;
250+ }
251+ NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
252+
253+ return 0;
254+fail:
255+ bnxt_qplib_disable_nq(nq);
256+ return rc;
257+}
258+
259+void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
260+{
261+ if (nq->hwq.max_elements)
262+ bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
263+}
264+
265+int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
266+{
267+ nq->pdev = pdev;
268+ if (!nq->hwq.max_elements ||
269+ nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
270+ nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
271+
272+ if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
273+ &nq->hwq.max_elements,
274+ BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
275+ PAGE_SIZE, HWQ_TYPE_L2_CMPL))
276+ return -ENOMEM;
277+
278+ nq->budget = 8;
279+ return 0;
280+}
281+
282+/* QP */
283+int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
284+{
285+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
286+ struct cmdq_create_qp1 req;
287+ struct creq_create_qp1_resp *resp;
288+ struct bnxt_qplib_pbl *pbl;
289+ struct bnxt_qplib_q *sq = &qp->sq;
290+ struct bnxt_qplib_q *rq = &qp->rq;
291+ int rc;
292+ u16 cmd_flags = 0;
293+ u32 qp_flags = 0;
294+
295+ RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
296+
297+ /* General */
298+ req.type = qp->type;
299+ req.dpi = cpu_to_le32(qp->dpi->dpi);
300+ req.qp_handle = cpu_to_le64(qp->qp_handle);
301+
302+ /* SQ */
303+ sq->hwq.max_elements = sq->max_wqe;
304+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
305+ &sq->hwq.max_elements,
306+ BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
307+ PAGE_SIZE, HWQ_TYPE_QUEUE);
308+ if (rc)
309+ goto exit;
310+
311+ sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
312+ if (!sq->swq) {
313+ rc = -ENOMEM;
314+ goto fail_sq;
315+ }
316+ pbl = &sq->hwq.pbl[PBL_LVL_0];
317+ req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
318+ req.sq_pg_size_sq_lvl =
319+ ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
320+ << CMDQ_CREATE_QP1_SQ_LVL_SFT) |
321+ (pbl->pg_size == ROCE_PG_SIZE_4K ?
322+ CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
323+ pbl->pg_size == ROCE_PG_SIZE_8K ?
324+ CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
325+ pbl->pg_size == ROCE_PG_SIZE_64K ?
326+ CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
327+ pbl->pg_size == ROCE_PG_SIZE_2M ?
328+ CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
329+ pbl->pg_size == ROCE_PG_SIZE_8M ?
330+ CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
331+ pbl->pg_size == ROCE_PG_SIZE_1G ?
332+ CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
333+ CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
334+
335+ if (qp->scq)
336+ req.scq_cid = cpu_to_le32(qp->scq->id);
337+
338+ qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
339+
340+ /* RQ */
341+ if (rq->max_wqe) {
342+ rq->hwq.max_elements = qp->rq.max_wqe;
343+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
344+ &rq->hwq.max_elements,
345+ BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
346+ PAGE_SIZE, HWQ_TYPE_QUEUE);
347+ if (rc)
348+ goto fail_sq;
349+
350+ rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
351+ GFP_KERNEL);
352+ if (!rq->swq) {
353+ rc = -ENOMEM;
354+ goto fail_rq;
355+ }
356+ pbl = &rq->hwq.pbl[PBL_LVL_0];
357+ req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
358+ req.rq_pg_size_rq_lvl =
359+ ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
360+ CMDQ_CREATE_QP1_RQ_LVL_SFT) |
361+ (pbl->pg_size == ROCE_PG_SIZE_4K ?
362+ CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
363+ pbl->pg_size == ROCE_PG_SIZE_8K ?
364+ CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
365+ pbl->pg_size == ROCE_PG_SIZE_64K ?
366+ CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
367+ pbl->pg_size == ROCE_PG_SIZE_2M ?
368+ CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
369+ pbl->pg_size == ROCE_PG_SIZE_8M ?
370+ CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
371+ pbl->pg_size == ROCE_PG_SIZE_1G ?
372+ CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
373+ CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
374+ if (qp->rcq)
375+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
376+ }
377+
378+ /* Header buffer - allow hdr_buf pass in */
379+ rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
380+ if (rc) {
381+ rc = -ENOMEM;
382+ goto fail;
383+ }
384+ req.qp_flags = cpu_to_le32(qp_flags);
385+ req.sq_size = cpu_to_le32(sq->hwq.max_elements);
386+ req.rq_size = cpu_to_le32(rq->hwq.max_elements);
387+
388+ req.sq_fwo_sq_sge =
389+ cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
390+ CMDQ_CREATE_QP1_SQ_SGE_SFT);
391+ req.rq_fwo_rq_sge =
392+ cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
393+ CMDQ_CREATE_QP1_RQ_SGE_SFT);
394+
395+ req.pd_id = cpu_to_le32(qp->pd->id);
396+
397+ resp = (struct creq_create_qp1_resp *)
398+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
399+ NULL, 0);
400+ if (!resp) {
401+ dev_err(&res->pdev->dev, "QPLIB: FP: CREATE_QP1 send failed");
402+ rc = -EINVAL;
403+ goto fail;
404+ }
405+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
406+ /* Cmd timed out */
407+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 timed out");
408+ rc = -ETIMEDOUT;
409+ goto fail;
410+ }
411+ if (resp->status ||
412+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
413+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 failed ");
414+ dev_err(&rcfw->pdev->dev,
415+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
416+ resp->status, le16_to_cpu(req.cookie),
417+ le16_to_cpu(resp->cookie));
418+ rc = -EINVAL;
419+ goto fail;
420+ }
421+ qp->id = le32_to_cpu(resp->xid);
422+ qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
423+ sq->flush_in_progress = false;
424+ rq->flush_in_progress = false;
425+
426+ return 0;
427+
428+fail:
429+ bnxt_qplib_free_qp_hdr_buf(res, qp);
430+fail_rq:
431+ bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
432+ kfree(rq->swq);
433+fail_sq:
434+ bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
435+ kfree(sq->swq);
436+exit:
437+ return rc;
438+}
439+
440+int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
441+{
442+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
443+ struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
444+ struct cmdq_create_qp req;
445+ struct creq_create_qp_resp *resp;
446+ struct bnxt_qplib_pbl *pbl;
447+ struct sq_psn_search **psn_search_ptr;
448+ unsigned long int psn_search, poff = 0;
449+ struct bnxt_qplib_q *sq = &qp->sq;
450+ struct bnxt_qplib_q *rq = &qp->rq;
451+ struct bnxt_qplib_hwq *xrrq;
452+ int i, rc, req_size, psn_sz;
453+ u16 cmd_flags = 0, max_ssge;
454+ u32 sw_prod, qp_flags = 0;
455+
456+ RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
457+
458+ /* General */
459+ req.type = qp->type;
460+ req.dpi = cpu_to_le32(qp->dpi->dpi);
461+ req.qp_handle = cpu_to_le64(qp->qp_handle);
462+
463+ /* SQ */
464+ psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
465+ sizeof(struct sq_psn_search) : 0;
466+ sq->hwq.max_elements = sq->max_wqe;
467+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
468+ sq->nmap, &sq->hwq.max_elements,
469+ BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
470+ psn_sz,
471+ PAGE_SIZE, HWQ_TYPE_QUEUE);
472+ if (rc)
473+ goto exit;
474+
475+ sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
476+ if (!sq->swq) {
477+ rc = -ENOMEM;
478+ goto fail_sq;
479+ }
480+ hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
481+ if (psn_sz) {
482+ psn_search_ptr = (struct sq_psn_search **)
483+ &hw_sq_send_ptr[get_sqe_pg
484+ (sq->hwq.max_elements)];
485+ psn_search = (unsigned long int)
486+ &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
487+ [get_sqe_idx(sq->hwq.max_elements)];
488+ if (psn_search & ~PAGE_MASK) {
489+ /* If the psn_search does not start on a page boundary,
490+ * then calculate the offset
491+ */
492+ poff = (psn_search & ~PAGE_MASK) /
493+ BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
494+ }
495+ for (i = 0; i < sq->hwq.max_elements; i++)
496+ sq->swq[i].psn_search =
497+ &psn_search_ptr[get_psne_pg(i + poff)]
498+ [get_psne_idx(i + poff)];
499+ }
500+ pbl = &sq->hwq.pbl[PBL_LVL_0];
501+ req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
502+ req.sq_pg_size_sq_lvl =
503+ ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
504+ << CMDQ_CREATE_QP_SQ_LVL_SFT) |
505+ (pbl->pg_size == ROCE_PG_SIZE_4K ?
506+ CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
507+ pbl->pg_size == ROCE_PG_SIZE_8K ?
508+ CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
509+ pbl->pg_size == ROCE_PG_SIZE_64K ?
510+ CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
511+ pbl->pg_size == ROCE_PG_SIZE_2M ?
512+ CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
513+ pbl->pg_size == ROCE_PG_SIZE_8M ?
514+ CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
515+ pbl->pg_size == ROCE_PG_SIZE_1G ?
516+ CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
517+ CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
518+
519+ /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
520+ hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
521+ for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
522+ hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
523+ [get_sqe_idx(sw_prod)];
524+ hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
525+ }
526+
527+ if (qp->scq)
528+ req.scq_cid = cpu_to_le32(qp->scq->id);
529+
530+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
531+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
532+ if (qp->sig_type)
533+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
534+
535+ /* RQ */
536+ if (rq->max_wqe) {
537+ rq->hwq.max_elements = rq->max_wqe;
538+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
539+ rq->nmap, &rq->hwq.max_elements,
540+ BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
541+ PAGE_SIZE, HWQ_TYPE_QUEUE);
542+ if (rc)
543+ goto fail_sq;
544+
545+ rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
546+ GFP_KERNEL);
547+ if (!rq->swq) {
548+ rc = -ENOMEM;
549+ goto fail_rq;
550+ }
551+ pbl = &rq->hwq.pbl[PBL_LVL_0];
552+ req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
553+ req.rq_pg_size_rq_lvl =
554+ ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
555+ CMDQ_CREATE_QP_RQ_LVL_SFT) |
556+ (pbl->pg_size == ROCE_PG_SIZE_4K ?
557+ CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
558+ pbl->pg_size == ROCE_PG_SIZE_8K ?
559+ CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
560+ pbl->pg_size == ROCE_PG_SIZE_64K ?
561+ CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
562+ pbl->pg_size == ROCE_PG_SIZE_2M ?
563+ CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
564+ pbl->pg_size == ROCE_PG_SIZE_8M ?
565+ CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
566+ pbl->pg_size == ROCE_PG_SIZE_1G ?
567+ CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
568+ CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
569+ }
570+
571+ if (qp->rcq)
572+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
573+ req.qp_flags = cpu_to_le32(qp_flags);
574+ req.sq_size = cpu_to_le32(sq->hwq.max_elements);
575+ req.rq_size = cpu_to_le32(rq->hwq.max_elements);
576+ qp->sq_hdr_buf = NULL;
577+ qp->rq_hdr_buf = NULL;
578+
579+ rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
580+ if (rc)
581+ goto fail_rq;
582+
583+ /* CTRL-22434: Irrespective of the requested SGE count on the SQ
584+ * always create the QP with max send sges possible if the requested
585+ * inline size is greater than 0.
586+ */
587+ max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
588+ req.sq_fwo_sq_sge = cpu_to_le16(
589+ ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
590+ << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
591+ req.rq_fwo_rq_sge = cpu_to_le16(
592+ ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
593+ << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
594+ /* ORRQ and IRRQ */
595+ if (psn_sz) {
596+ xrrq = &qp->orrq;
597+ xrrq->max_elements =
598+ ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
599+ req_size = xrrq->max_elements *
600+ BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
601+ req_size &= ~(PAGE_SIZE - 1);
602+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
603+ &xrrq->max_elements,
604+ BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
605+ 0, req_size, HWQ_TYPE_CTX);
606+ if (rc)
607+ goto fail_buf_free;
608+ pbl = &xrrq->pbl[PBL_LVL_0];
609+ req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
610+
611+ xrrq = &qp->irrq;
612+ xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
613+ qp->max_dest_rd_atomic);
614+ req_size = xrrq->max_elements *
615+ BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
616+ req_size &= ~(PAGE_SIZE - 1);
617+
618+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
619+ &xrrq->max_elements,
620+ BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
621+ 0, req_size, HWQ_TYPE_CTX);
622+ if (rc)
623+ goto fail_orrq;
624+
625+ pbl = &xrrq->pbl[PBL_LVL_0];
626+ req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
627+ }
628+ req.pd_id = cpu_to_le32(qp->pd->id);
629+
630+ resp = (struct creq_create_qp_resp *)
631+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
632+ NULL, 0);
633+ if (!resp) {
634+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP send failed");
635+ rc = -EINVAL;
636+ goto fail;
637+ }
638+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
639+ /* Cmd timed out */
640+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP timed out");
641+ rc = -ETIMEDOUT;
642+ goto fail;
643+ }
644+ if (resp->status ||
645+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
646+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP failed ");
647+ dev_err(&rcfw->pdev->dev,
648+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
649+ resp->status, le16_to_cpu(req.cookie),
650+ le16_to_cpu(resp->cookie));
651+ rc = -EINVAL;
652+ goto fail;
653+ }
654+ qp->id = le32_to_cpu(resp->xid);
655+ qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
656+ sq->flush_in_progress = false;
657+ rq->flush_in_progress = false;
658+
659+ return 0;
660+
661+fail:
662+ if (qp->irrq.max_elements)
663+ bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
664+fail_orrq:
665+ if (qp->orrq.max_elements)
666+ bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
667+fail_buf_free:
668+ bnxt_qplib_free_qp_hdr_buf(res, qp);
669+fail_rq:
670+ bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
671+ kfree(rq->swq);
672+fail_sq:
673+ bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
674+ kfree(sq->swq);
675+exit:
676+ return rc;
677+}
678+
679+static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
680+{
681+ switch (qp->state) {
682+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
683+ /* INIT->RTR, configure the path_mtu to the default
684+ * 2048 if not being requested
685+ */
686+ if (!(qp->modify_flags &
687+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
688+ qp->modify_flags |=
689+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
690+ qp->path_mtu =
691+ CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
692+ }
693+ qp->modify_flags &=
694+ ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
695+ /* Bono FW require the max_dest_rd_atomic to be >= 1 */
696+ if (qp->max_dest_rd_atomic < 1)
697+ qp->max_dest_rd_atomic = 1;
698+ qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
699+ /* Bono FW 20.6.5 requires SGID_INDEX configuration */
700+ if (!(qp->modify_flags &
701+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
702+ qp->modify_flags |=
703+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
704+ qp->ah.sgid_index = 0;
705+ }
706+ break;
707+ default:
708+ break;
709+ }
710+}
711+
712+static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
713+{
714+ switch (qp->state) {
715+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
716+ /* Bono FW requires the max_rd_atomic to be >= 1 */
717+ if (qp->max_rd_atomic < 1)
718+ qp->max_rd_atomic = 1;
719+ /* Bono FW does not allow PKEY_INDEX,
720+ * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
721+ * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
722+ * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
723+ * modification
724+ */
725+ qp->modify_flags &=
726+ ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
727+ CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
728+ CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
729+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
730+ CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
731+ CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
732+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
733+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
734+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
735+ CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
736+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
737+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
738+ break;
739+ default:
740+ break;
741+ }
742+}
743+
744+static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
745+{
746+ switch (qp->cur_qp_state) {
747+ case CMDQ_MODIFY_QP_NEW_STATE_RESET:
748+ break;
749+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
750+ __modify_flags_from_init_state(qp);
751+ break;
752+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
753+ __modify_flags_from_rtr_state(qp);
754+ break;
755+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
756+ break;
757+ case CMDQ_MODIFY_QP_NEW_STATE_SQD:
758+ break;
759+ case CMDQ_MODIFY_QP_NEW_STATE_SQE:
760+ break;
761+ case CMDQ_MODIFY_QP_NEW_STATE_ERR:
762+ break;
763+ default:
764+ break;
765+ }
766+}
767+
768+int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
769+{
770+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
771+ struct cmdq_modify_qp req;
772+ struct creq_modify_qp_resp *resp;
773+ u16 cmd_flags = 0, pkey;
774+ u32 temp32[4];
775+ u32 bmask;
776+
777+ RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
778+
779+ /* Filter out the qp_attr_mask based on the state->new transition */
780+ __filter_modify_flags(qp);
781+ bmask = qp->modify_flags;
782+ req.modify_mask = cpu_to_le32(qp->modify_flags);
783+ req.qp_cid = cpu_to_le32(qp->id);
784+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
785+ req.network_type_en_sqd_async_notify_new_state =
786+ (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
787+ (qp->en_sqd_async_notify ?
788+ CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
789+ }
790+ req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
791+
792+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
793+ req.access = qp->access;
794+
795+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
796+ if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
797+ qp->pkey_index, &pkey))
798+ req.pkey = cpu_to_le16(pkey);
799+ }
800+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
801+ req.qkey = cpu_to_le32(qp->qkey);
802+
803+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
804+ memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
805+ req.dgid[0] = cpu_to_le32(temp32[0]);
806+ req.dgid[1] = cpu_to_le32(temp32[1]);
807+ req.dgid[2] = cpu_to_le32(temp32[2]);
808+ req.dgid[3] = cpu_to_le32(temp32[3]);
809+ }
810+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
811+ req.flow_label = cpu_to_le32(qp->ah.flow_label);
812+
813+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
814+ req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
815+ [qp->ah.sgid_index]);
816+
817+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
818+ req.hop_limit = qp->ah.hop_limit;
819+
820+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
821+ req.traffic_class = qp->ah.traffic_class;
822+
823+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
824+ memcpy(req.dest_mac, qp->ah.dmac, 6);
825+
826+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
827+ req.path_mtu = qp->path_mtu;
828+
829+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
830+ req.timeout = qp->timeout;
831+
832+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
833+ req.retry_cnt = qp->retry_cnt;
834+
835+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
836+ req.rnr_retry = qp->rnr_retry;
837+
838+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
839+ req.min_rnr_timer = qp->min_rnr_timer;
840+
841+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
842+ req.rq_psn = cpu_to_le32(qp->rq.psn);
843+
844+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
845+ req.sq_psn = cpu_to_le32(qp->sq.psn);
846+
847+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
848+ req.max_rd_atomic =
849+ ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
850+
851+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
852+ req.max_dest_rd_atomic =
853+ IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
854+
855+ req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
856+ req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
857+ req.sq_sge = cpu_to_le16(qp->sq.max_sge);
858+ req.rq_sge = cpu_to_le16(qp->rq.max_sge);
859+ req.max_inline_data = cpu_to_le32(qp->max_inline_data);
860+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
861+ req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
862+
863+ req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
864+
865+ resp = (struct creq_modify_qp_resp *)
866+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
867+ NULL, 0);
868+ if (!resp) {
869+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP send failed");
870+ return -EINVAL;
871+ }
872+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
873+ /* Cmd timed out */
874+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP timed out");
875+ return -ETIMEDOUT;
876+ }
877+ if (resp->status ||
878+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
879+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP failed ");
880+ dev_err(&rcfw->pdev->dev,
881+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
882+ resp->status, le16_to_cpu(req.cookie),
883+ le16_to_cpu(resp->cookie));
884+ return -EINVAL;
885+ }
886+ qp->cur_qp_state = qp->state;
887+ return 0;
888+}
889+
890+int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
891+{
892+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
893+ struct cmdq_query_qp req;
894+ struct creq_query_qp_resp *resp;
895+ struct creq_query_qp_resp_sb *sb;
896+ u16 cmd_flags = 0;
897+ u32 temp32[4];
898+ int i;
899+
900+ RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
901+
902+ req.qp_cid = cpu_to_le32(qp->id);
903+ req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
904+ resp = (struct creq_query_qp_resp *)
905+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
906+ (void **)&sb, 0);
907+ if (!resp) {
908+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP send failed");
909+ return -EINVAL;
910+ }
911+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
912+ /* Cmd timed out */
913+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP timed out");
914+ return -ETIMEDOUT;
915+ }
916+ if (resp->status ||
917+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
918+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP failed ");
919+ dev_err(&rcfw->pdev->dev,
920+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
921+ resp->status, le16_to_cpu(req.cookie),
922+ le16_to_cpu(resp->cookie));
923+ return -EINVAL;
924+ }
925+ /* Extract the context from the side buffer */
926+ qp->state = sb->en_sqd_async_notify_state &
927+ CREQ_QUERY_QP_RESP_SB_STATE_MASK;
928+ qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
929+ CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
930+ true : false;
931+ qp->access = sb->access;
932+ qp->pkey_index = le16_to_cpu(sb->pkey);
933+ qp->qkey = le32_to_cpu(sb->qkey);
934+
935+ temp32[0] = le32_to_cpu(sb->dgid[0]);
936+ temp32[1] = le32_to_cpu(sb->dgid[1]);
937+ temp32[2] = le32_to_cpu(sb->dgid[2]);
938+ temp32[3] = le32_to_cpu(sb->dgid[3]);
939+ memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
940+
941+ qp->ah.flow_label = le32_to_cpu(sb->flow_label);
942+
943+ qp->ah.sgid_index = 0;
944+ for (i = 0; i < res->sgid_tbl.max; i++) {
945+ if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
946+ qp->ah.sgid_index = i;
947+ break;
948+ }
949+ }
950+ if (i == res->sgid_tbl.max)
951+ dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
952+
953+ qp->ah.hop_limit = sb->hop_limit;
954+ qp->ah.traffic_class = sb->traffic_class;
955+ memcpy(qp->ah.dmac, sb->dest_mac, 6);
956+ qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
957+ CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
958+ CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
959+ qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
960+ CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
961+ CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
962+ qp->timeout = sb->timeout;
963+ qp->retry_cnt = sb->retry_cnt;
964+ qp->rnr_retry = sb->rnr_retry;
965+ qp->min_rnr_timer = sb->min_rnr_timer;
966+ qp->rq.psn = le32_to_cpu(sb->rq_psn);
967+ qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
968+ qp->sq.psn = le32_to_cpu(sb->sq_psn);
969+ qp->max_dest_rd_atomic =
970+ IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
971+ qp->sq.max_wqe = qp->sq.hwq.max_elements;
972+ qp->rq.max_wqe = qp->rq.hwq.max_elements;
973+ qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
974+ qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
975+ qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
976+ qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
977+ memcpy(qp->smac, sb->src_mac, 6);
978+ qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
979+ return 0;
980+}
981+
982+static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
983+{
984+ struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
985+ struct cq_base *hw_cqe, **hw_cqe_ptr;
986+ int i;
987+
988+ for (i = 0; i < cq_hwq->max_elements; i++) {
989+ hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
990+ hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
991+ if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
992+ continue;
993+ switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
994+ case CQ_BASE_CQE_TYPE_REQ:
995+ case CQ_BASE_CQE_TYPE_TERMINAL:
996+ {
997+ struct cq_req *cqe = (struct cq_req *)hw_cqe;
998+
999+ if (qp == le64_to_cpu(cqe->qp_handle))
1000+ cqe->qp_handle = 0;
1001+ break;
1002+ }
1003+ case CQ_BASE_CQE_TYPE_RES_RC:
1004+ case CQ_BASE_CQE_TYPE_RES_UD:
1005+ case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1006+ {
1007+ struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1008+
1009+ if (qp == le64_to_cpu(cqe->qp_handle))
1010+ cqe->qp_handle = 0;
1011+ break;
1012+ }
1013+ default:
1014+ break;
1015+ }
1016+ }
1017+}
1018+
1019+int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1020+ struct bnxt_qplib_qp *qp)
1021+{
1022+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1023+ struct cmdq_destroy_qp req;
1024+ struct creq_destroy_qp_resp *resp;
1025+ unsigned long flags;
1026+ u16 cmd_flags = 0;
1027+
1028+ RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1029+
1030+ req.qp_cid = cpu_to_le32(qp->id);
1031+ resp = (struct creq_destroy_qp_resp *)
1032+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1033+ NULL, 0);
1034+ if (!resp) {
1035+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP send failed");
1036+ return -EINVAL;
1037+ }
1038+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1039+ /* Cmd timed out */
1040+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP timed out");
1041+ return -ETIMEDOUT;
1042+ }
1043+ if (resp->status ||
1044+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1045+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP failed ");
1046+ dev_err(&rcfw->pdev->dev,
1047+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1048+ resp->status, le16_to_cpu(req.cookie),
1049+ le16_to_cpu(resp->cookie));
1050+ return -EINVAL;
1051+ }
1052+
1053+ /* Must walk the associated CQs to nullified the QP ptr */
1054+ spin_lock_irqsave(&qp->scq->hwq.lock, flags);
1055+
1056+ __clean_cq(qp->scq, (u64)(unsigned long)qp);
1057+
1058+ if (qp->rcq && qp->rcq != qp->scq) {
1059+ spin_lock(&qp->rcq->hwq.lock);
1060+ __clean_cq(qp->rcq, (u64)(unsigned long)qp);
1061+ spin_unlock(&qp->rcq->hwq.lock);
1062+ }
1063+
1064+ spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
1065+
1066+ bnxt_qplib_free_qp_hdr_buf(res, qp);
1067+ bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
1068+ kfree(qp->sq.swq);
1069+
1070+ bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
1071+ kfree(qp->rq.swq);
1072+
1073+ if (qp->irrq.max_elements)
1074+ bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1075+ if (qp->orrq.max_elements)
1076+ bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1077+
1078+ return 0;
1079+}
1080+
1081+void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1082+ struct bnxt_qplib_sge *sge)
1083+{
1084+ struct bnxt_qplib_q *sq = &qp->sq;
1085+ u32 sw_prod;
1086+
1087+ memset(sge, 0, sizeof(*sge));
1088+
1089+ if (qp->sq_hdr_buf) {
1090+ sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1091+ sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1092+ sw_prod * qp->sq_hdr_buf_size);
1093+ sge->lkey = 0xFFFFFFFF;
1094+ sge->size = qp->sq_hdr_buf_size;
1095+ return qp->sq_hdr_buf + sw_prod * sge->size;
1096+ }
1097+ return NULL;
1098+}
1099+
1100+u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1101+{
1102+ struct bnxt_qplib_q *rq = &qp->rq;
1103+
1104+ return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1105+}
1106+
1107+dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1108+{
1109+ return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1110+}
1111+
1112+void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1113+ struct bnxt_qplib_sge *sge)
1114+{
1115+ struct bnxt_qplib_q *rq = &qp->rq;
1116+ u32 sw_prod;
1117+
1118+ memset(sge, 0, sizeof(*sge));
1119+
1120+ if (qp->rq_hdr_buf) {
1121+ sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1122+ sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1123+ sw_prod * qp->rq_hdr_buf_size);
1124+ sge->lkey = 0xFFFFFFFF;
1125+ sge->size = qp->rq_hdr_buf_size;
1126+ return qp->rq_hdr_buf + sw_prod * sge->size;
1127+ }
1128+ return NULL;
1129+}
1130+
1131+void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1132+{
1133+ struct bnxt_qplib_q *sq = &qp->sq;
1134+ struct dbr_dbr db_msg = { 0 };
1135+ u32 sw_prod;
1136+
1137+ sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1138+
1139+ db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1140+ DBR_DBR_INDEX_MASK);
1141+ db_msg.type_xid =
1142+ cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1143+ DBR_DBR_TYPE_SQ);
1144+ /* Flush all the WQE writes to HW */
1145+ wmb();
1146+ __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1147+}
1148+
1149+int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1150+ struct bnxt_qplib_swqe *wqe)
1151+{
1152+ struct bnxt_qplib_q *sq = &qp->sq;
1153+ struct bnxt_qplib_swq *swq;
1154+ struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1155+ struct sq_sge *hw_sge;
1156+ u32 sw_prod;
1157+ u8 wqe_size16;
1158+ int i, rc = 0, data_len = 0, pkt_num = 0;
1159+ __le32 temp32;
1160+
1161+ if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1162+ rc = -EINVAL;
1163+ goto done;
1164+ }
1165+ if (HWQ_CMP((sq->hwq.prod + 1), &sq->hwq) ==
1166+ HWQ_CMP(sq->hwq.cons, &sq->hwq)) {
1167+ rc = -ENOMEM;
1168+ goto done;
1169+ }
1170+ sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1171+ swq = &sq->swq[sw_prod];
1172+ swq->wr_id = wqe->wr_id;
1173+ swq->type = wqe->type;
1174+ swq->flags = wqe->flags;
1175+ if (qp->sig_type)
1176+ swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1177+ swq->start_psn = sq->psn & BTH_PSN_MASK;
1178+
1179+ hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1180+ hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1181+ [get_sqe_idx(sw_prod)];
1182+
1183+ memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1184+
1185+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1186+ /* Copy the inline data */
1187+ if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1188+ dev_warn(&sq->hwq.pdev->dev,
1189+ "QPLIB: Inline data length > 96 detected");
1190+ data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1191+ } else {
1192+ data_len = wqe->inline_len;
1193+ }
1194+ memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1195+ wqe_size16 = (data_len + 15) >> 4;
1196+ } else {
1197+ for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1198+ i < wqe->num_sge; i++, hw_sge++) {
1199+ hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1200+ hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1201+ hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1202+ data_len += wqe->sg_list[i].size;
1203+ }
1204+ /* Each SGE entry = 1 WQE size16 */
1205+ wqe_size16 = wqe->num_sge;
1206+ }
1207+
1208+ /* Specifics */
1209+ switch (wqe->type) {
1210+ case BNXT_QPLIB_SWQE_TYPE_SEND:
1211+ if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1212+ /* Assemble info for Raw Ethertype QPs */
1213+ struct sq_send_raweth_qp1 *sqe =
1214+ (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1215+
1216+ sqe->wqe_type = wqe->type;
1217+ sqe->flags = wqe->flags;
1218+ sqe->wqe_size = wqe_size16 +
1219+ ((offsetof(typeof(*sqe), data) + 15) >> 4);
1220+ sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1221+ sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1222+ sqe->length = cpu_to_le32(data_len);
1223+ sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1224+ SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1225+ SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1226+
1227+ break;
1228+ }
1229+ /* else, just fall thru */
1230+ case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1231+ case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1232+ {
1233+ struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1234+
1235+ sqe->wqe_type = wqe->type;
1236+ sqe->flags = wqe->flags;
1237+ sqe->wqe_size = wqe_size16 +
1238+ ((offsetof(typeof(*sqe), data) + 15) >> 4);
1239+ sqe->inv_key_or_imm_data = cpu_to_le32(
1240+ wqe->send.inv_key);
1241+ if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1242+ sqe->q_key = cpu_to_le32(wqe->send.q_key);
1243+ sqe->dst_qp = cpu_to_le32(
1244+ wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1245+ sqe->length = cpu_to_le32(data_len);
1246+ sqe->avid = cpu_to_le32(wqe->send.avid &
1247+ SQ_SEND_AVID_MASK);
1248+ sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1249+ } else {
1250+ sqe->length = cpu_to_le32(data_len);
1251+ sqe->dst_qp = 0;
1252+ sqe->avid = 0;
1253+ if (qp->mtu)
1254+ pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1255+ if (!pkt_num)
1256+ pkt_num = 1;
1257+ sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1258+ }
1259+ break;
1260+ }
1261+ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1262+ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1263+ case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1264+ {
1265+ struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1266+
1267+ sqe->wqe_type = wqe->type;
1268+ sqe->flags = wqe->flags;
1269+ sqe->wqe_size = wqe_size16 +
1270+ ((offsetof(typeof(*sqe), data) + 15) >> 4);
1271+ sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1272+ sqe->length = cpu_to_le32((u32)data_len);
1273+ sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1274+ sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1275+ if (qp->mtu)
1276+ pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1277+ if (!pkt_num)
1278+ pkt_num = 1;
1279+ sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1280+ break;
1281+ }
1282+ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1283+ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1284+ {
1285+ struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1286+
1287+ sqe->wqe_type = wqe->type;
1288+ sqe->flags = wqe->flags;
1289+ sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1290+ sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1291+ sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1292+ sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1293+ if (qp->mtu)
1294+ pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1295+ if (!pkt_num)
1296+ pkt_num = 1;
1297+ sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1298+ break;
1299+ }
1300+ case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1301+ {
1302+ struct sq_localinvalidate *sqe =
1303+ (struct sq_localinvalidate *)hw_sq_send_hdr;
1304+
1305+ sqe->wqe_type = wqe->type;
1306+ sqe->flags = wqe->flags;
1307+ sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1308+
1309+ break;
1310+ }
1311+ case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1312+ {
1313+ struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1314+
1315+ sqe->wqe_type = wqe->type;
1316+ sqe->flags = wqe->flags;
1317+ sqe->access_cntl = wqe->frmr.access_cntl |
1318+ SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1319+ sqe->zero_based_page_size_log =
1320+ (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1321+ SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1322+ (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1323+ sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1324+ temp32 = cpu_to_le32(wqe->frmr.length);
1325+ memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1326+ sqe->numlevels_pbl_page_size_log =
1327+ ((wqe->frmr.pbl_pg_sz_log <<
1328+ SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1329+ SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1330+ ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1331+ SQ_FR_PMR_NUMLEVELS_MASK);
1332+
1333+ for (i = 0; i < wqe->frmr.page_list_len; i++)
1334+ wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1335+ wqe->frmr.page_list[i] |
1336+ PTU_PTE_VALID);
1337+ sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1338+ sqe->va = cpu_to_le64(wqe->frmr.va);
1339+
1340+ break;
1341+ }
1342+ case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1343+ {
1344+ struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1345+
1346+ sqe->wqe_type = wqe->type;
1347+ sqe->flags = wqe->flags;
1348+ sqe->access_cntl = wqe->bind.access_cntl;
1349+ sqe->mw_type_zero_based = wqe->bind.mw_type |
1350+ (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1351+ sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1352+ sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1353+ sqe->va = cpu_to_le64(wqe->bind.va);
1354+ temp32 = cpu_to_le32(wqe->bind.length);
1355+ memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1356+ break;
1357+ }
1358+ default:
1359+ /* Bad wqe, return error */
1360+ rc = -EINVAL;
1361+ goto done;
1362+ }
1363+ swq->next_psn = sq->psn & BTH_PSN_MASK;
1364+ if (swq->psn_search) {
1365+ swq->psn_search->opcode_start_psn = cpu_to_le32(
1366+ ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1367+ SQ_PSN_SEARCH_START_PSN_MASK) |
1368+ ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1369+ SQ_PSN_SEARCH_OPCODE_MASK));
1370+ swq->psn_search->flags_next_psn = cpu_to_le32(
1371+ ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1372+ SQ_PSN_SEARCH_NEXT_PSN_MASK));
1373+ }
1374+
1375+ sq->hwq.prod++;
1376+done:
1377+ return rc;
1378+}
1379+
1380+void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1381+{
1382+ struct bnxt_qplib_q *rq = &qp->rq;
1383+ struct dbr_dbr db_msg = { 0 };
1384+ u32 sw_prod;
1385+
1386+ sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1387+ db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1388+ DBR_DBR_INDEX_MASK);
1389+ db_msg.type_xid =
1390+ cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1391+ DBR_DBR_TYPE_RQ);
1392+
1393+ /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1394+ wmb();
1395+ __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1396+}
1397+
1398+int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1399+ struct bnxt_qplib_swqe *wqe)
1400+{
1401+ struct bnxt_qplib_q *rq = &qp->rq;
1402+ struct rq_wqe *rqe, **rqe_ptr;
1403+ struct sq_sge *hw_sge;
1404+ u32 sw_prod;
1405+ int i, rc = 0;
1406+
1407+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1408+ dev_err(&rq->hwq.pdev->dev,
1409+ "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1410+ qp->id, qp->state);
1411+ rc = -EINVAL;
1412+ goto done;
1413+ }
1414+ if (HWQ_CMP((rq->hwq.prod + 1), &rq->hwq) ==
1415+ HWQ_CMP(rq->hwq.cons, &rq->hwq)) {
1416+ dev_err(&rq->hwq.pdev->dev,
1417+ "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1418+ rc = -EINVAL;
1419+ goto done;
1420+ }
1421+ sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1422+ rq->swq[sw_prod].wr_id = wqe->wr_id;
1423+
1424+ rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1425+ rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1426+
1427+ memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1428+
1429+ /* Calculate wqe_size16 and data_len */
1430+ for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1431+ i < wqe->num_sge; i++, hw_sge++) {
1432+ hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1433+ hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1434+ hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1435+ }
1436+ rqe->wqe_type = wqe->type;
1437+ rqe->flags = wqe->flags;
1438+ rqe->wqe_size = wqe->num_sge +
1439+ ((offsetof(typeof(*rqe), data) + 15) >> 4);
1440+
1441+ /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1442+ rqe->wr_id[0] = cpu_to_le32(sw_prod);
1443+
1444+ rq->hwq.prod++;
1445+done:
1446+ return rc;
1447+}
1448+
1449+/* CQ */
1450+
1451+/* Spinlock must be held */
1452+static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1453+{
1454+ struct dbr_dbr db_msg = { 0 };
1455+
1456+ db_msg.type_xid =
1457+ cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1458+ DBR_DBR_TYPE_CQ_ARMENA);
1459+ /* Flush memory writes before enabling the CQ */
1460+ wmb();
1461+ __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1462+}
1463+
1464+static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1465+{
1466+ struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1467+ struct dbr_dbr db_msg = { 0 };
1468+ u32 sw_cons;
1469+
1470+ /* Ring DB */
1471+ sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1472+ db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1473+ DBR_DBR_INDEX_MASK);
1474+ db_msg.type_xid =
1475+ cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1476+ arm_type);
1477+ /* flush memory writes before arming the CQ */
1478+ wmb();
1479+ __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1480+}
1481+
1482+int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1483+{
1484+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1485+ struct cmdq_create_cq req;
1486+ struct creq_create_cq_resp *resp;
1487+ struct bnxt_qplib_pbl *pbl;
1488+ u16 cmd_flags = 0;
1489+ int rc;
1490+
1491+ cq->hwq.max_elements = cq->max_wqe;
1492+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1493+ cq->nmap, &cq->hwq.max_elements,
1494+ BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1495+ PAGE_SIZE, HWQ_TYPE_QUEUE);
1496+ if (rc)
1497+ goto exit;
1498+
1499+ RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1500+
1501+ if (!cq->dpi) {
1502+ dev_err(&rcfw->pdev->dev,
1503+ "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1504+ return -EINVAL;
1505+ }
1506+ req.dpi = cpu_to_le32(cq->dpi->dpi);
1507+ req.cq_handle = cpu_to_le64(cq->cq_handle);
1508+
1509+ req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1510+ pbl = &cq->hwq.pbl[PBL_LVL_0];
1511+ req.pg_size_lvl = cpu_to_le32(
1512+ ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1513+ CMDQ_CREATE_CQ_LVL_SFT) |
1514+ (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1515+ pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1516+ pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1517+ pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1518+ pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1519+ pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1520+ CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1521+
1522+ req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1523+
1524+ req.cq_fco_cnq_id = cpu_to_le32(
1525+ (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1526+ CMDQ_CREATE_CQ_CNQ_ID_SFT);
1527+
1528+ resp = (struct creq_create_cq_resp *)
1529+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1530+ NULL, 0);
1531+ if (!resp) {
1532+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ send failed");
1533+ return -EINVAL;
1534+ }
1535+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1536+ /* Cmd timed out */
1537+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ timed out");
1538+ rc = -ETIMEDOUT;
1539+ goto fail;
1540+ }
1541+ if (resp->status ||
1542+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1543+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ failed ");
1544+ dev_err(&rcfw->pdev->dev,
1545+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1546+ resp->status, le16_to_cpu(req.cookie),
1547+ le16_to_cpu(resp->cookie));
1548+ rc = -EINVAL;
1549+ goto fail;
1550+ }
1551+ cq->id = le32_to_cpu(resp->xid);
1552+ cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1553+ cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1554+ init_waitqueue_head(&cq->waitq);
1555+
1556+ bnxt_qplib_arm_cq_enable(cq);
1557+ return 0;
1558+
1559+fail:
1560+ bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1561+exit:
1562+ return rc;
1563+}
1564+
1565+int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1566+{
1567+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1568+ struct cmdq_destroy_cq req;
1569+ struct creq_destroy_cq_resp *resp;
1570+ u16 cmd_flags = 0;
1571+
1572+ RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1573+
1574+ req.cq_cid = cpu_to_le32(cq->id);
1575+ resp = (struct creq_destroy_cq_resp *)
1576+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1577+ NULL, 0);
1578+ if (!resp) {
1579+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ send failed");
1580+ return -EINVAL;
1581+ }
1582+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1583+ /* Cmd timed out */
1584+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ timed out");
1585+ return -ETIMEDOUT;
1586+ }
1587+ if (resp->status ||
1588+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1589+ dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ failed ");
1590+ dev_err(&rcfw->pdev->dev,
1591+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1592+ resp->status, le16_to_cpu(req.cookie),
1593+ le16_to_cpu(resp->cookie));
1594+ return -EINVAL;
1595+ }
1596+ bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1597+ return 0;
1598+}
1599+
1600+static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
1601+ struct bnxt_qplib_cqe **pcqe, int *budget)
1602+{
1603+ u32 sw_prod, sw_cons;
1604+ struct bnxt_qplib_cqe *cqe;
1605+ int rc = 0;
1606+
1607+ /* Now complete all outstanding SQEs with FLUSHED_ERR */
1608+ sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1609+ cqe = *pcqe;
1610+ while (*budget) {
1611+ sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1612+ if (sw_cons == sw_prod) {
1613+ sq->flush_in_progress = false;
1614+ break;
1615+ }
1616+ memset(cqe, 0, sizeof(*cqe));
1617+ cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
1618+ cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1619+ cqe->qp_handle = (u64)(unsigned long)qp;
1620+ cqe->wr_id = sq->swq[sw_cons].wr_id;
1621+ cqe->src_qp = qp->id;
1622+ cqe->type = sq->swq[sw_cons].type;
1623+ cqe++;
1624+ (*budget)--;
1625+ sq->hwq.cons++;
1626+ }
1627+ *pcqe = cqe;
1628+ if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
1629+ /* Out of budget */
1630+ rc = -EAGAIN;
1631+
1632+ return rc;
1633+}
1634+
1635+static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
1636+ int opcode, struct bnxt_qplib_cqe **pcqe, int *budget)
1637+{
1638+ struct bnxt_qplib_cqe *cqe;
1639+ u32 sw_prod, sw_cons;
1640+ int rc = 0;
1641+
1642+ /* Flush the rest of the RQ */
1643+ sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1644+ cqe = *pcqe;
1645+ while (*budget) {
1646+ sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
1647+ if (sw_cons == sw_prod)
1648+ break;
1649+ memset(cqe, 0, sizeof(*cqe));
1650+ cqe->status =
1651+ CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
1652+ cqe->opcode = opcode;
1653+ cqe->qp_handle = (unsigned long)qp;
1654+ cqe->wr_id = rq->swq[sw_cons].wr_id;
1655+ cqe++;
1656+ (*budget)--;
1657+ rq->hwq.cons++;
1658+ }
1659+ *pcqe = cqe;
1660+ if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
1661+ /* Out of budget */
1662+ rc = -EAGAIN;
1663+
1664+ return rc;
1665+}
1666+
1667+static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
1668+ struct cq_req *hwcqe,
1669+ struct bnxt_qplib_cqe **pcqe, int *budget)
1670+{
1671+ struct bnxt_qplib_qp *qp;
1672+ struct bnxt_qplib_q *sq;
1673+ struct bnxt_qplib_cqe *cqe;
1674+ u32 sw_cons, cqe_cons;
1675+ int rc = 0;
1676+
1677+ qp = (struct bnxt_qplib_qp *)((unsigned long)
1678+ le64_to_cpu(hwcqe->qp_handle));
1679+ if (!qp) {
1680+ dev_err(&cq->hwq.pdev->dev,
1681+ "QPLIB: FP: Process Req qp is NULL");
1682+ return -EINVAL;
1683+ }
1684+ sq = &qp->sq;
1685+
1686+ cqe_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
1687+ if (cqe_cons > sq->hwq.max_elements) {
1688+ dev_err(&cq->hwq.pdev->dev,
1689+ "QPLIB: FP: CQ Process req reported ");
1690+ dev_err(&cq->hwq.pdev->dev,
1691+ "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1692+ cqe_cons, sq->hwq.max_elements);
1693+ return -EINVAL;
1694+ }
1695+ /* If we were in the middle of flushing the SQ, continue */
1696+ if (sq->flush_in_progress)
1697+ goto flush;
1698+
1699+ /* Require to walk the sq's swq to fabricate CQEs for all previously
1700+ * signaled SWQEs due to CQE aggregation from the current sq cons
1701+ * to the cqe_cons
1702+ */
1703+ cqe = *pcqe;
1704+ while (*budget) {
1705+ sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1706+ if (sw_cons == cqe_cons)
1707+ break;
1708+ memset(cqe, 0, sizeof(*cqe));
1709+ cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1710+ cqe->qp_handle = (u64)(unsigned long)qp;
1711+ cqe->src_qp = qp->id;
1712+ cqe->wr_id = sq->swq[sw_cons].wr_id;
1713+ cqe->type = sq->swq[sw_cons].type;
1714+
1715+ /* For the last CQE, check for status. For errors, regardless
1716+ * of the request being signaled or not, it must complete with
1717+ * the hwcqe error status
1718+ */
1719+ if (HWQ_CMP((sw_cons + 1), &sq->hwq) == cqe_cons &&
1720+ hwcqe->status != CQ_REQ_STATUS_OK) {
1721+ cqe->status = hwcqe->status;
1722+ dev_err(&cq->hwq.pdev->dev,
1723+ "QPLIB: FP: CQ Processed Req ");
1724+ dev_err(&cq->hwq.pdev->dev,
1725+ "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
1726+ sw_cons, cqe->wr_id, cqe->status);
1727+ cqe++;
1728+ (*budget)--;
1729+ sq->flush_in_progress = true;
1730+ /* Must block new posting of SQ and RQ */
1731+ qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1732+ } else {
1733+ if (sq->swq[sw_cons].flags &
1734+ SQ_SEND_FLAGS_SIGNAL_COMP) {
1735+ cqe->status = CQ_REQ_STATUS_OK;
1736+ cqe++;
1737+ (*budget)--;
1738+ }
1739+ }
1740+ sq->hwq.cons++;
1741+ }
1742+ *pcqe = cqe;
1743+ if (!*budget && HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_cons) {
1744+ /* Out of budget */
1745+ rc = -EAGAIN;
1746+ goto done;
1747+ }
1748+ if (!sq->flush_in_progress)
1749+ goto done;
1750+flush:
1751+ /* Require to walk the sq's swq to fabricate CQEs for all
1752+ * previously posted SWQEs due to the error CQE received
1753+ */
1754+ rc = __flush_sq(sq, qp, pcqe, budget);
1755+ if (!rc)
1756+ sq->flush_in_progress = false;
1757+done:
1758+ return rc;
1759+}
1760+
1761+static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
1762+ struct cq_res_rc *hwcqe,
1763+ struct bnxt_qplib_cqe **pcqe,
1764+ int *budget)
1765+{
1766+ struct bnxt_qplib_qp *qp;
1767+ struct bnxt_qplib_q *rq;
1768+ struct bnxt_qplib_cqe *cqe;
1769+ u32 wr_id_idx;
1770+ int rc = 0;
1771+
1772+ qp = (struct bnxt_qplib_qp *)((unsigned long)
1773+ le64_to_cpu(hwcqe->qp_handle));
1774+ if (!qp) {
1775+ dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
1776+ return -EINVAL;
1777+ }
1778+ cqe = *pcqe;
1779+ cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1780+ cqe->length = le32_to_cpu(hwcqe->length);
1781+ cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
1782+ cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
1783+ cqe->flags = le16_to_cpu(hwcqe->flags);
1784+ cqe->status = hwcqe->status;
1785+ cqe->qp_handle = (u64)(unsigned long)qp;
1786+
1787+ wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
1788+ CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
1789+ rq = &qp->rq;
1790+ if (wr_id_idx > rq->hwq.max_elements) {
1791+ dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC ");
1792+ dev_err(&cq->hwq.pdev->dev,
1793+ "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
1794+ wr_id_idx, rq->hwq.max_elements);
1795+ return -EINVAL;
1796+ }
1797+ if (rq->flush_in_progress)
1798+ goto flush_rq;
1799+
1800+ cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1801+ cqe++;
1802+ (*budget)--;
1803+ rq->hwq.cons++;
1804+ *pcqe = cqe;
1805+
1806+ if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1807+ rq->flush_in_progress = true;
1808+flush_rq:
1809+ rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget);
1810+ if (!rc)
1811+ rq->flush_in_progress = false;
1812+ }
1813+ return rc;
1814+}
1815+
1816+static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
1817+ struct cq_res_ud *hwcqe,
1818+ struct bnxt_qplib_cqe **pcqe,
1819+ int *budget)
1820+{
1821+ struct bnxt_qplib_qp *qp;
1822+ struct bnxt_qplib_q *rq;
1823+ struct bnxt_qplib_cqe *cqe;
1824+ u32 wr_id_idx;
1825+ int rc = 0;
1826+
1827+ qp = (struct bnxt_qplib_qp *)((unsigned long)
1828+ le64_to_cpu(hwcqe->qp_handle));
1829+ if (!qp) {
1830+ dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
1831+ return -EINVAL;
1832+ }
1833+ cqe = *pcqe;
1834+ cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1835+ cqe->length = le32_to_cpu(hwcqe->length);
1836+ cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
1837+ cqe->flags = le16_to_cpu(hwcqe->flags);
1838+ cqe->status = hwcqe->status;
1839+ cqe->qp_handle = (u64)(unsigned long)qp;
1840+ memcpy(cqe->smac, hwcqe->src_mac, 6);
1841+ wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
1842+ & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
1843+ cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
1844+ ((le32_to_cpu(
1845+ hwcqe->src_qp_high_srq_or_rq_wr_id) &
1846+ CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
1847+
1848+ rq = &qp->rq;
1849+ if (wr_id_idx > rq->hwq.max_elements) {
1850+ dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD ");
1851+ dev_err(&cq->hwq.pdev->dev,
1852+ "QPLIB: wr_id idx %#x exceeded RQ max %#x",
1853+ wr_id_idx, rq->hwq.max_elements);
1854+ return -EINVAL;
1855+ }
1856+ if (rq->flush_in_progress)
1857+ goto flush_rq;
1858+
1859+ cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1860+ cqe++;
1861+ (*budget)--;
1862+ rq->hwq.cons++;
1863+ *pcqe = cqe;
1864+
1865+ if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1866+ rq->flush_in_progress = true;
1867+flush_rq:
1868+ rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget);
1869+ if (!rc)
1870+ rq->flush_in_progress = false;
1871+ }
1872+ return rc;
1873+}
1874+
1875+static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
1876+ struct cq_res_raweth_qp1 *hwcqe,
1877+ struct bnxt_qplib_cqe **pcqe,
1878+ int *budget)
1879+{
1880+ struct bnxt_qplib_qp *qp;
1881+ struct bnxt_qplib_q *rq;
1882+ struct bnxt_qplib_cqe *cqe;
1883+ u32 wr_id_idx;
1884+ int rc = 0;
1885+
1886+ qp = (struct bnxt_qplib_qp *)((unsigned long)
1887+ le64_to_cpu(hwcqe->qp_handle));
1888+ if (!qp) {
1889+ dev_err(&cq->hwq.pdev->dev,
1890+ "QPLIB: process_cq Raw/QP1 qp is NULL");
1891+ return -EINVAL;
1892+ }
1893+ cqe = *pcqe;
1894+ cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1895+ cqe->flags = le16_to_cpu(hwcqe->flags);
1896+ cqe->qp_handle = (u64)(unsigned long)qp;
1897+
1898+ wr_id_idx =
1899+ le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
1900+ & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
1901+ cqe->src_qp = qp->id;
1902+ if (qp->id == 1 && !cqe->length) {
1903+ /* Add workaround for the length misdetection */
1904+ cqe->length = 296;
1905+ } else {
1906+ cqe->length = le16_to_cpu(hwcqe->length);
1907+ }
1908+ cqe->pkey_index = qp->pkey_index;
1909+ memcpy(cqe->smac, qp->smac, 6);
1910+
1911+ cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
1912+ cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
1913+
1914+ rq = &qp->rq;
1915+ if (wr_id_idx > rq->hwq.max_elements) {
1916+ dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
1917+ dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
1918+ wr_id_idx, rq->hwq.max_elements);
1919+ return -EINVAL;
1920+ }
1921+ if (rq->flush_in_progress)
1922+ goto flush_rq;
1923+
1924+ cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1925+ cqe++;
1926+ (*budget)--;
1927+ rq->hwq.cons++;
1928+ *pcqe = cqe;
1929+
1930+ if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1931+ rq->flush_in_progress = true;
1932+flush_rq:
1933+ rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe,
1934+ budget);
1935+ if (!rc)
1936+ rq->flush_in_progress = false;
1937+ }
1938+ return rc;
1939+}
1940+
1941+static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
1942+ struct cq_terminal *hwcqe,
1943+ struct bnxt_qplib_cqe **pcqe,
1944+ int *budget)
1945+{
1946+ struct bnxt_qplib_qp *qp;
1947+ struct bnxt_qplib_q *sq, *rq;
1948+ struct bnxt_qplib_cqe *cqe;
1949+ u32 sw_cons = 0, cqe_cons;
1950+ int rc = 0;
1951+ u8 opcode = 0;
1952+
1953+ /* Check the Status */
1954+ if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
1955+ dev_warn(&cq->hwq.pdev->dev,
1956+ "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
1957+ hwcqe->status);
1958+
1959+ qp = (struct bnxt_qplib_qp *)((unsigned long)
1960+ le64_to_cpu(hwcqe->qp_handle));
1961+ if (!qp) {
1962+ dev_err(&cq->hwq.pdev->dev,
1963+ "QPLIB: FP: CQ Process terminal qp is NULL");
1964+ return -EINVAL;
1965+ }
1966+ /* Must block new posting of SQ and RQ */
1967+ qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1968+
1969+ sq = &qp->sq;
1970+ rq = &qp->rq;
1971+
1972+ cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
1973+ if (cqe_cons == 0xFFFF)
1974+ goto do_rq;
1975+
1976+ if (cqe_cons > sq->hwq.max_elements) {
1977+ dev_err(&cq->hwq.pdev->dev,
1978+ "QPLIB: FP: CQ Process terminal reported ");
1979+ dev_err(&cq->hwq.pdev->dev,
1980+ "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1981+ cqe_cons, sq->hwq.max_elements);
1982+ goto do_rq;
1983+ }
1984+ /* If we were in the middle of flushing, continue */
1985+ if (sq->flush_in_progress)
1986+ goto flush_sq;
1987+
1988+ /* Terminal CQE can also include aggregated successful CQEs prior.
1989+ * So we must complete all CQEs from the current sq's cons to the
1990+ * cq_cons with status OK
1991+ */
1992+ cqe = *pcqe;
1993+ while (*budget) {
1994+ sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1995+ if (sw_cons == cqe_cons)
1996+ break;
1997+ if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
1998+ memset(cqe, 0, sizeof(*cqe));
1999+ cqe->status = CQ_REQ_STATUS_OK;
2000+ cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2001+ cqe->qp_handle = (u64)(unsigned long)qp;
2002+ cqe->src_qp = qp->id;
2003+ cqe->wr_id = sq->swq[sw_cons].wr_id;
2004+ cqe->type = sq->swq[sw_cons].type;
2005+ cqe++;
2006+ (*budget)--;
2007+ }
2008+ sq->hwq.cons++;
2009+ }
2010+ *pcqe = cqe;
2011+ if (!(*budget) && sw_cons != cqe_cons) {
2012+ /* Out of budget */
2013+ rc = -EAGAIN;
2014+ goto sq_done;
2015+ }
2016+ sq->flush_in_progress = true;
2017+flush_sq:
2018+ rc = __flush_sq(sq, qp, pcqe, budget);
2019+ if (!rc)
2020+ sq->flush_in_progress = false;
2021+sq_done:
2022+ if (rc)
2023+ return rc;
2024+do_rq:
2025+ cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2026+ if (cqe_cons == 0xFFFF) {
2027+ goto done;
2028+ } else if (cqe_cons > rq->hwq.max_elements) {
2029+ dev_err(&cq->hwq.pdev->dev,
2030+ "QPLIB: FP: CQ Processed terminal ");
2031+ dev_err(&cq->hwq.pdev->dev,
2032+ "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2033+ cqe_cons, rq->hwq.max_elements);
2034+ goto done;
2035+ }
2036+ /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2037+ * from the current rq->cons to the rq->prod regardless what the
2038+ * rq->cons the terminal CQE indicates
2039+ */
2040+ rq->flush_in_progress = true;
2041+ switch (qp->type) {
2042+ case CMDQ_CREATE_QP1_TYPE_GSI:
2043+ opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2044+ break;
2045+ case CMDQ_CREATE_QP_TYPE_RC:
2046+ opcode = CQ_BASE_CQE_TYPE_RES_RC;
2047+ break;
2048+ case CMDQ_CREATE_QP_TYPE_UD:
2049+ opcode = CQ_BASE_CQE_TYPE_RES_UD;
2050+ break;
2051+ }
2052+
2053+ rc = __flush_rq(rq, qp, opcode, pcqe, budget);
2054+ if (!rc)
2055+ rq->flush_in_progress = false;
2056+done:
2057+ return rc;
2058+}
2059+
2060+static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2061+ struct cq_cutoff *hwcqe)
2062+{
2063+ /* Check the Status */
2064+ if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2065+ dev_err(&cq->hwq.pdev->dev,
2066+ "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2067+ hwcqe->status);
2068+ return -EINVAL;
2069+ }
2070+ clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2071+ wake_up_interruptible(&cq->waitq);
2072+
2073+ return 0;
2074+}
2075+
2076+int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2077+ int num_cqes)
2078+{
2079+ struct cq_base *hw_cqe, **hw_cqe_ptr;
2080+ unsigned long flags;
2081+ u32 sw_cons, raw_cons;
2082+ int budget, rc = 0;
2083+
2084+ spin_lock_irqsave(&cq->hwq.lock, flags);
2085+ raw_cons = cq->hwq.cons;
2086+ budget = num_cqes;
2087+
2088+ while (budget) {
2089+ sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2090+ hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2091+ hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2092+
2093+ /* Check for Valid bit */
2094+ if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2095+ break;
2096+
2097+ /* From the device's respective CQE format to qplib_wc*/
2098+ switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2099+ case CQ_BASE_CQE_TYPE_REQ:
2100+ rc = bnxt_qplib_cq_process_req(cq,
2101+ (struct cq_req *)hw_cqe,
2102+ &cqe, &budget);
2103+ break;
2104+ case CQ_BASE_CQE_TYPE_RES_RC:
2105+ rc = bnxt_qplib_cq_process_res_rc(cq,
2106+ (struct cq_res_rc *)
2107+ hw_cqe, &cqe,
2108+ &budget);
2109+ break;
2110+ case CQ_BASE_CQE_TYPE_RES_UD:
2111+ rc = bnxt_qplib_cq_process_res_ud
2112+ (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2113+ &budget);
2114+ break;
2115+ case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2116+ rc = bnxt_qplib_cq_process_res_raweth_qp1
2117+ (cq, (struct cq_res_raweth_qp1 *)
2118+ hw_cqe, &cqe, &budget);
2119+ break;
2120+ case CQ_BASE_CQE_TYPE_TERMINAL:
2121+ rc = bnxt_qplib_cq_process_terminal
2122+ (cq, (struct cq_terminal *)hw_cqe,
2123+ &cqe, &budget);
2124+ break;
2125+ case CQ_BASE_CQE_TYPE_CUT_OFF:
2126+ bnxt_qplib_cq_process_cutoff
2127+ (cq, (struct cq_cutoff *)hw_cqe);
2128+ /* Done processing this CQ */
2129+ goto exit;
2130+ default:
2131+ dev_err(&cq->hwq.pdev->dev,
2132+ "QPLIB: process_cq unknown type 0x%lx",
2133+ hw_cqe->cqe_type_toggle &
2134+ CQ_BASE_CQE_TYPE_MASK);
2135+ rc = -EINVAL;
2136+ break;
2137+ }
2138+ if (rc < 0) {
2139+ if (rc == -EAGAIN)
2140+ break;
2141+ /* Error while processing the CQE, just skip to the
2142+ * next one
2143+ */
2144+ dev_err(&cq->hwq.pdev->dev,
2145+ "QPLIB: process_cqe error rc = 0x%x", rc);
2146+ }
2147+ raw_cons++;
2148+ }
2149+ if (cq->hwq.cons != raw_cons) {
2150+ cq->hwq.cons = raw_cons;
2151+ bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2152+ }
2153+exit:
2154+ spin_unlock_irqrestore(&cq->hwq.lock, flags);
2155+ return num_cqes - budget;
2156+}
2157+
2158+void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2159+{
2160+ unsigned long flags;
2161+
2162+ spin_lock_irqsave(&cq->hwq.lock, flags);
2163+ if (arm_type)
2164+ bnxt_qplib_arm_cq(cq, arm_type);
2165+
2166+ spin_unlock_irqrestore(&cq->hwq.lock, flags);
2167+}
drivers/infiniband/hw/bnxt_re/qplib_fp.h+439−0
@@ -0,0 +1,439 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: Fast Path Operators (header)
37+ */
38+
39+#ifndef __BNXT_QPLIB_FP_H__
40+#define __BNXT_QPLIB_FP_H__
41+
42+struct bnxt_qplib_sge {
43+ u64 addr;
44+ u32 lkey;
45+ u32 size;
46+};
47+
48+#define BNXT_QPLIB_MAX_SQE_ENTRY_SIZE sizeof(struct sq_send)
49+
50+#define SQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_SQE_ENTRY_SIZE)
51+#define SQE_MAX_IDX_PER_PG (SQE_CNT_PER_PG - 1)
52+
53+static inline u32 get_sqe_pg(u32 val)
54+{
55+ return ((val & ~SQE_MAX_IDX_PER_PG) / SQE_CNT_PER_PG);
56+}
57+
58+static inline u32 get_sqe_idx(u32 val)
59+{
60+ return (val & SQE_MAX_IDX_PER_PG);
61+}
62+
63+#define BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE sizeof(struct sq_psn_search)
64+
65+#define PSNE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE)
66+#define PSNE_MAX_IDX_PER_PG (PSNE_CNT_PER_PG - 1)
67+
68+static inline u32 get_psne_pg(u32 val)
69+{
70+ return ((val & ~PSNE_MAX_IDX_PER_PG) / PSNE_CNT_PER_PG);
71+}
72+
73+static inline u32 get_psne_idx(u32 val)
74+{
75+ return (val & PSNE_MAX_IDX_PER_PG);
76+}
77+
78+#define BNXT_QPLIB_QP_MAX_SGL 6
79+
80+struct bnxt_qplib_swq {
81+ u64 wr_id;
82+ u8 type;
83+ u8 flags;
84+ u32 start_psn;
85+ u32 next_psn;
86+ struct sq_psn_search *psn_search;
87+};
88+
89+struct bnxt_qplib_swqe {
90+ /* General */
91+ u64 wr_id;
92+ u8 reqs_type;
93+ u8 type;
94+#define BNXT_QPLIB_SWQE_TYPE_SEND 0
95+#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1
96+#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2
97+#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4
98+#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5
99+#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6
100+#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8
101+#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11
102+#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12
103+#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13
104+#define BNXT_QPLIB_SWQE_TYPE_REG_MR 13
105+#define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14
106+#define BNXT_QPLIB_SWQE_TYPE_RECV 128
107+#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129
108+ u8 flags;
109+#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0)
110+#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1)
111+#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2)
112+#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3)
113+#define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4)
114+ struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL];
115+ int num_sge;
116+ /* Max inline data is 96 bytes */
117+ u32 inline_len;
118+#define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96
119+ u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH];
120+
121+ union {
122+ /* Send, with imm, inval key */
123+ struct {
124+ union {
125+ __be32 imm_data;
126+ u32 inv_key;
127+ };
128+ u32 q_key;
129+ u32 dst_qp;
130+ u16 avid;
131+ } send;
132+
133+ /* Send Raw Ethernet and QP1 */
134+ struct {
135+ u16 lflags;
136+ u16 cfa_action;
137+ u32 cfa_meta;
138+ } rawqp1;
139+
140+ /* RDMA write, with imm, read */
141+ struct {
142+ union {
143+ __be32 imm_data;
144+ u32 inv_key;
145+ };
146+ u64 remote_va;
147+ u32 r_key;
148+ } rdma;
149+
150+ /* Atomic cmp/swap, fetch/add */
151+ struct {
152+ u64 remote_va;
153+ u32 r_key;
154+ u64 swap_data;
155+ u64 cmp_data;
156+ } atomic;
157+
158+ /* Local Invalidate */
159+ struct {
160+ u32 inv_l_key;
161+ } local_inv;
162+
163+ /* FR-PMR */
164+ struct {
165+ u8 access_cntl;
166+ u8 pg_sz_log;
167+ bool zero_based;
168+ u32 l_key;
169+ u32 length;
170+ u8 pbl_pg_sz_log;
171+#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0
172+#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1
173+#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4
174+#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6
175+#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8
176+#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9
177+#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10
178+#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18
179+ u8 levels;
180+#define PAGE_SHIFT_4K 12
181+ __le64 *pbl_ptr;
182+ dma_addr_t pbl_dma_ptr;
183+ u64 *page_list;
184+ u16 page_list_len;
185+ u64 va;
186+ } frmr;
187+
188+ /* Bind */
189+ struct {
190+ u8 access_cntl;
191+#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0)
192+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1)
193+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2)
194+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3)
195+#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4)
196+ bool zero_based;
197+ u8 mw_type;
198+ u32 parent_l_key;
199+ u32 r_key;
200+ u64 va;
201+ u32 length;
202+ } bind;
203+ };
204+};
205+
206+#define BNXT_QPLIB_MAX_RQE_ENTRY_SIZE sizeof(struct rq_wqe)
207+
208+#define RQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_RQE_ENTRY_SIZE)
209+#define RQE_MAX_IDX_PER_PG (RQE_CNT_PER_PG - 1)
210+#define RQE_PG(x) (((x) & ~RQE_MAX_IDX_PER_PG) / RQE_CNT_PER_PG)
211+#define RQE_IDX(x) ((x) & RQE_MAX_IDX_PER_PG)
212+
213+struct bnxt_qplib_q {
214+ struct bnxt_qplib_hwq hwq;
215+ struct bnxt_qplib_swq *swq;
216+ struct scatterlist *sglist;
217+ u32 nmap;
218+ u32 max_wqe;
219+ u16 max_sge;
220+ u32 psn;
221+ bool flush_in_progress;
222+};
223+
224+struct bnxt_qplib_qp {
225+ struct bnxt_qplib_pd *pd;
226+ struct bnxt_qplib_dpi *dpi;
227+ u64 qp_handle;
228+ u32 id;
229+ u8 type;
230+ u8 sig_type;
231+ u32 modify_flags;
232+ u8 state;
233+ u8 cur_qp_state;
234+ u32 max_inline_data;
235+ u32 mtu;
236+ u8 path_mtu;
237+ bool en_sqd_async_notify;
238+ u16 pkey_index;
239+ u32 qkey;
240+ u32 dest_qp_id;
241+ u8 access;
242+ u8 timeout;
243+ u8 retry_cnt;
244+ u8 rnr_retry;
245+ u32 min_rnr_timer;
246+ u32 max_rd_atomic;
247+ u32 max_dest_rd_atomic;
248+ u32 dest_qpn;
249+ u8 smac[6];
250+ u16 vlan_id;
251+ u8 nw_type;
252+ struct bnxt_qplib_ah ah;
253+
254+#define BTH_PSN_MASK ((1 << 24) - 1)
255+ /* SQ */
256+ struct bnxt_qplib_q sq;
257+ /* RQ */
258+ struct bnxt_qplib_q rq;
259+ /* SRQ */
260+ struct bnxt_qplib_srq *srq;
261+ /* CQ */
262+ struct bnxt_qplib_cq *scq;
263+ struct bnxt_qplib_cq *rcq;
264+ /* IRRQ and ORRQ */
265+ struct bnxt_qplib_hwq irrq;
266+ struct bnxt_qplib_hwq orrq;
267+ /* Header buffer for QP1 */
268+ int sq_hdr_buf_size;
269+ int rq_hdr_buf_size;
270+/*
271+ * Buffer space for ETH(14), IP or GRH(40), UDP header(8)
272+ * and ib_bth + ib_deth (20).
273+ * Max required is 82 when RoCE V2 is enabled
274+ */
275+#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86
276+ /* Ethernet header = 14 */
277+ /* ib_grh = 40 (provided by MAD) */
278+ /* ib_bth + ib_deth = 20 */
279+ /* MAD = 256 (provided by MAD) */
280+ /* iCRC = 4 */
281+#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14
282+#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512
283+#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20
284+#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40
285+#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20
286+ void *sq_hdr_buf;
287+ dma_addr_t sq_hdr_buf_map;
288+ void *rq_hdr_buf;
289+ dma_addr_t rq_hdr_buf_map;
290+};
291+
292+#define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base)
293+
294+#define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE)
295+#define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1)
296+#define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG)
297+#define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG)
298+
299+#define ROCE_CQE_CMP_V 0
300+#define CQE_CMP_VALID(hdr, raw_cons, cp_bit) \
301+ (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \
302+ !((raw_cons) & (cp_bit)))
303+
304+struct bnxt_qplib_cqe {
305+ u8 status;
306+ u8 type;
307+ u8 opcode;
308+ u32 length;
309+ u64 wr_id;
310+ union {
311+ __be32 immdata;
312+ u32 invrkey;
313+ };
314+ u64 qp_handle;
315+ u64 mr_handle;
316+ u16 flags;
317+ u8 smac[6];
318+ u32 src_qp;
319+ u16 raweth_qp1_flags;
320+ u16 raweth_qp1_errors;
321+ u16 raweth_qp1_cfa_code;
322+ u32 raweth_qp1_flags2;
323+ u32 raweth_qp1_metadata;
324+ u8 raweth_qp1_payload_offset;
325+ u16 pkey_index;
326+};
327+
328+#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01
329+struct bnxt_qplib_cq {
330+ struct bnxt_qplib_dpi *dpi;
331+ void __iomem *dbr_base;
332+ u32 max_wqe;
333+ u32 id;
334+ u16 count;
335+ u16 period;
336+ struct bnxt_qplib_hwq hwq;
337+ u32 cnq_hw_ring_id;
338+ bool resize_in_progress;
339+ struct scatterlist *sghead;
340+ u32 nmap;
341+ u64 cq_handle;
342+
343+#define CQ_RESIZE_WAIT_TIME_MS 500
344+ unsigned long flags;
345+#define CQ_FLAGS_RESIZE_IN_PROG 1
346+ wait_queue_head_t waitq;
347+};
348+
349+#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq)
350+#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq)
351+#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2)
352+#define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1)
353+#define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1)
354+#define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1)
355+
356+#define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base)
357+
358+#define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE)
359+#define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1)
360+#define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG)
361+#define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG)
362+
363+#define NQE_CMP_VALID(hdr, raw_cons, cp_bit) \
364+ (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \
365+ !((raw_cons) & (cp_bit)))
366+
367+#define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024)
368+
369+#define NQ_CONS_PCI_BAR_REGION 2
370+#define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT)
371+#define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID
372+#define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK
373+#define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \
374+ NQ_DB_IDX_VALID)
375+#define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \
376+ NQ_DB_IDX_VALID | \
377+ NQ_DB_IRQ_DIS)
378+#define NQ_DB_REARM(db, raw_cons, cp_bit) \
379+ writel(NQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db)
380+#define NQ_DB(db, raw_cons, cp_bit) \
381+ writel(NQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db)
382+
383+struct bnxt_qplib_nq {
384+ struct pci_dev *pdev;
385+
386+ int vector;
387+ int budget;
388+ bool requested;
389+ struct tasklet_struct worker;
390+ struct bnxt_qplib_hwq hwq;
391+
392+ u16 bar_reg;
393+ u16 bar_reg_off;
394+ u16 ring_id;
395+ void __iomem *bar_reg_iomem;
396+
397+ int (*cqn_handler)
398+ (struct bnxt_qplib_nq *nq,
399+ struct bnxt_qplib_cq *cq);
400+ int (*srqn_handler)
401+ (struct bnxt_qplib_nq *nq,
402+ void *srq,
403+ u8 event);
404+};
405+
406+void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq);
407+int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
408+ int msix_vector, int bar_reg_offset,
409+ int (*cqn_handler)(struct bnxt_qplib_nq *nq,
410+ struct bnxt_qplib_cq *cq),
411+ int (*srqn_handler)(struct bnxt_qplib_nq *nq,
412+ void *srq,
413+ u8 event));
414+int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
415+int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
416+int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
417+int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
418+int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
419+void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
420+ struct bnxt_qplib_sge *sge);
421+void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
422+ struct bnxt_qplib_sge *sge);
423+u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp);
424+dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp,
425+ u32 index);
426+void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp);
427+int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
428+ struct bnxt_qplib_swqe *wqe);
429+void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp);
430+int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
431+ struct bnxt_qplib_swqe *wqe);
432+int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
433+int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
434+int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
435+ int num);
436+void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
437+void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq);
438+int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq);
439+#endif /* __BNXT_QPLIB_FP_H__ */
drivers/infiniband/hw/bnxt_re/qplib_rcfw.c+694−0
@@ -0,0 +1,694 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: RDMA Controller HW interface
37+ */
38+#include <linux/interrupt.h>
39+#include <linux/spinlock.h>
40+#include <linux/pci.h>
41+#include <linux/prefetch.h>
42+#include "roce_hsi.h"
43+#include "qplib_res.h"
44+#include "qplib_rcfw.h"
45+static void bnxt_qplib_service_creq(unsigned long data);
46+
47+/* Hardware communication channel */
48+int bnxt_qplib_rcfw_wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
49+{
50+ u16 cbit;
51+ int rc;
52+
53+ cookie &= RCFW_MAX_COOKIE_VALUE;
54+ cbit = cookie % RCFW_MAX_OUTSTANDING_CMD;
55+ if (!test_bit(cbit, rcfw->cmdq_bitmap))
56+ dev_warn(&rcfw->pdev->dev,
57+ "QPLIB: CMD bit %d for cookie 0x%x is not set?",
58+ cbit, cookie);
59+
60+ rc = wait_event_timeout(rcfw->waitq,
61+ !test_bit(cbit, rcfw->cmdq_bitmap),
62+ msecs_to_jiffies(RCFW_CMD_WAIT_TIME_MS));
63+ if (!rc) {
64+ dev_warn(&rcfw->pdev->dev,
65+ "QPLIB: Bono Error: timeout %d msec, msg {0x%x}\n",
66+ RCFW_CMD_WAIT_TIME_MS, cookie);
67+ }
68+
69+ return rc;
70+};
71+
72+int bnxt_qplib_rcfw_block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
73+{
74+ u32 count = -1;
75+ u16 cbit;
76+
77+ cookie &= RCFW_MAX_COOKIE_VALUE;
78+ cbit = cookie % RCFW_MAX_OUTSTANDING_CMD;
79+ if (!test_bit(cbit, rcfw->cmdq_bitmap))
80+ goto done;
81+ do {
82+ bnxt_qplib_service_creq((unsigned long)rcfw);
83+ } while (test_bit(cbit, rcfw->cmdq_bitmap) && --count);
84+done:
85+ return count;
86+};
87+
88+void *bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
89+ struct cmdq_base *req, void **crsbe,
90+ u8 is_block)
91+{
92+ struct bnxt_qplib_crsq *crsq = &rcfw->crsq;
93+ struct bnxt_qplib_cmdqe *cmdqe, **cmdq_ptr;
94+ struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq;
95+ struct bnxt_qplib_hwq *crsb = &rcfw->crsb;
96+ struct bnxt_qplib_crsqe *crsqe = NULL;
97+ struct bnxt_qplib_crsbe **crsb_ptr;
98+ u32 sw_prod, cmdq_prod;
99+ u8 retry_cnt = 0xFF;
100+ dma_addr_t dma_addr;
101+ unsigned long flags;
102+ u32 size, opcode;
103+ u16 cookie, cbit;
104+ int pg, idx;
105+ u8 *preq;
106+
107+retry:
108+ opcode = req->opcode;
109+ if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) &&
110+ (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &&
111+ opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW)) {
112+ dev_err(&rcfw->pdev->dev,
113+ "QPLIB: RCFW not initialized, reject opcode 0x%x",
114+ opcode);
115+ return NULL;
116+ }
117+
118+ if (test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) &&
119+ opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {
120+ dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!");
121+ return NULL;
122+ }
123+
124+ /* Cmdq are in 16-byte units, each request can consume 1 or more
125+ * cmdqe
126+ */
127+ spin_lock_irqsave(&cmdq->lock, flags);
128+ if (req->cmd_size > cmdq->max_elements -
129+ ((HWQ_CMP(cmdq->prod, cmdq) - HWQ_CMP(cmdq->cons, cmdq)) &
130+ (cmdq->max_elements - 1))) {
131+ dev_err(&rcfw->pdev->dev, "QPLIB: RCFW: CMDQ is full!");
132+ spin_unlock_irqrestore(&cmdq->lock, flags);
133+
134+ if (!retry_cnt--)
135+ return NULL;
136+ goto retry;
137+ }
138+
139+ retry_cnt = 0xFF;
140+
141+ cookie = atomic_inc_return(&rcfw->seq_num) & RCFW_MAX_COOKIE_VALUE;
142+ cbit = cookie % RCFW_MAX_OUTSTANDING_CMD;
143+ if (is_block)
144+ cookie |= RCFW_CMD_IS_BLOCKING;
145+ req->cookie = cpu_to_le16(cookie);
146+ if (test_and_set_bit(cbit, rcfw->cmdq_bitmap)) {
147+ dev_err(&rcfw->pdev->dev,
148+ "QPLIB: RCFW MAX outstanding cmd reached!");
149+ atomic_dec(&rcfw->seq_num);
150+ spin_unlock_irqrestore(&cmdq->lock, flags);
151+
152+ if (!retry_cnt--)
153+ return NULL;
154+ goto retry;
155+ }
156+ /* Reserve a resp buffer slot if requested */
157+ if (req->resp_size && crsbe) {
158+ spin_lock(&crsb->lock);
159+ sw_prod = HWQ_CMP(crsb->prod, crsb);
160+ crsb_ptr = (struct bnxt_qplib_crsbe **)crsb->pbl_ptr;
161+ *crsbe = (void *)&crsb_ptr[get_crsb_pg(sw_prod)]
162+ [get_crsb_idx(sw_prod)];
163+ bnxt_qplib_crsb_dma_next(crsb->pbl_dma_ptr, sw_prod, &dma_addr);
164+ req->resp_addr = cpu_to_le64(dma_addr);
165+ crsb->prod++;
166+ spin_unlock(&crsb->lock);
167+
168+ req->resp_size = (sizeof(struct bnxt_qplib_crsbe) +
169+ BNXT_QPLIB_CMDQE_UNITS - 1) /
170+ BNXT_QPLIB_CMDQE_UNITS;
171+ }
172+ cmdq_ptr = (struct bnxt_qplib_cmdqe **)cmdq->pbl_ptr;
173+ preq = (u8 *)req;
174+ size = req->cmd_size * BNXT_QPLIB_CMDQE_UNITS;
175+ do {
176+ pg = 0;
177+ idx = 0;
178+
179+ /* Locate the next cmdq slot */
180+ sw_prod = HWQ_CMP(cmdq->prod, cmdq);
181+ cmdqe = &cmdq_ptr[get_cmdq_pg(sw_prod)][get_cmdq_idx(sw_prod)];
182+ if (!cmdqe) {
183+ dev_err(&rcfw->pdev->dev,
184+ "QPLIB: RCFW request failed with no cmdqe!");
185+ goto done;
186+ }
187+ /* Copy a segment of the req cmd to the cmdq */
188+ memset(cmdqe, 0, sizeof(*cmdqe));
189+ memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe)));
190+ preq += min_t(u32, size, sizeof(*cmdqe));
191+ size -= min_t(u32, size, sizeof(*cmdqe));
192+ cmdq->prod++;
193+ } while (size > 0);
194+
195+ cmdq_prod = cmdq->prod;
196+ if (rcfw->flags & FIRMWARE_FIRST_FLAG) {
197+ /* The very first doorbell write is required to set this flag
198+ * which prompts the FW to reset its internal pointers
199+ */
200+ cmdq_prod |= FIRMWARE_FIRST_FLAG;
201+ rcfw->flags &= ~FIRMWARE_FIRST_FLAG;
202+ }
203+ sw_prod = HWQ_CMP(crsq->prod, crsq);
204+ crsqe = &crsq->crsq[sw_prod];
205+ memset(crsqe, 0, sizeof(*crsqe));
206+ crsq->prod++;
207+ crsqe->req_size = req->cmd_size;
208+
209+ /* ring CMDQ DB */
210+ writel(cmdq_prod, rcfw->cmdq_bar_reg_iomem +
211+ rcfw->cmdq_bar_reg_prod_off);
212+ writel(RCFW_CMDQ_TRIG_VAL, rcfw->cmdq_bar_reg_iomem +
213+ rcfw->cmdq_bar_reg_trig_off);
214+done:
215+ spin_unlock_irqrestore(&cmdq->lock, flags);
216+ /* Return the CREQ response pointer */
217+ return crsqe ? &crsqe->qp_event : NULL;
218+}
219+
220+/* Completions */
221+static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw,
222+ struct creq_func_event *func_event)
223+{
224+ switch (func_event->event) {
225+ case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
226+ break;
227+ case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
228+ break;
229+ case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
230+ break;
231+ case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
232+ break;
233+ case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
234+ break;
235+ case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
236+ break;
237+ case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
238+ break;
239+ case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
240+ /* SRQ ctx error, call srq_handler??
241+ * But there's no SRQ handle!
242+ */
243+ break;
244+ case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
245+ break;
246+ case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
247+ break;
248+ case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
249+ break;
250+ case CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST:
251+ break;
252+ case CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED:
253+ break;
254+ default:
255+ return -EINVAL;
256+ }
257+ return 0;
258+}
259+
260+static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
261+ struct creq_qp_event *qp_event)
262+{
263+ struct bnxt_qplib_crsq *crsq = &rcfw->crsq;
264+ struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq;
265+ struct bnxt_qplib_crsqe *crsqe;
266+ u16 cbit, cookie, blocked = 0;
267+ unsigned long flags;
268+ u32 sw_cons;
269+
270+ switch (qp_event->event) {
271+ case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
272+ dev_dbg(&rcfw->pdev->dev,
273+ "QPLIB: Received QP error notification");
274+ break;
275+ default:
276+ /* Command Response */
277+ spin_lock_irqsave(&cmdq->lock, flags);
278+ sw_cons = HWQ_CMP(crsq->cons, crsq);
279+ crsqe = &crsq->crsq[sw_cons];
280+ crsq->cons++;
281+ memcpy(&crsqe->qp_event, qp_event, sizeof(crsqe->qp_event));
282+
283+ cookie = le16_to_cpu(crsqe->qp_event.cookie);
284+ blocked = cookie & RCFW_CMD_IS_BLOCKING;
285+ cookie &= RCFW_MAX_COOKIE_VALUE;
286+ cbit = cookie % RCFW_MAX_OUTSTANDING_CMD;
287+ if (!test_and_clear_bit(cbit, rcfw->cmdq_bitmap))
288+ dev_warn(&rcfw->pdev->dev,
289+ "QPLIB: CMD bit %d was not requested", cbit);
290+
291+ cmdq->cons += crsqe->req_size;
292+ spin_unlock_irqrestore(&cmdq->lock, flags);
293+ if (!blocked)
294+ wake_up(&rcfw->waitq);
295+ break;
296+ }
297+ return 0;
298+}
299+
300+/* SP - CREQ Completion handlers */
301+static void bnxt_qplib_service_creq(unsigned long data)
302+{
303+ struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data;
304+ struct bnxt_qplib_hwq *creq = &rcfw->creq;
305+ struct creq_base *creqe, **creq_ptr;
306+ u32 sw_cons, raw_cons;
307+ unsigned long flags;
308+ u32 type;
309+
310+ /* Service the CREQ until empty */
311+ spin_lock_irqsave(&creq->lock, flags);
312+ raw_cons = creq->cons;
313+ while (1) {
314+ sw_cons = HWQ_CMP(raw_cons, creq);
315+ creq_ptr = (struct creq_base **)creq->pbl_ptr;
316+ creqe = &creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)];
317+ if (!CREQ_CMP_VALID(creqe, raw_cons, creq->max_elements))
318+ break;
319+
320+ type = creqe->type & CREQ_BASE_TYPE_MASK;
321+ switch (type) {
322+ case CREQ_BASE_TYPE_QP_EVENT:
323+ if (!bnxt_qplib_process_qp_event
324+ (rcfw, (struct creq_qp_event *)creqe))
325+ rcfw->creq_qp_event_processed++;
326+ else {
327+ dev_warn(&rcfw->pdev->dev, "QPLIB: crsqe with");
328+ dev_warn(&rcfw->pdev->dev,
329+ "QPLIB: type = 0x%x not handled",
330+ type);
331+ }
332+ break;
333+ case CREQ_BASE_TYPE_FUNC_EVENT:
334+ if (!bnxt_qplib_process_func_event
335+ (rcfw, (struct creq_func_event *)creqe))
336+ rcfw->creq_func_event_processed++;
337+ else
338+ dev_warn
339+ (&rcfw->pdev->dev, "QPLIB:aeqe:%#x Not handled",
340+ type);
341+ break;
342+ default:
343+ dev_warn(&rcfw->pdev->dev, "QPLIB: creqe with ");
344+ dev_warn(&rcfw->pdev->dev,
345+ "QPLIB: op_event = 0x%x not handled", type);
346+ break;
347+ }
348+ raw_cons++;
349+ }
350+ if (creq->cons != raw_cons) {
351+ creq->cons = raw_cons;
352+ CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, raw_cons,
353+ creq->max_elements);
354+ }
355+ spin_unlock_irqrestore(&creq->lock, flags);
356+}
357+
358+static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
359+{
360+ struct bnxt_qplib_rcfw *rcfw = dev_instance;
361+ struct bnxt_qplib_hwq *creq = &rcfw->creq;
362+ struct creq_base **creq_ptr;
363+ u32 sw_cons;
364+
365+ /* Prefetch the CREQ element */
366+ sw_cons = HWQ_CMP(creq->cons, creq);
367+ creq_ptr = (struct creq_base **)rcfw->creq.pbl_ptr;
368+ prefetch(&creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]);
369+
370+ tasklet_schedule(&rcfw->worker);
371+
372+ return IRQ_HANDLED;
373+}
374+
375+/* RCFW */
376+int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
377+{
378+ struct creq_deinitialize_fw_resp *resp;
379+ struct cmdq_deinitialize_fw req;
380+ u16 cmd_flags = 0;
381+
382+ RCFW_CMD_PREP(req, DEINITIALIZE_FW, cmd_flags);
383+ resp = (struct creq_deinitialize_fw_resp *)
384+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
385+ NULL, 0);
386+ if (!resp)
387+ return -EINVAL;
388+
389+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie)))
390+ return -ETIMEDOUT;
391+
392+ if (resp->status ||
393+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie))
394+ return -EFAULT;
395+
396+ clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags);
397+ return 0;
398+}
399+
400+static int __get_pbl_pg_idx(struct bnxt_qplib_pbl *pbl)
401+{
402+ return (pbl->pg_size == ROCE_PG_SIZE_4K ?
403+ CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K :
404+ pbl->pg_size == ROCE_PG_SIZE_8K ?
405+ CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K :
406+ pbl->pg_size == ROCE_PG_SIZE_64K ?
407+ CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K :
408+ pbl->pg_size == ROCE_PG_SIZE_2M ?
409+ CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M :
410+ pbl->pg_size == ROCE_PG_SIZE_8M ?
411+ CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M :
412+ pbl->pg_size == ROCE_PG_SIZE_1G ?
413+ CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G :
414+ CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K);
415+}
416+
417+int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
418+ struct bnxt_qplib_ctx *ctx, int is_virtfn)
419+{
420+ struct creq_initialize_fw_resp *resp;
421+ struct cmdq_initialize_fw req;
422+ u16 cmd_flags = 0, level;
423+
424+ RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags);
425+
426+ /*
427+ * VFs need not setup the HW context area, PF
428+ * shall setup this area for VF. Skipping the
429+ * HW programming
430+ */
431+ if (is_virtfn)
432+ goto skip_ctx_setup;
433+
434+ level = ctx->qpc_tbl.level;
435+ req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) |
436+ __get_pbl_pg_idx(&ctx->qpc_tbl.pbl[level]);
437+ level = ctx->mrw_tbl.level;
438+ req.mrw_pg_size_mrw_lvl = (level << CMDQ_INITIALIZE_FW_MRW_LVL_SFT) |
439+ __get_pbl_pg_idx(&ctx->mrw_tbl.pbl[level]);
440+ level = ctx->srqc_tbl.level;
441+ req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) |
442+ __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]);
443+ level = ctx->cq_tbl.level;
444+ req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) |
445+ __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]);
446+ level = ctx->srqc_tbl.level;
447+ req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) |
448+ __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]);
449+ level = ctx->cq_tbl.level;
450+ req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) |
451+ __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]);
452+ level = ctx->tim_tbl.level;
453+ req.tim_pg_size_tim_lvl = (level << CMDQ_INITIALIZE_FW_TIM_LVL_SFT) |
454+ __get_pbl_pg_idx(&ctx->tim_tbl.pbl[level]);
455+ level = ctx->tqm_pde_level;
456+ req.tqm_pg_size_tqm_lvl = (level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) |
457+ __get_pbl_pg_idx(&ctx->tqm_pde.pbl[level]);
458+
459+ req.qpc_page_dir =
460+ cpu_to_le64(ctx->qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
461+ req.mrw_page_dir =
462+ cpu_to_le64(ctx->mrw_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
463+ req.srq_page_dir =
464+ cpu_to_le64(ctx->srqc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
465+ req.cq_page_dir =
466+ cpu_to_le64(ctx->cq_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
467+ req.tim_page_dir =
468+ cpu_to_le64(ctx->tim_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
469+ req.tqm_page_dir =
470+ cpu_to_le64(ctx->tqm_pde.pbl[PBL_LVL_0].pg_map_arr[0]);
471+
472+ req.number_of_qp = cpu_to_le32(ctx->qpc_tbl.max_elements);
473+ req.number_of_mrw = cpu_to_le32(ctx->mrw_tbl.max_elements);
474+ req.number_of_srq = cpu_to_le32(ctx->srqc_tbl.max_elements);
475+ req.number_of_cq = cpu_to_le32(ctx->cq_tbl.max_elements);
476+
477+ req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf);
478+ req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf);
479+ req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf);
480+ req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf);
481+ req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf);
482+
483+skip_ctx_setup:
484+ req.stat_ctx_id = cpu_to_le32(ctx->stats.fw_id);
485+ resp = (struct creq_initialize_fw_resp *)
486+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
487+ NULL, 0);
488+ if (!resp) {
489+ dev_err(&rcfw->pdev->dev,
490+ "QPLIB: RCFW: INITIALIZE_FW send failed");
491+ return -EINVAL;
492+ }
493+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
494+ /* Cmd timed out */
495+ dev_err(&rcfw->pdev->dev,
496+ "QPLIB: RCFW: INITIALIZE_FW timed out");
497+ return -ETIMEDOUT;
498+ }
499+ if (resp->status ||
500+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
501+ dev_err(&rcfw->pdev->dev,
502+ "QPLIB: RCFW: INITIALIZE_FW failed");
503+ return -EINVAL;
504+ }
505+ set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags);
506+ return 0;
507+}
508+
509+void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
510+{
511+ bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->crsb);
512+ kfree(rcfw->crsq.crsq);
513+ bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->cmdq);
514+ bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->creq);
515+
516+ rcfw->pdev = NULL;
517+}
518+
519+int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
520+ struct bnxt_qplib_rcfw *rcfw)
521+{
522+ rcfw->pdev = pdev;
523+ rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT;
524+ if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL, 0,
525+ &rcfw->creq.max_elements,
526+ BNXT_QPLIB_CREQE_UNITS, 0, PAGE_SIZE,
527+ HWQ_TYPE_L2_CMPL)) {
528+ dev_err(&rcfw->pdev->dev,
529+ "QPLIB: HW channel CREQ allocation failed");
530+ goto fail;
531+ }
532+ rcfw->cmdq.max_elements = BNXT_QPLIB_CMDQE_MAX_CNT;
533+ if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->cmdq, NULL, 0,
534+ &rcfw->cmdq.max_elements,
535+ BNXT_QPLIB_CMDQE_UNITS, 0, PAGE_SIZE,
536+ HWQ_TYPE_CTX)) {
537+ dev_err(&rcfw->pdev->dev,
538+ "QPLIB: HW channel CMDQ allocation failed");
539+ goto fail;
540+ }
541+
542+ rcfw->crsq.max_elements = rcfw->cmdq.max_elements;
543+ rcfw->crsq.crsq = kcalloc(rcfw->crsq.max_elements,
544+ sizeof(*rcfw->crsq.crsq), GFP_KERNEL);
545+ if (!rcfw->crsq.crsq)
546+ goto fail;
547+
548+ rcfw->crsb.max_elements = BNXT_QPLIB_CRSBE_MAX_CNT;
549+ if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->crsb, NULL, 0,
550+ &rcfw->crsb.max_elements,
551+ BNXT_QPLIB_CRSBE_UNITS, 0, PAGE_SIZE,
552+ HWQ_TYPE_CTX)) {
553+ dev_err(&rcfw->pdev->dev,
554+ "QPLIB: HW channel CRSB allocation failed");
555+ goto fail;
556+ }
557+ return 0;
558+
559+fail:
560+ bnxt_qplib_free_rcfw_channel(rcfw);
561+ return -ENOMEM;
562+}
563+
564+void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
565+{
566+ unsigned long indx;
567+
568+ /* Make sure the HW channel is stopped! */
569+ synchronize_irq(rcfw->vector);
570+ tasklet_disable(&rcfw->worker);
571+ tasklet_kill(&rcfw->worker);
572+
573+ if (rcfw->requested) {
574+ free_irq(rcfw->vector, rcfw);
575+ rcfw->requested = false;
576+ }
577+ if (rcfw->cmdq_bar_reg_iomem)
578+ iounmap(rcfw->cmdq_bar_reg_iomem);
579+ rcfw->cmdq_bar_reg_iomem = NULL;
580+
581+ if (rcfw->creq_bar_reg_iomem)
582+ iounmap(rcfw->creq_bar_reg_iomem);
583+ rcfw->creq_bar_reg_iomem = NULL;
584+
585+ indx = find_first_bit(rcfw->cmdq_bitmap, rcfw->bmap_size);
586+ if (indx != rcfw->bmap_size)
587+ dev_err(&rcfw->pdev->dev,
588+ "QPLIB: disabling RCFW with pending cmd-bit %lx", indx);
589+ kfree(rcfw->cmdq_bitmap);
590+ rcfw->bmap_size = 0;
591+
592+ rcfw->aeq_handler = NULL;
593+ rcfw->vector = 0;
594+}
595+
596+int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
597+ struct bnxt_qplib_rcfw *rcfw,
598+ int msix_vector,
599+ int cp_bar_reg_off, int virt_fn,
600+ int (*aeq_handler)(struct bnxt_qplib_rcfw *,
601+ struct creq_func_event *))
602+{
603+ resource_size_t res_base;
604+ struct cmdq_init init;
605+ u16 bmap_size;
606+ int rc;
607+
608+ /* General */
609+ atomic_set(&rcfw->seq_num, 0);
610+ rcfw->flags = FIRMWARE_FIRST_FLAG;
611+ bmap_size = BITS_TO_LONGS(RCFW_MAX_OUTSTANDING_CMD *
612+ sizeof(unsigned long));
613+ rcfw->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL);
614+ if (!rcfw->cmdq_bitmap)
615+ return -ENOMEM;
616+ rcfw->bmap_size = bmap_size;
617+
618+ /* CMDQ */
619+ rcfw->cmdq_bar_reg = RCFW_COMM_PCI_BAR_REGION;
620+ res_base = pci_resource_start(pdev, rcfw->cmdq_bar_reg);
621+ if (!res_base)
622+ return -ENOMEM;
623+
624+ rcfw->cmdq_bar_reg_iomem = ioremap_nocache(res_base +
625+ RCFW_COMM_BASE_OFFSET,
626+ RCFW_COMM_SIZE);
627+ if (!rcfw->cmdq_bar_reg_iomem) {
628+ dev_err(&rcfw->pdev->dev,
629+ "QPLIB: CMDQ BAR region %d mapping failed",
630+ rcfw->cmdq_bar_reg);
631+ return -ENOMEM;
632+ }
633+
634+ rcfw->cmdq_bar_reg_prod_off = virt_fn ? RCFW_VF_COMM_PROD_OFFSET :
635+ RCFW_PF_COMM_PROD_OFFSET;
636+
637+ rcfw->cmdq_bar_reg_trig_off = RCFW_COMM_TRIG_OFFSET;
638+
639+ /* CRSQ */
640+ rcfw->crsq.prod = 0;
641+ rcfw->crsq.cons = 0;
642+
643+ /* CREQ */
644+ rcfw->creq_bar_reg = RCFW_COMM_CONS_PCI_BAR_REGION;
645+ res_base = pci_resource_start(pdev, rcfw->creq_bar_reg);
646+ if (!res_base)
647+ dev_err(&rcfw->pdev->dev,
648+ "QPLIB: CREQ BAR region %d resc start is 0!",
649+ rcfw->creq_bar_reg);
650+ rcfw->creq_bar_reg_iomem = ioremap_nocache(res_base + cp_bar_reg_off,
651+ 4);
652+ if (!rcfw->creq_bar_reg_iomem) {
653+ dev_err(&rcfw->pdev->dev,
654+ "QPLIB: CREQ BAR region %d mapping failed",
655+ rcfw->creq_bar_reg);
656+ return -ENOMEM;
657+ }
658+ rcfw->creq_qp_event_processed = 0;
659+ rcfw->creq_func_event_processed = 0;
660+
661+ rcfw->vector = msix_vector;
662+ if (aeq_handler)
663+ rcfw->aeq_handler = aeq_handler;
664+
665+ tasklet_init(&rcfw->worker, bnxt_qplib_service_creq,
666+ (unsigned long)rcfw);
667+
668+ rcfw->requested = false;
669+ rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0,
670+ "bnxt_qplib_creq", rcfw);
671+ if (rc) {
672+ dev_err(&rcfw->pdev->dev,
673+ "QPLIB: Failed to request IRQ for CREQ rc = 0x%x", rc);
674+ bnxt_qplib_disable_rcfw_channel(rcfw);
675+ return rc;
676+ }
677+ rcfw->requested = true;
678+
679+ init_waitqueue_head(&rcfw->waitq);
680+
681+ CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, 0, rcfw->creq.max_elements);
682+
683+ init.cmdq_pbl = cpu_to_le64(rcfw->cmdq.pbl[PBL_LVL_0].pg_map_arr[0]);
684+ init.cmdq_size_cmdq_lvl = cpu_to_le16(
685+ ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) &
686+ CMDQ_INIT_CMDQ_SIZE_MASK) |
687+ ((rcfw->cmdq.level << CMDQ_INIT_CMDQ_LVL_SFT) &
688+ CMDQ_INIT_CMDQ_LVL_MASK));
689+ init.creq_ring_id = cpu_to_le16(rcfw->creq_ring_id);
690+
691+ /* Write to the Bono mailbox register */
692+ __iowrite32_copy(rcfw->cmdq_bar_reg_iomem, &init, sizeof(init) / 4);
693+ return 0;
694+}
drivers/infiniband/hw/bnxt_re/qplib_rcfw.h+231−0
@@ -0,0 +1,231 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: RDMA Controller HW interface (header)
37+ */
38+
39+#ifndef __BNXT_QPLIB_RCFW_H__
40+#define __BNXT_QPLIB_RCFW_H__
41+
42+#define RCFW_CMDQ_TRIG_VAL 1
43+#define RCFW_COMM_PCI_BAR_REGION 0
44+#define RCFW_COMM_CONS_PCI_BAR_REGION 2
45+#define RCFW_COMM_BASE_OFFSET 0x600
46+#define RCFW_PF_COMM_PROD_OFFSET 0xc
47+#define RCFW_VF_COMM_PROD_OFFSET 0xc
48+#define RCFW_COMM_TRIG_OFFSET 0x100
49+#define RCFW_COMM_SIZE 0x104
50+
51+#define RCFW_DBR_PCI_BAR_REGION 2
52+
53+#define RCFW_CMD_PREP(req, CMD, cmd_flags) \
54+ do { \
55+ memset(&(req), 0, sizeof((req))); \
56+ (req).opcode = CMDQ_BASE_OPCODE_##CMD; \
57+ (req).cmd_size = (sizeof((req)) + \
58+ BNXT_QPLIB_CMDQE_UNITS - 1) / \
59+ BNXT_QPLIB_CMDQE_UNITS; \
60+ (req).flags = cpu_to_le16(cmd_flags); \
61+ } while (0)
62+
63+#define RCFW_CMD_WAIT_TIME_MS 20000 /* 20 Seconds timeout */
64+
65+/* CMDQ elements */
66+#define BNXT_QPLIB_CMDQE_MAX_CNT 256
67+#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe)
68+#define BNXT_QPLIB_CMDQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CMDQE_UNITS)
69+
70+#define MAX_CMDQ_IDX (BNXT_QPLIB_CMDQE_MAX_CNT - 1)
71+#define MAX_CMDQ_IDX_PER_PG (BNXT_QPLIB_CMDQE_CNT_PER_PG - 1)
72+
73+#define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT
74+#define RCFW_MAX_COOKIE_VALUE 0x7FFF
75+#define RCFW_CMD_IS_BLOCKING 0x8000
76+
77+/* Cmdq contains a fix number of a 16-Byte slots */
78+struct bnxt_qplib_cmdqe {
79+ u8 data[16];
80+};
81+
82+static inline u32 get_cmdq_pg(u32 val)
83+{
84+ return (val & ~MAX_CMDQ_IDX_PER_PG) / BNXT_QPLIB_CMDQE_CNT_PER_PG;
85+}
86+
87+static inline u32 get_cmdq_idx(u32 val)
88+{
89+ return val & MAX_CMDQ_IDX_PER_PG;
90+}
91+
92+/* Crsq buf is 1024-Byte */
93+struct bnxt_qplib_crsbe {
94+ u8 data[1024];
95+};
96+
97+/* CRSQ SB */
98+#define BNXT_QPLIB_CRSBE_MAX_CNT 4
99+#define BNXT_QPLIB_CRSBE_UNITS sizeof(struct bnxt_qplib_crsbe)
100+#define BNXT_QPLIB_CRSBE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CRSBE_UNITS)
101+
102+#define MAX_CRSB_IDX (BNXT_QPLIB_CRSBE_MAX_CNT - 1)
103+#define MAX_CRSB_IDX_PER_PG (BNXT_QPLIB_CRSBE_CNT_PER_PG - 1)
104+
105+static inline u32 get_crsb_pg(u32 val)
106+{
107+ return (val & ~MAX_CRSB_IDX_PER_PG) / BNXT_QPLIB_CRSBE_CNT_PER_PG;
108+}
109+
110+static inline u32 get_crsb_idx(u32 val)
111+{
112+ return val & MAX_CRSB_IDX_PER_PG;
113+}
114+
115+static inline void bnxt_qplib_crsb_dma_next(dma_addr_t *pg_map_arr,
116+ u32 prod, dma_addr_t *dma_addr)
117+{
118+ *dma_addr = pg_map_arr[(prod) / BNXT_QPLIB_CRSBE_CNT_PER_PG];
119+ *dma_addr += ((prod) % BNXT_QPLIB_CRSBE_CNT_PER_PG) *
120+ BNXT_QPLIB_CRSBE_UNITS;
121+}
122+
123+/* CREQ */
124+/* Allocate 1 per QP for async error notification for now */
125+#define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024)
126+#define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */
127+#define BNXT_QPLIB_CREQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CREQE_UNITS)
128+
129+#define MAX_CREQ_IDX (BNXT_QPLIB_CREQE_MAX_CNT - 1)
130+#define MAX_CREQ_IDX_PER_PG (BNXT_QPLIB_CREQE_CNT_PER_PG - 1)
131+
132+static inline u32 get_creq_pg(u32 val)
133+{
134+ return (val & ~MAX_CREQ_IDX_PER_PG) / BNXT_QPLIB_CREQE_CNT_PER_PG;
135+}
136+
137+static inline u32 get_creq_idx(u32 val)
138+{
139+ return val & MAX_CREQ_IDX_PER_PG;
140+}
141+
142+#define BNXT_QPLIB_CREQE_PER_PG (PAGE_SIZE / sizeof(struct creq_base))
143+
144+#define CREQ_CMP_VALID(hdr, raw_cons, cp_bit) \
145+ (!!((hdr)->v & CREQ_BASE_V) == \
146+ !((raw_cons) & (cp_bit)))
147+
148+#define CREQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT)
149+#define CREQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID
150+#define CREQ_DB_IRQ_DIS CMPL_DOORBELL_MASK
151+#define CREQ_DB_CP_FLAGS_REARM (CREQ_DB_KEY_CP | \
152+ CREQ_DB_IDX_VALID)
153+#define CREQ_DB_CP_FLAGS (CREQ_DB_KEY_CP | \
154+ CREQ_DB_IDX_VALID | \
155+ CREQ_DB_IRQ_DIS)
156+#define CREQ_DB_REARM(db, raw_cons, cp_bit) \
157+ writel(CREQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db)
158+#define CREQ_DB(db, raw_cons, cp_bit) \
159+ writel(CREQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db)
160+
161+/* HWQ */
162+struct bnxt_qplib_crsqe {
163+ struct creq_qp_event qp_event;
164+ u32 req_size;
165+};
166+
167+struct bnxt_qplib_crsq {
168+ struct bnxt_qplib_crsqe *crsq;
169+ u32 prod;
170+ u32 cons;
171+ u32 max_elements;
172+};
173+
174+/* RCFW Communication Channels */
175+struct bnxt_qplib_rcfw {
176+ struct pci_dev *pdev;
177+ int vector;
178+ struct tasklet_struct worker;
179+ bool requested;
180+ unsigned long *cmdq_bitmap;
181+ u32 bmap_size;
182+ unsigned long flags;
183+#define FIRMWARE_INITIALIZED_FLAG 1
184+#define FIRMWARE_FIRST_FLAG BIT(31)
185+ wait_queue_head_t waitq;
186+ int (*aeq_handler)(struct bnxt_qplib_rcfw *,
187+ struct creq_func_event *);
188+ atomic_t seq_num;
189+
190+ /* Bar region info */
191+ void __iomem *cmdq_bar_reg_iomem;
192+ u16 cmdq_bar_reg;
193+ u16 cmdq_bar_reg_prod_off;
194+ u16 cmdq_bar_reg_trig_off;
195+ u16 creq_ring_id;
196+ u16 creq_bar_reg;
197+ void __iomem *creq_bar_reg_iomem;
198+
199+ /* Cmd-Resp and Async Event notification queue */
200+ struct bnxt_qplib_hwq creq;
201+ u64 creq_qp_event_processed;
202+ u64 creq_func_event_processed;
203+
204+ /* Actual Cmd and Resp Queues */
205+ struct bnxt_qplib_hwq cmdq;
206+ struct bnxt_qplib_crsq crsq;
207+ struct bnxt_qplib_hwq crsb;
208+};
209+
210+void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
211+int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
212+ struct bnxt_qplib_rcfw *rcfw);
213+void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
214+int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
215+ struct bnxt_qplib_rcfw *rcfw,
216+ int msix_vector,
217+ int cp_bar_reg_off, int virt_fn,
218+ int (*aeq_handler)
219+ (struct bnxt_qplib_rcfw *,
220+ struct creq_func_event *));
221+
222+int bnxt_qplib_rcfw_block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie);
223+int bnxt_qplib_rcfw_wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie);
224+void *bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
225+ struct cmdq_base *req, void **crsbe,
226+ u8 is_block);
227+
228+int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw);
229+int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
230+ struct bnxt_qplib_ctx *ctx, int is_virtfn);
231+#endif /* __BNXT_QPLIB_RCFW_H__ */
drivers/infiniband/hw/bnxt_re/qplib_res.c+825−0
@@ -0,0 +1,825 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: QPLib resource manager
37+ */
38+
39+#include <linux/spinlock.h>
40+#include <linux/pci.h>
41+#include <linux/interrupt.h>
42+#include <linux/inetdevice.h>
43+#include <linux/dma-mapping.h>
44+#include <linux/if_vlan.h>
45+#include "roce_hsi.h"
46+#include "qplib_res.h"
47+#include "qplib_sp.h"
48+#include "qplib_rcfw.h"
49+
50+static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
51+ struct bnxt_qplib_stats *stats);
52+static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
53+ struct bnxt_qplib_stats *stats);
54+
55+/* PBL */
56+static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
57+ bool is_umem)
58+{
59+ int i;
60+
61+ if (!is_umem) {
62+ for (i = 0; i < pbl->pg_count; i++) {
63+ if (pbl->pg_arr[i])
64+ dma_free_coherent(&pdev->dev, pbl->pg_size,
65+ (void *)((unsigned long)
66+ pbl->pg_arr[i] &
67+ PAGE_MASK),
68+ pbl->pg_map_arr[i]);
69+ else
70+ dev_warn(&pdev->dev,
71+ "QPLIB: PBL free pg_arr[%d] empty?!",
72+ i);
73+ pbl->pg_arr[i] = NULL;
74+ }
75+ }
76+ kfree(pbl->pg_arr);
77+ pbl->pg_arr = NULL;
78+ kfree(pbl->pg_map_arr);
79+ pbl->pg_map_arr = NULL;
80+ pbl->pg_count = 0;
81+ pbl->pg_size = 0;
82+}
83+
84+static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
85+ struct scatterlist *sghead, u32 pages, u32 pg_size)
86+{
87+ struct scatterlist *sg;
88+ bool is_umem = false;
89+ int i;
90+
91+ /* page ptr arrays */
92+ pbl->pg_arr = kcalloc(pages, sizeof(void *), GFP_KERNEL);
93+ if (!pbl->pg_arr)
94+ return -ENOMEM;
95+
96+ pbl->pg_map_arr = kcalloc(pages, sizeof(dma_addr_t), GFP_KERNEL);
97+ if (!pbl->pg_map_arr) {
98+ kfree(pbl->pg_arr);
99+ pbl->pg_arr = NULL;
100+ return -ENOMEM;
101+ }
102+ pbl->pg_count = 0;
103+ pbl->pg_size = pg_size;
104+
105+ if (!sghead) {
106+ for (i = 0; i < pages; i++) {
107+ pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
108+ pbl->pg_size,
109+ &pbl->pg_map_arr[i],
110+ GFP_KERNEL);
111+ if (!pbl->pg_arr[i])
112+ goto fail;
113+ memset(pbl->pg_arr[i], 0, pbl->pg_size);
114+ pbl->pg_count++;
115+ }
116+ } else {
117+ i = 0;
118+ is_umem = true;
119+ for_each_sg(sghead, sg, pages, i) {
120+ pbl->pg_map_arr[i] = sg_dma_address(sg);
121+ pbl->pg_arr[i] = sg_virt(sg);
122+ if (!pbl->pg_arr[i])
123+ goto fail;
124+
125+ pbl->pg_count++;
126+ }
127+ }
128+
129+ return 0;
130+
131+fail:
132+ __free_pbl(pdev, pbl, is_umem);
133+ return -ENOMEM;
134+}
135+
136+/* HWQ */
137+void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)
138+{
139+ int i;
140+
141+ if (!hwq->max_elements)
142+ return;
143+ if (hwq->level >= PBL_LVL_MAX)
144+ return;
145+
146+ for (i = 0; i < hwq->level + 1; i++) {
147+ if (i == hwq->level)
148+ __free_pbl(pdev, &hwq->pbl[i], hwq->is_user);
149+ else
150+ __free_pbl(pdev, &hwq->pbl[i], false);
151+ }
152+
153+ hwq->level = PBL_LVL_MAX;
154+ hwq->max_elements = 0;
155+ hwq->element_size = 0;
156+ hwq->prod = 0;
157+ hwq->cons = 0;
158+ hwq->cp_bit = 0;
159+}
160+
161+/* All HWQs are power of 2 in size */
162+int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
163+ struct scatterlist *sghead, int nmap,
164+ u32 *elements, u32 element_size, u32 aux,
165+ u32 pg_size, enum bnxt_qplib_hwq_type hwq_type)
166+{
167+ u32 pages, slots, size, aux_pages = 0, aux_size = 0;
168+ dma_addr_t *src_phys_ptr, **dst_virt_ptr;
169+ int i, rc;
170+
171+ hwq->level = PBL_LVL_MAX;
172+
173+ slots = roundup_pow_of_two(*elements);
174+ if (aux) {
175+ aux_size = roundup_pow_of_two(aux);
176+ aux_pages = (slots * aux_size) / pg_size;
177+ if ((slots * aux_size) % pg_size)
178+ aux_pages++;
179+ }
180+ size = roundup_pow_of_two(element_size);
181+
182+ if (!sghead) {
183+ hwq->is_user = false;
184+ pages = (slots * size) / pg_size + aux_pages;
185+ if ((slots * size) % pg_size)
186+ pages++;
187+ if (!pages)
188+ return -EINVAL;
189+ } else {
190+ hwq->is_user = true;
191+ pages = nmap;
192+ }
193+
194+ /* Alloc the 1st memory block; can be a PDL/PTL/PBL */
195+ if (sghead && (pages == MAX_PBL_LVL_0_PGS))
196+ rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], sghead,
197+ pages, pg_size);
198+ else
199+ rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], NULL, 1, pg_size);
200+ if (rc)
201+ goto fail;
202+
203+ hwq->level = PBL_LVL_0;
204+
205+ if (pages > MAX_PBL_LVL_0_PGS) {
206+ if (pages > MAX_PBL_LVL_1_PGS) {
207+ /* 2 levels of indirection */
208+ rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], NULL,
209+ MAX_PBL_LVL_1_PGS_FOR_LVL_2, pg_size);
210+ if (rc)
211+ goto fail;
212+ /* Fill in lvl0 PBL */
213+ dst_virt_ptr =
214+ (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
215+ src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
216+ for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
217+ dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
218+ src_phys_ptr[i] | PTU_PDE_VALID;
219+ hwq->level = PBL_LVL_1;
220+
221+ rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_2], sghead,
222+ pages, pg_size);
223+ if (rc)
224+ goto fail;
225+
226+ /* Fill in lvl1 PBL */
227+ dst_virt_ptr =
228+ (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
229+ src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
230+ for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
231+ dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
232+ src_phys_ptr[i] | PTU_PTE_VALID;
233+ }
234+ if (hwq_type == HWQ_TYPE_QUEUE) {
235+ /* Find the last pg of the size */
236+ i = hwq->pbl[PBL_LVL_2].pg_count;
237+ dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
238+ PTU_PTE_LAST;
239+ if (i > 1)
240+ dst_virt_ptr[PTR_PG(i - 2)]
241+ [PTR_IDX(i - 2)] |=
242+ PTU_PTE_NEXT_TO_LAST;
243+ }
244+ hwq->level = PBL_LVL_2;
245+ } else {
246+ u32 flag = hwq_type == HWQ_TYPE_L2_CMPL ? 0 :
247+ PTU_PTE_VALID;
248+
249+ /* 1 level of indirection */
250+ rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], sghead,
251+ pages, pg_size);
252+ if (rc)
253+ goto fail;
254+ /* Fill in lvl0 PBL */
255+ dst_virt_ptr =
256+ (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
257+ src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
258+ for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) {
259+ dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
260+ src_phys_ptr[i] | flag;
261+ }
262+ if (hwq_type == HWQ_TYPE_QUEUE) {
263+ /* Find the last pg of the size */
264+ i = hwq->pbl[PBL_LVL_1].pg_count;
265+ dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
266+ PTU_PTE_LAST;
267+ if (i > 1)
268+ dst_virt_ptr[PTR_PG(i - 2)]
269+ [PTR_IDX(i - 2)] |=
270+ PTU_PTE_NEXT_TO_LAST;
271+ }
272+ hwq->level = PBL_LVL_1;
273+ }
274+ }
275+ hwq->pdev = pdev;
276+ spin_lock_init(&hwq->lock);
277+ hwq->prod = 0;
278+ hwq->cons = 0;
279+ *elements = hwq->max_elements = slots;
280+ hwq->element_size = size;
281+
282+ /* For direct access to the elements */
283+ hwq->pbl_ptr = hwq->pbl[hwq->level].pg_arr;
284+ hwq->pbl_dma_ptr = hwq->pbl[hwq->level].pg_map_arr;
285+
286+ return 0;
287+
288+fail:
289+ bnxt_qplib_free_hwq(pdev, hwq);
290+ return -ENOMEM;
291+}
292+
293+/* Context Tables */
294+void bnxt_qplib_free_ctx(struct pci_dev *pdev,
295+ struct bnxt_qplib_ctx *ctx)
296+{
297+ int i;
298+
299+ bnxt_qplib_free_hwq(pdev, &ctx->qpc_tbl);
300+ bnxt_qplib_free_hwq(pdev, &ctx->mrw_tbl);
301+ bnxt_qplib_free_hwq(pdev, &ctx->srqc_tbl);
302+ bnxt_qplib_free_hwq(pdev, &ctx->cq_tbl);
303+ bnxt_qplib_free_hwq(pdev, &ctx->tim_tbl);
304+ for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
305+ bnxt_qplib_free_hwq(pdev, &ctx->tqm_tbl[i]);
306+ bnxt_qplib_free_hwq(pdev, &ctx->tqm_pde);
307+ bnxt_qplib_free_stats_ctx(pdev, &ctx->stats);
308+}
309+
310+/*
311+ * Routine: bnxt_qplib_alloc_ctx
312+ * Description:
313+ * Context tables are memories which are used by the chip fw.
314+ * The 6 tables defined are:
315+ * QPC ctx - holds QP states
316+ * MRW ctx - holds memory region and window
317+ * SRQ ctx - holds shared RQ states
318+ * CQ ctx - holds completion queue states
319+ * TQM ctx - holds Tx Queue Manager context
320+ * TIM ctx - holds timer context
321+ * Depending on the size of the tbl requested, either a 1 Page Buffer List
322+ * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
323+ * instead.
324+ * Table might be employed as follows:
325+ * For 0 < ctx size <= 1 PAGE, 0 level of ind is used
326+ * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
327+ * For 512 < ctx size <= MAX, 2 levels of ind is used
328+ * Returns:
329+ * 0 if success, else -ERRORS
330+ */
331+int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
332+ struct bnxt_qplib_ctx *ctx,
333+ bool virt_fn)
334+{
335+ int i, j, k, rc = 0;
336+ int fnz_idx = -1;
337+ __le64 **pbl_ptr;
338+
339+ if (virt_fn)
340+ goto stats_alloc;
341+
342+ /* QPC Tables */
343+ ctx->qpc_tbl.max_elements = ctx->qpc_count;
344+ rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->qpc_tbl, NULL, 0,
345+ &ctx->qpc_tbl.max_elements,
346+ BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE, 0,
347+ PAGE_SIZE, HWQ_TYPE_CTX);
348+ if (rc)
349+ goto fail;
350+
351+ /* MRW Tables */
352+ ctx->mrw_tbl.max_elements = ctx->mrw_count;
353+ rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->mrw_tbl, NULL, 0,
354+ &ctx->mrw_tbl.max_elements,
355+ BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE, 0,
356+ PAGE_SIZE, HWQ_TYPE_CTX);
357+ if (rc)
358+ goto fail;
359+
360+ /* SRQ Tables */
361+ ctx->srqc_tbl.max_elements = ctx->srqc_count;
362+ rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->srqc_tbl, NULL, 0,
363+ &ctx->srqc_tbl.max_elements,
364+ BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE, 0,
365+ PAGE_SIZE, HWQ_TYPE_CTX);
366+ if (rc)
367+ goto fail;
368+
369+ /* CQ Tables */
370+ ctx->cq_tbl.max_elements = ctx->cq_count;
371+ rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->cq_tbl, NULL, 0,
372+ &ctx->cq_tbl.max_elements,
373+ BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE, 0,
374+ PAGE_SIZE, HWQ_TYPE_CTX);
375+ if (rc)
376+ goto fail;
377+
378+ /* TQM Buffer */
379+ ctx->tqm_pde.max_elements = 512;
380+ rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_pde, NULL, 0,
381+ &ctx->tqm_pde.max_elements, sizeof(u64),
382+ 0, PAGE_SIZE, HWQ_TYPE_CTX);
383+ if (rc)
384+ goto fail;
385+
386+ for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
387+ if (!ctx->tqm_count[i])
388+ continue;
389+ ctx->tqm_tbl[i].max_elements = ctx->qpc_count *
390+ ctx->tqm_count[i];
391+ rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_tbl[i], NULL, 0,
392+ &ctx->tqm_tbl[i].max_elements, 1,
393+ 0, PAGE_SIZE, HWQ_TYPE_CTX);
394+ if (rc)
395+ goto fail;
396+ }
397+ pbl_ptr = (__le64 **)ctx->tqm_pde.pbl_ptr;
398+ for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
399+ i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
400+ if (!ctx->tqm_tbl[i].max_elements)
401+ continue;
402+ if (fnz_idx == -1)
403+ fnz_idx = i;
404+ switch (ctx->tqm_tbl[i].level) {
405+ case PBL_LVL_2:
406+ for (k = 0; k < ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_count;
407+ k++)
408+ pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)] =
409+ cpu_to_le64(
410+ ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_map_arr[k]
411+ | PTU_PTE_VALID);
412+ break;
413+ case PBL_LVL_1:
414+ case PBL_LVL_0:
415+ default:
416+ pbl_ptr[PTR_PG(j)][PTR_IDX(j)] = cpu_to_le64(
417+ ctx->tqm_tbl[i].pbl[PBL_LVL_0].pg_map_arr[0] |
418+ PTU_PTE_VALID);
419+ break;
420+ }
421+ }
422+ if (fnz_idx == -1)
423+ fnz_idx = 0;
424+ ctx->tqm_pde_level = ctx->tqm_tbl[fnz_idx].level == PBL_LVL_2 ?
425+ PBL_LVL_2 : ctx->tqm_tbl[fnz_idx].level + 1;
426+
427+ /* TIM Buffer */
428+ ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
429+ rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tim_tbl, NULL, 0,
430+ &ctx->tim_tbl.max_elements, 1,
431+ 0, PAGE_SIZE, HWQ_TYPE_CTX);
432+ if (rc)
433+ goto fail;
434+
435+stats_alloc:
436+ /* Stats */
437+ rc = bnxt_qplib_alloc_stats_ctx(pdev, &ctx->stats);
438+ if (rc)
439+ goto fail;
440+
441+ return 0;
442+
443+fail:
444+ bnxt_qplib_free_ctx(pdev, ctx);
445+ return rc;
446+}
447+
448+/* GUID */
449+void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid)
450+{
451+ u8 mac[ETH_ALEN];
452+
453+ /* MAC-48 to EUI-64 mapping */
454+ memcpy(mac, dev_addr, ETH_ALEN);
455+ guid[0] = mac[0] ^ 2;
456+ guid[1] = mac[1];
457+ guid[2] = mac[2];
458+ guid[3] = 0xff;
459+ guid[4] = 0xfe;
460+ guid[5] = mac[3];
461+ guid[6] = mac[4];
462+ guid[7] = mac[5];
463+}
464+
465+static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,
466+ struct bnxt_qplib_sgid_tbl *sgid_tbl)
467+{
468+ kfree(sgid_tbl->tbl);
469+ kfree(sgid_tbl->hw_id);
470+ kfree(sgid_tbl->ctx);
471+ sgid_tbl->tbl = NULL;
472+ sgid_tbl->hw_id = NULL;
473+ sgid_tbl->ctx = NULL;
474+ sgid_tbl->max = 0;
475+ sgid_tbl->active = 0;
476+}
477+
478+static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,
479+ struct bnxt_qplib_sgid_tbl *sgid_tbl,
480+ u16 max)
481+{
482+ sgid_tbl->tbl = kcalloc(max, sizeof(struct bnxt_qplib_gid), GFP_KERNEL);
483+ if (!sgid_tbl->tbl)
484+ return -ENOMEM;
485+
486+ sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL);
487+ if (!sgid_tbl->hw_id)
488+ goto out_free1;
489+
490+ sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
491+ if (!sgid_tbl->ctx)
492+ goto out_free2;
493+
494+ sgid_tbl->max = max;
495+ return 0;
496+out_free2:
497+ kfree(sgid_tbl->hw_id);
498+ sgid_tbl->hw_id = NULL;
499+out_free1:
500+ kfree(sgid_tbl->tbl);
501+ sgid_tbl->tbl = NULL;
502+ return -ENOMEM;
503+};
504+
505+static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
506+ struct bnxt_qplib_sgid_tbl *sgid_tbl)
507+{
508+ int i;
509+
510+ for (i = 0; i < sgid_tbl->max; i++) {
511+ if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
512+ sizeof(bnxt_qplib_gid_zero)))
513+ bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i], true);
514+ }
515+ memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max);
516+ memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
517+ sgid_tbl->active = 0;
518+}
519+
520+static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
521+ struct net_device *netdev)
522+{
523+ memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max);
524+ memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
525+}
526+
527+static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res,
528+ struct bnxt_qplib_pkey_tbl *pkey_tbl)
529+{
530+ if (!pkey_tbl->tbl)
531+ dev_dbg(&res->pdev->dev, "QPLIB: PKEY tbl not present");
532+ else
533+ kfree(pkey_tbl->tbl);
534+
535+ pkey_tbl->tbl = NULL;
536+ pkey_tbl->max = 0;
537+ pkey_tbl->active = 0;
538+}
539+
540+static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res,
541+ struct bnxt_qplib_pkey_tbl *pkey_tbl,
542+ u16 max)
543+{
544+ pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL);
545+ if (!pkey_tbl->tbl)
546+ return -ENOMEM;
547+
548+ pkey_tbl->max = max;
549+ return 0;
550+};
551+
552+/* PDs */
553+int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd)
554+{
555+ u32 bit_num;
556+
557+ bit_num = find_first_bit(pdt->tbl, pdt->max);
558+ if (bit_num == pdt->max)
559+ return -ENOMEM;
560+
561+ /* Found unused PD */
562+ clear_bit(bit_num, pdt->tbl);
563+ pd->id = bit_num;
564+ return 0;
565+}
566+
567+int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
568+ struct bnxt_qplib_pd_tbl *pdt,
569+ struct bnxt_qplib_pd *pd)
570+{
571+ if (test_and_set_bit(pd->id, pdt->tbl)) {
572+ dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d",
573+ pd->id);
574+ return -EINVAL;
575+ }
576+ pd->id = 0;
577+ return 0;
578+}
579+
580+static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
581+{
582+ kfree(pdt->tbl);
583+ pdt->tbl = NULL;
584+ pdt->max = 0;
585+}
586+
587+static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
588+ struct bnxt_qplib_pd_tbl *pdt,
589+ u32 max)
590+{
591+ u32 bytes;
592+
593+ bytes = max >> 3;
594+ if (!bytes)
595+ bytes = 1;
596+ pdt->tbl = kmalloc(bytes, GFP_KERNEL);
597+ if (!pdt->tbl)
598+ return -ENOMEM;
599+
600+ pdt->max = max;
601+ memset((u8 *)pdt->tbl, 0xFF, bytes);
602+
603+ return 0;
604+}
605+
606+/* DPIs */
607+int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,
608+ struct bnxt_qplib_dpi *dpi,
609+ void *app)
610+{
611+ u32 bit_num;
612+
613+ bit_num = find_first_bit(dpit->tbl, dpit->max);
614+ if (bit_num == dpit->max)
615+ return -ENOMEM;
616+
617+ /* Found unused DPI */
618+ clear_bit(bit_num, dpit->tbl);
619+ dpit->app_tbl[bit_num] = app;
620+
621+ dpi->dpi = bit_num;
622+ dpi->dbr = dpit->dbr_bar_reg_iomem + (bit_num * PAGE_SIZE);
623+ dpi->umdbr = dpit->unmapped_dbr + (bit_num * PAGE_SIZE);
624+
625+ return 0;
626+}
627+
628+int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
629+ struct bnxt_qplib_dpi_tbl *dpit,
630+ struct bnxt_qplib_dpi *dpi)
631+{
632+ if (dpi->dpi >= dpit->max) {
633+ dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d", dpi->dpi);
634+ return -EINVAL;
635+ }
636+ if (test_and_set_bit(dpi->dpi, dpit->tbl)) {
637+ dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d",
638+ dpi->dpi);
639+ return -EINVAL;
640+ }
641+ if (dpit->app_tbl)
642+ dpit->app_tbl[dpi->dpi] = NULL;
643+ memset(dpi, 0, sizeof(*dpi));
644+
645+ return 0;
646+}
647+
648+static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res *res,
649+ struct bnxt_qplib_dpi_tbl *dpit)
650+{
651+ kfree(dpit->tbl);
652+ kfree(dpit->app_tbl);
653+ if (dpit->dbr_bar_reg_iomem)
654+ pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
655+ memset(dpit, 0, sizeof(*dpit));
656+}
657+
658+static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
659+ struct bnxt_qplib_dpi_tbl *dpit,
660+ u32 dbr_offset)
661+{
662+ u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION;
663+ resource_size_t bar_reg_base;
664+ u32 dbr_len, bytes;
665+
666+ if (dpit->dbr_bar_reg_iomem) {
667+ dev_err(&res->pdev->dev,
668+ "QPLIB: DBR BAR region %d already mapped", dbr_bar_reg);
669+ return -EALREADY;
670+ }
671+
672+ bar_reg_base = pci_resource_start(res->pdev, dbr_bar_reg);
673+ if (!bar_reg_base) {
674+ dev_err(&res->pdev->dev,
675+ "QPLIB: BAR region %d resc start failed", dbr_bar_reg);
676+ return -ENOMEM;
677+ }
678+
679+ dbr_len = pci_resource_len(res->pdev, dbr_bar_reg) - dbr_offset;
680+ if (!dbr_len || ((dbr_len & (PAGE_SIZE - 1)) != 0)) {
681+ dev_err(&res->pdev->dev, "QPLIB: Invalid DBR length %d",
682+ dbr_len);
683+ return -ENOMEM;
684+ }
685+
686+ dpit->dbr_bar_reg_iomem = ioremap_nocache(bar_reg_base + dbr_offset,
687+ dbr_len);
688+ if (!dpit->dbr_bar_reg_iomem) {
689+ dev_err(&res->pdev->dev,
690+ "QPLIB: FP: DBR BAR region %d mapping failed",
691+ dbr_bar_reg);
692+ return -ENOMEM;
693+ }
694+
695+ dpit->unmapped_dbr = bar_reg_base + dbr_offset;
696+ dpit->max = dbr_len / PAGE_SIZE;
697+
698+ dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL);
699+ if (!dpit->app_tbl) {
700+ pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
701+ dev_err(&res->pdev->dev,
702+ "QPLIB: DPI app tbl allocation failed");
703+ return -ENOMEM;
704+ }
705+
706+ bytes = dpit->max >> 3;
707+ if (!bytes)
708+ bytes = 1;
709+
710+ dpit->tbl = kmalloc(bytes, GFP_KERNEL);
711+ if (!dpit->tbl) {
712+ pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
713+ kfree(dpit->app_tbl);
714+ dpit->app_tbl = NULL;
715+ dev_err(&res->pdev->dev,
716+ "QPLIB: DPI tbl allocation failed for size = %d",
717+ bytes);
718+ return -ENOMEM;
719+ }
720+
721+ memset((u8 *)dpit->tbl, 0xFF, bytes);
722+
723+ return 0;
724+}
725+
726+/* PKEYs */
727+static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl)
728+{
729+ memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
730+ pkey_tbl->active = 0;
731+}
732+
733+static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res,
734+ struct bnxt_qplib_pkey_tbl *pkey_tbl)
735+{
736+ u16 pkey = 0xFFFF;
737+
738+ memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
739+
740+ /* pkey default = 0xFFFF */
741+ bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false);
742+}
743+
744+/* Stats */
745+static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
746+ struct bnxt_qplib_stats *stats)
747+{
748+ if (stats->dma) {
749+ dma_free_coherent(&pdev->dev, stats->size,
750+ stats->dma, stats->dma_map);
751+ }
752+ memset(stats, 0, sizeof(*stats));
753+ stats->fw_id = -1;
754+}
755+
756+static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
757+ struct bnxt_qplib_stats *stats)
758+{
759+ memset(stats, 0, sizeof(*stats));
760+ stats->fw_id = -1;
761+ stats->size = sizeof(struct ctx_hw_stats);
762+ stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
763+ &stats->dma_map, GFP_KERNEL);
764+ if (!stats->dma) {
765+ dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed");
766+ return -ENOMEM;
767+ }
768+ return 0;
769+}
770+
771+void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)
772+{
773+ bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl);
774+ bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
775+}
776+
777+int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
778+{
779+ bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
780+ bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl);
781+
782+ return 0;
783+}
784+
785+void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
786+{
787+ bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl);
788+ bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
789+ bnxt_qplib_free_pd_tbl(&res->pd_tbl);
790+ bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
791+
792+ res->netdev = NULL;
793+ res->pdev = NULL;
794+}
795+
796+int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,
797+ struct net_device *netdev,
798+ struct bnxt_qplib_dev_attr *dev_attr)
799+{
800+ int rc = 0;
801+
802+ res->pdev = pdev;
803+ res->netdev = netdev;
804+
805+ rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid);
806+ if (rc)
807+ goto fail;
808+
809+ rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey);
810+ if (rc)
811+ goto fail;
812+
813+ rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);
814+ if (rc)
815+ goto fail;
816+
817+ rc = bnxt_qplib_alloc_dpi_tbl(res, &res->dpi_tbl, dev_attr->l2_db_size);
818+ if (rc)
819+ goto fail;
820+
821+ return 0;
822+fail:
823+ bnxt_qplib_free_res(res);
824+ return rc;
825+}
drivers/infiniband/hw/bnxt_re/qplib_res.h+223−0
@@ -0,0 +1,223 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: QPLib resource manager (header)
37+ */
38+
39+#ifndef __BNXT_QPLIB_RES_H__
40+#define __BNXT_QPLIB_RES_H__
41+
42+extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;
43+
44+#define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *))
45+#define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1)
46+#define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG)
47+#define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG)
48+
49+#define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1))
50+
51+enum bnxt_qplib_hwq_type {
52+ HWQ_TYPE_CTX,
53+ HWQ_TYPE_QUEUE,
54+ HWQ_TYPE_L2_CMPL
55+};
56+
57+#define MAX_PBL_LVL_0_PGS 1
58+#define MAX_PBL_LVL_1_PGS 512
59+#define MAX_PBL_LVL_1_PGS_SHIFT 9
60+#define MAX_PBL_LVL_1_PGS_FOR_LVL_2 256
61+#define MAX_PBL_LVL_2_PGS (256 * 512)
62+
63+enum bnxt_qplib_pbl_lvl {
64+ PBL_LVL_0,
65+ PBL_LVL_1,
66+ PBL_LVL_2,
67+ PBL_LVL_MAX
68+};
69+
70+#define ROCE_PG_SIZE_4K (4 * 1024)
71+#define ROCE_PG_SIZE_8K (8 * 1024)
72+#define ROCE_PG_SIZE_64K (64 * 1024)
73+#define ROCE_PG_SIZE_2M (2 * 1024 * 1024)
74+#define ROCE_PG_SIZE_8M (8 * 1024 * 1024)
75+#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024)
76+
77+struct bnxt_qplib_pbl {
78+ u32 pg_count;
79+ u32 pg_size;
80+ void **pg_arr;
81+ dma_addr_t *pg_map_arr;
82+};
83+
84+struct bnxt_qplib_hwq {
85+ struct pci_dev *pdev;
86+ /* lock to protect qplib_hwq */
87+ spinlock_t lock;
88+ struct bnxt_qplib_pbl pbl[PBL_LVL_MAX];
89+ enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */
90+ /* ptr for easy access to the PBL entries */
91+ void **pbl_ptr;
92+ /* ptr for easy access to the dma_addr */
93+ dma_addr_t *pbl_dma_ptr;
94+ u32 max_elements;
95+ u16 element_size; /* Size of each entry */
96+
97+ u32 prod; /* raw */
98+ u32 cons; /* raw */
99+ u8 cp_bit;
100+ u8 is_user;
101+};
102+
103+/* Tables */
104+struct bnxt_qplib_pd_tbl {
105+ unsigned long *tbl;
106+ u32 max;
107+};
108+
109+struct bnxt_qplib_sgid_tbl {
110+ struct bnxt_qplib_gid *tbl;
111+ u16 *hw_id;
112+ u16 max;
113+ u16 active;
114+ void *ctx;
115+};
116+
117+struct bnxt_qplib_pkey_tbl {
118+ u16 *tbl;
119+ u16 max;
120+ u16 active;
121+};
122+
123+struct bnxt_qplib_dpi {
124+ u32 dpi;
125+ void __iomem *dbr;
126+ u64 umdbr;
127+};
128+
129+struct bnxt_qplib_dpi_tbl {
130+ void **app_tbl;
131+ unsigned long *tbl;
132+ u16 max;
133+ void __iomem *dbr_bar_reg_iomem;
134+ u64 unmapped_dbr;
135+};
136+
137+struct bnxt_qplib_stats {
138+ dma_addr_t dma_map;
139+ void *dma;
140+ u32 size;
141+ u32 fw_id;
142+};
143+
144+struct bnxt_qplib_vf_res {
145+ u32 max_qp_per_vf;
146+ u32 max_mrw_per_vf;
147+ u32 max_srq_per_vf;
148+ u32 max_cq_per_vf;
149+ u32 max_gid_per_vf;
150+};
151+
152+#define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448
153+#define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64
154+#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64
155+#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128
156+
157+struct bnxt_qplib_ctx {
158+ u32 qpc_count;
159+ struct bnxt_qplib_hwq qpc_tbl;
160+ u32 mrw_count;
161+ struct bnxt_qplib_hwq mrw_tbl;
162+ u32 srqc_count;
163+ struct bnxt_qplib_hwq srqc_tbl;
164+ u32 cq_count;
165+ struct bnxt_qplib_hwq cq_tbl;
166+ struct bnxt_qplib_hwq tim_tbl;
167+#define MAX_TQM_ALLOC_REQ 32
168+#define MAX_TQM_ALLOC_BLK_SIZE 8
169+ u8 tqm_count[MAX_TQM_ALLOC_REQ];
170+ struct bnxt_qplib_hwq tqm_pde;
171+ u32 tqm_pde_level;
172+ struct bnxt_qplib_hwq tqm_tbl[MAX_TQM_ALLOC_REQ];
173+ struct bnxt_qplib_stats stats;
174+ struct bnxt_qplib_vf_res vf_res;
175+};
176+
177+struct bnxt_qplib_res {
178+ struct pci_dev *pdev;
179+ struct net_device *netdev;
180+
181+ struct bnxt_qplib_rcfw *rcfw;
182+
183+ struct bnxt_qplib_pd_tbl pd_tbl;
184+ struct bnxt_qplib_sgid_tbl sgid_tbl;
185+ struct bnxt_qplib_pkey_tbl pkey_tbl;
186+ struct bnxt_qplib_dpi_tbl dpi_tbl;
187+};
188+
189+#define to_bnxt_qplib(ptr, type, member) \
190+ container_of(ptr, type, member)
191+
192+struct bnxt_qplib_pd;
193+struct bnxt_qplib_dev_attr;
194+
195+void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq);
196+int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
197+ struct scatterlist *sl, int nmap, u32 *elements,
198+ u32 elements_per_page, u32 aux, u32 pg_size,
199+ enum bnxt_qplib_hwq_type hwq_type);
200+void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid);
201+int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pd_tbl,
202+ struct bnxt_qplib_pd *pd);
203+int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
204+ struct bnxt_qplib_pd_tbl *pd_tbl,
205+ struct bnxt_qplib_pd *pd);
206+int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,
207+ struct bnxt_qplib_dpi *dpi,
208+ void *app);
209+int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
210+ struct bnxt_qplib_dpi_tbl *dpi_tbl,
211+ struct bnxt_qplib_dpi *dpi);
212+void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res);
213+int bnxt_qplib_init_res(struct bnxt_qplib_res *res);
214+void bnxt_qplib_free_res(struct bnxt_qplib_res *res);
215+int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,
216+ struct net_device *netdev,
217+ struct bnxt_qplib_dev_attr *dev_attr);
218+void bnxt_qplib_free_ctx(struct pci_dev *pdev,
219+ struct bnxt_qplib_ctx *ctx);
220+int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
221+ struct bnxt_qplib_ctx *ctx,
222+ bool virt_fn);
223+#endif /* __BNXT_QPLIB_RES_H__ */
drivers/infiniband/hw/bnxt_re/qplib_sp.c+838−0
@@ -0,0 +1,838 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: Slow Path Operators
37+ */
38+
39+#include <linux/interrupt.h>
40+#include <linux/spinlock.h>
41+#include <linux/sched.h>
42+#include <linux/pci.h>
43+
44+#include "roce_hsi.h"
45+
46+#include "qplib_res.h"
47+#include "qplib_rcfw.h"
48+#include "qplib_sp.h"
49+
50+const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,
51+ 0, 0, 0, 0, 0, 0, 0, 0 } };
52+
53+/* Device */
54+int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
55+ struct bnxt_qplib_dev_attr *attr)
56+{
57+ struct cmdq_query_func req;
58+ struct creq_query_func_resp *resp;
59+ struct creq_query_func_resp_sb *sb;
60+ u16 cmd_flags = 0;
61+ u32 temp;
62+ u8 *tqm_alloc;
63+ int i;
64+
65+ RCFW_CMD_PREP(req, QUERY_FUNC, cmd_flags);
66+
67+ req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
68+ resp = (struct creq_query_func_resp *)
69+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void **)&sb,
70+ 0);
71+ if (!resp) {
72+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC send failed");
73+ return -EINVAL;
74+ }
75+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
76+ /* Cmd timed out */
77+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC timed out");
78+ return -ETIMEDOUT;
79+ }
80+ if (resp->status ||
81+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
82+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC failed ");
83+ dev_err(&rcfw->pdev->dev,
84+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
85+ resp->status, le16_to_cpu(req.cookie),
86+ le16_to_cpu(resp->cookie));
87+ return -EINVAL;
88+ }
89+ /* Extract the context from the side buffer */
90+ attr->max_qp = le32_to_cpu(sb->max_qp);
91+ attr->max_qp_rd_atom =
92+ sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
93+ BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom;
94+ attr->max_qp_init_rd_atom =
95+ sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
96+ BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom;
97+ attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr);
98+ attr->max_qp_sges = sb->max_sge;
99+ attr->max_cq = le32_to_cpu(sb->max_cq);
100+ attr->max_cq_wqes = le32_to_cpu(sb->max_cqe);
101+ attr->max_cq_sges = attr->max_qp_sges;
102+ attr->max_mr = le32_to_cpu(sb->max_mr);
103+ attr->max_mw = le32_to_cpu(sb->max_mw);
104+
105+ attr->max_mr_size = le64_to_cpu(sb->max_mr_size);
106+ attr->max_pd = 64 * 1024;
107+ attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp);
108+ attr->max_ah = le32_to_cpu(sb->max_ah);
109+
110+ attr->max_fmr = le32_to_cpu(sb->max_fmr);
111+ attr->max_map_per_fmr = sb->max_map_per_fmr;
112+
113+ attr->max_srq = le16_to_cpu(sb->max_srq);
114+ attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
115+ attr->max_srq_sges = sb->max_srq_sge;
116+ /* Bono only reports 1 PKEY for now, but it can support > 1 */
117+ attr->max_pkey = le32_to_cpu(sb->max_pkeys);
118+
119+ attr->max_inline_data = le32_to_cpu(sb->max_inline_data);
120+ attr->l2_db_size = (sb->l2_db_space_size + 1) * PAGE_SIZE;
121+ attr->max_sgid = le32_to_cpu(sb->max_gid);
122+
123+ strlcpy(attr->fw_ver, "20.6.28.0", sizeof(attr->fw_ver));
124+
125+ for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) {
126+ temp = le32_to_cpu(sb->tqm_alloc_reqs[i]);
127+ tqm_alloc = (u8 *)&temp;
128+ attr->tqm_alloc_reqs[i * 4] = *tqm_alloc;
129+ attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc);
130+ attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc);
131+ attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc);
132+ }
133+ return 0;
134+}
135+
136+/* SGID */
137+int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
138+ struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
139+ struct bnxt_qplib_gid *gid)
140+{
141+ if (index > sgid_tbl->max) {
142+ dev_err(&res->pdev->dev,
143+ "QPLIB: Index %d exceeded SGID table max (%d)",
144+ index, sgid_tbl->max);
145+ return -EINVAL;
146+ }
147+ memcpy(gid, &sgid_tbl->tbl[index], sizeof(*gid));
148+ return 0;
149+}
150+
151+int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
152+ struct bnxt_qplib_gid *gid, bool update)
153+{
154+ struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
155+ struct bnxt_qplib_res,
156+ sgid_tbl);
157+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
158+ int index;
159+
160+ if (!sgid_tbl) {
161+ dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated");
162+ return -EINVAL;
163+ }
164+ /* Do we need a sgid_lock here? */
165+ if (!sgid_tbl->active) {
166+ dev_err(&res->pdev->dev,
167+ "QPLIB: SGID table has no active entries");
168+ return -ENOMEM;
169+ }
170+ for (index = 0; index < sgid_tbl->max; index++) {
171+ if (!memcmp(&sgid_tbl->tbl[index], gid, sizeof(*gid)))
172+ break;
173+ }
174+ if (index == sgid_tbl->max) {
175+ dev_warn(&res->pdev->dev, "GID not found in the SGID table");
176+ return 0;
177+ }
178+ /* Remove GID from the SGID table */
179+ if (update) {
180+ struct cmdq_delete_gid req;
181+ struct creq_delete_gid_resp *resp;
182+ u16 cmd_flags = 0;
183+
184+ RCFW_CMD_PREP(req, DELETE_GID, cmd_flags);
185+ if (sgid_tbl->hw_id[index] == 0xFFFF) {
186+ dev_err(&res->pdev->dev,
187+ "QPLIB: GID entry contains an invalid HW id");
188+ return -EINVAL;
189+ }
190+ req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]);
191+ resp = (struct creq_delete_gid_resp *)
192+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, NULL,
193+ 0);
194+ if (!resp) {
195+ dev_err(&res->pdev->dev,
196+ "QPLIB: SP: DELETE_GID send failed");
197+ return -EINVAL;
198+ }
199+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw,
200+ le16_to_cpu(req.cookie))) {
201+ /* Cmd timed out */
202+ dev_err(&res->pdev->dev,
203+ "QPLIB: SP: DELETE_GID timed out");
204+ return -ETIMEDOUT;
205+ }
206+ if (resp->status ||
207+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
208+ dev_err(&res->pdev->dev,
209+ "QPLIB: SP: DELETE_GID failed ");
210+ dev_err(&res->pdev->dev,
211+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
212+ resp->status, le16_to_cpu(req.cookie),
213+ le16_to_cpu(resp->cookie));
214+ return -EINVAL;
215+ }
216+ }
217+ memcpy(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero,
218+ sizeof(bnxt_qplib_gid_zero));
219+ sgid_tbl->active--;
220+ dev_dbg(&res->pdev->dev,
221+ "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x",
222+ index, sgid_tbl->hw_id[index], sgid_tbl->active);
223+ sgid_tbl->hw_id[index] = (u16)-1;
224+
225+ /* unlock */
226+ return 0;
227+}
228+
229+int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
230+ struct bnxt_qplib_gid *gid, u8 *smac, u16 vlan_id,
231+ bool update, u32 *index)
232+{
233+ struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
234+ struct bnxt_qplib_res,
235+ sgid_tbl);
236+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
237+ int i, free_idx, rc = 0;
238+
239+ if (!sgid_tbl) {
240+ dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated");
241+ return -EINVAL;
242+ }
243+ /* Do we need a sgid_lock here? */
244+ if (sgid_tbl->active == sgid_tbl->max) {
245+ dev_err(&res->pdev->dev, "QPLIB: SGID table is full");
246+ return -ENOMEM;
247+ }
248+ free_idx = sgid_tbl->max;
249+ for (i = 0; i < sgid_tbl->max; i++) {
250+ if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid))) {
251+ dev_dbg(&res->pdev->dev,
252+ "QPLIB: SGID entry already exist in entry %d!",
253+ i);
254+ *index = i;
255+ return -EALREADY;
256+ } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
257+ sizeof(bnxt_qplib_gid_zero)) &&
258+ free_idx == sgid_tbl->max) {
259+ free_idx = i;
260+ }
261+ }
262+ if (free_idx == sgid_tbl->max) {
263+ dev_err(&res->pdev->dev,
264+ "QPLIB: SGID table is FULL but count is not MAX??");
265+ return -ENOMEM;
266+ }
267+ if (update) {
268+ struct cmdq_add_gid req;
269+ struct creq_add_gid_resp *resp;
270+ u16 cmd_flags = 0;
271+ u32 temp32[4];
272+ u16 temp16[3];
273+
274+ RCFW_CMD_PREP(req, ADD_GID, cmd_flags);
275+
276+ memcpy(temp32, gid->data, sizeof(struct bnxt_qplib_gid));
277+ req.gid[0] = cpu_to_be32(temp32[3]);
278+ req.gid[1] = cpu_to_be32(temp32[2]);
279+ req.gid[2] = cpu_to_be32(temp32[1]);
280+ req.gid[3] = cpu_to_be32(temp32[0]);
281+ if (vlan_id != 0xFFFF)
282+ req.vlan = cpu_to_le16((vlan_id &
283+ CMDQ_ADD_GID_VLAN_VLAN_ID_MASK) |
284+ CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
285+ CMDQ_ADD_GID_VLAN_VLAN_EN);
286+
287+ /* MAC in network format */
288+ memcpy(temp16, smac, 6);
289+ req.src_mac[0] = cpu_to_be16(temp16[0]);
290+ req.src_mac[1] = cpu_to_be16(temp16[1]);
291+ req.src_mac[2] = cpu_to_be16(temp16[2]);
292+
293+ resp = (struct creq_add_gid_resp *)
294+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
295+ NULL, 0);
296+ if (!resp) {
297+ dev_err(&res->pdev->dev,
298+ "QPLIB: SP: ADD_GID send failed");
299+ return -EINVAL;
300+ }
301+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw,
302+ le16_to_cpu(req.cookie))) {
303+ /* Cmd timed out */
304+ dev_err(&res->pdev->dev,
305+ "QPIB: SP: ADD_GID timed out");
306+ return -ETIMEDOUT;
307+ }
308+ if (resp->status ||
309+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
310+ dev_err(&res->pdev->dev, "QPLIB: SP: ADD_GID failed ");
311+ dev_err(&res->pdev->dev,
312+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
313+ resp->status, le16_to_cpu(req.cookie),
314+ le16_to_cpu(resp->cookie));
315+ return -EINVAL;
316+ }
317+ sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp->xid);
318+ }
319+ /* Add GID to the sgid_tbl */
320+ memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid));
321+ sgid_tbl->active++;
322+ dev_dbg(&res->pdev->dev,
323+ "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x",
324+ free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active);
325+
326+ *index = free_idx;
327+ /* unlock */
328+ return rc;
329+}
330+
331+/* pkeys */
332+int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,
333+ struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,
334+ u16 *pkey)
335+{
336+ if (index == 0xFFFF) {
337+ *pkey = 0xFFFF;
338+ return 0;
339+ }
340+ if (index > pkey_tbl->max) {
341+ dev_err(&res->pdev->dev,
342+ "QPLIB: Index %d exceeded PKEY table max (%d)",
343+ index, pkey_tbl->max);
344+ return -EINVAL;
345+ }
346+ memcpy(pkey, &pkey_tbl->tbl[index], sizeof(*pkey));
347+ return 0;
348+}
349+
350+int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res,
351+ struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
352+ bool update)
353+{
354+ int i, rc = 0;
355+
356+ if (!pkey_tbl) {
357+ dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated");
358+ return -EINVAL;
359+ }
360+
361+ /* Do we need a pkey_lock here? */
362+ if (!pkey_tbl->active) {
363+ dev_err(&res->pdev->dev,
364+ "QPLIB: PKEY table has no active entries");
365+ return -ENOMEM;
366+ }
367+ for (i = 0; i < pkey_tbl->max; i++) {
368+ if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey)))
369+ break;
370+ }
371+ if (i == pkey_tbl->max) {
372+ dev_err(&res->pdev->dev,
373+ "QPLIB: PKEY 0x%04x not found in the pkey table",
374+ *pkey);
375+ return -ENOMEM;
376+ }
377+ memset(&pkey_tbl->tbl[i], 0, sizeof(*pkey));
378+ pkey_tbl->active--;
379+
380+ /* unlock */
381+ return rc;
382+}
383+
384+int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,
385+ struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
386+ bool update)
387+{
388+ int i, free_idx, rc = 0;
389+
390+ if (!pkey_tbl) {
391+ dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated");
392+ return -EINVAL;
393+ }
394+
395+ /* Do we need a pkey_lock here? */
396+ if (pkey_tbl->active == pkey_tbl->max) {
397+ dev_err(&res->pdev->dev, "QPLIB: PKEY table is full");
398+ return -ENOMEM;
399+ }
400+ free_idx = pkey_tbl->max;
401+ for (i = 0; i < pkey_tbl->max; i++) {
402+ if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey)))
403+ return -EALREADY;
404+ else if (!pkey_tbl->tbl[i] && free_idx == pkey_tbl->max)
405+ free_idx = i;
406+ }
407+ if (free_idx == pkey_tbl->max) {
408+ dev_err(&res->pdev->dev,
409+ "QPLIB: PKEY table is FULL but count is not MAX??");
410+ return -ENOMEM;
411+ }
412+ /* Add PKEY to the pkey_tbl */
413+ memcpy(&pkey_tbl->tbl[free_idx], pkey, sizeof(*pkey));
414+ pkey_tbl->active++;
415+
416+ /* unlock */
417+ return rc;
418+}
419+
420+/* AH */
421+int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah)
422+{
423+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
424+ struct cmdq_create_ah req;
425+ struct creq_create_ah_resp *resp;
426+ u16 cmd_flags = 0;
427+ u32 temp32[4];
428+ u16 temp16[3];
429+
430+ RCFW_CMD_PREP(req, CREATE_AH, cmd_flags);
431+
432+ memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid));
433+ req.dgid[0] = cpu_to_le32(temp32[0]);
434+ req.dgid[1] = cpu_to_le32(temp32[1]);
435+ req.dgid[2] = cpu_to_le32(temp32[2]);
436+ req.dgid[3] = cpu_to_le32(temp32[3]);
437+
438+ req.type = ah->nw_type;
439+ req.hop_limit = ah->hop_limit;
440+ req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]);
441+ req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label &
442+ CMDQ_CREATE_AH_FLOW_LABEL_MASK) |
443+ CMDQ_CREATE_AH_DEST_VLAN_ID_MASK);
444+ req.pd_id = cpu_to_le32(ah->pd->id);
445+ req.traffic_class = ah->traffic_class;
446+
447+ /* MAC in network format */
448+ memcpy(temp16, ah->dmac, 6);
449+ req.dest_mac[0] = cpu_to_le16(temp16[0]);
450+ req.dest_mac[1] = cpu_to_le16(temp16[1]);
451+ req.dest_mac[2] = cpu_to_le16(temp16[2]);
452+
453+ resp = (struct creq_create_ah_resp *)
454+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
455+ NULL, 1);
456+ if (!resp) {
457+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH send failed");
458+ return -EINVAL;
459+ }
460+ if (!bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie))) {
461+ /* Cmd timed out */
462+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH timed out");
463+ return -ETIMEDOUT;
464+ }
465+ if (resp->status ||
466+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
467+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH failed ");
468+ dev_err(&rcfw->pdev->dev,
469+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
470+ resp->status, le16_to_cpu(req.cookie),
471+ le16_to_cpu(resp->cookie));
472+ return -EINVAL;
473+ }
474+ ah->id = le32_to_cpu(resp->xid);
475+ return 0;
476+}
477+
478+int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah)
479+{
480+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
481+ struct cmdq_destroy_ah req;
482+ struct creq_destroy_ah_resp *resp;
483+ u16 cmd_flags = 0;
484+
485+ /* Clean up the AH table in the device */
486+ RCFW_CMD_PREP(req, DESTROY_AH, cmd_flags);
487+
488+ req.ah_cid = cpu_to_le32(ah->id);
489+
490+ resp = (struct creq_destroy_ah_resp *)
491+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
492+ NULL, 1);
493+ if (!resp) {
494+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH send failed");
495+ return -EINVAL;
496+ }
497+ if (!bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie))) {
498+ /* Cmd timed out */
499+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH timed out");
500+ return -ETIMEDOUT;
501+ }
502+ if (resp->status ||
503+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
504+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH failed ");
505+ dev_err(&rcfw->pdev->dev,
506+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
507+ resp->status, le16_to_cpu(req.cookie),
508+ le16_to_cpu(resp->cookie));
509+ return -EINVAL;
510+ }
511+ return 0;
512+}
513+
514+/* MRW */
515+int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
516+{
517+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
518+ struct cmdq_deallocate_key req;
519+ struct creq_deallocate_key_resp *resp;
520+ u16 cmd_flags = 0;
521+
522+ if (mrw->lkey == 0xFFFFFFFF) {
523+ dev_info(&res->pdev->dev,
524+ "QPLIB: SP: Free a reserved lkey MRW");
525+ return 0;
526+ }
527+
528+ RCFW_CMD_PREP(req, DEALLOCATE_KEY, cmd_flags);
529+
530+ req.mrw_flags = mrw->type;
531+
532+ if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) ||
533+ (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) ||
534+ (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B))
535+ req.key = cpu_to_le32(mrw->rkey);
536+ else
537+ req.key = cpu_to_le32(mrw->lkey);
538+
539+ resp = (struct creq_deallocate_key_resp *)
540+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
541+ NULL, 0);
542+ if (!resp) {
543+ dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR send failed");
544+ return -EINVAL;
545+ }
546+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
547+ /* Cmd timed out */
548+ dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR timed out");
549+ return -ETIMEDOUT;
550+ }
551+ if (resp->status ||
552+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
553+ dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR failed ");
554+ dev_err(&res->pdev->dev,
555+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
556+ resp->status, le16_to_cpu(req.cookie),
557+ le16_to_cpu(resp->cookie));
558+ return -EINVAL;
559+ }
560+ /* Free the qplib's MRW memory */
561+ if (mrw->hwq.max_elements)
562+ bnxt_qplib_free_hwq(res->pdev, &mrw->hwq);
563+
564+ return 0;
565+}
566+
567+int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
568+{
569+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
570+ struct cmdq_allocate_mrw req;
571+ struct creq_allocate_mrw_resp *resp;
572+ u16 cmd_flags = 0;
573+ unsigned long tmp;
574+
575+ RCFW_CMD_PREP(req, ALLOCATE_MRW, cmd_flags);
576+
577+ req.pd_id = cpu_to_le32(mrw->pd->id);
578+ req.mrw_flags = mrw->type;
579+ if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR &&
580+ mrw->flags & BNXT_QPLIB_FR_PMR) ||
581+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A ||
582+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)
583+ req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY;
584+ tmp = (unsigned long)mrw;
585+ req.mrw_handle = cpu_to_le64(tmp);
586+
587+ resp = (struct creq_allocate_mrw_resp *)
588+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
589+ NULL, 0);
590+ if (!resp) {
591+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW send failed");
592+ return -EINVAL;
593+ }
594+ if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
595+ /* Cmd timed out */
596+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW timed out");
597+ return -ETIMEDOUT;
598+ }
599+ if (resp->status ||
600+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
601+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW failed ");
602+ dev_err(&rcfw->pdev->dev,
603+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
604+ resp->status, le16_to_cpu(req.cookie),
605+ le16_to_cpu(resp->cookie));
606+ return -EINVAL;
607+ }
608+ if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) ||
609+ (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) ||
610+ (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B))
611+ mrw->rkey = le32_to_cpu(resp->xid);
612+ else
613+ mrw->lkey = le32_to_cpu(resp->xid);
614+ return 0;
615+}
616+
617+int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
618+ bool block)
619+{
620+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
621+ struct cmdq_deregister_mr req;
622+ struct creq_deregister_mr_resp *resp;
623+ u16 cmd_flags = 0;
624+ int rc;
625+
626+ RCFW_CMD_PREP(req, DEREGISTER_MR, cmd_flags);
627+
628+ req.lkey = cpu_to_le32(mrw->lkey);
629+ resp = (struct creq_deregister_mr_resp *)
630+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
631+ NULL, block);
632+ if (!resp) {
633+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: DEREG_MR send failed");
634+ return -EINVAL;
635+ }
636+ if (block)
637+ rc = bnxt_qplib_rcfw_block_for_resp(rcfw,
638+ le16_to_cpu(req.cookie));
639+ else
640+ rc = bnxt_qplib_rcfw_wait_for_resp(rcfw,
641+ le16_to_cpu(req.cookie));
642+ if (!rc) {
643+ /* Cmd timed out */
644+ dev_err(&res->pdev->dev, "QPLIB: SP: DEREG_MR timed out");
645+ return -ETIMEDOUT;
646+ }
647+ if (resp->status ||
648+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
649+ dev_err(&rcfw->pdev->dev, "QPLIB: SP: DEREG_MR failed ");
650+ dev_err(&rcfw->pdev->dev,
651+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
652+ resp->status, le16_to_cpu(req.cookie),
653+ le16_to_cpu(resp->cookie));
654+ return -EINVAL;
655+ }
656+
657+ /* Free the qplib's MR memory */
658+ if (mrw->hwq.max_elements) {
659+ mrw->va = 0;
660+ mrw->total_size = 0;
661+ bnxt_qplib_free_hwq(res->pdev, &mrw->hwq);
662+ }
663+
664+ return 0;
665+}
666+
667+int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
668+ u64 *pbl_tbl, int num_pbls, bool block)
669+{
670+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
671+ struct cmdq_register_mr req;
672+ struct creq_register_mr_resp *resp;
673+ u16 cmd_flags = 0, level;
674+ int pg_ptrs, pages, i, rc;
675+ dma_addr_t **pbl_ptr;
676+ u32 pg_size;
677+
678+ if (num_pbls) {
679+ pg_ptrs = roundup_pow_of_two(num_pbls);
680+ pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT;
681+ if (!pages)
682+ pages++;
683+
684+ if (pages > MAX_PBL_LVL_1_PGS) {
685+ dev_err(&res->pdev->dev, "QPLIB: SP: Reg MR pages ");
686+ dev_err(&res->pdev->dev,
687+ "requested (0x%x) exceeded max (0x%x)",
688+ pages, MAX_PBL_LVL_1_PGS);
689+ return -ENOMEM;
690+ }
691+ /* Free the hwq if it already exist, must be a rereg */
692+ if (mr->hwq.max_elements)
693+ bnxt_qplib_free_hwq(res->pdev, &mr->hwq);
694+
695+ mr->hwq.max_elements = pages;
696+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL, 0,
697+ &mr->hwq.max_elements,
698+ PAGE_SIZE, 0, PAGE_SIZE,
699+ HWQ_TYPE_CTX);
700+ if (rc) {
701+ dev_err(&res->pdev->dev,
702+ "SP: Reg MR memory allocation failed");
703+ return -ENOMEM;
704+ }
705+ /* Write to the hwq */
706+ pbl_ptr = (dma_addr_t **)mr->hwq.pbl_ptr;
707+ for (i = 0; i < num_pbls; i++)
708+ pbl_ptr[PTR_PG(i)][PTR_IDX(i)] =
709+ (pbl_tbl[i] & PAGE_MASK) | PTU_PTE_VALID;
710+ }
711+
712+ RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags);
713+
714+ /* Configure the request */
715+ if (mr->hwq.level == PBL_LVL_MAX) {
716+ level = 0;
717+ req.pbl = 0;
718+ pg_size = PAGE_SIZE;
719+ } else {
720+ level = mr->hwq.level + 1;
721+ req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
722+ pg_size = mr->hwq.pbl[PBL_LVL_0].pg_size;
723+ }
724+ req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) |
725+ ((ilog2(pg_size) <<
726+ CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) &
727+ CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK);
728+ req.access = (mr->flags & 0xFFFF);
729+ req.va = cpu_to_le64(mr->va);
730+ req.key = cpu_to_le32(mr->lkey);
731+ req.mr_size = cpu_to_le64(mr->total_size);
732+
733+ resp = (struct creq_register_mr_resp *)
734+ bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
735+ NULL, block);
736+ if (!resp) {
737+ dev_err(&res->pdev->dev, "SP: REG_MR send failed");
738+ rc = -EINVAL;
739+ goto fail;
740+ }
741+ if (block)
742+ rc = bnxt_qplib_rcfw_block_for_resp(rcfw,
743+ le16_to_cpu(req.cookie));
744+ else
745+ rc = bnxt_qplib_rcfw_wait_for_resp(rcfw,
746+ le16_to_cpu(req.cookie));
747+ if (!rc) {
748+ /* Cmd timed out */
749+ dev_err(&res->pdev->dev, "SP: REG_MR timed out");
750+ rc = -ETIMEDOUT;
751+ goto fail;
752+ }
753+ if (resp->status ||
754+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
755+ dev_err(&res->pdev->dev, "QPLIB: SP: REG_MR failed ");
756+ dev_err(&res->pdev->dev,
757+ "QPLIB: SP: with status 0x%x cmdq 0x%x resp 0x%x",
758+ resp->status, le16_to_cpu(req.cookie),
759+ le16_to_cpu(resp->cookie));
760+ rc = -EINVAL;
761+ goto fail;
762+ }
763+ return 0;
764+
765+fail:
766+ if (mr->hwq.max_elements)
767+ bnxt_qplib_free_hwq(res->pdev, &mr->hwq);
768+ return rc;
769+}
770+
771+int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
772+ struct bnxt_qplib_frpl *frpl,
773+ int max_pg_ptrs)
774+{
775+ int pg_ptrs, pages, rc;
776+
777+ /* Re-calculate the max to fit the HWQ allocation model */
778+ pg_ptrs = roundup_pow_of_two(max_pg_ptrs);
779+ pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT;
780+ if (!pages)
781+ pages++;
782+
783+ if (pages > MAX_PBL_LVL_1_PGS)
784+ return -ENOMEM;
785+
786+ frpl->hwq.max_elements = pages;
787+ rc = bnxt_qplib_alloc_init_hwq(res->pdev, &frpl->hwq, NULL, 0,
788+ &frpl->hwq.max_elements, PAGE_SIZE, 0,
789+ PAGE_SIZE, HWQ_TYPE_CTX);
790+ if (!rc)
791+ frpl->max_pg_ptrs = pg_ptrs;
792+
793+ return rc;
794+}
795+
796+int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
797+ struct bnxt_qplib_frpl *frpl)
798+{
799+ bnxt_qplib_free_hwq(res->pdev, &frpl->hwq);
800+ return 0;
801+}
802+
803+int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids)
804+{
805+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
806+ struct cmdq_map_tc_to_cos req;
807+ struct creq_map_tc_to_cos_resp *resp;
808+ u16 cmd_flags = 0;
809+ int tleft;
810+
811+ RCFW_CMD_PREP(req, MAP_TC_TO_COS, cmd_flags);
812+ req.cos0 = cpu_to_le16(cids[0]);
813+ req.cos1 = cpu_to_le16(cids[1]);
814+
815+ resp = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, NULL, 0);
816+ if (!resp) {
817+ dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS send failed");
818+ return -EINVAL;
819+ }
820+
821+ tleft = bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie));
822+ if (!tleft) {
823+ dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS timed out");
824+ return -ETIMEDOUT;
825+ }
826+
827+ if (resp->status ||
828+ le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
829+ dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS failed ");
830+ dev_err(&res->pdev->dev,
831+ "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
832+ resp->status, le16_to_cpu(req.cookie),
833+ le16_to_cpu(resp->cookie));
834+ return -EINVAL;
835+ }
836+
837+ return 0;
838+}
drivers/infiniband/hw/bnxt_re/qplib_sp.h+160−0
@@ -0,0 +1,160 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: Slow Path Operators (header)
37+ *
38+ */
39+
40+#ifndef __BNXT_QPLIB_SP_H__
41+#define __BNXT_QPLIB_SP_H__
42+
43+struct bnxt_qplib_dev_attr {
44+ char fw_ver[32];
45+ u16 max_sgid;
46+ u16 max_mrw;
47+ u32 max_qp;
48+#define BNXT_QPLIB_MAX_OUT_RD_ATOM 126
49+ u32 max_qp_rd_atom;
50+ u32 max_qp_init_rd_atom;
51+ u32 max_qp_wqes;
52+ u32 max_qp_sges;
53+ u32 max_cq;
54+ u32 max_cq_wqes;
55+ u32 max_cq_sges;
56+ u32 max_mr;
57+ u64 max_mr_size;
58+ u32 max_pd;
59+ u32 max_mw;
60+ u32 max_raw_ethy_qp;
61+ u32 max_ah;
62+ u32 max_fmr;
63+ u32 max_map_per_fmr;
64+ u32 max_srq;
65+ u32 max_srq_wqes;
66+ u32 max_srq_sges;
67+ u32 max_pkey;
68+ u32 max_inline_data;
69+ u32 l2_db_size;
70+ u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ];
71+};
72+
73+struct bnxt_qplib_pd {
74+ u32 id;
75+};
76+
77+struct bnxt_qplib_gid {
78+ u8 data[16];
79+};
80+
81+struct bnxt_qplib_ah {
82+ struct bnxt_qplib_gid dgid;
83+ struct bnxt_qplib_pd *pd;
84+ u32 id;
85+ u8 sgid_index;
86+ /* For Query AH if the hw table and SW table are differnt */
87+ u8 host_sgid_index;
88+ u8 traffic_class;
89+ u32 flow_label;
90+ u8 hop_limit;
91+ u8 sl;
92+ u8 dmac[6];
93+ u16 vlan_id;
94+ u8 nw_type;
95+};
96+
97+struct bnxt_qplib_mrw {
98+ struct bnxt_qplib_pd *pd;
99+ int type;
100+ u32 flags;
101+#define BNXT_QPLIB_FR_PMR 0x80000000
102+ u32 lkey;
103+ u32 rkey;
104+#define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF
105+ u64 va;
106+ u64 total_size;
107+ u32 npages;
108+ u64 mr_handle;
109+ struct bnxt_qplib_hwq hwq;
110+};
111+
112+struct bnxt_qplib_frpl {
113+ int max_pg_ptrs;
114+ struct bnxt_qplib_hwq hwq;
115+};
116+
117+#define BNXT_QPLIB_ACCESS_LOCAL_WRITE BIT(0)
118+#define BNXT_QPLIB_ACCESS_REMOTE_READ BIT(1)
119+#define BNXT_QPLIB_ACCESS_REMOTE_WRITE BIT(2)
120+#define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC BIT(3)
121+#define BNXT_QPLIB_ACCESS_MW_BIND BIT(4)
122+#define BNXT_QPLIB_ACCESS_ZERO_BASED BIT(5)
123+#define BNXT_QPLIB_ACCESS_ON_DEMAND BIT(6)
124+
125+int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
126+ struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
127+ struct bnxt_qplib_gid *gid);
128+int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
129+ struct bnxt_qplib_gid *gid, bool update);
130+int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
131+ struct bnxt_qplib_gid *gid, u8 *mac, u16 vlan_id,
132+ bool update, u32 *index);
133+int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,
134+ struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,
135+ u16 *pkey);
136+int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res,
137+ struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
138+ bool update);
139+int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,
140+ struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
141+ bool update);
142+int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
143+ struct bnxt_qplib_dev_attr *attr);
144+int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah);
145+int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah);
146+int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res,
147+ struct bnxt_qplib_mrw *mrw);
148+int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
149+ bool block);
150+int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
151+ u64 *pbl_tbl, int num_pbls, bool block);
152+int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr);
153+int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res,
154+ struct bnxt_qplib_mrw *mr, int max);
155+int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
156+ struct bnxt_qplib_frpl *frpl, int max);
157+int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
158+ struct bnxt_qplib_frpl *frpl);
159+int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids);
160+#endif /* __BNXT_QPLIB_SP_H__*/
drivers/infiniband/hw/bnxt_re/roce_hsi.h+2821−0
@@ -0,0 +1,2821 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: RoCE HSI File - Autogenerated
37+ */
38+
39+#ifndef __BNXT_RE_HSI_H__
40+#define __BNXT_RE_HSI_H__
41+
42+/* include bnxt_hsi.h from bnxt_en driver */
43+#include "bnxt_hsi.h"
44+
45+/* CMP Door Bell Format (4 bytes) */
46+struct cmpl_doorbell {
47+ __le32 key_mask_valid_idx;
48+ #define CMPL_DOORBELL_IDX_MASK 0xffffffUL
49+ #define CMPL_DOORBELL_IDX_SFT 0
50+ #define CMPL_DOORBELL_RESERVED_MASK 0x3000000UL
51+ #define CMPL_DOORBELL_RESERVED_SFT 24
52+ #define CMPL_DOORBELL_IDX_VALID 0x4000000UL
53+ #define CMPL_DOORBELL_MASK 0x8000000UL
54+ #define CMPL_DOORBELL_KEY_MASK 0xf0000000UL
55+ #define CMPL_DOORBELL_KEY_SFT 28
56+ #define CMPL_DOORBELL_KEY_CMPL (0x2UL << 28)
57+};
58+
59+/* Status Door Bell Format (4 bytes) */
60+struct status_doorbell {
61+ __le32 key_idx;
62+ #define STATUS_DOORBELL_IDX_MASK 0xffffffUL
63+ #define STATUS_DOORBELL_IDX_SFT 0
64+ #define STATUS_DOORBELL_RESERVED_MASK 0xf000000UL
65+ #define STATUS_DOORBELL_RESERVED_SFT 24
66+ #define STATUS_DOORBELL_KEY_MASK 0xf0000000UL
67+ #define STATUS_DOORBELL_KEY_SFT 28
68+ #define STATUS_DOORBELL_KEY_STAT (0x3UL << 28)
69+};
70+
71+/* RoCE Host Structures */
72+
73+/* Doorbell Structures */
74+/* 64b Doorbell Format (8 bytes) */
75+struct dbr_dbr {
76+ __le32 index;
77+ #define DBR_DBR_INDEX_MASK 0xfffffUL
78+ #define DBR_DBR_INDEX_SFT 0
79+ #define DBR_DBR_RESERVED12_MASK 0xfff00000UL
80+ #define DBR_DBR_RESERVED12_SFT 20
81+ __le32 type_xid;
82+ #define DBR_DBR_XID_MASK 0xfffffUL
83+ #define DBR_DBR_XID_SFT 0
84+ #define DBR_DBR_RESERVED8_MASK 0xff00000UL
85+ #define DBR_DBR_RESERVED8_SFT 20
86+ #define DBR_DBR_TYPE_MASK 0xf0000000UL
87+ #define DBR_DBR_TYPE_SFT 28
88+ #define DBR_DBR_TYPE_SQ (0x0UL << 28)
89+ #define DBR_DBR_TYPE_RQ (0x1UL << 28)
90+ #define DBR_DBR_TYPE_SRQ (0x2UL << 28)
91+ #define DBR_DBR_TYPE_SRQ_ARM (0x3UL << 28)
92+ #define DBR_DBR_TYPE_CQ (0x4UL << 28)
93+ #define DBR_DBR_TYPE_CQ_ARMSE (0x5UL << 28)
94+ #define DBR_DBR_TYPE_CQ_ARMALL (0x6UL << 28)
95+ #define DBR_DBR_TYPE_CQ_ARMENA (0x7UL << 28)
96+ #define DBR_DBR_TYPE_SRQ_ARMENA (0x8UL << 28)
97+ #define DBR_DBR_TYPE_CQ_CUTOFF_ACK (0x9UL << 28)
98+ #define DBR_DBR_TYPE_NULL (0xfUL << 28)
99+};
100+
101+/* 32b Doorbell Format (4 bytes) */
102+struct dbr_dbr32 {
103+ __le32 type_abs_incr_xid;
104+ #define DBR_DBR32_XID_MASK 0xfffffUL
105+ #define DBR_DBR32_XID_SFT 0
106+ #define DBR_DBR32_RESERVED4_MASK 0xf00000UL
107+ #define DBR_DBR32_RESERVED4_SFT 20
108+ #define DBR_DBR32_INCR_MASK 0xf000000UL
109+ #define DBR_DBR32_INCR_SFT 24
110+ #define DBR_DBR32_ABS 0x10000000UL
111+ #define DBR_DBR32_TYPE_MASK 0xe0000000UL
112+ #define DBR_DBR32_TYPE_SFT 29
113+ #define DBR_DBR32_TYPE_SQ (0x0UL << 29)
114+};
115+
116+/* SQ WQE Structures */
117+/* Base SQ WQE (8 bytes) */
118+struct sq_base {
119+ u8 wqe_type;
120+ #define SQ_BASE_WQE_TYPE_SEND 0x0UL
121+ #define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD 0x1UL
122+ #define SQ_BASE_WQE_TYPE_SEND_W_INVALID 0x2UL
123+ #define SQ_BASE_WQE_TYPE_WRITE_WQE 0x4UL
124+ #define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
125+ #define SQ_BASE_WQE_TYPE_READ_WQE 0x6UL
126+ #define SQ_BASE_WQE_TYPE_ATOMIC_CS 0x8UL
127+ #define SQ_BASE_WQE_TYPE_ATOMIC_FA 0xbUL
128+ #define SQ_BASE_WQE_TYPE_LOCAL_INVALID 0xcUL
129+ #define SQ_BASE_WQE_TYPE_FR_PMR 0xdUL
130+ #define SQ_BASE_WQE_TYPE_BIND 0xeUL
131+ u8 unused_0[7];
132+};
133+
134+/* WQE SGE (16 bytes) */
135+struct sq_sge {
136+ __le64 va_or_pa;
137+ __le32 l_key;
138+ __le32 size;
139+};
140+
141+/* PSN Search Structure (8 bytes) */
142+struct sq_psn_search {
143+ __le32 opcode_start_psn;
144+ #define SQ_PSN_SEARCH_START_PSN_MASK 0xffffffUL
145+ #define SQ_PSN_SEARCH_START_PSN_SFT 0
146+ #define SQ_PSN_SEARCH_OPCODE_MASK 0xff000000UL
147+ #define SQ_PSN_SEARCH_OPCODE_SFT 24
148+ __le32 flags_next_psn;
149+ #define SQ_PSN_SEARCH_NEXT_PSN_MASK 0xffffffUL
150+ #define SQ_PSN_SEARCH_NEXT_PSN_SFT 0
151+ #define SQ_PSN_SEARCH_FLAGS_MASK 0xff000000UL
152+ #define SQ_PSN_SEARCH_FLAGS_SFT 24
153+};
154+
155+/* Send SQ WQE (40 bytes) */
156+struct sq_send {
157+ u8 wqe_type;
158+ #define SQ_SEND_WQE_TYPE_SEND 0x0UL
159+ #define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD 0x1UL
160+ #define SQ_SEND_WQE_TYPE_SEND_W_INVALID 0x2UL
161+ u8 flags;
162+ #define SQ_SEND_FLAGS_SIGNAL_COMP 0x1UL
163+ #define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
164+ #define SQ_SEND_FLAGS_UC_FENCE 0x4UL
165+ #define SQ_SEND_FLAGS_SE 0x8UL
166+ #define SQ_SEND_FLAGS_INLINE 0x10UL
167+ u8 wqe_size;
168+ u8 reserved8_1;
169+ __le32 inv_key_or_imm_data;
170+ __le32 length;
171+ __le32 q_key;
172+ __le32 dst_qp;
173+ #define SQ_SEND_DST_QP_MASK 0xffffffUL
174+ #define SQ_SEND_DST_QP_SFT 0
175+ #define SQ_SEND_RESERVED8_2_MASK 0xff000000UL
176+ #define SQ_SEND_RESERVED8_2_SFT 24
177+ __le32 avid;
178+ #define SQ_SEND_AVID_MASK 0xfffffUL
179+ #define SQ_SEND_AVID_SFT 0
180+ #define SQ_SEND_RESERVED_AVID_MASK 0xfff00000UL
181+ #define SQ_SEND_RESERVED_AVID_SFT 20
182+ __le64 reserved64;
183+ __le32 data[24];
184+};
185+
186+/* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */
187+struct sq_send_raweth_qp1 {
188+ u8 wqe_type;
189+ #define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND 0x0UL
190+ u8 flags;
191+ #define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP 0x1UL
192+ #define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
193+ #define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE 0x4UL
194+ #define SQ_SEND_RAWETH_QP1_FLAGS_SE 0x8UL
195+ #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE 0x10UL
196+ u8 wqe_size;
197+ u8 reserved8;
198+ __le16 lflags;
199+ #define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM 0x1UL
200+ #define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM 0x2UL
201+ #define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC 0x4UL
202+ #define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP 0x8UL
203+ #define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM 0x10UL
204+ #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_1 0x20UL
205+ #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_2 0x40UL
206+ #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_3 0x80UL
207+ #define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC 0x100UL
208+ #define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC 0x200UL
209+ __le16 cfa_action;
210+ __le32 length;
211+ __le32 reserved32_1;
212+ __le32 cfa_meta;
213+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK 0xfffUL
214+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT 0
215+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE 0x1000UL
216+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK 0xe000UL
217+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT 13
218+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK 0x70000UL
219+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT 16
220+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 (0x0UL << 16)
221+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 (0x1UL << 16)
222+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 (0x2UL << 16)
223+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 (0x3UL << 16)
224+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 (0x4UL << 16)
225+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG (0x5UL << 16)
226+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST \
227+ SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG
228+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK 0xff80000UL
229+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19
230+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK 0xf0000000UL
231+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT 28
232+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE (0x0UL << 28)
233+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG (0x1UL << 28)
234+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST \
235+ SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG
236+ __le32 reserved32_2;
237+ __le64 reserved64;
238+ __le32 data[24];
239+};
240+
241+/* RDMA SQ WQE (40 bytes) */
242+struct sq_rdma {
243+ u8 wqe_type;
244+ #define SQ_RDMA_WQE_TYPE_WRITE_WQE 0x4UL
245+ #define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
246+ #define SQ_RDMA_WQE_TYPE_READ_WQE 0x6UL
247+ u8 flags;
248+ #define SQ_RDMA_FLAGS_SIGNAL_COMP 0x1UL
249+ #define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
250+ #define SQ_RDMA_FLAGS_UC_FENCE 0x4UL
251+ #define SQ_RDMA_FLAGS_SE 0x8UL
252+ #define SQ_RDMA_FLAGS_INLINE 0x10UL
253+ u8 wqe_size;
254+ u8 reserved8;
255+ __le32 imm_data;
256+ __le32 length;
257+ __le32 reserved32_1;
258+ __le64 remote_va;
259+ __le32 remote_key;
260+ __le32 reserved32_2;
261+ __le32 data[24];
262+};
263+
264+/* Atomic SQ WQE (40 bytes) */
265+struct sq_atomic {
266+ u8 wqe_type;
267+ #define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS 0x8UL
268+ #define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA 0xbUL
269+ u8 flags;
270+ #define SQ_ATOMIC_FLAGS_SIGNAL_COMP 0x1UL
271+ #define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
272+ #define SQ_ATOMIC_FLAGS_UC_FENCE 0x4UL
273+ #define SQ_ATOMIC_FLAGS_SE 0x8UL
274+ #define SQ_ATOMIC_FLAGS_INLINE 0x10UL
275+ __le16 reserved16;
276+ __le32 remote_key;
277+ __le64 remote_va;
278+ __le64 swap_data;
279+ __le64 cmp_data;
280+ __le32 data[24];
281+};
282+
283+/* Local Invalidate SQ WQE (40 bytes) */
284+struct sq_localinvalidate {
285+ u8 wqe_type;
286+ #define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID 0xcUL
287+ u8 flags;
288+ #define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP 0x1UL
289+ #define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
290+ #define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE 0x4UL
291+ #define SQ_LOCALINVALIDATE_FLAGS_SE 0x8UL
292+ #define SQ_LOCALINVALIDATE_FLAGS_INLINE 0x10UL
293+ __le16 reserved16;
294+ __le32 inv_l_key;
295+ __le64 reserved64;
296+ __le32 reserved128[4];
297+ __le32 data[24];
298+};
299+
300+/* FR-PMR SQ WQE (40 bytes) */
301+struct sq_fr_pmr {
302+ u8 wqe_type;
303+ #define SQ_FR_PMR_WQE_TYPE_FR_PMR 0xdUL
304+ u8 flags;
305+ #define SQ_FR_PMR_FLAGS_SIGNAL_COMP 0x1UL
306+ #define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
307+ #define SQ_FR_PMR_FLAGS_UC_FENCE 0x4UL
308+ #define SQ_FR_PMR_FLAGS_SE 0x8UL
309+ #define SQ_FR_PMR_FLAGS_INLINE 0x10UL
310+ u8 access_cntl;
311+ #define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE 0x1UL
312+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ 0x2UL
313+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE 0x4UL
314+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
315+ #define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND 0x10UL
316+ u8 zero_based_page_size_log;
317+ #define SQ_FR_PMR_PAGE_SIZE_LOG_MASK 0x1fUL
318+ #define SQ_FR_PMR_PAGE_SIZE_LOG_SFT 0
319+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
320+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
321+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
322+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
323+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
324+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
325+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
326+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
327+ #define SQ_FR_PMR_ZERO_BASED 0x20UL
328+ #define SQ_FR_PMR_RESERVED2_MASK 0xc0UL
329+ #define SQ_FR_PMR_RESERVED2_SFT 6
330+ __le32 l_key;
331+ u8 length[5];
332+ u8 reserved8_1;
333+ u8 reserved8_2;
334+ u8 numlevels_pbl_page_size_log;
335+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK 0x1fUL
336+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT 0
337+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
338+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
339+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
340+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
341+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
342+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
343+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
344+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
345+ #define SQ_FR_PMR_RESERVED1 0x20UL
346+ #define SQ_FR_PMR_NUMLEVELS_MASK 0xc0UL
347+ #define SQ_FR_PMR_NUMLEVELS_SFT 6
348+ #define SQ_FR_PMR_NUMLEVELS_PHYSICAL (0x0UL << 6)
349+ #define SQ_FR_PMR_NUMLEVELS_LAYER1 (0x1UL << 6)
350+ #define SQ_FR_PMR_NUMLEVELS_LAYER2 (0x2UL << 6)
351+ __le64 pblptr;
352+ __le64 va;
353+ __le32 data[24];
354+};
355+
356+/* Bind SQ WQE (40 bytes) */
357+struct sq_bind {
358+ u8 wqe_type;
359+ #define SQ_BIND_WQE_TYPE_BIND 0xeUL
360+ u8 flags;
361+ #define SQ_BIND_FLAGS_SIGNAL_COMP 0x1UL
362+ #define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
363+ #define SQ_BIND_FLAGS_UC_FENCE 0x4UL
364+ #define SQ_BIND_FLAGS_SE 0x8UL
365+ #define SQ_BIND_FLAGS_INLINE 0x10UL
366+ u8 access_cntl;
367+ #define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE 0x1UL
368+ #define SQ_BIND_ACCESS_CNTL_REMOTE_READ 0x2UL
369+ #define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE 0x4UL
370+ #define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
371+ #define SQ_BIND_ACCESS_CNTL_WINDOW_BIND 0x10UL
372+ u8 reserved8_1;
373+ u8 mw_type_zero_based;
374+ #define SQ_BIND_ZERO_BASED 0x1UL
375+ #define SQ_BIND_MW_TYPE 0x2UL
376+ #define SQ_BIND_MW_TYPE_TYPE1 (0x0UL << 1)
377+ #define SQ_BIND_MW_TYPE_TYPE2 (0x1UL << 1)
378+ #define SQ_BIND_RESERVED6_MASK 0xfcUL
379+ #define SQ_BIND_RESERVED6_SFT 2
380+ u8 reserved8_2;
381+ __le16 reserved16;
382+ __le32 parent_l_key;
383+ __le32 l_key;
384+ __le64 va;
385+ u8 length[5];
386+ u8 data_reserved24[99];
387+ #define SQ_BIND_RESERVED24_MASK 0xffffff00UL
388+ #define SQ_BIND_RESERVED24_SFT 8
389+ #define SQ_BIND_DATA_MASK 0xffffffffUL
390+ #define SQ_BIND_DATA_SFT 0
391+};
392+
393+/* RQ/SRQ WQE Structures */
394+/* RQ/SRQ WQE (40 bytes) */
395+struct rq_wqe {
396+ u8 wqe_type;
397+ #define RQ_WQE_WQE_TYPE_RCV 0x80UL
398+ u8 flags;
399+ u8 wqe_size;
400+ u8 reserved8;
401+ __le32 reserved32;
402+ __le32 wr_id[2];
403+ #define RQ_WQE_WR_ID_MASK 0xfffffUL
404+ #define RQ_WQE_WR_ID_SFT 0
405+ #define RQ_WQE_RESERVED44_MASK 0xfff00000UL
406+ #define RQ_WQE_RESERVED44_SFT 20
407+ __le32 reserved128[4];
408+ __le32 data[24];
409+};
410+
411+/* CQ CQE Structures */
412+/* Base CQE (32 bytes) */
413+struct cq_base {
414+ __le64 reserved64_1;
415+ __le64 reserved64_2;
416+ __le64 reserved64_3;
417+ u8 cqe_type_toggle;
418+ #define CQ_BASE_TOGGLE 0x1UL
419+ #define CQ_BASE_CQE_TYPE_MASK 0x1eUL
420+ #define CQ_BASE_CQE_TYPE_SFT 1
421+ #define CQ_BASE_CQE_TYPE_REQ (0x0UL << 1)
422+ #define CQ_BASE_CQE_TYPE_RES_RC (0x1UL << 1)
423+ #define CQ_BASE_CQE_TYPE_RES_UD (0x2UL << 1)
424+ #define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
425+ #define CQ_BASE_CQE_TYPE_TERMINAL (0xeUL << 1)
426+ #define CQ_BASE_CQE_TYPE_CUT_OFF (0xfUL << 1)
427+ #define CQ_BASE_RESERVED3_MASK 0xe0UL
428+ #define CQ_BASE_RESERVED3_SFT 5
429+ u8 status;
430+ __le16 reserved16;
431+ __le32 reserved32;
432+};
433+
434+/* Requester CQ CQE (32 bytes) */
435+struct cq_req {
436+ __le64 qp_handle;
437+ __le16 sq_cons_idx;
438+ __le16 reserved16_1;
439+ __le32 reserved32_2;
440+ __le64 reserved64;
441+ u8 cqe_type_toggle;
442+ #define CQ_REQ_TOGGLE 0x1UL
443+ #define CQ_REQ_CQE_TYPE_MASK 0x1eUL
444+ #define CQ_REQ_CQE_TYPE_SFT 1
445+ #define CQ_REQ_CQE_TYPE_REQ (0x0UL << 1)
446+ #define CQ_REQ_RESERVED3_MASK 0xe0UL
447+ #define CQ_REQ_RESERVED3_SFT 5
448+ u8 status;
449+ #define CQ_REQ_STATUS_OK 0x0UL
450+ #define CQ_REQ_STATUS_BAD_RESPONSE_ERR 0x1UL
451+ #define CQ_REQ_STATUS_LOCAL_LENGTH_ERR 0x2UL
452+ #define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR 0x3UL
453+ #define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR 0x4UL
454+ #define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
455+ #define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL
456+ #define CQ_REQ_STATUS_REMOTE_ACCESS_ERR 0x7UL
457+ #define CQ_REQ_STATUS_REMOTE_OPERATION_ERR 0x8UL
458+ #define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR 0x9UL
459+ #define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR 0xaUL
460+ #define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR 0xbUL
461+ __le16 reserved16_2;
462+ __le32 reserved32_1;
463+};
464+
465+/* Responder RC CQE (32 bytes) */
466+struct cq_res_rc {
467+ __le32 length;
468+ __le32 imm_data_or_inv_r_key;
469+ __le64 qp_handle;
470+ __le64 mr_handle;
471+ u8 cqe_type_toggle;
472+ #define CQ_RES_RC_TOGGLE 0x1UL
473+ #define CQ_RES_RC_CQE_TYPE_MASK 0x1eUL
474+ #define CQ_RES_RC_CQE_TYPE_SFT 1
475+ #define CQ_RES_RC_CQE_TYPE_RES_RC (0x1UL << 1)
476+ #define CQ_RES_RC_RESERVED3_MASK 0xe0UL
477+ #define CQ_RES_RC_RESERVED3_SFT 5
478+ u8 status;
479+ #define CQ_RES_RC_STATUS_OK 0x0UL
480+ #define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR 0x1UL
481+ #define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR 0x2UL
482+ #define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR 0x3UL
483+ #define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
484+ #define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
485+ #define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL
486+ #define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
487+ #define CQ_RES_RC_STATUS_HW_FLUSH_ERR 0x8UL
488+ __le16 flags;
489+ #define CQ_RES_RC_FLAGS_SRQ 0x1UL
490+ #define CQ_RES_RC_FLAGS_SRQ_RQ (0x0UL << 0)
491+ #define CQ_RES_RC_FLAGS_SRQ_SRQ (0x1UL << 0)
492+ #define CQ_RES_RC_FLAGS_SRQ_LAST CQ_RES_RC_FLAGS_SRQ_SRQ
493+ #define CQ_RES_RC_FLAGS_IMM 0x2UL
494+ #define CQ_RES_RC_FLAGS_INV 0x4UL
495+ #define CQ_RES_RC_FLAGS_RDMA 0x8UL
496+ #define CQ_RES_RC_FLAGS_RDMA_SEND (0x0UL << 3)
497+ #define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE (0x1UL << 3)
498+ #define CQ_RES_RC_FLAGS_RDMA_LAST CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE
499+ __le32 srq_or_rq_wr_id;
500+ #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
501+ #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0
502+ #define CQ_RES_RC_RESERVED12_MASK 0xfff00000UL
503+ #define CQ_RES_RC_RESERVED12_SFT 20
504+};
505+
506+/* Responder UD CQE (32 bytes) */
507+struct cq_res_ud {
508+ __le32 length;
509+ #define CQ_RES_UD_LENGTH_MASK 0x3fffUL
510+ #define CQ_RES_UD_LENGTH_SFT 0
511+ #define CQ_RES_UD_RESERVED18_MASK 0xffffc000UL
512+ #define CQ_RES_UD_RESERVED18_SFT 14
513+ __le32 imm_data;
514+ __le64 qp_handle;
515+ __le16 src_mac[3];
516+ __le16 src_qp_low;
517+ u8 cqe_type_toggle;
518+ #define CQ_RES_UD_TOGGLE 0x1UL
519+ #define CQ_RES_UD_CQE_TYPE_MASK 0x1eUL
520+ #define CQ_RES_UD_CQE_TYPE_SFT 1
521+ #define CQ_RES_UD_CQE_TYPE_RES_UD (0x2UL << 1)
522+ #define CQ_RES_UD_RESERVED3_MASK 0xe0UL
523+ #define CQ_RES_UD_RESERVED3_SFT 5
524+ u8 status;
525+ #define CQ_RES_UD_STATUS_OK 0x0UL
526+ #define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR 0x1UL
527+ #define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
528+ #define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR 0x3UL
529+ #define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
530+ #define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
531+ #define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
532+ #define CQ_RES_UD_STATUS_HW_FLUSH_ERR 0x8UL
533+ __le16 flags;
534+ #define CQ_RES_UD_FLAGS_SRQ 0x1UL
535+ #define CQ_RES_UD_FLAGS_SRQ_RQ (0x0UL << 0)
536+ #define CQ_RES_UD_FLAGS_SRQ_SRQ (0x1UL << 0)
537+ #define CQ_RES_UD_FLAGS_SRQ_LAST CQ_RES_UD_FLAGS_SRQ_SRQ
538+ #define CQ_RES_UD_FLAGS_IMM 0x2UL
539+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK 0xcUL
540+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT 2
541+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 (0x0UL << 2)
542+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 2)
543+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 2)
544+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST \
545+ CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6
546+ __le32 src_qp_high_srq_or_rq_wr_id;
547+ #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
548+ #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0
549+ #define CQ_RES_UD_RESERVED4_MASK 0xf00000UL
550+ #define CQ_RES_UD_RESERVED4_SFT 20
551+ #define CQ_RES_UD_SRC_QP_HIGH_MASK 0xff000000UL
552+ #define CQ_RES_UD_SRC_QP_HIGH_SFT 24
553+};
554+
555+/* Responder RawEth and QP1 CQE (32 bytes) */
556+struct cq_res_raweth_qp1 {
557+ __le16 length;
558+ #define CQ_RES_RAWETH_QP1_LENGTH_MASK 0x3fffUL
559+ #define CQ_RES_RAWETH_QP1_LENGTH_SFT 0
560+ #define CQ_RES_RAWETH_QP1_RESERVED2_MASK 0xc000UL
561+ #define CQ_RES_RAWETH_QP1_RESERVED2_SFT 14
562+ __le16 raweth_qp1_flags;
563+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR 0x1UL
564+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_MASK 0x3eUL
565+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_SFT 1
566+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK 0x3c0UL
567+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT 6
568+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL << 6)
569+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP (0x1UL << 6)
570+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP (0x2UL << 6)
571+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP (0x3UL << 6)
572+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE (0x4UL << 6)
573+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE (0x5UL << 6)
574+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP (0x7UL << 6)
575+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP \
576+ (0x8UL << 6)
577+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP \
578+ (0x9UL << 6)
579+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST \
580+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP
581+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK 0x3ffUL
582+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT 0
583+ #define CQ_RES_RAWETH_QP1_RESERVED6_MASK 0xfc00UL
584+ #define CQ_RES_RAWETH_QP1_RESERVED6_SFT 10
585+ __le16 raweth_qp1_errors;
586+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_MASK 0xfUL
587+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_SFT 0
588+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR 0x10UL
589+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR 0x20UL
590+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL
591+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL
592+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR 0x100UL
593+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL
594+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9
595+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR \
596+ (0x0UL << 9)
597+ #define \
598+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \
599+ (0x1UL << 9)
600+ #define \
601+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \
602+ (0x2UL << 9)
603+ #define \
604+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \
605+ (0x3UL << 9)
606+ #define \
607+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \
608+ (0x4UL << 9)
609+ #define \
610+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \
611+ (0x5UL << 9)
612+ #define \
613+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \
614+ (0x6UL << 9)
615+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \
616+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL
617+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL
618+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12
619+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR \
620+ (0x0UL << 12)
621+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION \
622+ (0x1UL << 12)
623+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \
624+ (0x2UL << 12)
625+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL \
626+ (0x3UL << 12)
627+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \
628+ (0x4UL << 12)
629+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \
630+ (0x5UL << 12)
631+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \
632+ (0x6UL << 12)
633+ #define \
634+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL\
635+ (0x7UL << 12)
636+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \
637+ (0x8UL << 12)
638+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \
639+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN
640+ __le16 raweth_qp1_cfa_code;
641+ __le64 qp_handle;
642+ __le32 raweth_qp1_flags2;
643+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC 0x1UL
644+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC 0x2UL
645+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC 0x4UL
646+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC 0x8UL
647+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL
648+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4
649+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE \
650+ (0x0UL << 4)
651+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN \
652+ (0x1UL << 4)
653+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST\
654+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN
655+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE 0x100UL
656+ __le32 raweth_qp1_metadata;
657+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK 0xfffUL
658+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT 0
659+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE 0x1000UL
660+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK 0xe000UL
661+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT 13
662+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK 0xffff0000UL
663+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT 16
664+ u8 cqe_type_toggle;
665+ #define CQ_RES_RAWETH_QP1_TOGGLE 0x1UL
666+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK 0x1eUL
667+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT 1
668+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
669+ #define CQ_RES_RAWETH_QP1_RESERVED3_MASK 0xe0UL
670+ #define CQ_RES_RAWETH_QP1_RESERVED3_SFT 5
671+ u8 status;
672+ #define CQ_RES_RAWETH_QP1_STATUS_OK 0x0UL
673+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR 0x1UL
674+ #define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
675+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR 0x3UL
676+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
677+ #define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
678+ #define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
679+ #define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR 0x8UL
680+ __le16 flags;
681+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ 0x1UL
682+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ 0x0UL
683+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ 0x1UL
684+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST \
685+ CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ
686+ __le32 raweth_qp1_payload_offset_srq_or_rq_wr_id;
687+ #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
688+ #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT 0
689+ #define CQ_RES_RAWETH_QP1_RESERVED4_MASK 0xf00000UL
690+ #define CQ_RES_RAWETH_QP1_RESERVED4_SFT 20
691+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL
692+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24
693+};
694+
695+/* Terminal CQE (32 bytes) */
696+struct cq_terminal {
697+ __le64 qp_handle;
698+ __le16 sq_cons_idx;
699+ __le16 rq_cons_idx;
700+ __le32 reserved32_1;
701+ __le64 reserved64_3;
702+ u8 cqe_type_toggle;
703+ #define CQ_TERMINAL_TOGGLE 0x1UL
704+ #define CQ_TERMINAL_CQE_TYPE_MASK 0x1eUL
705+ #define CQ_TERMINAL_CQE_TYPE_SFT 1
706+ #define CQ_TERMINAL_CQE_TYPE_TERMINAL (0xeUL << 1)
707+ #define CQ_TERMINAL_RESERVED3_MASK 0xe0UL
708+ #define CQ_TERMINAL_RESERVED3_SFT 5
709+ u8 status;
710+ #define CQ_TERMINAL_STATUS_OK 0x0UL
711+ __le16 reserved16;
712+ __le32 reserved32_2;
713+};
714+
715+/* Cutoff CQE (32 bytes) */
716+struct cq_cutoff {
717+ __le64 reserved64_1;
718+ __le64 reserved64_2;
719+ __le64 reserved64_3;
720+ u8 cqe_type_toggle;
721+ #define CQ_CUTOFF_TOGGLE 0x1UL
722+ #define CQ_CUTOFF_CQE_TYPE_MASK 0x1eUL
723+ #define CQ_CUTOFF_CQE_TYPE_SFT 1
724+ #define CQ_CUTOFF_CQE_TYPE_CUT_OFF (0xfUL << 1)
725+ #define CQ_CUTOFF_RESERVED3_MASK 0xe0UL
726+ #define CQ_CUTOFF_RESERVED3_SFT 5
727+ u8 status;
728+ #define CQ_CUTOFF_STATUS_OK 0x0UL
729+ __le16 reserved16;
730+ __le32 reserved32;
731+};
732+
733+/* Notification Queue (NQ) Structures */
734+/* Base NQ Record (16 bytes) */
735+struct nq_base {
736+ __le16 info10_type;
737+ #define NQ_BASE_TYPE_MASK 0x3fUL
738+ #define NQ_BASE_TYPE_SFT 0
739+ #define NQ_BASE_TYPE_CQ_NOTIFICATION 0x30UL
740+ #define NQ_BASE_TYPE_SRQ_EVENT 0x32UL
741+ #define NQ_BASE_TYPE_DBQ_EVENT 0x34UL
742+ #define NQ_BASE_TYPE_QP_EVENT 0x38UL
743+ #define NQ_BASE_TYPE_FUNC_EVENT 0x3aUL
744+ #define NQ_BASE_INFO10_MASK 0xffc0UL
745+ #define NQ_BASE_INFO10_SFT 6
746+ __le16 info16;
747+ __le32 info32;
748+ __le32 info63_v[2];
749+ #define NQ_BASE_V 0x1UL
750+ #define NQ_BASE_INFO63_MASK 0xfffffffeUL
751+ #define NQ_BASE_INFO63_SFT 1
752+};
753+
754+/* Completion Queue Notification (16 bytes) */
755+struct nq_cn {
756+ __le16 type;
757+ #define NQ_CN_TYPE_MASK 0x3fUL
758+ #define NQ_CN_TYPE_SFT 0
759+ #define NQ_CN_TYPE_CQ_NOTIFICATION 0x30UL
760+ #define NQ_CN_RESERVED9_MASK 0xffc0UL
761+ #define NQ_CN_RESERVED9_SFT 6
762+ __le16 reserved16;
763+ __le32 cq_handle_low;
764+ __le32 v;
765+ #define NQ_CN_V 0x1UL
766+ #define NQ_CN_RESERVED31_MASK 0xfffffffeUL
767+ #define NQ_CN_RESERVED31_SFT 1
768+ __le32 cq_handle_high;
769+};
770+
771+/* SRQ Event Notification (16 bytes) */
772+struct nq_srq_event {
773+ u8 type;
774+ #define NQ_SRQ_EVENT_TYPE_MASK 0x3fUL
775+ #define NQ_SRQ_EVENT_TYPE_SFT 0
776+ #define NQ_SRQ_EVENT_TYPE_SRQ_EVENT 0x32UL
777+ #define NQ_SRQ_EVENT_RESERVED1_MASK 0xc0UL
778+ #define NQ_SRQ_EVENT_RESERVED1_SFT 6
779+ u8 event;
780+ #define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT 0x1UL
781+ __le16 reserved16;
782+ __le32 srq_handle_low;
783+ __le32 v;
784+ #define NQ_SRQ_EVENT_V 0x1UL
785+ #define NQ_SRQ_EVENT_RESERVED31_MASK 0xfffffffeUL
786+ #define NQ_SRQ_EVENT_RESERVED31_SFT 1
787+ __le32 srq_handle_high;
788+};
789+
790+/* DBQ Async Event Notification (16 bytes) */
791+struct nq_dbq_event {
792+ u8 type;
793+ #define NQ_DBQ_EVENT_TYPE_MASK 0x3fUL
794+ #define NQ_DBQ_EVENT_TYPE_SFT 0
795+ #define NQ_DBQ_EVENT_TYPE_DBQ_EVENT 0x34UL
796+ #define NQ_DBQ_EVENT_RESERVED1_MASK 0xc0UL
797+ #define NQ_DBQ_EVENT_RESERVED1_SFT 6
798+ u8 event;
799+ #define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT 0x1UL
800+ __le16 db_pfid;
801+ #define NQ_DBQ_EVENT_DB_PFID_MASK 0xfUL
802+ #define NQ_DBQ_EVENT_DB_PFID_SFT 0
803+ #define NQ_DBQ_EVENT_RESERVED12_MASK 0xfff0UL
804+ #define NQ_DBQ_EVENT_RESERVED12_SFT 4
805+ __le32 db_dpi;
806+ #define NQ_DBQ_EVENT_DB_DPI_MASK 0xfffffUL
807+ #define NQ_DBQ_EVENT_DB_DPI_SFT 0
808+ #define NQ_DBQ_EVENT_RESERVED12_2_MASK 0xfff00000UL
809+ #define NQ_DBQ_EVENT_RESERVED12_2_SFT 20
810+ __le32 v;
811+ #define NQ_DBQ_EVENT_V 0x1UL
812+ #define NQ_DBQ_EVENT_RESERVED32_MASK 0xfffffffeUL
813+ #define NQ_DBQ_EVENT_RESERVED32_SFT 1
814+ __le32 db_type_db_xid;
815+ #define NQ_DBQ_EVENT_DB_XID_MASK 0xfffffUL
816+ #define NQ_DBQ_EVENT_DB_XID_SFT 0
817+ #define NQ_DBQ_EVENT_RESERVED8_MASK 0xff00000UL
818+ #define NQ_DBQ_EVENT_RESERVED8_SFT 20
819+ #define NQ_DBQ_EVENT_DB_TYPE_MASK 0xf0000000UL
820+ #define NQ_DBQ_EVENT_DB_TYPE_SFT 28
821+};
822+
823+/* Read Request/Response Queue Structures */
824+/* Input Read Request Queue (IRRQ) Message (32 bytes) */
825+struct xrrq_irrq {
826+ __le16 credits_type;
827+ #define XRRQ_IRRQ_TYPE 0x1UL
828+ #define XRRQ_IRRQ_TYPE_READ_REQ 0x0UL
829+ #define XRRQ_IRRQ_TYPE_ATOMIC_REQ 0x1UL
830+ #define XRRQ_IRRQ_RESERVED10_MASK 0x7feUL
831+ #define XRRQ_IRRQ_RESERVED10_SFT 1
832+ #define XRRQ_IRRQ_CREDITS_MASK 0xf800UL
833+ #define XRRQ_IRRQ_CREDITS_SFT 11
834+ __le16 reserved16;
835+ __le32 reserved32;
836+ __le32 psn;
837+ #define XRRQ_IRRQ_PSN_MASK 0xffffffUL
838+ #define XRRQ_IRRQ_PSN_SFT 0
839+ #define XRRQ_IRRQ_RESERVED8_1_MASK 0xff000000UL
840+ #define XRRQ_IRRQ_RESERVED8_1_SFT 24
841+ __le32 msn;
842+ #define XRRQ_IRRQ_MSN_MASK 0xffffffUL
843+ #define XRRQ_IRRQ_MSN_SFT 0
844+ #define XRRQ_IRRQ_RESERVED8_2_MASK 0xff000000UL
845+ #define XRRQ_IRRQ_RESERVED8_2_SFT 24
846+ __le64 va_or_atomic_result;
847+ __le32 rdma_r_key;
848+ __le32 length;
849+};
850+
851+/* Output Read Request Queue (ORRQ) Message (32 bytes) */
852+struct xrrq_orrq {
853+ __le16 num_sges_type;
854+ #define XRRQ_ORRQ_TYPE 0x1UL
855+ #define XRRQ_ORRQ_TYPE_READ_REQ 0x0UL
856+ #define XRRQ_ORRQ_TYPE_ATOMIC_REQ 0x1UL
857+ #define XRRQ_ORRQ_RESERVED10_MASK 0x7feUL
858+ #define XRRQ_ORRQ_RESERVED10_SFT 1
859+ #define XRRQ_ORRQ_NUM_SGES_MASK 0xf800UL
860+ #define XRRQ_ORRQ_NUM_SGES_SFT 11
861+ __le16 reserved16;
862+ __le32 length;
863+ __le32 psn;
864+ #define XRRQ_ORRQ_PSN_MASK 0xffffffUL
865+ #define XRRQ_ORRQ_PSN_SFT 0
866+ #define XRRQ_ORRQ_RESERVED8_1_MASK 0xff000000UL
867+ #define XRRQ_ORRQ_RESERVED8_1_SFT 24
868+ __le32 end_psn;
869+ #define XRRQ_ORRQ_END_PSN_MASK 0xffffffUL
870+ #define XRRQ_ORRQ_END_PSN_SFT 0
871+ #define XRRQ_ORRQ_RESERVED8_2_MASK 0xff000000UL
872+ #define XRRQ_ORRQ_RESERVED8_2_SFT 24
873+ __le64 first_sge_phy_or_sing_sge_va;
874+ __le32 single_sge_l_key;
875+ __le32 single_sge_size;
876+};
877+
878+/* Page Buffer List Memory Structures (PBL) */
879+/* Page Table Entry (PTE) (8 bytes) */
880+struct ptu_pte {
881+ __le32 page_next_to_last_last_valid[2];
882+ #define PTU_PTE_VALID 0x1UL
883+ #define PTU_PTE_LAST 0x2UL
884+ #define PTU_PTE_NEXT_TO_LAST 0x4UL
885+ #define PTU_PTE_PAGE_MASK 0xfffff000UL
886+ #define PTU_PTE_PAGE_SFT 12
887+};
888+
889+/* Page Directory Entry (PDE) (8 bytes) */
890+struct ptu_pde {
891+ __le32 page_valid[2];
892+ #define PTU_PDE_VALID 0x1UL
893+ #define PTU_PDE_PAGE_MASK 0xfffff000UL
894+ #define PTU_PDE_PAGE_SFT 12
895+};
896+
897+/* RoCE Fastpath Host Structures */
898+/* Command Queue (CMDQ) Interface */
899+/* Init CMDQ (16 bytes) */
900+struct cmdq_init {
901+ __le64 cmdq_pbl;
902+ __le16 cmdq_size_cmdq_lvl;
903+ #define CMDQ_INIT_CMDQ_LVL_MASK 0x3UL
904+ #define CMDQ_INIT_CMDQ_LVL_SFT 0
905+ #define CMDQ_INIT_CMDQ_SIZE_MASK 0xfffcUL
906+ #define CMDQ_INIT_CMDQ_SIZE_SFT 2
907+ __le16 creq_ring_id;
908+ __le32 prod_idx;
909+};
910+
911+/* Update CMDQ producer index (16 bytes) */
912+struct cmdq_update {
913+ __le64 reserved64;
914+ __le32 reserved32;
915+ __le32 prod_idx;
916+};
917+
918+/* CMDQ common header structure (16 bytes) */
919+struct cmdq_base {
920+ u8 opcode;
921+ #define CMDQ_BASE_OPCODE_CREATE_QP 0x1UL
922+ #define CMDQ_BASE_OPCODE_DESTROY_QP 0x2UL
923+ #define CMDQ_BASE_OPCODE_MODIFY_QP 0x3UL
924+ #define CMDQ_BASE_OPCODE_QUERY_QP 0x4UL
925+ #define CMDQ_BASE_OPCODE_CREATE_SRQ 0x5UL
926+ #define CMDQ_BASE_OPCODE_DESTROY_SRQ 0x6UL
927+ #define CMDQ_BASE_OPCODE_QUERY_SRQ 0x8UL
928+ #define CMDQ_BASE_OPCODE_CREATE_CQ 0x9UL
929+ #define CMDQ_BASE_OPCODE_DESTROY_CQ 0xaUL
930+ #define CMDQ_BASE_OPCODE_RESIZE_CQ 0xcUL
931+ #define CMDQ_BASE_OPCODE_ALLOCATE_MRW 0xdUL
932+ #define CMDQ_BASE_OPCODE_DEALLOCATE_KEY 0xeUL
933+ #define CMDQ_BASE_OPCODE_REGISTER_MR 0xfUL
934+ #define CMDQ_BASE_OPCODE_DEREGISTER_MR 0x10UL
935+ #define CMDQ_BASE_OPCODE_ADD_GID 0x11UL
936+ #define CMDQ_BASE_OPCODE_DELETE_GID 0x12UL
937+ #define CMDQ_BASE_OPCODE_MODIFY_GID 0x17UL
938+ #define CMDQ_BASE_OPCODE_QUERY_GID 0x18UL
939+ #define CMDQ_BASE_OPCODE_CREATE_QP1 0x13UL
940+ #define CMDQ_BASE_OPCODE_DESTROY_QP1 0x14UL
941+ #define CMDQ_BASE_OPCODE_CREATE_AH 0x15UL
942+ #define CMDQ_BASE_OPCODE_DESTROY_AH 0x16UL
943+ #define CMDQ_BASE_OPCODE_INITIALIZE_FW 0x80UL
944+ #define CMDQ_BASE_OPCODE_DEINITIALIZE_FW 0x81UL
945+ #define CMDQ_BASE_OPCODE_STOP_FUNC 0x82UL
946+ #define CMDQ_BASE_OPCODE_QUERY_FUNC 0x83UL
947+ #define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES 0x84UL
948+ #define CMDQ_BASE_OPCODE_READ_CONTEXT 0x85UL
949+ #define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST 0x86UL
950+ #define CMDQ_BASE_OPCODE_READ_VF_MEMORY 0x87UL
951+ #define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST 0x88UL
952+ #define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY 0x89UL
953+ #define CMDQ_BASE_OPCODE_MAP_TC_TO_COS 0x8aUL
954+ #define CMDQ_BASE_OPCODE_QUERY_VERSION 0x8bUL
955+ #define CMDQ_BASE_OPCODE_MODIFY_CC 0x8cUL
956+ #define CMDQ_BASE_OPCODE_QUERY_CC 0x8dUL
957+ u8 cmd_size;
958+ __le16 flags;
959+ __le16 cookie;
960+ u8 resp_size;
961+ u8 reserved8;
962+ __le64 resp_addr;
963+};
964+
965+/* Create QP command (96 bytes) */
966+struct cmdq_create_qp {
967+ u8 opcode;
968+ #define CMDQ_CREATE_QP_OPCODE_CREATE_QP 0x1UL
969+ u8 cmd_size;
970+ __le16 flags;
971+ __le16 cookie;
972+ u8 resp_size;
973+ u8 reserved8;
974+ __le64 resp_addr;
975+ __le64 qp_handle;
976+ __le32 qp_flags;
977+ #define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED 0x1UL
978+ #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL
979+ #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
980+ #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL
981+ u8 type;
982+ #define CMDQ_CREATE_QP_TYPE_RC 0x2UL
983+ #define CMDQ_CREATE_QP_TYPE_UD 0x4UL
984+ #define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE 0x6UL
985+ u8 sq_pg_size_sq_lvl;
986+ #define CMDQ_CREATE_QP_SQ_LVL_MASK 0xfUL
987+ #define CMDQ_CREATE_QP_SQ_LVL_SFT 0
988+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_0 0x0UL
989+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_1 0x1UL
990+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_2 0x2UL
991+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK 0xf0UL
992+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT 4
993+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K (0x0UL << 4)
994+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K (0x1UL << 4)
995+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K (0x2UL << 4)
996+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M (0x3UL << 4)
997+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M (0x4UL << 4)
998+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G (0x5UL << 4)
999+ u8 rq_pg_size_rq_lvl;
1000+ #define CMDQ_CREATE_QP_RQ_LVL_MASK 0xfUL
1001+ #define CMDQ_CREATE_QP_RQ_LVL_SFT 0
1002+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_0 0x0UL
1003+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_1 0x1UL
1004+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_2 0x2UL
1005+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK 0xf0UL
1006+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT 4
1007+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K (0x0UL << 4)
1008+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K (0x1UL << 4)
1009+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K (0x2UL << 4)
1010+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M (0x3UL << 4)
1011+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M (0x4UL << 4)
1012+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G (0x5UL << 4)
1013+ u8 unused_0;
1014+ __le32 dpi;
1015+ __le32 sq_size;
1016+ __le32 rq_size;
1017+ __le16 sq_fwo_sq_sge;
1018+ #define CMDQ_CREATE_QP_SQ_SGE_MASK 0xfUL
1019+ #define CMDQ_CREATE_QP_SQ_SGE_SFT 0
1020+ #define CMDQ_CREATE_QP_SQ_FWO_MASK 0xfff0UL
1021+ #define CMDQ_CREATE_QP_SQ_FWO_SFT 4
1022+ __le16 rq_fwo_rq_sge;
1023+ #define CMDQ_CREATE_QP_RQ_SGE_MASK 0xfUL
1024+ #define CMDQ_CREATE_QP_RQ_SGE_SFT 0
1025+ #define CMDQ_CREATE_QP_RQ_FWO_MASK 0xfff0UL
1026+ #define CMDQ_CREATE_QP_RQ_FWO_SFT 4
1027+ __le32 scq_cid;
1028+ __le32 rcq_cid;
1029+ __le32 srq_cid;
1030+ __le32 pd_id;
1031+ __le64 sq_pbl;
1032+ __le64 rq_pbl;
1033+ __le64 irrq_addr;
1034+ __le64 orrq_addr;
1035+};
1036+
1037+/* Destroy QP command (24 bytes) */
1038+struct cmdq_destroy_qp {
1039+ u8 opcode;
1040+ #define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP 0x2UL
1041+ u8 cmd_size;
1042+ __le16 flags;
1043+ __le16 cookie;
1044+ u8 resp_size;
1045+ u8 reserved8;
1046+ __le64 resp_addr;
1047+ __le32 qp_cid;
1048+ __le32 unused_0;
1049+};
1050+
1051+/* Modify QP command (112 bytes) */
1052+struct cmdq_modify_qp {
1053+ u8 opcode;
1054+ #define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP 0x3UL
1055+ u8 cmd_size;
1056+ __le16 flags;
1057+ __le16 cookie;
1058+ u8 resp_size;
1059+ u8 reserved8;
1060+ __le64 resp_addr;
1061+ __le32 modify_mask;
1062+ #define CMDQ_MODIFY_QP_MODIFY_MASK_STATE 0x1UL
1063+ #define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY 0x2UL
1064+ #define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS 0x4UL
1065+ #define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY 0x8UL
1066+ #define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY 0x10UL
1067+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DGID 0x20UL
1068+ #define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL 0x40UL
1069+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX 0x80UL
1070+ #define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT 0x100UL
1071+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS 0x200UL
1072+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC 0x400UL
1073+ #define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU 0x1000UL
1074+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT 0x2000UL
1075+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT 0x4000UL
1076+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY 0x8000UL
1077+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN 0x10000UL
1078+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC 0x20000UL
1079+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER 0x40000UL
1080+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN 0x80000UL
1081+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC 0x100000UL
1082+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE 0x200000UL
1083+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE 0x400000UL
1084+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE 0x800000UL
1085+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE 0x1000000UL
1086+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA 0x2000000UL
1087+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID 0x4000000UL
1088+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC 0x8000000UL
1089+ #define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID 0x10000000UL
1090+ #define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC 0x20000000UL
1091+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN 0x40000000UL
1092+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP 0x80000000UL
1093+ __le32 qp_cid;
1094+ u8 network_type_en_sqd_async_notify_new_state;
1095+ #define CMDQ_MODIFY_QP_NEW_STATE_MASK 0xfUL
1096+ #define CMDQ_MODIFY_QP_NEW_STATE_SFT 0
1097+ #define CMDQ_MODIFY_QP_NEW_STATE_RESET 0x0UL
1098+ #define CMDQ_MODIFY_QP_NEW_STATE_INIT 0x1UL
1099+ #define CMDQ_MODIFY_QP_NEW_STATE_RTR 0x2UL
1100+ #define CMDQ_MODIFY_QP_NEW_STATE_RTS 0x3UL
1101+ #define CMDQ_MODIFY_QP_NEW_STATE_SQD 0x4UL
1102+ #define CMDQ_MODIFY_QP_NEW_STATE_SQE 0x5UL
1103+ #define CMDQ_MODIFY_QP_NEW_STATE_ERR 0x6UL
1104+ #define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY 0x10UL
1105+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK 0xc0UL
1106+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT 6
1107+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 (0x0UL << 6)
1108+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 (0x2UL << 6)
1109+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 (0x3UL << 6)
1110+ u8 access;
1111+ #define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE 0x1UL
1112+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE 0x2UL
1113+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ 0x4UL
1114+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC 0x8UL
1115+ __le16 pkey;
1116+ __le32 qkey;
1117+ __le32 dgid[4];
1118+ __le32 flow_label;
1119+ __le16 sgid_index;
1120+ u8 hop_limit;
1121+ u8 traffic_class;
1122+ __le16 dest_mac[3];
1123+ u8 tos_dscp_tos_ecn;
1124+ #define CMDQ_MODIFY_QP_TOS_ECN_MASK 0x3UL
1125+ #define CMDQ_MODIFY_QP_TOS_ECN_SFT 0
1126+ #define CMDQ_MODIFY_QP_TOS_DSCP_MASK 0xfcUL
1127+ #define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2
1128+ u8 path_mtu;
1129+ #define CMDQ_MODIFY_QP_PATH_MTU_MASK 0xf0UL
1130+ #define CMDQ_MODIFY_QP_PATH_MTU_SFT 4
1131+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_256 (0x0UL << 4)
1132+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_512 (0x1UL << 4)
1133+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 (0x2UL << 4)
1134+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 (0x3UL << 4)
1135+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 (0x4UL << 4)
1136+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 (0x5UL << 4)
1137+ u8 timeout;
1138+ u8 retry_cnt;
1139+ u8 rnr_retry;
1140+ u8 min_rnr_timer;
1141+ __le32 rq_psn;
1142+ __le32 sq_psn;
1143+ u8 max_rd_atomic;
1144+ u8 max_dest_rd_atomic;
1145+ __le16 enable_cc;
1146+ #define CMDQ_MODIFY_QP_ENABLE_CC 0x1UL
1147+ __le32 sq_size;
1148+ __le32 rq_size;
1149+ __le16 sq_sge;
1150+ __le16 rq_sge;
1151+ __le32 max_inline_data;
1152+ __le32 dest_qp_id;
1153+ __le32 unused_3;
1154+ __le16 src_mac[3];
1155+ __le16 vlan_pcp_vlan_dei_vlan_id;
1156+ #define CMDQ_MODIFY_QP_VLAN_ID_MASK 0xfffUL
1157+ #define CMDQ_MODIFY_QP_VLAN_ID_SFT 0
1158+ #define CMDQ_MODIFY_QP_VLAN_DEI 0x1000UL
1159+ #define CMDQ_MODIFY_QP_VLAN_PCP_MASK 0xe000UL
1160+ #define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13
1161+};
1162+
1163+/* Query QP command (24 bytes) */
1164+struct cmdq_query_qp {
1165+ u8 opcode;
1166+ #define CMDQ_QUERY_QP_OPCODE_QUERY_QP 0x4UL
1167+ u8 cmd_size;
1168+ __le16 flags;
1169+ __le16 cookie;
1170+ u8 resp_size;
1171+ u8 reserved8;
1172+ __le64 resp_addr;
1173+ __le32 qp_cid;
1174+ __le32 unused_0;
1175+};
1176+
1177+/* Create SRQ command (48 bytes) */
1178+struct cmdq_create_srq {
1179+ u8 opcode;
1180+ #define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ 0x5UL
1181+ u8 cmd_size;
1182+ __le16 flags;
1183+ __le16 cookie;
1184+ u8 resp_size;
1185+ u8 reserved8;
1186+ __le64 resp_addr;
1187+ __le64 srq_handle;
1188+ __le16 pg_size_lvl;
1189+ #define CMDQ_CREATE_SRQ_LVL_MASK 0x3UL
1190+ #define CMDQ_CREATE_SRQ_LVL_SFT 0
1191+ #define CMDQ_CREATE_SRQ_LVL_LVL_0 0x0UL
1192+ #define CMDQ_CREATE_SRQ_LVL_LVL_1 0x1UL
1193+ #define CMDQ_CREATE_SRQ_LVL_LVL_2 0x2UL
1194+ #define CMDQ_CREATE_SRQ_PG_SIZE_MASK 0x1cUL
1195+ #define CMDQ_CREATE_SRQ_PG_SIZE_SFT 2
1196+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K (0x0UL << 2)
1197+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K (0x1UL << 2)
1198+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K (0x2UL << 2)
1199+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M (0x3UL << 2)
1200+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M (0x4UL << 2)
1201+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G (0x5UL << 2)
1202+ __le16 eventq_id;
1203+ #define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK 0xfffUL
1204+ #define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0
1205+ __le16 srq_size;
1206+ __le16 srq_fwo;
1207+ __le32 dpi;
1208+ __le32 pd_id;
1209+ __le64 pbl;
1210+};
1211+
1212+/* Destroy SRQ command (24 bytes) */
1213+struct cmdq_destroy_srq {
1214+ u8 opcode;
1215+ #define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ 0x6UL
1216+ u8 cmd_size;
1217+ __le16 flags;
1218+ __le16 cookie;
1219+ u8 resp_size;
1220+ u8 reserved8;
1221+ __le64 resp_addr;
1222+ __le32 srq_cid;
1223+ __le32 unused_0;
1224+};
1225+
1226+/* Query SRQ command (24 bytes) */
1227+struct cmdq_query_srq {
1228+ u8 opcode;
1229+ #define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ 0x8UL
1230+ u8 cmd_size;
1231+ __le16 flags;
1232+ __le16 cookie;
1233+ u8 resp_size;
1234+ u8 reserved8;
1235+ __le64 resp_addr;
1236+ __le32 srq_cid;
1237+ __le32 unused_0;
1238+};
1239+
1240+/* Create CQ command (48 bytes) */
1241+struct cmdq_create_cq {
1242+ u8 opcode;
1243+ #define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ 0x9UL
1244+ u8 cmd_size;
1245+ __le16 flags;
1246+ __le16 cookie;
1247+ u8 resp_size;
1248+ u8 reserved8;
1249+ __le64 resp_addr;
1250+ __le64 cq_handle;
1251+ __le32 pg_size_lvl;
1252+ #define CMDQ_CREATE_CQ_LVL_MASK 0x3UL
1253+ #define CMDQ_CREATE_CQ_LVL_SFT 0
1254+ #define CMDQ_CREATE_CQ_LVL_LVL_0 0x0UL
1255+ #define CMDQ_CREATE_CQ_LVL_LVL_1 0x1UL
1256+ #define CMDQ_CREATE_CQ_LVL_LVL_2 0x2UL
1257+ #define CMDQ_CREATE_CQ_PG_SIZE_MASK 0x1cUL
1258+ #define CMDQ_CREATE_CQ_PG_SIZE_SFT 2
1259+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_4K (0x0UL << 2)
1260+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_8K (0x1UL << 2)
1261+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_64K (0x2UL << 2)
1262+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_2M (0x3UL << 2)
1263+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_8M (0x4UL << 2)
1264+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_1G (0x5UL << 2)
1265+ __le32 cq_fco_cnq_id;
1266+ #define CMDQ_CREATE_CQ_CNQ_ID_MASK 0xfffUL
1267+ #define CMDQ_CREATE_CQ_CNQ_ID_SFT 0
1268+ #define CMDQ_CREATE_CQ_CQ_FCO_MASK 0xfffff000UL
1269+ #define CMDQ_CREATE_CQ_CQ_FCO_SFT 12
1270+ __le32 dpi;
1271+ __le32 cq_size;
1272+ __le64 pbl;
1273+};
1274+
1275+/* Destroy CQ command (24 bytes) */
1276+struct cmdq_destroy_cq {
1277+ u8 opcode;
1278+ #define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ 0xaUL
1279+ u8 cmd_size;
1280+ __le16 flags;
1281+ __le16 cookie;
1282+ u8 resp_size;
1283+ u8 reserved8;
1284+ __le64 resp_addr;
1285+ __le32 cq_cid;
1286+ __le32 unused_0;
1287+};
1288+
1289+/* Resize CQ command (40 bytes) */
1290+struct cmdq_resize_cq {
1291+ u8 opcode;
1292+ #define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ 0xcUL
1293+ u8 cmd_size;
1294+ __le16 flags;
1295+ __le16 cookie;
1296+ u8 resp_size;
1297+ u8 reserved8;
1298+ __le64 resp_addr;
1299+ __le32 cq_cid;
1300+ __le32 new_cq_size_pg_size_lvl;
1301+ #define CMDQ_RESIZE_CQ_LVL_MASK 0x3UL
1302+ #define CMDQ_RESIZE_CQ_LVL_SFT 0
1303+ #define CMDQ_RESIZE_CQ_LVL_LVL_0 0x0UL
1304+ #define CMDQ_RESIZE_CQ_LVL_LVL_1 0x1UL
1305+ #define CMDQ_RESIZE_CQ_LVL_LVL_2 0x2UL
1306+ #define CMDQ_RESIZE_CQ_PG_SIZE_MASK 0x1cUL
1307+ #define CMDQ_RESIZE_CQ_PG_SIZE_SFT 2
1308+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K (0x0UL << 2)
1309+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K (0x1UL << 2)
1310+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K (0x2UL << 2)
1311+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M (0x3UL << 2)
1312+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M (0x4UL << 2)
1313+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G (0x5UL << 2)
1314+ #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK 0x1fffe0UL
1315+ #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5
1316+ __le64 new_pbl;
1317+ __le32 new_cq_fco;
1318+ __le32 unused_2;
1319+};
1320+
1321+/* Allocate MRW command (32 bytes) */
1322+struct cmdq_allocate_mrw {
1323+ u8 opcode;
1324+ #define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW 0xdUL
1325+ u8 cmd_size;
1326+ __le16 flags;
1327+ __le16 cookie;
1328+ u8 resp_size;
1329+ u8 reserved8;
1330+ __le64 resp_addr;
1331+ __le64 mrw_handle;
1332+ u8 mrw_flags;
1333+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK 0xfUL
1334+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT 0
1335+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR 0x0UL
1336+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR 0x1UL
1337+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 0x2UL
1338+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A 0x3UL
1339+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B 0x4UL
1340+ u8 access;
1341+ #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_MASK 0x1fUL
1342+ #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_SFT 0
1343+ #define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY 0x20UL
1344+ __le16 unused_1;
1345+ __le32 pd_id;
1346+};
1347+
1348+/* De-allocate key command (24 bytes) */
1349+struct cmdq_deallocate_key {
1350+ u8 opcode;
1351+ #define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY 0xeUL
1352+ u8 cmd_size;
1353+ __le16 flags;
1354+ __le16 cookie;
1355+ u8 resp_size;
1356+ u8 reserved8;
1357+ __le64 resp_addr;
1358+ u8 mrw_flags;
1359+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK 0xfUL
1360+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT 0
1361+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR 0x0UL
1362+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR 0x1UL
1363+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 0x2UL
1364+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A 0x3UL
1365+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B 0x4UL
1366+ u8 unused_1[3];
1367+ __le32 key;
1368+};
1369+
1370+/* Register MR command (48 bytes) */
1371+struct cmdq_register_mr {
1372+ u8 opcode;
1373+ #define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR 0xfUL
1374+ u8 cmd_size;
1375+ __le16 flags;
1376+ __le16 cookie;
1377+ u8 resp_size;
1378+ u8 reserved8;
1379+ __le64 resp_addr;
1380+ u8 log2_pg_size_lvl;
1381+ #define CMDQ_REGISTER_MR_LVL_MASK 0x3UL
1382+ #define CMDQ_REGISTER_MR_LVL_SFT 0
1383+ #define CMDQ_REGISTER_MR_LVL_LVL_0 0x0UL
1384+ #define CMDQ_REGISTER_MR_LVL_LVL_1 0x1UL
1385+ #define CMDQ_REGISTER_MR_LVL_LVL_2 0x2UL
1386+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK 0x7cUL
1387+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT 2
1388+ u8 access;
1389+ #define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE 0x1UL
1390+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ 0x2UL
1391+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE 0x4UL
1392+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC 0x8UL
1393+ #define CMDQ_REGISTER_MR_ACCESS_MW_BIND 0x10UL
1394+ #define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED 0x20UL
1395+ __le16 unused_1;
1396+ __le32 key;
1397+ __le64 pbl;
1398+ __le64 va;
1399+ __le64 mr_size;
1400+};
1401+
1402+/* Deregister MR command (24 bytes) */
1403+struct cmdq_deregister_mr {
1404+ u8 opcode;
1405+ #define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR 0x10UL
1406+ u8 cmd_size;
1407+ __le16 flags;
1408+ __le16 cookie;
1409+ u8 resp_size;
1410+ u8 reserved8;
1411+ __le64 resp_addr;
1412+ __le32 lkey;
1413+ __le32 unused_0;
1414+};
1415+
1416+/* Add GID command (48 bytes) */
1417+struct cmdq_add_gid {
1418+ u8 opcode;
1419+ #define CMDQ_ADD_GID_OPCODE_ADD_GID 0x11UL
1420+ u8 cmd_size;
1421+ __le16 flags;
1422+ __le16 cookie;
1423+ u8 resp_size;
1424+ u8 reserved8;
1425+ __le64 resp_addr;
1426+ __be32 gid[4];
1427+ __be16 src_mac[3];
1428+ __le16 vlan;
1429+ #define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK 0xfffUL
1430+ #define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT 0
1431+ #define CMDQ_ADD_GID_VLAN_TPID_MASK 0x7000UL
1432+ #define CMDQ_ADD_GID_VLAN_TPID_SFT 12
1433+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12)
1434+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_8100 (0x1UL << 12)
1435+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9100 (0x2UL << 12)
1436+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9200 (0x3UL << 12)
1437+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9300 (0x4UL << 12)
1438+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
1439+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
1440+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
1441+ #define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3
1442+ #define CMDQ_ADD_GID_VLAN_VLAN_EN 0x8000UL
1443+ __le16 ipid;
1444+ __le16 stats_ctx;
1445+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL
1446+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT 0
1447+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL
1448+ __le32 unused_0;
1449+};
1450+
1451+/* Delete GID command (24 bytes) */
1452+struct cmdq_delete_gid {
1453+ u8 opcode;
1454+ #define CMDQ_DELETE_GID_OPCODE_DELETE_GID 0x12UL
1455+ u8 cmd_size;
1456+ __le16 flags;
1457+ __le16 cookie;
1458+ u8 resp_size;
1459+ u8 reserved8;
1460+ __le64 resp_addr;
1461+ __le16 gid_index;
1462+ __le16 unused_0;
1463+ __le32 unused_1;
1464+};
1465+
1466+/* Modify GID command (48 bytes) */
1467+struct cmdq_modify_gid {
1468+ u8 opcode;
1469+ #define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID 0x17UL
1470+ u8 cmd_size;
1471+ __le16 flags;
1472+ __le16 cookie;
1473+ u8 resp_size;
1474+ u8 reserved8;
1475+ __le64 resp_addr;
1476+ __le32 gid[4];
1477+ __le16 src_mac[3];
1478+ __le16 vlan;
1479+ #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK 0xfffUL
1480+ #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT 0
1481+ #define CMDQ_MODIFY_GID_VLAN_TPID_MASK 0x7000UL
1482+ #define CMDQ_MODIFY_GID_VLAN_TPID_SFT 12
1483+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12)
1484+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 (0x1UL << 12)
1485+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 (0x2UL << 12)
1486+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 (0x3UL << 12)
1487+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 (0x4UL << 12)
1488+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
1489+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
1490+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
1491+ #define CMDQ_MODIFY_GID_VLAN_TPID_LAST \
1492+ CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3
1493+ #define CMDQ_MODIFY_GID_VLAN_VLAN_EN 0x8000UL
1494+ __le16 ipid;
1495+ __le16 gid_index;
1496+ __le16 stats_ctx;
1497+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL
1498+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT 0
1499+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL
1500+ __le16 unused_0;
1501+};
1502+
1503+/* Query GID command (24 bytes) */
1504+struct cmdq_query_gid {
1505+ u8 opcode;
1506+ #define CMDQ_QUERY_GID_OPCODE_QUERY_GID 0x18UL
1507+ u8 cmd_size;
1508+ __le16 flags;
1509+ __le16 cookie;
1510+ u8 resp_size;
1511+ u8 reserved8;
1512+ __le64 resp_addr;
1513+ __le16 gid_index;
1514+ __le16 unused_0;
1515+ __le32 unused_1;
1516+};
1517+
1518+/* Create QP1 command (80 bytes) */
1519+struct cmdq_create_qp1 {
1520+ u8 opcode;
1521+ #define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 0x13UL
1522+ u8 cmd_size;
1523+ __le16 flags;
1524+ __le16 cookie;
1525+ u8 resp_size;
1526+ u8 reserved8;
1527+ __le64 resp_addr;
1528+ __le64 qp_handle;
1529+ __le32 qp_flags;
1530+ #define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED 0x1UL
1531+ #define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION 0x2UL
1532+ #define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
1533+ u8 type;
1534+ #define CMDQ_CREATE_QP1_TYPE_GSI 0x1UL
1535+ u8 sq_pg_size_sq_lvl;
1536+ #define CMDQ_CREATE_QP1_SQ_LVL_MASK 0xfUL
1537+ #define CMDQ_CREATE_QP1_SQ_LVL_SFT 0
1538+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_0 0x0UL
1539+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_1 0x1UL
1540+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_2 0x2UL
1541+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK 0xf0UL
1542+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT 4
1543+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K (0x0UL << 4)
1544+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K (0x1UL << 4)
1545+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K (0x2UL << 4)
1546+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M (0x3UL << 4)
1547+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M (0x4UL << 4)
1548+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G (0x5UL << 4)
1549+ u8 rq_pg_size_rq_lvl;
1550+ #define CMDQ_CREATE_QP1_RQ_LVL_MASK 0xfUL
1551+ #define CMDQ_CREATE_QP1_RQ_LVL_SFT 0
1552+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_0 0x0UL
1553+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_1 0x1UL
1554+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_2 0x2UL
1555+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK 0xf0UL
1556+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT 4
1557+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K (0x0UL << 4)
1558+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K (0x1UL << 4)
1559+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K (0x2UL << 4)
1560+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M (0x3UL << 4)
1561+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M (0x4UL << 4)
1562+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G (0x5UL << 4)
1563+ u8 unused_0;
1564+ __le32 dpi;
1565+ __le32 sq_size;
1566+ __le32 rq_size;
1567+ __le16 sq_fwo_sq_sge;
1568+ #define CMDQ_CREATE_QP1_SQ_SGE_MASK 0xfUL
1569+ #define CMDQ_CREATE_QP1_SQ_SGE_SFT 0
1570+ #define CMDQ_CREATE_QP1_SQ_FWO_MASK 0xfff0UL
1571+ #define CMDQ_CREATE_QP1_SQ_FWO_SFT 4
1572+ __le16 rq_fwo_rq_sge;
1573+ #define CMDQ_CREATE_QP1_RQ_SGE_MASK 0xfUL
1574+ #define CMDQ_CREATE_QP1_RQ_SGE_SFT 0
1575+ #define CMDQ_CREATE_QP1_RQ_FWO_MASK 0xfff0UL
1576+ #define CMDQ_CREATE_QP1_RQ_FWO_SFT 4
1577+ __le32 scq_cid;
1578+ __le32 rcq_cid;
1579+ __le32 srq_cid;
1580+ __le32 pd_id;
1581+ __le64 sq_pbl;
1582+ __le64 rq_pbl;
1583+};
1584+
1585+/* Destroy QP1 command (24 bytes) */
1586+struct cmdq_destroy_qp1 {
1587+ u8 opcode;
1588+ #define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 0x14UL
1589+ u8 cmd_size;
1590+ __le16 flags;
1591+ __le16 cookie;
1592+ u8 resp_size;
1593+ u8 reserved8;
1594+ __le64 resp_addr;
1595+ __le32 qp1_cid;
1596+ __le32 unused_0;
1597+};
1598+
1599+/* Create AH command (64 bytes) */
1600+struct cmdq_create_ah {
1601+ u8 opcode;
1602+ #define CMDQ_CREATE_AH_OPCODE_CREATE_AH 0x15UL
1603+ u8 cmd_size;
1604+ __le16 flags;
1605+ __le16 cookie;
1606+ u8 resp_size;
1607+ u8 reserved8;
1608+ __le64 resp_addr;
1609+ __le64 ah_handle;
1610+ __le32 dgid[4];
1611+ u8 type;
1612+ #define CMDQ_CREATE_AH_TYPE_V1 0x0UL
1613+ #define CMDQ_CREATE_AH_TYPE_V2IPV4 0x2UL
1614+ #define CMDQ_CREATE_AH_TYPE_V2IPV6 0x3UL
1615+ u8 hop_limit;
1616+ __le16 sgid_index;
1617+ __le32 dest_vlan_id_flow_label;
1618+ #define CMDQ_CREATE_AH_FLOW_LABEL_MASK 0xfffffUL
1619+ #define CMDQ_CREATE_AH_FLOW_LABEL_SFT 0
1620+ #define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK 0xfff00000UL
1621+ #define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20
1622+ __le32 pd_id;
1623+ __le32 unused_0;
1624+ __le16 dest_mac[3];
1625+ u8 traffic_class;
1626+ u8 unused_1;
1627+};
1628+
1629+/* Destroy AH command (24 bytes) */
1630+struct cmdq_destroy_ah {
1631+ u8 opcode;
1632+ #define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH 0x16UL
1633+ u8 cmd_size;
1634+ __le16 flags;
1635+ __le16 cookie;
1636+ u8 resp_size;
1637+ u8 reserved8;
1638+ __le64 resp_addr;
1639+ __le32 ah_cid;
1640+ __le32 unused_0;
1641+};
1642+
1643+/* Initialize Firmware command (112 bytes) */
1644+struct cmdq_initialize_fw {
1645+ u8 opcode;
1646+ #define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW 0x80UL
1647+ u8 cmd_size;
1648+ __le16 flags;
1649+ __le16 cookie;
1650+ u8 resp_size;
1651+ u8 reserved8;
1652+ __le64 resp_addr;
1653+ u8 qpc_pg_size_qpc_lvl;
1654+ #define CMDQ_INITIALIZE_FW_QPC_LVL_MASK 0xfUL
1655+ #define CMDQ_INITIALIZE_FW_QPC_LVL_SFT 0
1656+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 0x0UL
1657+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 0x1UL
1658+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 0x2UL
1659+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK 0xf0UL
1660+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT 4
1661+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K (0x0UL << 4)
1662+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K (0x1UL << 4)
1663+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K (0x2UL << 4)
1664+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M (0x3UL << 4)
1665+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M (0x4UL << 4)
1666+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G (0x5UL << 4)
1667+ u8 mrw_pg_size_mrw_lvl;
1668+ #define CMDQ_INITIALIZE_FW_MRW_LVL_MASK 0xfUL
1669+ #define CMDQ_INITIALIZE_FW_MRW_LVL_SFT 0
1670+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 0x0UL
1671+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 0x1UL
1672+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 0x2UL
1673+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK 0xf0UL
1674+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT 4
1675+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K (0x0UL << 4)
1676+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K (0x1UL << 4)
1677+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K (0x2UL << 4)
1678+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M (0x3UL << 4)
1679+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M (0x4UL << 4)
1680+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G (0x5UL << 4)
1681+ u8 srq_pg_size_srq_lvl;
1682+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK 0xfUL
1683+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT 0
1684+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 0x0UL
1685+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 0x1UL
1686+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 0x2UL
1687+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK 0xf0UL
1688+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT 4
1689+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K (0x0UL << 4)
1690+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K (0x1UL << 4)
1691+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K (0x2UL << 4)
1692+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M (0x3UL << 4)
1693+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M (0x4UL << 4)
1694+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G (0x5UL << 4)
1695+ u8 cq_pg_size_cq_lvl;
1696+ #define CMDQ_INITIALIZE_FW_CQ_LVL_MASK 0xfUL
1697+ #define CMDQ_INITIALIZE_FW_CQ_LVL_SFT 0
1698+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 0x0UL
1699+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 0x1UL
1700+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 0x2UL
1701+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK 0xf0UL
1702+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT 4
1703+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K (0x0UL << 4)
1704+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K (0x1UL << 4)
1705+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K (0x2UL << 4)
1706+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M (0x3UL << 4)
1707+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M (0x4UL << 4)
1708+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G (0x5UL << 4)
1709+ u8 tqm_pg_size_tqm_lvl;
1710+ #define CMDQ_INITIALIZE_FW_TQM_LVL_MASK 0xfUL
1711+ #define CMDQ_INITIALIZE_FW_TQM_LVL_SFT 0
1712+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 0x0UL
1713+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 0x1UL
1714+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 0x2UL
1715+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK 0xf0UL
1716+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT 4
1717+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K (0x0UL << 4)
1718+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K (0x1UL << 4)
1719+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K (0x2UL << 4)
1720+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M (0x3UL << 4)
1721+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M (0x4UL << 4)
1722+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G (0x5UL << 4)
1723+ u8 tim_pg_size_tim_lvl;
1724+ #define CMDQ_INITIALIZE_FW_TIM_LVL_MASK 0xfUL
1725+ #define CMDQ_INITIALIZE_FW_TIM_LVL_SFT 0
1726+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 0x0UL
1727+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 0x1UL
1728+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 0x2UL
1729+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK 0xf0UL
1730+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT 4
1731+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K (0x0UL << 4)
1732+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K (0x1UL << 4)
1733+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K (0x2UL << 4)
1734+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M (0x3UL << 4)
1735+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M (0x4UL << 4)
1736+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G (0x5UL << 4)
1737+ __le16 reserved16;
1738+ __le64 qpc_page_dir;
1739+ __le64 mrw_page_dir;
1740+ __le64 srq_page_dir;
1741+ __le64 cq_page_dir;
1742+ __le64 tqm_page_dir;
1743+ __le64 tim_page_dir;
1744+ __le32 number_of_qp;
1745+ __le32 number_of_mrw;
1746+ __le32 number_of_srq;
1747+ __le32 number_of_cq;
1748+ __le32 max_qp_per_vf;
1749+ __le32 max_mrw_per_vf;
1750+ __le32 max_srq_per_vf;
1751+ __le32 max_cq_per_vf;
1752+ __le32 max_gid_per_vf;
1753+ __le32 stat_ctx_id;
1754+};
1755+
1756+/* De-initialize Firmware command (16 bytes) */
1757+struct cmdq_deinitialize_fw {
1758+ u8 opcode;
1759+ #define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW 0x81UL
1760+ u8 cmd_size;
1761+ __le16 flags;
1762+ __le16 cookie;
1763+ u8 resp_size;
1764+ u8 reserved8;
1765+ __le64 resp_addr;
1766+};
1767+
1768+/* Stop function command (16 bytes) */
1769+struct cmdq_stop_func {
1770+ u8 opcode;
1771+ #define CMDQ_STOP_FUNC_OPCODE_STOP_FUNC 0x82UL
1772+ u8 cmd_size;
1773+ __le16 flags;
1774+ __le16 cookie;
1775+ u8 resp_size;
1776+ u8 reserved8;
1777+ __le64 resp_addr;
1778+};
1779+
1780+/* Query function command (16 bytes) */
1781+struct cmdq_query_func {
1782+ u8 opcode;
1783+ #define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC 0x83UL
1784+ u8 cmd_size;
1785+ __le16 flags;
1786+ __le16 cookie;
1787+ u8 resp_size;
1788+ u8 reserved8;
1789+ __le64 resp_addr;
1790+};
1791+
1792+/* Set function resources command (16 bytes) */
1793+struct cmdq_set_func_resources {
1794+ u8 opcode;
1795+ #define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL
1796+ u8 cmd_size;
1797+ __le16 flags;
1798+ __le16 cookie;
1799+ u8 resp_size;
1800+ u8 reserved8;
1801+ __le64 resp_addr;
1802+};
1803+
1804+/* Read hardware resource context command (24 bytes) */
1805+struct cmdq_read_context {
1806+ u8 opcode;
1807+ #define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT 0x85UL
1808+ u8 cmd_size;
1809+ __le16 flags;
1810+ __le16 cookie;
1811+ u8 resp_size;
1812+ u8 reserved8;
1813+ __le64 resp_addr;
1814+ __le32 type_xid;
1815+ #define CMDQ_READ_CONTEXT_XID_MASK 0xffffffUL
1816+ #define CMDQ_READ_CONTEXT_XID_SFT 0
1817+ #define CMDQ_READ_CONTEXT_TYPE_MASK 0xff000000UL
1818+ #define CMDQ_READ_CONTEXT_TYPE_SFT 24
1819+ #define CMDQ_READ_CONTEXT_TYPE_QPC (0x0UL << 24)
1820+ #define CMDQ_READ_CONTEXT_TYPE_CQ (0x1UL << 24)
1821+ #define CMDQ_READ_CONTEXT_TYPE_MRW (0x2UL << 24)
1822+ #define CMDQ_READ_CONTEXT_TYPE_SRQ (0x3UL << 24)
1823+ __le32 unused_0;
1824+};
1825+
1826+/* Map TC to COS. Can only be issued from a PF (24 bytes) */
1827+struct cmdq_map_tc_to_cos {
1828+ u8 opcode;
1829+ #define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS 0x8aUL
1830+ u8 cmd_size;
1831+ __le16 flags;
1832+ __le16 cookie;
1833+ u8 resp_size;
1834+ u8 reserved8;
1835+ __le64 resp_addr;
1836+ __le16 cos0;
1837+ #define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE 0xffffUL
1838+ __le16 cos1;
1839+ #define CMDQ_MAP_TC_TO_COS_COS1_DISABLE 0x8000UL
1840+ #define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE 0xffffUL
1841+ __le32 unused_0;
1842+};
1843+
1844+/* Query version command (16 bytes) */
1845+struct cmdq_query_version {
1846+ u8 opcode;
1847+ #define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION 0x8bUL
1848+ u8 cmd_size;
1849+ __le16 flags;
1850+ __le16 cookie;
1851+ u8 resp_size;
1852+ u8 reserved8;
1853+ __le64 resp_addr;
1854+};
1855+
1856+/* Command-Response Event Queue (CREQ) Structures */
1857+/* Base CREQ Record (16 bytes) */
1858+struct creq_base {
1859+ u8 type;
1860+ #define CREQ_BASE_TYPE_MASK 0x3fUL
1861+ #define CREQ_BASE_TYPE_SFT 0
1862+ #define CREQ_BASE_TYPE_QP_EVENT 0x38UL
1863+ #define CREQ_BASE_TYPE_FUNC_EVENT 0x3aUL
1864+ #define CREQ_BASE_RESERVED2_MASK 0xc0UL
1865+ #define CREQ_BASE_RESERVED2_SFT 6
1866+ u8 reserved56[7];
1867+ u8 v;
1868+ #define CREQ_BASE_V 0x1UL
1869+ #define CREQ_BASE_RESERVED7_MASK 0xfeUL
1870+ #define CREQ_BASE_RESERVED7_SFT 1
1871+ u8 event;
1872+ __le16 reserved48[3];
1873+};
1874+
1875+/* RoCE Function Async Event Notification (16 bytes) */
1876+struct creq_func_event {
1877+ u8 type;
1878+ #define CREQ_FUNC_EVENT_TYPE_MASK 0x3fUL
1879+ #define CREQ_FUNC_EVENT_TYPE_SFT 0
1880+ #define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT 0x3aUL
1881+ #define CREQ_FUNC_EVENT_RESERVED2_MASK 0xc0UL
1882+ #define CREQ_FUNC_EVENT_RESERVED2_SFT 6
1883+ u8 reserved56[7];
1884+ u8 v;
1885+ #define CREQ_FUNC_EVENT_V 0x1UL
1886+ #define CREQ_FUNC_EVENT_RESERVED7_MASK 0xfeUL
1887+ #define CREQ_FUNC_EVENT_RESERVED7_SFT 1
1888+ u8 event;
1889+ #define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR 0x1UL
1890+ #define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR 0x2UL
1891+ #define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR 0x3UL
1892+ #define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR 0x4UL
1893+ #define CREQ_FUNC_EVENT_EVENT_CQ_ERROR 0x5UL
1894+ #define CREQ_FUNC_EVENT_EVENT_TQM_ERROR 0x6UL
1895+ #define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR 0x7UL
1896+ #define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR 0x8UL
1897+ #define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR 0x9UL
1898+ #define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR 0xaUL
1899+ #define CREQ_FUNC_EVENT_EVENT_TIM_ERROR 0xbUL
1900+ #define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST 0x80UL
1901+ #define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED 0x81UL
1902+ __le16 reserved48[3];
1903+};
1904+
1905+/* RoCE Slowpath Command Completion (16 bytes) */
1906+struct creq_qp_event {
1907+ u8 type;
1908+ #define CREQ_QP_EVENT_TYPE_MASK 0x3fUL
1909+ #define CREQ_QP_EVENT_TYPE_SFT 0
1910+ #define CREQ_QP_EVENT_TYPE_QP_EVENT 0x38UL
1911+ #define CREQ_QP_EVENT_RESERVED2_MASK 0xc0UL
1912+ #define CREQ_QP_EVENT_RESERVED2_SFT 6
1913+ u8 status;
1914+ __le16 cookie;
1915+ __le32 reserved32;
1916+ u8 v;
1917+ #define CREQ_QP_EVENT_V 0x1UL
1918+ #define CREQ_QP_EVENT_RESERVED7_MASK 0xfeUL
1919+ #define CREQ_QP_EVENT_RESERVED7_SFT 1
1920+ u8 event;
1921+ #define CREQ_QP_EVENT_EVENT_CREATE_QP 0x1UL
1922+ #define CREQ_QP_EVENT_EVENT_DESTROY_QP 0x2UL
1923+ #define CREQ_QP_EVENT_EVENT_MODIFY_QP 0x3UL
1924+ #define CREQ_QP_EVENT_EVENT_QUERY_QP 0x4UL
1925+ #define CREQ_QP_EVENT_EVENT_CREATE_SRQ 0x5UL
1926+ #define CREQ_QP_EVENT_EVENT_DESTROY_SRQ 0x6UL
1927+ #define CREQ_QP_EVENT_EVENT_QUERY_SRQ 0x8UL
1928+ #define CREQ_QP_EVENT_EVENT_CREATE_CQ 0x9UL
1929+ #define CREQ_QP_EVENT_EVENT_DESTROY_CQ 0xaUL
1930+ #define CREQ_QP_EVENT_EVENT_RESIZE_CQ 0xcUL
1931+ #define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW 0xdUL
1932+ #define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY 0xeUL
1933+ #define CREQ_QP_EVENT_EVENT_REGISTER_MR 0xfUL
1934+ #define CREQ_QP_EVENT_EVENT_DEREGISTER_MR 0x10UL
1935+ #define CREQ_QP_EVENT_EVENT_ADD_GID 0x11UL
1936+ #define CREQ_QP_EVENT_EVENT_DELETE_GID 0x12UL
1937+ #define CREQ_QP_EVENT_EVENT_MODIFY_GID 0x17UL
1938+ #define CREQ_QP_EVENT_EVENT_QUERY_GID 0x18UL
1939+ #define CREQ_QP_EVENT_EVENT_CREATE_QP1 0x13UL
1940+ #define CREQ_QP_EVENT_EVENT_DESTROY_QP1 0x14UL
1941+ #define CREQ_QP_EVENT_EVENT_CREATE_AH 0x15UL
1942+ #define CREQ_QP_EVENT_EVENT_DESTROY_AH 0x16UL
1943+ #define CREQ_QP_EVENT_EVENT_INITIALIZE_FW 0x80UL
1944+ #define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW 0x81UL
1945+ #define CREQ_QP_EVENT_EVENT_STOP_FUNC 0x82UL
1946+ #define CREQ_QP_EVENT_EVENT_QUERY_FUNC 0x83UL
1947+ #define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES 0x84UL
1948+ #define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS 0x8aUL
1949+ #define CREQ_QP_EVENT_EVENT_QUERY_VERSION 0x8bUL
1950+ #define CREQ_QP_EVENT_EVENT_MODIFY_CC 0x8cUL
1951+ #define CREQ_QP_EVENT_EVENT_QUERY_CC 0x8dUL
1952+ #define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION 0xc0UL
1953+ __le16 reserved48[3];
1954+};
1955+
1956+/* Create QP command response (16 bytes) */
1957+struct creq_create_qp_resp {
1958+ u8 type;
1959+ #define CREQ_CREATE_QP_RESP_TYPE_MASK 0x3fUL
1960+ #define CREQ_CREATE_QP_RESP_TYPE_SFT 0
1961+ #define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT 0x38UL
1962+ #define CREQ_CREATE_QP_RESP_RESERVED2_MASK 0xc0UL
1963+ #define CREQ_CREATE_QP_RESP_RESERVED2_SFT 6
1964+ u8 status;
1965+ __le16 cookie;
1966+ __le32 xid;
1967+ u8 v;
1968+ #define CREQ_CREATE_QP_RESP_V 0x1UL
1969+ #define CREQ_CREATE_QP_RESP_RESERVED7_MASK 0xfeUL
1970+ #define CREQ_CREATE_QP_RESP_RESERVED7_SFT 1
1971+ u8 event;
1972+ #define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP 0x1UL
1973+ __le16 reserved48[3];
1974+};
1975+
1976+/* Destroy QP command response (16 bytes) */
1977+struct creq_destroy_qp_resp {
1978+ u8 type;
1979+ #define CREQ_DESTROY_QP_RESP_TYPE_MASK 0x3fUL
1980+ #define CREQ_DESTROY_QP_RESP_TYPE_SFT 0
1981+ #define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT 0x38UL
1982+ #define CREQ_DESTROY_QP_RESP_RESERVED2_MASK 0xc0UL
1983+ #define CREQ_DESTROY_QP_RESP_RESERVED2_SFT 6
1984+ u8 status;
1985+ __le16 cookie;
1986+ __le32 xid;
1987+ u8 v;
1988+ #define CREQ_DESTROY_QP_RESP_V 0x1UL
1989+ #define CREQ_DESTROY_QP_RESP_RESERVED7_MASK 0xfeUL
1990+ #define CREQ_DESTROY_QP_RESP_RESERVED7_SFT 1
1991+ u8 event;
1992+ #define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP 0x2UL
1993+ __le16 reserved48[3];
1994+};
1995+
1996+/* Modify QP command response (16 bytes) */
1997+struct creq_modify_qp_resp {
1998+ u8 type;
1999+ #define CREQ_MODIFY_QP_RESP_TYPE_MASK 0x3fUL
2000+ #define CREQ_MODIFY_QP_RESP_TYPE_SFT 0
2001+ #define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT 0x38UL
2002+ #define CREQ_MODIFY_QP_RESP_RESERVED2_MASK 0xc0UL
2003+ #define CREQ_MODIFY_QP_RESP_RESERVED2_SFT 6
2004+ u8 status;
2005+ __le16 cookie;
2006+ __le32 xid;
2007+ u8 v;
2008+ #define CREQ_MODIFY_QP_RESP_V 0x1UL
2009+ #define CREQ_MODIFY_QP_RESP_RESERVED7_MASK 0xfeUL
2010+ #define CREQ_MODIFY_QP_RESP_RESERVED7_SFT 1
2011+ u8 event;
2012+ #define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP 0x3UL
2013+ __le16 reserved48[3];
2014+};
2015+
2016+/* Query QP command response (16 bytes) */
2017+struct creq_query_qp_resp {
2018+ u8 type;
2019+ #define CREQ_QUERY_QP_RESP_TYPE_MASK 0x3fUL
2020+ #define CREQ_QUERY_QP_RESP_TYPE_SFT 0
2021+ #define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT 0x38UL
2022+ #define CREQ_QUERY_QP_RESP_RESERVED2_MASK 0xc0UL
2023+ #define CREQ_QUERY_QP_RESP_RESERVED2_SFT 6
2024+ u8 status;
2025+ __le16 cookie;
2026+ __le32 size;
2027+ u8 v;
2028+ #define CREQ_QUERY_QP_RESP_V 0x1UL
2029+ #define CREQ_QUERY_QP_RESP_RESERVED7_MASK 0xfeUL
2030+ #define CREQ_QUERY_QP_RESP_RESERVED7_SFT 1
2031+ u8 event;
2032+ #define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP 0x4UL
2033+ __le16 reserved48[3];
2034+};
2035+
2036+/* Query QP command response side buffer structure (104 bytes) */
2037+struct creq_query_qp_resp_sb {
2038+ u8 opcode;
2039+ #define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP 0x4UL
2040+ u8 status;
2041+ __le16 cookie;
2042+ __le16 flags;
2043+ u8 resp_size;
2044+ u8 reserved8;
2045+ __le32 xid;
2046+ u8 en_sqd_async_notify_state;
2047+ #define CREQ_QUERY_QP_RESP_SB_STATE_MASK 0xfUL
2048+ #define CREQ_QUERY_QP_RESP_SB_STATE_SFT 0
2049+ #define CREQ_QUERY_QP_RESP_SB_STATE_RESET 0x0UL
2050+ #define CREQ_QUERY_QP_RESP_SB_STATE_INIT 0x1UL
2051+ #define CREQ_QUERY_QP_RESP_SB_STATE_RTR 0x2UL
2052+ #define CREQ_QUERY_QP_RESP_SB_STATE_RTS 0x3UL
2053+ #define CREQ_QUERY_QP_RESP_SB_STATE_SQD 0x4UL
2054+ #define CREQ_QUERY_QP_RESP_SB_STATE_SQE 0x5UL
2055+ #define CREQ_QUERY_QP_RESP_SB_STATE_ERR 0x6UL
2056+ #define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY 0x10UL
2057+ u8 access;
2058+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE 0x1UL
2059+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE 0x2UL
2060+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ 0x4UL
2061+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC 0x8UL
2062+ __le16 pkey;
2063+ __le32 qkey;
2064+ __le32 reserved32;
2065+ __le32 dgid[4];
2066+ __le32 flow_label;
2067+ __le16 sgid_index;
2068+ u8 hop_limit;
2069+ u8 traffic_class;
2070+ __le16 dest_mac[3];
2071+ __le16 path_mtu_dest_vlan_id;
2072+ #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK 0xfffUL
2073+ #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0
2074+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK 0xf000UL
2075+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT 12
2076+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 (0x0UL << 12)
2077+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 (0x1UL << 12)
2078+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 (0x2UL << 12)
2079+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 (0x3UL << 12)
2080+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 (0x4UL << 12)
2081+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 (0x5UL << 12)
2082+ u8 timeout;
2083+ u8 retry_cnt;
2084+ u8 rnr_retry;
2085+ u8 min_rnr_timer;
2086+ __le32 rq_psn;
2087+ __le32 sq_psn;
2088+ u8 max_rd_atomic;
2089+ u8 max_dest_rd_atomic;
2090+ u8 tos_dscp_tos_ecn;
2091+ #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK 0x3UL
2092+ #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT 0
2093+ #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK 0xfcUL
2094+ #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2
2095+ u8 enable_cc;
2096+ #define CREQ_QUERY_QP_RESP_SB_ENABLE_CC 0x1UL
2097+ #define CREQ_QUERY_QP_RESP_SB_RESERVED7_MASK 0xfeUL
2098+ #define CREQ_QUERY_QP_RESP_SB_RESERVED7_SFT 1
2099+ __le32 sq_size;
2100+ __le32 rq_size;
2101+ __le16 sq_sge;
2102+ __le16 rq_sge;
2103+ __le32 max_inline_data;
2104+ __le32 dest_qp_id;
2105+ __le32 unused_1;
2106+ __le16 src_mac[3];
2107+ __le16 vlan_pcp_vlan_dei_vlan_id;
2108+ #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK 0xfffUL
2109+ #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT 0
2110+ #define CREQ_QUERY_QP_RESP_SB_VLAN_DEI 0x1000UL
2111+ #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK 0xe000UL
2112+ #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13
2113+};
2114+
2115+/* Create SRQ command response (16 bytes) */
2116+struct creq_create_srq_resp {
2117+ u8 type;
2118+ #define CREQ_CREATE_SRQ_RESP_TYPE_MASK 0x3fUL
2119+ #define CREQ_CREATE_SRQ_RESP_TYPE_SFT 0
2120+ #define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT 0x38UL
2121+ #define CREQ_CREATE_SRQ_RESP_RESERVED2_MASK 0xc0UL
2122+ #define CREQ_CREATE_SRQ_RESP_RESERVED2_SFT 6
2123+ u8 status;
2124+ __le16 cookie;
2125+ __le32 xid;
2126+ u8 v;
2127+ #define CREQ_CREATE_SRQ_RESP_V 0x1UL
2128+ #define CREQ_CREATE_SRQ_RESP_RESERVED7_MASK 0xfeUL
2129+ #define CREQ_CREATE_SRQ_RESP_RESERVED7_SFT 1
2130+ u8 event;
2131+ #define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ 0x5UL
2132+ __le16 reserved48[3];
2133+};
2134+
2135+/* Destroy SRQ command response (16 bytes) */
2136+struct creq_destroy_srq_resp {
2137+ u8 type;
2138+ #define CREQ_DESTROY_SRQ_RESP_TYPE_MASK 0x3fUL
2139+ #define CREQ_DESTROY_SRQ_RESP_TYPE_SFT 0
2140+ #define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT 0x38UL
2141+ #define CREQ_DESTROY_SRQ_RESP_RESERVED2_MASK 0xc0UL
2142+ #define CREQ_DESTROY_SRQ_RESP_RESERVED2_SFT 6
2143+ u8 status;
2144+ __le16 cookie;
2145+ __le32 xid;
2146+ u8 v;
2147+ #define CREQ_DESTROY_SRQ_RESP_V 0x1UL
2148+ #define CREQ_DESTROY_SRQ_RESP_RESERVED7_MASK 0xfeUL
2149+ #define CREQ_DESTROY_SRQ_RESP_RESERVED7_SFT 1
2150+ u8 event;
2151+ #define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ 0x6UL
2152+ __le16 enable_for_arm[3];
2153+ #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK 0x30000UL
2154+ #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16
2155+ #define CREQ_DESTROY_SRQ_RESP_RESERVED46_MASK 0xfffc0000UL
2156+ #define CREQ_DESTROY_SRQ_RESP_RESERVED46_SFT 18
2157+};
2158+
2159+/* Query SRQ command response (16 bytes) */
2160+struct creq_query_srq_resp {
2161+ u8 type;
2162+ #define CREQ_QUERY_SRQ_RESP_TYPE_MASK 0x3fUL
2163+ #define CREQ_QUERY_SRQ_RESP_TYPE_SFT 0
2164+ #define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT 0x38UL
2165+ #define CREQ_QUERY_SRQ_RESP_RESERVED2_MASK 0xc0UL
2166+ #define CREQ_QUERY_SRQ_RESP_RESERVED2_SFT 6
2167+ u8 status;
2168+ __le16 cookie;
2169+ __le32 size;
2170+ u8 v;
2171+ #define CREQ_QUERY_SRQ_RESP_V 0x1UL
2172+ #define CREQ_QUERY_SRQ_RESP_RESERVED7_MASK 0xfeUL
2173+ #define CREQ_QUERY_SRQ_RESP_RESERVED7_SFT 1
2174+ u8 event;
2175+ #define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ 0x8UL
2176+ __le16 reserved48[3];
2177+};
2178+
2179+/* Query SRQ command response side buffer structure (24 bytes) */
2180+struct creq_query_srq_resp_sb {
2181+ u8 opcode;
2182+ #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ 0x8UL
2183+ u8 status;
2184+ __le16 cookie;
2185+ __le16 flags;
2186+ u8 resp_size;
2187+ u8 reserved8;
2188+ __le32 xid;
2189+ __le16 srq_limit;
2190+ __le16 reserved16;
2191+ __le32 data[4];
2192+};
2193+
2194+/* Create CQ command Response (16 bytes) */
2195+struct creq_create_cq_resp {
2196+ u8 type;
2197+ #define CREQ_CREATE_CQ_RESP_TYPE_MASK 0x3fUL
2198+ #define CREQ_CREATE_CQ_RESP_TYPE_SFT 0
2199+ #define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT 0x38UL
2200+ #define CREQ_CREATE_CQ_RESP_RESERVED2_MASK 0xc0UL
2201+ #define CREQ_CREATE_CQ_RESP_RESERVED2_SFT 6
2202+ u8 status;
2203+ __le16 cookie;
2204+ __le32 xid;
2205+ u8 v;
2206+ #define CREQ_CREATE_CQ_RESP_V 0x1UL
2207+ #define CREQ_CREATE_CQ_RESP_RESERVED7_MASK 0xfeUL
2208+ #define CREQ_CREATE_CQ_RESP_RESERVED7_SFT 1
2209+ u8 event;
2210+ #define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ 0x9UL
2211+ __le16 reserved48[3];
2212+};
2213+
2214+/* Destroy CQ command response (16 bytes) */
2215+struct creq_destroy_cq_resp {
2216+ u8 type;
2217+ #define CREQ_DESTROY_CQ_RESP_TYPE_MASK 0x3fUL
2218+ #define CREQ_DESTROY_CQ_RESP_TYPE_SFT 0
2219+ #define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT 0x38UL
2220+ #define CREQ_DESTROY_CQ_RESP_RESERVED2_MASK 0xc0UL
2221+ #define CREQ_DESTROY_CQ_RESP_RESERVED2_SFT 6
2222+ u8 status;
2223+ __le16 cookie;
2224+ __le32 xid;
2225+ u8 v;
2226+ #define CREQ_DESTROY_CQ_RESP_V 0x1UL
2227+ #define CREQ_DESTROY_CQ_RESP_RESERVED7_MASK 0xfeUL
2228+ #define CREQ_DESTROY_CQ_RESP_RESERVED7_SFT 1
2229+ u8 event;
2230+ #define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ 0xaUL
2231+ __le16 cq_arm_lvl;
2232+ #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK 0x3UL
2233+ #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0
2234+ #define CREQ_DESTROY_CQ_RESP_RESERVED14_MASK 0xfffcUL
2235+ #define CREQ_DESTROY_CQ_RESP_RESERVED14_SFT 2
2236+ __le16 total_cnq_events;
2237+ __le16 reserved16;
2238+};
2239+
2240+/* Resize CQ command response (16 bytes) */
2241+struct creq_resize_cq_resp {
2242+ u8 type;
2243+ #define CREQ_RESIZE_CQ_RESP_TYPE_MASK 0x3fUL
2244+ #define CREQ_RESIZE_CQ_RESP_TYPE_SFT 0
2245+ #define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT 0x38UL
2246+ #define CREQ_RESIZE_CQ_RESP_RESERVED2_MASK 0xc0UL
2247+ #define CREQ_RESIZE_CQ_RESP_RESERVED2_SFT 6
2248+ u8 status;
2249+ __le16 cookie;
2250+ __le32 xid;
2251+ u8 v;
2252+ #define CREQ_RESIZE_CQ_RESP_V 0x1UL
2253+ #define CREQ_RESIZE_CQ_RESP_RESERVED7_MASK 0xfeUL
2254+ #define CREQ_RESIZE_CQ_RESP_RESERVED7_SFT 1
2255+ u8 event;
2256+ #define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ 0xcUL
2257+ __le16 reserved48[3];
2258+};
2259+
2260+/* Allocate MRW command response (16 bytes) */
2261+struct creq_allocate_mrw_resp {
2262+ u8 type;
2263+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK 0x3fUL
2264+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT 0
2265+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT 0x38UL
2266+ #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_MASK 0xc0UL
2267+ #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_SFT 6
2268+ u8 status;
2269+ __le16 cookie;
2270+ __le32 xid;
2271+ u8 v;
2272+ #define CREQ_ALLOCATE_MRW_RESP_V 0x1UL
2273+ #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_MASK 0xfeUL
2274+ #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_SFT 1
2275+ u8 event;
2276+ #define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW 0xdUL
2277+ __le16 reserved48[3];
2278+};
2279+
2280+/* De-allocate key command response (16 bytes) */
2281+struct creq_deallocate_key_resp {
2282+ u8 type;
2283+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK 0x3fUL
2284+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT 0
2285+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT 0x38UL
2286+ #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_MASK 0xc0UL
2287+ #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_SFT 6
2288+ u8 status;
2289+ __le16 cookie;
2290+ __le32 xid;
2291+ u8 v;
2292+ #define CREQ_DEALLOCATE_KEY_RESP_V 0x1UL
2293+ #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_MASK 0xfeUL
2294+ #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_SFT 1
2295+ u8 event;
2296+ #define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY 0xeUL
2297+ __le16 reserved16;
2298+ __le32 bound_window_info;
2299+};
2300+
2301+/* Register MR command response (16 bytes) */
2302+struct creq_register_mr_resp {
2303+ u8 type;
2304+ #define CREQ_REGISTER_MR_RESP_TYPE_MASK 0x3fUL
2305+ #define CREQ_REGISTER_MR_RESP_TYPE_SFT 0
2306+ #define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL
2307+ #define CREQ_REGISTER_MR_RESP_RESERVED2_MASK 0xc0UL
2308+ #define CREQ_REGISTER_MR_RESP_RESERVED2_SFT 6
2309+ u8 status;
2310+ __le16 cookie;
2311+ __le32 xid;
2312+ u8 v;
2313+ #define CREQ_REGISTER_MR_RESP_V 0x1UL
2314+ #define CREQ_REGISTER_MR_RESP_RESERVED7_MASK 0xfeUL
2315+ #define CREQ_REGISTER_MR_RESP_RESERVED7_SFT 1
2316+ u8 event;
2317+ #define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR 0xfUL
2318+ __le16 reserved48[3];
2319+};
2320+
2321+/* Deregister MR command response (16 bytes) */
2322+struct creq_deregister_mr_resp {
2323+ u8 type;
2324+ #define CREQ_DEREGISTER_MR_RESP_TYPE_MASK 0x3fUL
2325+ #define CREQ_DEREGISTER_MR_RESP_TYPE_SFT 0
2326+ #define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL
2327+ #define CREQ_DEREGISTER_MR_RESP_RESERVED2_MASK 0xc0UL
2328+ #define CREQ_DEREGISTER_MR_RESP_RESERVED2_SFT 6
2329+ u8 status;
2330+ __le16 cookie;
2331+ __le32 xid;
2332+ u8 v;
2333+ #define CREQ_DEREGISTER_MR_RESP_V 0x1UL
2334+ #define CREQ_DEREGISTER_MR_RESP_RESERVED7_MASK 0xfeUL
2335+ #define CREQ_DEREGISTER_MR_RESP_RESERVED7_SFT 1
2336+ u8 event;
2337+ #define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR 0x10UL
2338+ __le16 reserved16;
2339+ __le32 bound_windows;
2340+};
2341+
2342+/* Add GID command response (16 bytes) */
2343+struct creq_add_gid_resp {
2344+ u8 type;
2345+ #define CREQ_ADD_GID_RESP_TYPE_MASK 0x3fUL
2346+ #define CREQ_ADD_GID_RESP_TYPE_SFT 0
2347+ #define CREQ_ADD_GID_RESP_TYPE_QP_EVENT 0x38UL
2348+ #define CREQ_ADD_GID_RESP_RESERVED2_MASK 0xc0UL
2349+ #define CREQ_ADD_GID_RESP_RESERVED2_SFT 6
2350+ u8 status;
2351+ __le16 cookie;
2352+ __le32 xid;
2353+ u8 v;
2354+ #define CREQ_ADD_GID_RESP_V 0x1UL
2355+ #define CREQ_ADD_GID_RESP_RESERVED7_MASK 0xfeUL
2356+ #define CREQ_ADD_GID_RESP_RESERVED7_SFT 1
2357+ u8 event;
2358+ #define CREQ_ADD_GID_RESP_EVENT_ADD_GID 0x11UL
2359+ __le16 reserved48[3];
2360+};
2361+
2362+/* Delete GID command response (16 bytes) */
2363+struct creq_delete_gid_resp {
2364+ u8 type;
2365+ #define CREQ_DELETE_GID_RESP_TYPE_MASK 0x3fUL
2366+ #define CREQ_DELETE_GID_RESP_TYPE_SFT 0
2367+ #define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT 0x38UL
2368+ #define CREQ_DELETE_GID_RESP_RESERVED2_MASK 0xc0UL
2369+ #define CREQ_DELETE_GID_RESP_RESERVED2_SFT 6
2370+ u8 status;
2371+ __le16 cookie;
2372+ __le32 xid;
2373+ u8 v;
2374+ #define CREQ_DELETE_GID_RESP_V 0x1UL
2375+ #define CREQ_DELETE_GID_RESP_RESERVED7_MASK 0xfeUL
2376+ #define CREQ_DELETE_GID_RESP_RESERVED7_SFT 1
2377+ u8 event;
2378+ #define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID 0x12UL
2379+ __le16 reserved48[3];
2380+};
2381+
2382+/* Modify GID command response (16 bytes) */
2383+struct creq_modify_gid_resp {
2384+ u8 type;
2385+ #define CREQ_MODIFY_GID_RESP_TYPE_MASK 0x3fUL
2386+ #define CREQ_MODIFY_GID_RESP_TYPE_SFT 0
2387+ #define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT 0x38UL
2388+ #define CREQ_MODIFY_GID_RESP_RESERVED2_MASK 0xc0UL
2389+ #define CREQ_MODIFY_GID_RESP_RESERVED2_SFT 6
2390+ u8 status;
2391+ __le16 cookie;
2392+ __le32 xid;
2393+ u8 v;
2394+ #define CREQ_MODIFY_GID_RESP_V 0x1UL
2395+ #define CREQ_MODIFY_GID_RESP_RESERVED7_MASK 0xfeUL
2396+ #define CREQ_MODIFY_GID_RESP_RESERVED7_SFT 1
2397+ u8 event;
2398+ #define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID 0x11UL
2399+ __le16 reserved48[3];
2400+};
2401+
2402+/* Query GID command response (16 bytes) */
2403+struct creq_query_gid_resp {
2404+ u8 type;
2405+ #define CREQ_QUERY_GID_RESP_TYPE_MASK 0x3fUL
2406+ #define CREQ_QUERY_GID_RESP_TYPE_SFT 0
2407+ #define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT 0x38UL
2408+ #define CREQ_QUERY_GID_RESP_RESERVED2_MASK 0xc0UL
2409+ #define CREQ_QUERY_GID_RESP_RESERVED2_SFT 6
2410+ u8 status;
2411+ __le16 cookie;
2412+ __le32 size;
2413+ u8 v;
2414+ #define CREQ_QUERY_GID_RESP_V 0x1UL
2415+ #define CREQ_QUERY_GID_RESP_RESERVED7_MASK 0xfeUL
2416+ #define CREQ_QUERY_GID_RESP_RESERVED7_SFT 1
2417+ u8 event;
2418+ #define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID 0x18UL
2419+ __le16 reserved48[3];
2420+};
2421+
2422+/* Query GID command response side buffer structure (40 bytes) */
2423+struct creq_query_gid_resp_sb {
2424+ u8 opcode;
2425+ #define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID 0x18UL
2426+ u8 status;
2427+ __le16 cookie;
2428+ __le16 flags;
2429+ u8 resp_size;
2430+ u8 reserved8;
2431+ __le32 gid[4];
2432+ __le16 src_mac[3];
2433+ __le16 vlan;
2434+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK 0xfffUL
2435+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT 0
2436+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK 0x7000UL
2437+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT 12
2438+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 (0x0UL << 12)
2439+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 (0x1UL << 12)
2440+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 (0x2UL << 12)
2441+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 (0x3UL << 12)
2442+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 (0x4UL << 12)
2443+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
2444+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
2445+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
2446+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST \
2447+ CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3
2448+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN 0x8000UL
2449+ __le16 ipid;
2450+ __le16 gid_index;
2451+ __le32 unused_0;
2452+};
2453+
2454+/* Create QP1 command response (16 bytes) */
2455+struct creq_create_qp1_resp {
2456+ u8 type;
2457+ #define CREQ_CREATE_QP1_RESP_TYPE_MASK 0x3fUL
2458+ #define CREQ_CREATE_QP1_RESP_TYPE_SFT 0
2459+ #define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT 0x38UL
2460+ #define CREQ_CREATE_QP1_RESP_RESERVED2_MASK 0xc0UL
2461+ #define CREQ_CREATE_QP1_RESP_RESERVED2_SFT 6
2462+ u8 status;
2463+ __le16 cookie;
2464+ __le32 xid;
2465+ u8 v;
2466+ #define CREQ_CREATE_QP1_RESP_V 0x1UL
2467+ #define CREQ_CREATE_QP1_RESP_RESERVED7_MASK 0xfeUL
2468+ #define CREQ_CREATE_QP1_RESP_RESERVED7_SFT 1
2469+ u8 event;
2470+ #define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 0x13UL
2471+ __le16 reserved48[3];
2472+};
2473+
2474+/* Destroy QP1 command response (16 bytes) */
2475+struct creq_destroy_qp1_resp {
2476+ u8 type;
2477+ #define CREQ_DESTROY_QP1_RESP_TYPE_MASK 0x3fUL
2478+ #define CREQ_DESTROY_QP1_RESP_TYPE_SFT 0
2479+ #define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT 0x38UL
2480+ #define CREQ_DESTROY_QP1_RESP_RESERVED2_MASK 0xc0UL
2481+ #define CREQ_DESTROY_QP1_RESP_RESERVED2_SFT 6
2482+ u8 status;
2483+ __le16 cookie;
2484+ __le32 xid;
2485+ u8 v;
2486+ #define CREQ_DESTROY_QP1_RESP_V 0x1UL
2487+ #define CREQ_DESTROY_QP1_RESP_RESERVED7_MASK 0xfeUL
2488+ #define CREQ_DESTROY_QP1_RESP_RESERVED7_SFT 1
2489+ u8 event;
2490+ #define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 0x14UL
2491+ __le16 reserved48[3];
2492+};
2493+
2494+/* Create AH command response (16 bytes) */
2495+struct creq_create_ah_resp {
2496+ u8 type;
2497+ #define CREQ_CREATE_AH_RESP_TYPE_MASK 0x3fUL
2498+ #define CREQ_CREATE_AH_RESP_TYPE_SFT 0
2499+ #define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT 0x38UL
2500+ #define CREQ_CREATE_AH_RESP_RESERVED2_MASK 0xc0UL
2501+ #define CREQ_CREATE_AH_RESP_RESERVED2_SFT 6
2502+ u8 status;
2503+ __le16 cookie;
2504+ __le32 xid;
2505+ u8 v;
2506+ #define CREQ_CREATE_AH_RESP_V 0x1UL
2507+ #define CREQ_CREATE_AH_RESP_RESERVED7_MASK 0xfeUL
2508+ #define CREQ_CREATE_AH_RESP_RESERVED7_SFT 1
2509+ u8 event;
2510+ #define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH 0x15UL
2511+ __le16 reserved48[3];
2512+};
2513+
2514+/* Destroy AH command response (16 bytes) */
2515+struct creq_destroy_ah_resp {
2516+ u8 type;
2517+ #define CREQ_DESTROY_AH_RESP_TYPE_MASK 0x3fUL
2518+ #define CREQ_DESTROY_AH_RESP_TYPE_SFT 0
2519+ #define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT 0x38UL
2520+ #define CREQ_DESTROY_AH_RESP_RESERVED2_MASK 0xc0UL
2521+ #define CREQ_DESTROY_AH_RESP_RESERVED2_SFT 6
2522+ u8 status;
2523+ __le16 cookie;
2524+ __le32 xid;
2525+ u8 v;
2526+ #define CREQ_DESTROY_AH_RESP_V 0x1UL
2527+ #define CREQ_DESTROY_AH_RESP_RESERVED7_MASK 0xfeUL
2528+ #define CREQ_DESTROY_AH_RESP_RESERVED7_SFT 1
2529+ u8 event;
2530+ #define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH 0x16UL
2531+ __le16 reserved48[3];
2532+};
2533+
2534+/* Initialize Firmware command response (16 bytes) */
2535+struct creq_initialize_fw_resp {
2536+ u8 type;
2537+ #define CREQ_INITIALIZE_FW_RESP_TYPE_MASK 0x3fUL
2538+ #define CREQ_INITIALIZE_FW_RESP_TYPE_SFT 0
2539+ #define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL
2540+ #define CREQ_INITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL
2541+ #define CREQ_INITIALIZE_FW_RESP_RESERVED2_SFT 6
2542+ u8 status;
2543+ __le16 cookie;
2544+ __le32 reserved32;
2545+ u8 v;
2546+ #define CREQ_INITIALIZE_FW_RESP_V 0x1UL
2547+ #define CREQ_INITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL
2548+ #define CREQ_INITIALIZE_FW_RESP_RESERVED7_SFT 1
2549+ u8 event;
2550+ #define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW 0x80UL
2551+ __le16 reserved48[3];
2552+};
2553+
2554+/* De-initialize Firmware command response (16 bytes) */
2555+struct creq_deinitialize_fw_resp {
2556+ u8 type;
2557+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK 0x3fUL
2558+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT 0
2559+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL
2560+ #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL
2561+ #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_SFT 6
2562+ u8 status;
2563+ __le16 cookie;
2564+ __le32 reserved32;
2565+ u8 v;
2566+ #define CREQ_DEINITIALIZE_FW_RESP_V 0x1UL
2567+ #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL
2568+ #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_SFT 1
2569+ u8 event;
2570+ #define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW 0x81UL
2571+ __le16 reserved48[3];
2572+};
2573+
2574+/* Stop function command response (16 bytes) */
2575+struct creq_stop_func_resp {
2576+ u8 type;
2577+ #define CREQ_STOP_FUNC_RESP_TYPE_MASK 0x3fUL
2578+ #define CREQ_STOP_FUNC_RESP_TYPE_SFT 0
2579+ #define CREQ_STOP_FUNC_RESP_TYPE_QP_EVENT 0x38UL
2580+ #define CREQ_STOP_FUNC_RESP_RESERVED2_MASK 0xc0UL
2581+ #define CREQ_STOP_FUNC_RESP_RESERVED2_SFT 6
2582+ u8 status;
2583+ __le16 cookie;
2584+ __le32 reserved32;
2585+ u8 v;
2586+ #define CREQ_STOP_FUNC_RESP_V 0x1UL
2587+ #define CREQ_STOP_FUNC_RESP_RESERVED7_MASK 0xfeUL
2588+ #define CREQ_STOP_FUNC_RESP_RESERVED7_SFT 1
2589+ u8 event;
2590+ #define CREQ_STOP_FUNC_RESP_EVENT_STOP_FUNC 0x82UL
2591+ __le16 reserved48[3];
2592+};
2593+
2594+/* Query function command response (16 bytes) */
2595+struct creq_query_func_resp {
2596+ u8 type;
2597+ #define CREQ_QUERY_FUNC_RESP_TYPE_MASK 0x3fUL
2598+ #define CREQ_QUERY_FUNC_RESP_TYPE_SFT 0
2599+ #define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT 0x38UL
2600+ #define CREQ_QUERY_FUNC_RESP_RESERVED2_MASK 0xc0UL
2601+ #define CREQ_QUERY_FUNC_RESP_RESERVED2_SFT 6
2602+ u8 status;
2603+ __le16 cookie;
2604+ __le32 size;
2605+ u8 v;
2606+ #define CREQ_QUERY_FUNC_RESP_V 0x1UL
2607+ #define CREQ_QUERY_FUNC_RESP_RESERVED7_MASK 0xfeUL
2608+ #define CREQ_QUERY_FUNC_RESP_RESERVED7_SFT 1
2609+ u8 event;
2610+ #define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC 0x83UL
2611+ __le16 reserved48[3];
2612+};
2613+
2614+/* Query function command response side buffer structure (88 bytes) */
2615+struct creq_query_func_resp_sb {
2616+ u8 opcode;
2617+ #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC 0x83UL
2618+ u8 status;
2619+ __le16 cookie;
2620+ __le16 flags;
2621+ u8 resp_size;
2622+ u8 reserved8;
2623+ __le64 max_mr_size;
2624+ __le32 max_qp;
2625+ __le16 max_qp_wr;
2626+ __le16 dev_cap_flags;
2627+ #define CREQ_QUERY_FUNC_RESP_SB_DEV_CAP_FLAGS_RESIZE_QP 0x1UL
2628+ __le32 max_cq;
2629+ __le32 max_cqe;
2630+ __le32 max_pd;
2631+ u8 max_sge;
2632+ u8 max_srq_sge;
2633+ u8 max_qp_rd_atom;
2634+ u8 max_qp_init_rd_atom;
2635+ __le32 max_mr;
2636+ __le32 max_mw;
2637+ __le32 max_raw_eth_qp;
2638+ __le32 max_ah;
2639+ __le32 max_fmr;
2640+ __le32 max_srq_wr;
2641+ __le32 max_pkeys;
2642+ __le32 max_inline_data;
2643+ u8 max_map_per_fmr;
2644+ u8 l2_db_space_size;
2645+ __le16 max_srq;
2646+ __le32 max_gid;
2647+ __le32 tqm_alloc_reqs[8];
2648+};
2649+
2650+/* Set resources command response (16 bytes) */
2651+struct creq_set_func_resources_resp {
2652+ u8 type;
2653+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK 0x3fUL
2654+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT 0
2655+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT 0x38UL
2656+ #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_MASK 0xc0UL
2657+ #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_SFT 6
2658+ u8 status;
2659+ __le16 cookie;
2660+ __le32 reserved32;
2661+ u8 v;
2662+ #define CREQ_SET_FUNC_RESOURCES_RESP_V 0x1UL
2663+ #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_MASK 0xfeUL
2664+ #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_SFT 1
2665+ u8 event;
2666+ #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL
2667+ __le16 reserved48[3];
2668+};
2669+
2670+/* Map TC to COS response (16 bytes) */
2671+struct creq_map_tc_to_cos_resp {
2672+ u8 type;
2673+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK 0x3fUL
2674+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT 0
2675+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT 0x38UL
2676+ #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_MASK 0xc0UL
2677+ #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_SFT 6
2678+ u8 status;
2679+ __le16 cookie;
2680+ __le32 reserved32;
2681+ u8 v;
2682+ #define CREQ_MAP_TC_TO_COS_RESP_V 0x1UL
2683+ #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_MASK 0xfeUL
2684+ #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_SFT 1
2685+ u8 event;
2686+ #define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS 0x8aUL
2687+ __le16 reserved48[3];
2688+};
2689+
2690+/* Query version response (16 bytes) */
2691+struct creq_query_version_resp {
2692+ u8 type;
2693+ #define CREQ_QUERY_VERSION_RESP_TYPE_MASK 0x3fUL
2694+ #define CREQ_QUERY_VERSION_RESP_TYPE_SFT 0
2695+ #define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT 0x38UL
2696+ #define CREQ_QUERY_VERSION_RESP_RESERVED2_MASK 0xc0UL
2697+ #define CREQ_QUERY_VERSION_RESP_RESERVED2_SFT 6
2698+ u8 status;
2699+ __le16 cookie;
2700+ u8 fw_maj;
2701+ u8 fw_minor;
2702+ u8 fw_bld;
2703+ u8 fw_rsvd;
2704+ u8 v;
2705+ #define CREQ_QUERY_VERSION_RESP_V 0x1UL
2706+ #define CREQ_QUERY_VERSION_RESP_RESERVED7_MASK 0xfeUL
2707+ #define CREQ_QUERY_VERSION_RESP_RESERVED7_SFT 1
2708+ u8 event;
2709+ #define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION 0x8bUL
2710+ __le16 reserved16;
2711+ u8 intf_maj;
2712+ u8 intf_minor;
2713+ u8 intf_bld;
2714+ u8 intf_rsvd;
2715+};
2716+
2717+/* Modify congestion control command response (16 bytes) */
2718+struct creq_modify_cc_resp {
2719+ u8 type;
2720+ #define CREQ_MODIFY_CC_RESP_TYPE_MASK 0x3fUL
2721+ #define CREQ_MODIFY_CC_RESP_TYPE_SFT 0
2722+ #define CREQ_MODIFY_CC_RESP_TYPE_QP_EVENT 0x38UL
2723+ #define CREQ_MODIFY_CC_RESP_RESERVED2_MASK 0xc0UL
2724+ #define CREQ_MODIFY_CC_RESP_RESERVED2_SFT 6
2725+ u8 status;
2726+ __le16 cookie;
2727+ __le32 reserved32;
2728+ u8 v;
2729+ #define CREQ_MODIFY_CC_RESP_V 0x1UL
2730+ #define CREQ_MODIFY_CC_RESP_RESERVED7_MASK 0xfeUL
2731+ #define CREQ_MODIFY_CC_RESP_RESERVED7_SFT 1
2732+ u8 event;
2733+ #define CREQ_MODIFY_CC_RESP_EVENT_MODIFY_CC 0x8cUL
2734+ __le16 reserved48[3];
2735+};
2736+
2737+/* Query congestion control command response (16 bytes) */
2738+struct creq_query_cc_resp {
2739+ u8 type;
2740+ #define CREQ_QUERY_CC_RESP_TYPE_MASK 0x3fUL
2741+ #define CREQ_QUERY_CC_RESP_TYPE_SFT 0
2742+ #define CREQ_QUERY_CC_RESP_TYPE_QP_EVENT 0x38UL
2743+ #define CREQ_QUERY_CC_RESP_RESERVED2_MASK 0xc0UL
2744+ #define CREQ_QUERY_CC_RESP_RESERVED2_SFT 6
2745+ u8 status;
2746+ __le16 cookie;
2747+ __le32 size;
2748+ u8 v;
2749+ #define CREQ_QUERY_CC_RESP_V 0x1UL
2750+ #define CREQ_QUERY_CC_RESP_RESERVED7_MASK 0xfeUL
2751+ #define CREQ_QUERY_CC_RESP_RESERVED7_SFT 1
2752+ u8 event;
2753+ #define CREQ_QUERY_CC_RESP_EVENT_QUERY_CC 0x8dUL
2754+ __le16 reserved48[3];
2755+};
2756+
2757+/* Query congestion control command response side buffer structure (32 bytes) */
2758+struct creq_query_cc_resp_sb {
2759+ u8 opcode;
2760+ #define CREQ_QUERY_CC_RESP_SB_OPCODE_QUERY_CC 0x8dUL
2761+ u8 status;
2762+ __le16 cookie;
2763+ __le16 flags;
2764+ u8 resp_size;
2765+ u8 reserved8;
2766+ u8 enable_cc;
2767+ #define CREQ_QUERY_CC_RESP_SB_ENABLE_CC 0x1UL
2768+ u8 g;
2769+ #define CREQ_QUERY_CC_RESP_SB_G_MASK 0x7UL
2770+ #define CREQ_QUERY_CC_RESP_SB_G_SFT 0
2771+ u8 num_phases_per_state;
2772+ __le16 init_cr;
2773+ u8 unused_2;
2774+ __le16 unused_3;
2775+ u8 unused_4;
2776+ __le16 init_tr;
2777+ u8 tos_dscp_tos_ecn;
2778+ #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_MASK 0x3UL
2779+ #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_SFT 0
2780+ #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_MASK 0xfcUL
2781+ #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_SFT 2
2782+ __le64 reserved64;
2783+ __le64 reserved64_1;
2784+};
2785+
2786+/* QP error notification event (16 bytes) */
2787+struct creq_qp_error_notification {
2788+ u8 type;
2789+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK 0x3fUL
2790+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT 0
2791+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT 0x38UL
2792+ #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_MASK 0xc0UL
2793+ #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_SFT 6
2794+ u8 status;
2795+ u8 req_slow_path_state;
2796+ u8 req_err_state_reason;
2797+ __le32 xid;
2798+ u8 v;
2799+ #define CREQ_QP_ERROR_NOTIFICATION_V 0x1UL
2800+ #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_MASK 0xfeUL
2801+ #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_SFT 1
2802+ u8 event;
2803+ #define CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION 0xc0UL
2804+ u8 res_slow_path_state;
2805+ u8 res_err_state_reason;
2806+ __le16 sq_cons_idx;
2807+ __le16 rq_cons_idx;
2808+};
2809+
2810+/* RoCE Slowpath HSI Specification 1.6.0 */
2811+#define ROCE_SP_HSI_VERSION_MAJOR 1
2812+#define ROCE_SP_HSI_VERSION_MINOR 6
2813+#define ROCE_SP_HSI_VERSION_UPDATE 0
2814+
2815+#define ROCE_SP_HSI_VERSION_STR "1.6.0"
2816+/*
2817+ * Following is the signature for ROCE_SP_HSI message field that indicates not
2818+ * applicable (All F's). Need to cast it the size of the field if needed.
2819+ */
2820+#define ROCE_SP_HSI_NA_SIGNATURE ((__le32)(-1))
2821+#endif /* __BNXT_RE_HSI_H__ */
include/uapi/rdma/bnxt_re-abi.h+89−0
@@ -0,0 +1,89 @@
1+/*
2+ * Broadcom NetXtreme-E RoCE driver.
3+ *
4+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6+ *
7+ * This software is available to you under a choice of one of two
8+ * licenses. You may choose to be licensed under the terms of the GNU
9+ * General Public License (GPL) Version 2, available from the file
10+ * COPYING in the main directory of this source tree, or the
11+ * BSD license below:
12+ *
13+ * Redistribution and use in source and binary forms, with or without
14+ * modification, are permitted provided that the following conditions
15+ * are met:
16+ *
17+ * 1. Redistributions of source code must retain the above copyright
18+ * notice, this list of conditions and the following disclaimer.
19+ * 2. Redistributions in binary form must reproduce the above copyright
20+ * notice, this list of conditions and the following disclaimer in
21+ * the documentation and/or other materials provided with the
22+ * distribution.
23+ *
24+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35+ *
36+ * Description: Uverbs ABI header file
37+ */
38+
39+#ifndef __BNXT_RE_UVERBS_ABI_H__
40+#define __BNXT_RE_UVERBS_ABI_H__
41+
42+#define BNXT_RE_ABI_VERSION 1
43+
44+struct bnxt_re_uctx_resp {
45+ __u32 dev_id;
46+ __u32 max_qp;
47+ __u32 pg_size;
48+ __u32 cqe_sz;
49+ __u32 max_cqd;
50+ __u32 rsvd;
51+};
52+
53+struct bnxt_re_pd_resp {
54+ __u32 pdid;
55+ __u32 dpi;
56+ __u64 dbr;
57+};
58+
59+struct bnxt_re_cq_req {
60+ __u64 cq_va;
61+ __u64 cq_handle;
62+};
63+
64+struct bnxt_re_cq_resp {
65+ __u32 cqid;
66+ __u32 tail;
67+ __u32 phase;
68+ __u32 rsvd;
69+};
70+
71+struct bnxt_re_qp_req {
72+ __u64 qpsva;
73+ __u64 qprva;
74+ __u64 qp_handle;
75+};
76+
77+struct bnxt_re_qp_resp {
78+ __u32 qpid;
79+ __u32 rsvd;
80+};
81+
82+enum bnxt_re_shpg_offt {
83+ BNXT_RE_BEG_RESV_OFFT = 0x00,
84+ BNXT_RE_AVID_OFFT = 0x10,
85+ BNXT_RE_AVID_SIZE = 0x04,
86+ BNXT_RE_END_RESV_OFFT = 0xFF0
87+};
88+
89+#endif /* __BNXT_RE_UVERBS_ABI_H__*/

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t