History

RDMA/bnxt_re: Add bnxt_re RoCE driver

This patch introduces the RoCE driver for the Broadcom NetXtreme-E 10/25/40/50G RoCE HCAs. The RoCE driver is a two part driver that relies on the parent bnxt_en NIC driver to operate. The changes needed in the bnxt_en driver have already been incorporated via Dave Miller's net tree into the mainline kernel. The vendor official git repository for this driver is available on github as: https://github.com/Broadcom/linux-rdma-nxt/

Selvin Xavier committed 9 years ago

Commit 1ac5a401 parent 24dc83114 files · +13347 −0Browse files

Unified · Split

14 files
  1. drivers/infiniband/hw/bnxt_re/bnxt_re.h +146 −0
  2. drivers/infiniband/hw/bnxt_re/ib_verbs.c +3202 −0
  3. drivers/infiniband/hw/bnxt_re/ib_verbs.h +197 −0
  4. drivers/infiniband/hw/bnxt_re/main.c +1315 −0
  5. drivers/infiniband/hw/bnxt_re/qplib_fp.c +2167 −0
  6. drivers/infiniband/hw/bnxt_re/qplib_fp.h +439 −0
  7. drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +694 −0
  8. drivers/infiniband/hw/bnxt_re/qplib_rcfw.h +231 −0
  9. drivers/infiniband/hw/bnxt_re/qplib_res.c +825 −0
  10. drivers/infiniband/hw/bnxt_re/qplib_res.h +223 −0
  11. drivers/infiniband/hw/bnxt_re/qplib_sp.c +838 −0
  12. drivers/infiniband/hw/bnxt_re/qplib_sp.h +160 −0
  13. drivers/infiniband/hw/bnxt_re/roce_hsi.h +2821 −0
  14. include/uapi/rdma/bnxt_re-abi.h +89 −0
drivers/infiniband/hw/bnxt_re/bnxt_re.h+146−0
@@ -0,0 +1,146 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Slow Path Operators (header)
37 *
38 */
39
40#ifndef __BNXT_RE_H__
41#define __BNXT_RE_H__
42#define ROCE_DRV_MODULE_NAME "bnxt_re"
43#define ROCE_DRV_MODULE_VERSION "1.0.0"
44
45#define BNXT_RE_DESC "Broadcom NetXtreme-C/E RoCE Driver"
46
47#define BNXT_RE_PAGE_SIZE_4K BIT(12)
48#define BNXT_RE_PAGE_SIZE_8K BIT(13)
49#define BNXT_RE_PAGE_SIZE_64K BIT(16)
50#define BNXT_RE_PAGE_SIZE_2M BIT(21)
51#define BNXT_RE_PAGE_SIZE_8M BIT(23)
52#define BNXT_RE_PAGE_SIZE_1G BIT(30)
53
54#define BNXT_RE_MAX_QPC_COUNT (64 * 1024)
55#define BNXT_RE_MAX_MRW_COUNT (64 * 1024)
56#define BNXT_RE_MAX_SRQC_COUNT (64 * 1024)
57#define BNXT_RE_MAX_CQ_COUNT (64 * 1024)
58
59struct bnxt_re_work {
60 struct work_struct work;
61 unsigned long event;
62 struct bnxt_re_dev *rdev;
63 struct net_device *vlan_dev;
64};
65
66struct bnxt_re_sqp_entries {
67 struct bnxt_qplib_sge sge;
68 u64 wrid;
69 /* For storing the actual qp1 cqe */
70 struct bnxt_qplib_cqe cqe;
71 struct bnxt_re_qp *qp1_qp;
72};
73
74#define BNXT_RE_MIN_MSIX 2
75#define BNXT_RE_MAX_MSIX 16
76#define BNXT_RE_AEQ_IDX 0
77#define BNXT_RE_NQ_IDX 1
78
79struct bnxt_re_dev {
80 struct ib_device ibdev;
81 struct list_head list;
82 unsigned long flags;
83#define BNXT_RE_FLAG_NETDEV_REGISTERED 0
84#define BNXT_RE_FLAG_IBDEV_REGISTERED 1
85#define BNXT_RE_FLAG_GOT_MSIX 2
86#define BNXT_RE_FLAG_RCFW_CHANNEL_EN 8
87#define BNXT_RE_FLAG_QOS_WORK_REG 16
88 struct net_device *netdev;
89 unsigned int version, major, minor;
90 struct bnxt_en_dev *en_dev;
91 struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX];
92 int num_msix;
93
94 int id;
95
96 struct delayed_work worker;
97 u8 cur_prio_map;
98
99 /* FP Notification Queue (CQ & SRQ) */
100 struct tasklet_struct nq_task;
101
102 /* RCFW Channel */
103 struct bnxt_qplib_rcfw rcfw;
104
105 /* NQ */
106 struct bnxt_qplib_nq nq;
107
108 /* Device Resources */
109 struct bnxt_qplib_dev_attr dev_attr;
110 struct bnxt_qplib_ctx qplib_ctx;
111 struct bnxt_qplib_res qplib_res;
112 struct bnxt_qplib_dpi dpi_privileged;
113
114 atomic_t qp_count;
115 struct mutex qp_lock; /* protect qp list */
116 struct list_head qp_list;
117
118 atomic_t cq_count;
119 atomic_t srq_count;
120 atomic_t mr_count;
121 atomic_t mw_count;
122 /* Max of 2 lossless traffic class supported per port */
123 u16 cosq[2];
124
125 /* QP for for handling QP1 packets */
126 u32 sqp_id;
127 struct bnxt_re_qp *qp1_sqp;
128 struct bnxt_re_ah *sqp_ah;
129 struct bnxt_re_sqp_entries sqp_tbl[1024];
130};
131
132#define to_bnxt_re_dev(ptr, member) \
133 container_of((ptr), struct bnxt_re_dev, member)
134
135#define BNXT_RE_ROCE_V1_PACKET 0
136#define BNXT_RE_ROCEV2_IPV4_PACKET 2
137#define BNXT_RE_ROCEV2_IPV6_PACKET 3
138
139static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev)
140{
141 if (rdev)
142 return &rdev->ibdev.dev;
143 return NULL;
144}
145
146#endif
drivers/infiniband/hw/bnxt_re/ib_verbs.c+3202−0
@@ -0,0 +1,3202 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: IB Verbs interpreter
37 */
38
39#include <linux/interrupt.h>
40#include <linux/types.h>
41#include <linux/pci.h>
42#include <linux/netdevice.h>
43#include <linux/if_ether.h>
44
45#include <rdma/ib_verbs.h>
46#include <rdma/ib_user_verbs.h>
47#include <rdma/ib_umem.h>
48#include <rdma/ib_addr.h>
49#include <rdma/ib_mad.h>
50#include <rdma/ib_cache.h>
51
52#include "bnxt_ulp.h"
53
54#include "roce_hsi.h"
55#include "qplib_res.h"
56#include "qplib_sp.h"
57#include "qplib_fp.h"
58#include "qplib_rcfw.h"
59
60#include "bnxt_re.h"
61#include "ib_verbs.h"
62#include <rdma/bnxt_re-abi.h>
63
64static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
65 struct bnxt_qplib_sge *sg_list, int num)
66{
67 int i, total = 0;
68
69 for (i = 0; i < num; i++) {
70 sg_list[i].addr = ib_sg_list[i].addr;
71 sg_list[i].lkey = ib_sg_list[i].lkey;
72 sg_list[i].size = ib_sg_list[i].length;
73 total += sg_list[i].size;
74 }
75 return total;
76}
77
78/* Device */
79struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num)
80{
81 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
82 struct net_device *netdev = NULL;
83
84 rcu_read_lock();
85 if (rdev)
86 netdev = rdev->netdev;
87 if (netdev)
88 dev_hold(netdev);
89
90 rcu_read_unlock();
91 return netdev;
92}
93
94int bnxt_re_query_device(struct ib_device *ibdev,
95 struct ib_device_attr *ib_attr,
96 struct ib_udata *udata)
97{
98 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
99 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
100
101 memset(ib_attr, 0, sizeof(*ib_attr));
102
103 ib_attr->fw_ver = (u64)(unsigned long)(dev_attr->fw_ver);
104 bnxt_qplib_get_guid(rdev->netdev->dev_addr,
105 (u8 *)&ib_attr->sys_image_guid);
106 ib_attr->max_mr_size = ~0ull;
107 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_8K |
108 BNXT_RE_PAGE_SIZE_64K | BNXT_RE_PAGE_SIZE_2M |
109 BNXT_RE_PAGE_SIZE_8M | BNXT_RE_PAGE_SIZE_1G;
110
111 ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
112 ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
113 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
114 ib_attr->max_qp = dev_attr->max_qp;
115 ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
116 ib_attr->device_cap_flags =
117 IB_DEVICE_CURR_QP_STATE_MOD
118 | IB_DEVICE_RC_RNR_NAK_GEN
119 | IB_DEVICE_SHUTDOWN_PORT
120 | IB_DEVICE_SYS_IMAGE_GUID
121 | IB_DEVICE_LOCAL_DMA_LKEY
122 | IB_DEVICE_RESIZE_MAX_WR
123 | IB_DEVICE_PORT_ACTIVE_EVENT
124 | IB_DEVICE_N_NOTIFY_CQ
125 | IB_DEVICE_MEM_WINDOW
126 | IB_DEVICE_MEM_WINDOW_TYPE_2B
127 | IB_DEVICE_MEM_MGT_EXTENSIONS;
128 ib_attr->max_sge = dev_attr->max_qp_sges;
129 ib_attr->max_sge_rd = dev_attr->max_qp_sges;
130 ib_attr->max_cq = dev_attr->max_cq;
131 ib_attr->max_cqe = dev_attr->max_cq_wqes;
132 ib_attr->max_mr = dev_attr->max_mr;
133 ib_attr->max_pd = dev_attr->max_pd;
134 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
135 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_rd_atom;
136 ib_attr->atomic_cap = IB_ATOMIC_HCA;
137 ib_attr->masked_atomic_cap = IB_ATOMIC_HCA;
138
139 ib_attr->max_ee_rd_atom = 0;
140 ib_attr->max_res_rd_atom = 0;
141 ib_attr->max_ee_init_rd_atom = 0;
142 ib_attr->max_ee = 0;
143 ib_attr->max_rdd = 0;
144 ib_attr->max_mw = dev_attr->max_mw;
145 ib_attr->max_raw_ipv6_qp = 0;
146 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
147 ib_attr->max_mcast_grp = 0;
148 ib_attr->max_mcast_qp_attach = 0;
149 ib_attr->max_total_mcast_qp_attach = 0;
150 ib_attr->max_ah = dev_attr->max_ah;
151
152 ib_attr->max_fmr = dev_attr->max_fmr;
153 ib_attr->max_map_per_fmr = 1; /* ? */
154
155 ib_attr->max_srq = dev_attr->max_srq;
156 ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
157 ib_attr->max_srq_sge = dev_attr->max_srq_sges;
158
159 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
160
161 ib_attr->max_pkeys = 1;
162 ib_attr->local_ca_ack_delay = 0;
163 return 0;
164}
165
166int bnxt_re_modify_device(struct ib_device *ibdev,
167 int device_modify_mask,
168 struct ib_device_modify *device_modify)
169{
170 switch (device_modify_mask) {
171 case IB_DEVICE_MODIFY_SYS_IMAGE_GUID:
172 /* Modify the GUID requires the modification of the GID table */
173 /* GUID should be made as READ-ONLY */
174 break;
175 case IB_DEVICE_MODIFY_NODE_DESC:
176 /* Node Desc should be made as READ-ONLY */
177 break;
178 default:
179 break;
180 }
181 return 0;
182}
183
184static void __to_ib_speed_width(struct net_device *netdev, u8 *speed, u8 *width)
185{
186 struct ethtool_link_ksettings lksettings;
187 u32 espeed;
188
189 if (netdev->ethtool_ops && netdev->ethtool_ops->get_link_ksettings) {
190 memset(&lksettings, 0, sizeof(lksettings));
191 rtnl_lock();
192 netdev->ethtool_ops->get_link_ksettings(netdev, &lksettings);
193 rtnl_unlock();
194 espeed = lksettings.base.speed;
195 } else {
196 espeed = SPEED_UNKNOWN;
197 }
198 switch (espeed) {
199 case SPEED_1000:
200 *speed = IB_SPEED_SDR;
201 *width = IB_WIDTH_1X;
202 break;
203 case SPEED_10000:
204 *speed = IB_SPEED_QDR;
205 *width = IB_WIDTH_1X;
206 break;
207 case SPEED_20000:
208 *speed = IB_SPEED_DDR;
209 *width = IB_WIDTH_4X;
210 break;
211 case SPEED_25000:
212 *speed = IB_SPEED_EDR;
213 *width = IB_WIDTH_1X;
214 break;
215 case SPEED_40000:
216 *speed = IB_SPEED_QDR;
217 *width = IB_WIDTH_4X;
218 break;
219 case SPEED_50000:
220 break;
221 default:
222 *speed = IB_SPEED_SDR;
223 *width = IB_WIDTH_1X;
224 break;
225 }
226}
227
228/* Port */
229int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
230 struct ib_port_attr *port_attr)
231{
232 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
233 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
234
235 memset(port_attr, 0, sizeof(*port_attr));
236
237 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
238 port_attr->state = IB_PORT_ACTIVE;
239 port_attr->phys_state = 5;
240 } else {
241 port_attr->state = IB_PORT_DOWN;
242 port_attr->phys_state = 3;
243 }
244 port_attr->max_mtu = IB_MTU_4096;
245 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
246 port_attr->gid_tbl_len = dev_attr->max_sgid;
247 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
248 IB_PORT_DEVICE_MGMT_SUP |
249 IB_PORT_VENDOR_CLASS_SUP |
250 IB_PORT_IP_BASED_GIDS;
251
252 /* Max MSG size set to 2G for now */
253 port_attr->max_msg_sz = 0x80000000;
254 port_attr->bad_pkey_cntr = 0;
255 port_attr->qkey_viol_cntr = 0;
256 port_attr->pkey_tbl_len = dev_attr->max_pkey;
257 port_attr->lid = 0;
258 port_attr->sm_lid = 0;
259 port_attr->lmc = 0;
260 port_attr->max_vl_num = 4;
261 port_attr->sm_sl = 0;
262 port_attr->subnet_timeout = 0;
263 port_attr->init_type_reply = 0;
264 /* call the underlying netdev's ethtool hooks to query speed settings
265 * for which we acquire rtnl_lock _only_ if it's registered with
266 * IB stack to avoid race in the NETDEV_UNREG path
267 */
268 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
269 __to_ib_speed_width(rdev->netdev, &port_attr->active_speed,
270 &port_attr->active_width);
271 return 0;
272}
273
274int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
275 int port_modify_mask,
276 struct ib_port_modify *port_modify)
277{
278 switch (port_modify_mask) {
279 case IB_PORT_SHUTDOWN:
280 break;
281 case IB_PORT_INIT_TYPE:
282 break;
283 case IB_PORT_RESET_QKEY_CNTR:
284 break;
285 default:
286 break;
287 }
288 return 0;
289}
290
291int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
292 struct ib_port_immutable *immutable)
293{
294 struct ib_port_attr port_attr;
295
296 if (bnxt_re_query_port(ibdev, port_num, &port_attr))
297 return -EINVAL;
298
299 immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
300 immutable->gid_tbl_len = port_attr.gid_tbl_len;
301 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
302 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
303 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
304 return 0;
305}
306
307int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
308 u16 index, u16 *pkey)
309{
310 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
311
312 /* Ignore port_num */
313
314 memset(pkey, 0, sizeof(*pkey));
315 return bnxt_qplib_get_pkey(&rdev->qplib_res,
316 &rdev->qplib_res.pkey_tbl, index, pkey);
317}
318
319int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
320 int index, union ib_gid *gid)
321{
322 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
323 int rc = 0;
324
325 /* Ignore port_num */
326 memset(gid, 0, sizeof(*gid));
327 rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
328 &rdev->qplib_res.sgid_tbl, index,
329 (struct bnxt_qplib_gid *)gid);
330 return rc;
331}
332
333int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
334 unsigned int index, void **context)
335{
336 int rc = 0;
337 struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
338 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
339 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
340
341 /* Delete the entry from the hardware */
342 ctx = *context;
343 if (!ctx)
344 return -EINVAL;
345
346 if (sgid_tbl && sgid_tbl->active) {
347 if (ctx->idx >= sgid_tbl->max)
348 return -EINVAL;
349 ctx->refcnt--;
350 if (!ctx->refcnt) {
351 rc = bnxt_qplib_del_sgid
352 (sgid_tbl,
353 &sgid_tbl->tbl[ctx->idx], true);
354 if (rc)
355 dev_err(rdev_to_dev(rdev),
356 "Failed to remove GID: %#x", rc);
357 ctx_tbl = sgid_tbl->ctx;
358 ctx_tbl[ctx->idx] = NULL;
359 kfree(ctx);
360 }
361 } else {
362 return -EINVAL;
363 }
364 return rc;
365}
366
367int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
368 unsigned int index, const union ib_gid *gid,
369 const struct ib_gid_attr *attr, void **context)
370{
371 int rc;
372 u32 tbl_idx = 0;
373 u16 vlan_id = 0xFFFF;
374 struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
375 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
376 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
377
378 if ((attr->ndev) && is_vlan_dev(attr->ndev))
379 vlan_id = vlan_dev_vlan_id(attr->ndev);
380
381 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)gid,
382 rdev->qplib_res.netdev->dev_addr,
383 vlan_id, true, &tbl_idx);
384 if (rc == -EALREADY) {
385 ctx_tbl = sgid_tbl->ctx;
386 ctx_tbl[tbl_idx]->refcnt++;
387 *context = ctx_tbl[tbl_idx];
388 return 0;
389 }
390
391 if (rc < 0) {
392 dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc);
393 return rc;
394 }
395
396 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
397 if (!ctx)
398 return -ENOMEM;
399 ctx_tbl = sgid_tbl->ctx;
400 ctx->idx = tbl_idx;
401 ctx->refcnt = 1;
402 ctx_tbl[tbl_idx] = ctx;
403
404 return rc;
405}
406
407enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
408 u8 port_num)
409{
410 return IB_LINK_LAYER_ETHERNET;
411}
412
413/* Protection Domains */
414int bnxt_re_dealloc_pd(struct ib_pd *ib_pd)
415{
416 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
417 struct bnxt_re_dev *rdev = pd->rdev;
418 int rc;
419
420 if (ib_pd->uobject && pd->dpi.dbr) {
421 struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
422 struct bnxt_re_ucontext *ucntx;
423
424 /* Free DPI only if this is the first PD allocated by the
425 * application and mark the context dpi as NULL
426 */
427 ucntx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
428
429 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
430 &rdev->qplib_res.dpi_tbl,
431 &pd->dpi);
432 if (rc)
433 dev_err(rdev_to_dev(rdev), "Failed to deallocate HW DPI");
434 /* Don't fail, continue*/
435 ucntx->dpi = NULL;
436 }
437
438 rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res,
439 &rdev->qplib_res.pd_tbl,
440 &pd->qplib_pd);
441 if (rc) {
442 dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD");
443 return rc;
444 }
445
446 kfree(pd);
447 return 0;
448}
449
450struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev,
451 struct ib_ucontext *ucontext,
452 struct ib_udata *udata)
453{
454 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
455 struct bnxt_re_ucontext *ucntx = container_of(ucontext,
456 struct bnxt_re_ucontext,
457 ib_uctx);
458 struct bnxt_re_pd *pd;
459 int rc;
460
461 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
462 if (!pd)
463 return ERR_PTR(-ENOMEM);
464
465 pd->rdev = rdev;
466 if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) {
467 dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD");
468 rc = -ENOMEM;
469 goto fail;
470 }
471
472 if (udata) {
473 struct bnxt_re_pd_resp resp;
474
475 if (!ucntx->dpi) {
476 /* Allocate DPI in alloc_pd to avoid failing of
477 * ibv_devinfo and family of application when DPIs
478 * are depleted.
479 */
480 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
481 &pd->dpi, ucntx)) {
482 rc = -ENOMEM;
483 goto dbfail;
484 }
485 ucntx->dpi = &pd->dpi;
486 }
487
488 resp.pdid = pd->qplib_pd.id;
489 /* Still allow mapping this DBR to the new user PD. */
490 resp.dpi = ucntx->dpi->dpi;
491 resp.dbr = (u64)ucntx->dpi->umdbr;
492
493 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
494 if (rc) {
495 dev_err(rdev_to_dev(rdev),
496 "Failed to copy user response\n");
497 goto dbfail;
498 }
499 }
500
501 return &pd->ib_pd;
502dbfail:
503 (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
504 &pd->qplib_pd);
505fail:
506 kfree(pd);
507 return ERR_PTR(rc);
508}
509
510/* Address Handles */
511int bnxt_re_destroy_ah(struct ib_ah *ib_ah)
512{
513 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
514 struct bnxt_re_dev *rdev = ah->rdev;
515 int rc;
516
517 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah);
518 if (rc) {
519 dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH");
520 return rc;
521 }
522 kfree(ah);
523 return 0;
524}
525
526struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
527 struct ib_ah_attr *ah_attr,
528 struct ib_udata *udata)
529{
530 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
531 struct bnxt_re_dev *rdev = pd->rdev;
532 struct bnxt_re_ah *ah;
533 int rc;
534 u16 vlan_tag;
535 u8 nw_type;
536
537 struct ib_gid_attr sgid_attr;
538
539 if (!(ah_attr->ah_flags & IB_AH_GRH)) {
540 dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set");
541 return ERR_PTR(-EINVAL);
542 }
543 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
544 if (!ah)
545 return ERR_PTR(-ENOMEM);
546
547 ah->rdev = rdev;
548 ah->qplib_ah.pd = &pd->qplib_pd;
549
550 /* Supply the configuration for the HW */
551 memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw,
552 sizeof(union ib_gid));
553 /*
554 * If RoCE V2 is enabled, stack will have two entries for
555 * each GID entry. Avoiding this duplicte entry in HW. Dividing
556 * the GID index by 2 for RoCE V2
557 */
558 ah->qplib_ah.sgid_index = ah_attr->grh.sgid_index / 2;
559 ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index;
560 ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class;
561 ah->qplib_ah.flow_label = ah_attr->grh.flow_label;
562 ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit;
563 ah->qplib_ah.sl = ah_attr->sl;
564 if (ib_pd->uobject &&
565 !rdma_is_multicast_addr((struct in6_addr *)
566 ah_attr->grh.dgid.raw) &&
567 !rdma_link_local_addr((struct in6_addr *)
568 ah_attr->grh.dgid.raw)) {
569 union ib_gid sgid;
570
571 rc = ib_get_cached_gid(&rdev->ibdev, 1,
572 ah_attr->grh.sgid_index, &sgid,
573 &sgid_attr);
574 if (rc) {
575 dev_err(rdev_to_dev(rdev),
576 "Failed to query gid at index %d",
577 ah_attr->grh.sgid_index);
578 goto fail;
579 }
580 if (sgid_attr.ndev) {
581 if (is_vlan_dev(sgid_attr.ndev))
582 vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev);
583 dev_put(sgid_attr.ndev);
584 }
585 /* Get network header type for this GID */
586 nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
587 switch (nw_type) {
588 case RDMA_NETWORK_IPV4:
589 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
590 break;
591 case RDMA_NETWORK_IPV6:
592 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
593 break;
594 default:
595 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1;
596 break;
597 }
598 rc = rdma_addr_find_l2_eth_by_grh(&sgid, &ah_attr->grh.dgid,
599 ah_attr->dmac, &vlan_tag,
600 &sgid_attr.ndev->ifindex,
601 NULL);
602 if (rc) {
603 dev_err(rdev_to_dev(rdev), "Failed to get dmac\n");
604 goto fail;
605 }
606 }
607
608 memcpy(ah->qplib_ah.dmac, ah_attr->dmac, ETH_ALEN);
609 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
610 if (rc) {
611 dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH");
612 goto fail;
613 }
614
615 /* Write AVID to shared page. */
616 if (ib_pd->uobject) {
617 struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
618 struct bnxt_re_ucontext *uctx;
619 unsigned long flag;
620 u32 *wrptr;
621
622 uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
623 spin_lock_irqsave(&uctx->sh_lock, flag);
624 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
625 *wrptr = ah->qplib_ah.id;
626 wmb(); /* make sure cache is updated. */
627 spin_unlock_irqrestore(&uctx->sh_lock, flag);
628 }
629
630 return &ah->ib_ah;
631
632fail:
633 kfree(ah);
634 return ERR_PTR(rc);
635}
636
637int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
638{
639 return 0;
640}
641
642int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
643{
644 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
645
646 memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data,
647 sizeof(union ib_gid));
648 ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index;
649 ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class;
650 ah_attr->sl = ah->qplib_ah.sl;
651 memcpy(ah_attr->dmac, ah->qplib_ah.dmac, ETH_ALEN);
652 ah_attr->ah_flags = IB_AH_GRH;
653 ah_attr->port_num = 1;
654 ah_attr->static_rate = 0;
655 return 0;
656}
657
658/* Queue Pairs */
659int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
660{
661 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
662 struct bnxt_re_dev *rdev = qp->rdev;
663 int rc;
664
665 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
666 if (rc) {
667 dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP");
668 return rc;
669 }
670 if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) {
671 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
672 &rdev->sqp_ah->qplib_ah);
673 if (rc) {
674 dev_err(rdev_to_dev(rdev),
675 "Failed to destroy HW AH for shadow QP");
676 return rc;
677 }
678
679 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res,
680 &rdev->qp1_sqp->qplib_qp);
681 if (rc) {
682 dev_err(rdev_to_dev(rdev),
683 "Failed to destroy Shadow QP");
684 return rc;
685 }
686 mutex_lock(&rdev->qp_lock);
687 list_del(&rdev->qp1_sqp->list);
688 atomic_dec(&rdev->qp_count);
689 mutex_unlock(&rdev->qp_lock);
690
691 kfree(rdev->sqp_ah);
692 kfree(rdev->qp1_sqp);
693 }
694
695 if (qp->rumem && !IS_ERR(qp->rumem))
696 ib_umem_release(qp->rumem);
697 if (qp->sumem && !IS_ERR(qp->sumem))
698 ib_umem_release(qp->sumem);
699
700 mutex_lock(&rdev->qp_lock);
701 list_del(&qp->list);
702 atomic_dec(&rdev->qp_count);
703 mutex_unlock(&rdev->qp_lock);
704 kfree(qp);
705 return 0;
706}
707
708static u8 __from_ib_qp_type(enum ib_qp_type type)
709{
710 switch (type) {
711 case IB_QPT_GSI:
712 return CMDQ_CREATE_QP1_TYPE_GSI;
713 case IB_QPT_RC:
714 return CMDQ_CREATE_QP_TYPE_RC;
715 case IB_QPT_UD:
716 return CMDQ_CREATE_QP_TYPE_UD;
717 default:
718 return IB_QPT_MAX;
719 }
720}
721
722static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
723 struct bnxt_re_qp *qp, struct ib_udata *udata)
724{
725 struct bnxt_re_qp_req ureq;
726 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
727 struct ib_umem *umem;
728 int bytes = 0;
729 struct ib_ucontext *context = pd->ib_pd.uobject->context;
730 struct bnxt_re_ucontext *cntx = container_of(context,
731 struct bnxt_re_ucontext,
732 ib_uctx);
733 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
734 return -EFAULT;
735
736 bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
737 /* Consider mapping PSN search memory only for RC QPs. */
738 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC)
739 bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search));
740 bytes = PAGE_ALIGN(bytes);
741 umem = ib_umem_get(context, ureq.qpsva, bytes,
742 IB_ACCESS_LOCAL_WRITE, 1);
743 if (IS_ERR(umem))
744 return PTR_ERR(umem);
745
746 qp->sumem = umem;
747 qplib_qp->sq.sglist = umem->sg_head.sgl;
748 qplib_qp->sq.nmap = umem->nmap;
749 qplib_qp->qp_handle = ureq.qp_handle;
750
751 if (!qp->qplib_qp.srq) {
752 bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
753 bytes = PAGE_ALIGN(bytes);
754 umem = ib_umem_get(context, ureq.qprva, bytes,
755 IB_ACCESS_LOCAL_WRITE, 1);
756 if (IS_ERR(umem))
757 goto rqfail;
758 qp->rumem = umem;
759 qplib_qp->rq.sglist = umem->sg_head.sgl;
760 qplib_qp->rq.nmap = umem->nmap;
761 }
762
763 qplib_qp->dpi = cntx->dpi;
764 return 0;
765rqfail:
766 ib_umem_release(qp->sumem);
767 qp->sumem = NULL;
768 qplib_qp->sq.sglist = NULL;
769 qplib_qp->sq.nmap = 0;
770
771 return PTR_ERR(umem);
772}
773
774static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
775 (struct bnxt_re_pd *pd,
776 struct bnxt_qplib_res *qp1_res,
777 struct bnxt_qplib_qp *qp1_qp)
778{
779 struct bnxt_re_dev *rdev = pd->rdev;
780 struct bnxt_re_ah *ah;
781 union ib_gid sgid;
782 int rc;
783
784 ah = kzalloc(sizeof(*ah), GFP_KERNEL);
785 if (!ah)
786 return NULL;
787
788 memset(ah, 0, sizeof(*ah));
789 ah->rdev = rdev;
790 ah->qplib_ah.pd = &pd->qplib_pd;
791
792 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
793 if (rc)
794 goto fail;
795
796 /* supply the dgid data same as sgid */
797 memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
798 sizeof(union ib_gid));
799 ah->qplib_ah.sgid_index = 0;
800
801 ah->qplib_ah.traffic_class = 0;
802 ah->qplib_ah.flow_label = 0;
803 ah->qplib_ah.hop_limit = 1;
804 ah->qplib_ah.sl = 0;
805 /* Have DMAC same as SMAC */
806 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
807
808 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
809 if (rc) {
810 dev_err(rdev_to_dev(rdev),
811 "Failed to allocate HW AH for Shadow QP");
812 goto fail;
813 }
814
815 return ah;
816
817fail:
818 kfree(ah);
819 return NULL;
820}
821
822static struct bnxt_re_qp *bnxt_re_create_shadow_qp
823 (struct bnxt_re_pd *pd,
824 struct bnxt_qplib_res *qp1_res,
825 struct bnxt_qplib_qp *qp1_qp)
826{
827 struct bnxt_re_dev *rdev = pd->rdev;
828 struct bnxt_re_qp *qp;
829 int rc;
830
831 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
832 if (!qp)
833 return NULL;
834
835 memset(qp, 0, sizeof(*qp));
836 qp->rdev = rdev;
837
838 /* Initialize the shadow QP structure from the QP1 values */
839 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
840
841 qp->qplib_qp.pd = &pd->qplib_pd;
842 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
843 qp->qplib_qp.type = IB_QPT_UD;
844
845 qp->qplib_qp.max_inline_data = 0;
846 qp->qplib_qp.sig_type = true;
847
848 /* Shadow QP SQ depth should be same as QP1 RQ depth */
849 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
850 qp->qplib_qp.sq.max_sge = 2;
851
852 qp->qplib_qp.scq = qp1_qp->scq;
853 qp->qplib_qp.rcq = qp1_qp->rcq;
854
855 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
856 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
857
858 qp->qplib_qp.mtu = qp1_qp->mtu;
859
860 qp->qplib_qp.sq_hdr_buf_size = 0;
861 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
862 qp->qplib_qp.dpi = &rdev->dpi_privileged;
863
864 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
865 if (rc)
866 goto fail;
867
868 rdev->sqp_id = qp->qplib_qp.id;
869
870 spin_lock_init(&qp->sq_lock);
871 INIT_LIST_HEAD(&qp->list);
872 mutex_lock(&rdev->qp_lock);
873 list_add_tail(&qp->list, &rdev->qp_list);
874 atomic_inc(&rdev->qp_count);
875 mutex_unlock(&rdev->qp_lock);
876 return qp;
877fail:
878 kfree(qp);
879 return NULL;
880}
881
882struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
883 struct ib_qp_init_attr *qp_init_attr,
884 struct ib_udata *udata)
885{
886 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
887 struct bnxt_re_dev *rdev = pd->rdev;
888 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
889 struct bnxt_re_qp *qp;
890 struct bnxt_re_cq *cq;
891 int rc, entries;
892
893 if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
894 (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) ||
895 (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) ||
896 (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) ||
897 (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data))
898 return ERR_PTR(-EINVAL);
899
900 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
901 if (!qp)
902 return ERR_PTR(-ENOMEM);
903
904 qp->rdev = rdev;
905 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
906 qp->qplib_qp.pd = &pd->qplib_pd;
907 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
908 qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type);
909 if (qp->qplib_qp.type == IB_QPT_MAX) {
910 dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported",
911 qp->qplib_qp.type);
912 rc = -EINVAL;
913 goto fail;
914 }
915 qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data;
916 qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type ==
917 IB_SIGNAL_ALL_WR) ? true : false);
918
919 entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1);
920 qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
921 dev_attr->max_qp_wqes + 1);
922
923 qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge;
924 if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges)
925 qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges;
926
927 if (qp_init_attr->send_cq) {
928 cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq,
929 ib_cq);
930 if (!cq) {
931 dev_err(rdev_to_dev(rdev), "Send CQ not found");
932 rc = -EINVAL;
933 goto fail;
934 }
935 qp->qplib_qp.scq = &cq->qplib_cq;
936 }
937
938 if (qp_init_attr->recv_cq) {
939 cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq,
940 ib_cq);
941 if (!cq) {
942 dev_err(rdev_to_dev(rdev), "Receive CQ not found");
943 rc = -EINVAL;
944 goto fail;
945 }
946 qp->qplib_qp.rcq = &cq->qplib_cq;
947 }
948
949 if (qp_init_attr->srq) {
950 dev_err(rdev_to_dev(rdev), "SRQ not supported");
951 rc = -ENOTSUPP;
952 goto fail;
953 } else {
954 /* Allocate 1 more than what's provided so posting max doesn't
955 * mean empty
956 */
957 entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1);
958 qp->qplib_qp.rq.max_wqe = min_t(u32, entries,
959 dev_attr->max_qp_wqes + 1);
960
961 qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge;
962 if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges)
963 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
964 }
965
966 qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
967
968 if (qp_init_attr->qp_type == IB_QPT_GSI) {
969 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
970 if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges)
971 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
972 qp->qplib_qp.sq.max_sge++;
973 if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges)
974 qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges;
975
976 qp->qplib_qp.rq_hdr_buf_size =
977 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
978
979 qp->qplib_qp.sq_hdr_buf_size =
980 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
981 qp->qplib_qp.dpi = &rdev->dpi_privileged;
982 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp);
983 if (rc) {
984 dev_err(rdev_to_dev(rdev), "Failed to create HW QP1");
985 goto fail;
986 }
987 /* Create a shadow QP to handle the QP1 traffic */
988 rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res,
989 &qp->qplib_qp);
990 if (!rdev->qp1_sqp) {
991 rc = -EINVAL;
992 dev_err(rdev_to_dev(rdev),
993 "Failed to create Shadow QP for QP1");
994 goto qp_destroy;
995 }
996 rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
997 &qp->qplib_qp);
998 if (!rdev->sqp_ah) {
999 bnxt_qplib_destroy_qp(&rdev->qplib_res,
1000 &rdev->qp1_sqp->qplib_qp);
1001 rc = -EINVAL;
1002 dev_err(rdev_to_dev(rdev),
1003 "Failed to create AH entry for ShadowQP");
1004 goto qp_destroy;
1005 }
1006
1007 } else {
1008 qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom;
1009 qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1010 if (udata) {
1011 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1012 if (rc)
1013 goto fail;
1014 } else {
1015 qp->qplib_qp.dpi = &rdev->dpi_privileged;
1016 }
1017
1018 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1019 if (rc) {
1020 dev_err(rdev_to_dev(rdev), "Failed to create HW QP");
1021 goto fail;
1022 }
1023 }
1024
1025 qp->ib_qp.qp_num = qp->qplib_qp.id;
1026 spin_lock_init(&qp->sq_lock);
1027
1028 if (udata) {
1029 struct bnxt_re_qp_resp resp;
1030
1031 resp.qpid = qp->ib_qp.qp_num;
1032 resp.rsvd = 0;
1033 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1034 if (rc) {
1035 dev_err(rdev_to_dev(rdev), "Failed to copy QP udata");
1036 goto qp_destroy;
1037 }
1038 }
1039 INIT_LIST_HEAD(&qp->list);
1040 mutex_lock(&rdev->qp_lock);
1041 list_add_tail(&qp->list, &rdev->qp_list);
1042 atomic_inc(&rdev->qp_count);
1043 mutex_unlock(&rdev->qp_lock);
1044
1045 return &qp->ib_qp;
1046qp_destroy:
1047 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1048fail:
1049 kfree(qp);
1050 return ERR_PTR(rc);
1051}
1052
1053static u8 __from_ib_qp_state(enum ib_qp_state state)
1054{
1055 switch (state) {
1056 case IB_QPS_RESET:
1057 return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1058 case IB_QPS_INIT:
1059 return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1060 case IB_QPS_RTR:
1061 return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1062 case IB_QPS_RTS:
1063 return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1064 case IB_QPS_SQD:
1065 return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1066 case IB_QPS_SQE:
1067 return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1068 case IB_QPS_ERR:
1069 default:
1070 return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1071 }
1072}
1073
1074static enum ib_qp_state __to_ib_qp_state(u8 state)
1075{
1076 switch (state) {
1077 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1078 return IB_QPS_RESET;
1079 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1080 return IB_QPS_INIT;
1081 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1082 return IB_QPS_RTR;
1083 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1084 return IB_QPS_RTS;
1085 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1086 return IB_QPS_SQD;
1087 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1088 return IB_QPS_SQE;
1089 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1090 default:
1091 return IB_QPS_ERR;
1092 }
1093}
1094
1095static u32 __from_ib_mtu(enum ib_mtu mtu)
1096{
1097 switch (mtu) {
1098 case IB_MTU_256:
1099 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1100 case IB_MTU_512:
1101 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1102 case IB_MTU_1024:
1103 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1104 case IB_MTU_2048:
1105 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1106 case IB_MTU_4096:
1107 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1108 default:
1109 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1110 }
1111}
1112
1113static enum ib_mtu __to_ib_mtu(u32 mtu)
1114{
1115 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1116 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1117 return IB_MTU_256;
1118 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1119 return IB_MTU_512;
1120 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1121 return IB_MTU_1024;
1122 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1123 return IB_MTU_2048;
1124 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1125 return IB_MTU_4096;
1126 default:
1127 return IB_MTU_2048;
1128 }
1129}
1130
1131static int __from_ib_access_flags(int iflags)
1132{
1133 int qflags = 0;
1134
1135 if (iflags & IB_ACCESS_LOCAL_WRITE)
1136 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1137 if (iflags & IB_ACCESS_REMOTE_READ)
1138 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
1139 if (iflags & IB_ACCESS_REMOTE_WRITE)
1140 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
1141 if (iflags & IB_ACCESS_REMOTE_ATOMIC)
1142 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
1143 if (iflags & IB_ACCESS_MW_BIND)
1144 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
1145 if (iflags & IB_ZERO_BASED)
1146 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
1147 if (iflags & IB_ACCESS_ON_DEMAND)
1148 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
1149 return qflags;
1150};
1151
1152static enum ib_access_flags __to_ib_access_flags(int qflags)
1153{
1154 enum ib_access_flags iflags = 0;
1155
1156 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
1157 iflags |= IB_ACCESS_LOCAL_WRITE;
1158 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
1159 iflags |= IB_ACCESS_REMOTE_WRITE;
1160 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
1161 iflags |= IB_ACCESS_REMOTE_READ;
1162 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
1163 iflags |= IB_ACCESS_REMOTE_ATOMIC;
1164 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
1165 iflags |= IB_ACCESS_MW_BIND;
1166 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
1167 iflags |= IB_ZERO_BASED;
1168 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
1169 iflags |= IB_ACCESS_ON_DEMAND;
1170 return iflags;
1171};
1172
1173static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1174 struct bnxt_re_qp *qp1_qp,
1175 int qp_attr_mask)
1176{
1177 struct bnxt_re_qp *qp = rdev->qp1_sqp;
1178 int rc = 0;
1179
1180 if (qp_attr_mask & IB_QP_STATE) {
1181 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1182 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1183 }
1184 if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1185 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1186 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1187 }
1188
1189 if (qp_attr_mask & IB_QP_QKEY) {
1190 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1191 /* Using a Random QKEY */
1192 qp->qplib_qp.qkey = 0x81818181;
1193 }
1194 if (qp_attr_mask & IB_QP_SQ_PSN) {
1195 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1196 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1197 }
1198
1199 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1200 if (rc)
1201 dev_err(rdev_to_dev(rdev),
1202 "Failed to modify Shadow QP for QP1");
1203 return rc;
1204}
1205
1206int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1207 int qp_attr_mask, struct ib_udata *udata)
1208{
1209 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1210 struct bnxt_re_dev *rdev = qp->rdev;
1211 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1212 enum ib_qp_state curr_qp_state, new_qp_state;
1213 int rc, entries;
1214 int status;
1215 union ib_gid sgid;
1216 struct ib_gid_attr sgid_attr;
1217 u8 nw_type;
1218
1219 qp->qplib_qp.modify_flags = 0;
1220 if (qp_attr_mask & IB_QP_STATE) {
1221 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1222 new_qp_state = qp_attr->qp_state;
1223 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
1224 ib_qp->qp_type, qp_attr_mask,
1225 IB_LINK_LAYER_ETHERNET)) {
1226 dev_err(rdev_to_dev(rdev),
1227 "Invalid attribute mask: %#x specified ",
1228 qp_attr_mask);
1229 dev_err(rdev_to_dev(rdev),
1230 "for qpn: %#x type: %#x",
1231 ib_qp->qp_num, ib_qp->qp_type);
1232 dev_err(rdev_to_dev(rdev),
1233 "curr_qp_state=0x%x, new_qp_state=0x%x\n",
1234 curr_qp_state, new_qp_state);
1235 return -EINVAL;
1236 }
1237 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1238 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
1239 }
1240 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1241 qp->qplib_qp.modify_flags |=
1242 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1243 qp->qplib_qp.en_sqd_async_notify = true;
1244 }
1245 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1246 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1247 qp->qplib_qp.access =
1248 __from_ib_access_flags(qp_attr->qp_access_flags);
1249 /* LOCAL_WRITE access must be set to allow RC receive */
1250 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1251 }
1252 if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1253 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1254 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1255 }
1256 if (qp_attr_mask & IB_QP_QKEY) {
1257 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1258 qp->qplib_qp.qkey = qp_attr->qkey;
1259 }
1260 if (qp_attr_mask & IB_QP_AV) {
1261 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1262 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1263 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1264 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1265 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1266 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1267 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1268 memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw,
1269 sizeof(qp->qplib_qp.ah.dgid.data));
1270 qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label;
1271 /* If RoCE V2 is enabled, stack will have two entries for
1272 * each GID entry. Avoiding this duplicte entry in HW. Dividing
1273 * the GID index by 2 for RoCE V2
1274 */
1275 qp->qplib_qp.ah.sgid_index =
1276 qp_attr->ah_attr.grh.sgid_index / 2;
1277 qp->qplib_qp.ah.host_sgid_index =
1278 qp_attr->ah_attr.grh.sgid_index;
1279 qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit;
1280 qp->qplib_qp.ah.traffic_class =
1281 qp_attr->ah_attr.grh.traffic_class;
1282 qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl;
1283 ether_addr_copy(qp->qplib_qp.ah.dmac, qp_attr->ah_attr.dmac);
1284
1285 status = ib_get_cached_gid(&rdev->ibdev, 1,
1286 qp_attr->ah_attr.grh.sgid_index,
1287 &sgid, &sgid_attr);
1288 if (!status && sgid_attr.ndev) {
1289 memcpy(qp->qplib_qp.smac, sgid_attr.ndev->dev_addr,
1290 ETH_ALEN);
1291 dev_put(sgid_attr.ndev);
1292 nw_type = ib_gid_to_network_type(sgid_attr.gid_type,
1293 &sgid);
1294 switch (nw_type) {
1295 case RDMA_NETWORK_IPV4:
1296 qp->qplib_qp.nw_type =
1297 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
1298 break;
1299 case RDMA_NETWORK_IPV6:
1300 qp->qplib_qp.nw_type =
1301 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
1302 break;
1303 default:
1304 qp->qplib_qp.nw_type =
1305 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
1306 break;
1307 }
1308 }
1309 }
1310
1311 if (qp_attr_mask & IB_QP_PATH_MTU) {
1312 qp->qplib_qp.modify_flags |=
1313 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1314 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
1315 } else if (qp_attr->qp_state == IB_QPS_RTR) {
1316 qp->qplib_qp.modify_flags |=
1317 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1318 qp->qplib_qp.path_mtu =
1319 __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
1320 }
1321
1322 if (qp_attr_mask & IB_QP_TIMEOUT) {
1323 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
1324 qp->qplib_qp.timeout = qp_attr->timeout;
1325 }
1326 if (qp_attr_mask & IB_QP_RETRY_CNT) {
1327 qp->qplib_qp.modify_flags |=
1328 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
1329 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
1330 }
1331 if (qp_attr_mask & IB_QP_RNR_RETRY) {
1332 qp->qplib_qp.modify_flags |=
1333 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
1334 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
1335 }
1336 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
1337 qp->qplib_qp.modify_flags |=
1338 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
1339 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
1340 }
1341 if (qp_attr_mask & IB_QP_RQ_PSN) {
1342 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
1343 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
1344 }
1345 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1346 qp->qplib_qp.modify_flags |=
1347 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
1348 qp->qplib_qp.max_rd_atomic = qp_attr->max_rd_atomic;
1349 }
1350 if (qp_attr_mask & IB_QP_SQ_PSN) {
1351 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1352 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
1353 }
1354 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1355 qp->qplib_qp.modify_flags |=
1356 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
1357 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
1358 }
1359 if (qp_attr_mask & IB_QP_CAP) {
1360 qp->qplib_qp.modify_flags |=
1361 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
1362 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
1363 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
1364 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
1365 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
1366 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
1367 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
1368 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
1369 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
1370 (qp_attr->cap.max_inline_data >=
1371 dev_attr->max_inline_data)) {
1372 dev_err(rdev_to_dev(rdev),
1373 "Create QP failed - max exceeded");
1374 return -EINVAL;
1375 }
1376 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
1377 qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
1378 dev_attr->max_qp_wqes + 1);
1379 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
1380 if (qp->qplib_qp.rq.max_wqe) {
1381 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
1382 qp->qplib_qp.rq.max_wqe =
1383 min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1384 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
1385 } else {
1386 /* SRQ was used prior, just ignore the RQ caps */
1387 }
1388 }
1389 if (qp_attr_mask & IB_QP_DEST_QPN) {
1390 qp->qplib_qp.modify_flags |=
1391 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
1392 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
1393 }
1394 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1395 if (rc) {
1396 dev_err(rdev_to_dev(rdev), "Failed to modify HW QP");
1397 return rc;
1398 }
1399 if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp)
1400 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
1401 return rc;
1402}
1403
1404int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1405 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1406{
1407 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1408 struct bnxt_re_dev *rdev = qp->rdev;
1409 struct bnxt_qplib_qp qplib_qp;
1410 int rc;
1411
1412 memset(&qplib_qp, 0, sizeof(struct bnxt_qplib_qp));
1413 qplib_qp.id = qp->qplib_qp.id;
1414 qplib_qp.ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
1415
1416 rc = bnxt_qplib_query_qp(&rdev->qplib_res, &qplib_qp);
1417 if (rc) {
1418 dev_err(rdev_to_dev(rdev), "Failed to query HW QP");
1419 return rc;
1420 }
1421 qp_attr->qp_state = __to_ib_qp_state(qplib_qp.state);
1422 qp_attr->en_sqd_async_notify = qplib_qp.en_sqd_async_notify ? 1 : 0;
1423 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp.access);
1424 qp_attr->pkey_index = qplib_qp.pkey_index;
1425 qp_attr->qkey = qplib_qp.qkey;
1426 memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp.ah.dgid.data,
1427 sizeof(qplib_qp.ah.dgid.data));
1428 qp_attr->ah_attr.grh.flow_label = qplib_qp.ah.flow_label;
1429 qp_attr->ah_attr.grh.sgid_index = qplib_qp.ah.host_sgid_index;
1430 qp_attr->ah_attr.grh.hop_limit = qplib_qp.ah.hop_limit;
1431 qp_attr->ah_attr.grh.traffic_class = qplib_qp.ah.traffic_class;
1432 qp_attr->ah_attr.sl = qplib_qp.ah.sl;
1433 ether_addr_copy(qp_attr->ah_attr.dmac, qplib_qp.ah.dmac);
1434 qp_attr->path_mtu = __to_ib_mtu(qplib_qp.path_mtu);
1435 qp_attr->timeout = qplib_qp.timeout;
1436 qp_attr->retry_cnt = qplib_qp.retry_cnt;
1437 qp_attr->rnr_retry = qplib_qp.rnr_retry;
1438 qp_attr->min_rnr_timer = qplib_qp.min_rnr_timer;
1439 qp_attr->rq_psn = qplib_qp.rq.psn;
1440 qp_attr->max_rd_atomic = qplib_qp.max_rd_atomic;
1441 qp_attr->sq_psn = qplib_qp.sq.psn;
1442 qp_attr->max_dest_rd_atomic = qplib_qp.max_dest_rd_atomic;
1443 qp_init_attr->sq_sig_type = qplib_qp.sig_type ? IB_SIGNAL_ALL_WR :
1444 IB_SIGNAL_REQ_WR;
1445 qp_attr->dest_qp_num = qplib_qp.dest_qpn;
1446
1447 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
1448 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
1449 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
1450 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
1451 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
1452 qp_init_attr->cap = qp_attr->cap;
1453
1454 return 0;
1455}
1456
1457/* Routine for sending QP1 packets for RoCE V1 an V2
1458 */
1459static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
1460 struct ib_send_wr *wr,
1461 struct bnxt_qplib_swqe *wqe,
1462 int payload_size)
1463{
1464 struct ib_device *ibdev = &qp->rdev->ibdev;
1465 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
1466 ib_ah);
1467 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
1468 struct bnxt_qplib_sge sge;
1469 union ib_gid sgid;
1470 u8 nw_type;
1471 u16 ether_type;
1472 struct ib_gid_attr sgid_attr;
1473 union ib_gid dgid;
1474 bool is_eth = false;
1475 bool is_vlan = false;
1476 bool is_grh = false;
1477 bool is_udp = false;
1478 u8 ip_version = 0;
1479 u16 vlan_id = 0xFFFF;
1480 void *buf;
1481 int i, rc = 0, size;
1482
1483 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
1484
1485 rc = ib_get_cached_gid(ibdev, 1,
1486 qplib_ah->host_sgid_index, &sgid,
1487 &sgid_attr);
1488 if (rc) {
1489 dev_err(rdev_to_dev(qp->rdev),
1490 "Failed to query gid at index %d",
1491 qplib_ah->host_sgid_index);
1492 return rc;
1493 }
1494 if (sgid_attr.ndev) {
1495 if (is_vlan_dev(sgid_attr.ndev))
1496 vlan_id = vlan_dev_vlan_id(sgid_attr.ndev);
1497 dev_put(sgid_attr.ndev);
1498 }
1499 /* Get network header type for this GID */
1500 nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
1501 switch (nw_type) {
1502 case RDMA_NETWORK_IPV4:
1503 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
1504 break;
1505 case RDMA_NETWORK_IPV6:
1506 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
1507 break;
1508 default:
1509 nw_type = BNXT_RE_ROCE_V1_PACKET;
1510 break;
1511 }
1512 memcpy(&dgid.raw, &qplib_ah->dgid, 16);
1513 is_udp = sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
1514 if (is_udp) {
1515 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) {
1516 ip_version = 4;
1517 ether_type = ETH_P_IP;
1518 } else {
1519 ip_version = 6;
1520 ether_type = ETH_P_IPV6;
1521 }
1522 is_grh = false;
1523 } else {
1524 ether_type = ETH_P_IBOE;
1525 is_grh = true;
1526 }
1527
1528 is_eth = true;
1529 is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false;
1530
1531 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
1532 ip_version, is_udp, 0, &qp->qp1_hdr);
1533
1534 /* ETH */
1535 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
1536 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
1537
1538 /* For vlan, check the sgid for vlan existence */
1539
1540 if (!is_vlan) {
1541 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
1542 } else {
1543 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
1544 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
1545 }
1546
1547 if (is_grh || (ip_version == 6)) {
1548 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid));
1549 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
1550 sizeof(sgid));
1551 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit;
1552 }
1553
1554 if (ip_version == 4) {
1555 qp->qp1_hdr.ip4.tos = 0;
1556 qp->qp1_hdr.ip4.id = 0;
1557 qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
1558 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
1559
1560 memcpy(&qp->qp1_hdr.ip4.saddr, sgid.raw + 12, 4);
1561 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
1562 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
1563 }
1564
1565 if (is_udp) {
1566 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
1567 qp->qp1_hdr.udp.sport = htons(0x8CD1);
1568 qp->qp1_hdr.udp.csum = 0;
1569 }
1570
1571 /* BTH */
1572 if (wr->opcode == IB_WR_SEND_WITH_IMM) {
1573 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
1574 qp->qp1_hdr.immediate_present = 1;
1575 } else {
1576 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
1577 }
1578 if (wr->send_flags & IB_SEND_SOLICITED)
1579 qp->qp1_hdr.bth.solicited_event = 1;
1580 /* pad_count */
1581 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
1582
1583 /* P_key for QP1 is for all members */
1584 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
1585 qp->qp1_hdr.bth.destination_qpn = IB_QP1;
1586 qp->qp1_hdr.bth.ack_req = 0;
1587 qp->send_psn++;
1588 qp->send_psn &= BTH_PSN_MASK;
1589 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
1590 /* DETH */
1591 /* Use the priviledged Q_Key for QP1 */
1592 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
1593 qp->qp1_hdr.deth.source_qpn = IB_QP1;
1594
1595 /* Pack the QP1 to the transmit buffer */
1596 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
1597 if (buf) {
1598 size = ib_ud_header_pack(&qp->qp1_hdr, buf);
1599 for (i = wqe->num_sge; i; i--) {
1600 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
1601 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
1602 wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
1603 }
1604
1605 /*
1606 * Max Header buf size for IPV6 RoCE V2 is 86,
1607 * which is same as the QP1 SQ header buffer.
1608 * Header buf size for IPV4 RoCE V2 can be 66.
1609 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
1610 * Subtract 20 bytes from QP1 SQ header buf size
1611 */
1612 if (is_udp && ip_version == 4)
1613 sge.size -= 20;
1614 /*
1615 * Max Header buf size for RoCE V1 is 78.
1616 * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
1617 * Subtract 8 bytes from QP1 SQ header buf size
1618 */
1619 if (!is_udp)
1620 sge.size -= 8;
1621
1622 /* Subtract 4 bytes for non vlan packets */
1623 if (!is_vlan)
1624 sge.size -= 4;
1625
1626 wqe->sg_list[0].addr = sge.addr;
1627 wqe->sg_list[0].lkey = sge.lkey;
1628 wqe->sg_list[0].size = sge.size;
1629 wqe->num_sge++;
1630
1631 } else {
1632 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!");
1633 rc = -ENOMEM;
1634 }
1635 return rc;
1636}
1637
1638/* For the MAD layer, it only provides the recv SGE the size of
1639 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH,
1640 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire
1641 * receive packet (334 bytes) with no VLAN and then copy the GRH
1642 * and the MAD datagram out to the provided SGE.
1643 */
1644static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
1645 struct ib_recv_wr *wr,
1646 struct bnxt_qplib_swqe *wqe,
1647 int payload_size)
1648{
1649 struct bnxt_qplib_sge ref, sge;
1650 u32 rq_prod_index;
1651 struct bnxt_re_sqp_entries *sqp_entry;
1652
1653 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
1654
1655 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
1656 return -ENOMEM;
1657
1658 /* Create 1 SGE to receive the entire
1659 * ethernet packet
1660 */
1661 /* Save the reference from ULP */
1662 ref.addr = wqe->sg_list[0].addr;
1663 ref.lkey = wqe->sg_list[0].lkey;
1664 ref.size = wqe->sg_list[0].size;
1665
1666 sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index];
1667
1668 /* SGE 1 */
1669 wqe->sg_list[0].addr = sge.addr;
1670 wqe->sg_list[0].lkey = sge.lkey;
1671 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1672 sge.size -= wqe->sg_list[0].size;
1673
1674 sqp_entry->sge.addr = ref.addr;
1675 sqp_entry->sge.lkey = ref.lkey;
1676 sqp_entry->sge.size = ref.size;
1677 /* Store the wrid for reporting completion */
1678 sqp_entry->wrid = wqe->wr_id;
1679 /* change the wqe->wrid to table index */
1680 wqe->wr_id = rq_prod_index;
1681 return 0;
1682}
1683
1684static int is_ud_qp(struct bnxt_re_qp *qp)
1685{
1686 return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD;
1687}
1688
1689static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
1690 struct ib_send_wr *wr,
1691 struct bnxt_qplib_swqe *wqe)
1692{
1693 struct bnxt_re_ah *ah = NULL;
1694
1695 if (is_ud_qp(qp)) {
1696 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
1697 wqe->send.q_key = ud_wr(wr)->remote_qkey;
1698 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
1699 wqe->send.avid = ah->qplib_ah.id;
1700 }
1701 switch (wr->opcode) {
1702 case IB_WR_SEND:
1703 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
1704 break;
1705 case IB_WR_SEND_WITH_IMM:
1706 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
1707 wqe->send.imm_data = wr->ex.imm_data;
1708 break;
1709 case IB_WR_SEND_WITH_INV:
1710 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
1711 wqe->send.inv_key = wr->ex.invalidate_rkey;
1712 break;
1713 default:
1714 return -EINVAL;
1715 }
1716 if (wr->send_flags & IB_SEND_SIGNALED)
1717 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1718 if (wr->send_flags & IB_SEND_FENCE)
1719 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1720 if (wr->send_flags & IB_SEND_SOLICITED)
1721 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1722 if (wr->send_flags & IB_SEND_INLINE)
1723 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
1724
1725 return 0;
1726}
1727
1728static int bnxt_re_build_rdma_wqe(struct ib_send_wr *wr,
1729 struct bnxt_qplib_swqe *wqe)
1730{
1731 switch (wr->opcode) {
1732 case IB_WR_RDMA_WRITE:
1733 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
1734 break;
1735 case IB_WR_RDMA_WRITE_WITH_IMM:
1736 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
1737 wqe->rdma.imm_data = wr->ex.imm_data;
1738 break;
1739 case IB_WR_RDMA_READ:
1740 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
1741 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
1742 break;
1743 default:
1744 return -EINVAL;
1745 }
1746 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
1747 wqe->rdma.r_key = rdma_wr(wr)->rkey;
1748 if (wr->send_flags & IB_SEND_SIGNALED)
1749 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1750 if (wr->send_flags & IB_SEND_FENCE)
1751 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1752 if (wr->send_flags & IB_SEND_SOLICITED)
1753 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1754 if (wr->send_flags & IB_SEND_INLINE)
1755 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
1756
1757 return 0;
1758}
1759
1760static int bnxt_re_build_atomic_wqe(struct ib_send_wr *wr,
1761 struct bnxt_qplib_swqe *wqe)
1762{
1763 switch (wr->opcode) {
1764 case IB_WR_ATOMIC_CMP_AND_SWP:
1765 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
1766 wqe->atomic.swap_data = atomic_wr(wr)->swap;
1767 break;
1768 case IB_WR_ATOMIC_FETCH_AND_ADD:
1769 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
1770 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
1771 break;
1772 default:
1773 return -EINVAL;
1774 }
1775 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
1776 wqe->atomic.r_key = atomic_wr(wr)->rkey;
1777 if (wr->send_flags & IB_SEND_SIGNALED)
1778 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1779 if (wr->send_flags & IB_SEND_FENCE)
1780 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1781 if (wr->send_flags & IB_SEND_SOLICITED)
1782 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1783 return 0;
1784}
1785
1786static int bnxt_re_build_inv_wqe(struct ib_send_wr *wr,
1787 struct bnxt_qplib_swqe *wqe)
1788{
1789 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
1790 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
1791
1792 if (wr->send_flags & IB_SEND_SIGNALED)
1793 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1794 if (wr->send_flags & IB_SEND_FENCE)
1795 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1796 if (wr->send_flags & IB_SEND_SOLICITED)
1797 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1798
1799 return 0;
1800}
1801
1802static int bnxt_re_build_reg_wqe(struct ib_reg_wr *wr,
1803 struct bnxt_qplib_swqe *wqe)
1804{
1805 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
1806 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
1807 int access = wr->access;
1808
1809 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
1810 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
1811 wqe->frmr.page_list = mr->pages;
1812 wqe->frmr.page_list_len = mr->npages;
1813 wqe->frmr.levels = qplib_frpl->hwq.level + 1;
1814 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
1815
1816 if (wr->wr.send_flags & IB_SEND_FENCE)
1817 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1818 if (wr->wr.send_flags & IB_SEND_SIGNALED)
1819 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1820
1821 if (access & IB_ACCESS_LOCAL_WRITE)
1822 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1823 if (access & IB_ACCESS_REMOTE_READ)
1824 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
1825 if (access & IB_ACCESS_REMOTE_WRITE)
1826 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
1827 if (access & IB_ACCESS_REMOTE_ATOMIC)
1828 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
1829 if (access & IB_ACCESS_MW_BIND)
1830 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
1831
1832 wqe->frmr.l_key = wr->key;
1833 wqe->frmr.length = wr->mr->length;
1834 wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1;
1835 wqe->frmr.va = wr->mr->iova;
1836 return 0;
1837}
1838
1839static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
1840 struct ib_send_wr *wr,
1841 struct bnxt_qplib_swqe *wqe)
1842{
1843 /* Copy the inline data to the data field */
1844 u8 *in_data;
1845 u32 i, sge_len;
1846 void *sge_addr;
1847
1848 in_data = wqe->inline_data;
1849 for (i = 0; i < wr->num_sge; i++) {
1850 sge_addr = (void *)(unsigned long)
1851 wr->sg_list[i].addr;
1852 sge_len = wr->sg_list[i].length;
1853
1854 if ((sge_len + wqe->inline_len) >
1855 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1856 dev_err(rdev_to_dev(rdev),
1857 "Inline data size requested > supported value");
1858 return -EINVAL;
1859 }
1860 sge_len = wr->sg_list[i].length;
1861
1862 memcpy(in_data, sge_addr, sge_len);
1863 in_data += wr->sg_list[i].length;
1864 wqe->inline_len += wr->sg_list[i].length;
1865 }
1866 return wqe->inline_len;
1867}
1868
1869static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
1870 struct ib_send_wr *wr,
1871 struct bnxt_qplib_swqe *wqe)
1872{
1873 int payload_sz = 0;
1874
1875 if (wr->send_flags & IB_SEND_INLINE)
1876 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
1877 else
1878 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
1879 wqe->num_sge);
1880
1881 return payload_sz;
1882}
1883
1884static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
1885 struct bnxt_re_qp *qp,
1886 struct ib_send_wr *wr)
1887{
1888 struct bnxt_qplib_swqe wqe;
1889 int rc = 0, payload_sz = 0;
1890 unsigned long flags;
1891
1892 spin_lock_irqsave(&qp->sq_lock, flags);
1893 memset(&wqe, 0, sizeof(wqe));
1894 while (wr) {
1895 /* House keeping */
1896 memset(&wqe, 0, sizeof(wqe));
1897
1898 /* Common */
1899 wqe.num_sge = wr->num_sge;
1900 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
1901 dev_err(rdev_to_dev(rdev),
1902 "Limit exceeded for Send SGEs");
1903 rc = -EINVAL;
1904 goto bad;
1905 }
1906
1907 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
1908 if (payload_sz < 0) {
1909 rc = -EINVAL;
1910 goto bad;
1911 }
1912 wqe.wr_id = wr->wr_id;
1913
1914 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
1915
1916 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
1917 if (!rc)
1918 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
1919bad:
1920 if (rc) {
1921 dev_err(rdev_to_dev(rdev),
1922 "Post send failed opcode = %#x rc = %d",
1923 wr->opcode, rc);
1924 break;
1925 }
1926 wr = wr->next;
1927 }
1928 bnxt_qplib_post_send_db(&qp->qplib_qp);
1929 spin_unlock_irqrestore(&qp->sq_lock, flags);
1930 return rc;
1931}
1932
1933int bnxt_re_post_send(struct ib_qp *ib_qp, struct ib_send_wr *wr,
1934 struct ib_send_wr **bad_wr)
1935{
1936 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1937 struct bnxt_qplib_swqe wqe;
1938 int rc = 0, payload_sz = 0;
1939 unsigned long flags;
1940
1941 spin_lock_irqsave(&qp->sq_lock, flags);
1942 while (wr) {
1943 /* House keeping */
1944 memset(&wqe, 0, sizeof(wqe));
1945
1946 /* Common */
1947 wqe.num_sge = wr->num_sge;
1948 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
1949 dev_err(rdev_to_dev(qp->rdev),
1950 "Limit exceeded for Send SGEs");
1951 rc = -EINVAL;
1952 goto bad;
1953 }
1954
1955 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
1956 if (payload_sz < 0) {
1957 rc = -EINVAL;
1958 goto bad;
1959 }
1960 wqe.wr_id = wr->wr_id;
1961
1962 switch (wr->opcode) {
1963 case IB_WR_SEND:
1964 case IB_WR_SEND_WITH_IMM:
1965 if (ib_qp->qp_type == IB_QPT_GSI) {
1966 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
1967 payload_sz);
1968 if (rc)
1969 goto bad;
1970 wqe.rawqp1.lflags |=
1971 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
1972 }
1973 switch (wr->send_flags) {
1974 case IB_SEND_IP_CSUM:
1975 wqe.rawqp1.lflags |=
1976 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
1977 break;
1978 default:
1979 break;
1980 }
1981 /* Fall thru to build the wqe */
1982 case IB_WR_SEND_WITH_INV:
1983 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
1984 break;
1985 case IB_WR_RDMA_WRITE:
1986 case IB_WR_RDMA_WRITE_WITH_IMM:
1987 case IB_WR_RDMA_READ:
1988 rc = bnxt_re_build_rdma_wqe(wr, &wqe);
1989 break;
1990 case IB_WR_ATOMIC_CMP_AND_SWP:
1991 case IB_WR_ATOMIC_FETCH_AND_ADD:
1992 rc = bnxt_re_build_atomic_wqe(wr, &wqe);
1993 break;
1994 case IB_WR_RDMA_READ_WITH_INV:
1995 dev_err(rdev_to_dev(qp->rdev),
1996 "RDMA Read with Invalidate is not supported");
1997 rc = -EINVAL;
1998 goto bad;
1999 case IB_WR_LOCAL_INV:
2000 rc = bnxt_re_build_inv_wqe(wr, &wqe);
2001 break;
2002 case IB_WR_REG_MR:
2003 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2004 break;
2005 default:
2006 /* Unsupported WRs */
2007 dev_err(rdev_to_dev(qp->rdev),
2008 "WR (%#x) is not supported", wr->opcode);
2009 rc = -EINVAL;
2010 goto bad;
2011 }
2012 if (!rc)
2013 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2014bad:
2015 if (rc) {
2016 dev_err(rdev_to_dev(qp->rdev),
2017 "post_send failed op:%#x qps = %#x rc = %d\n",
2018 wr->opcode, qp->qplib_qp.state, rc);
2019 *bad_wr = wr;
2020 break;
2021 }
2022 wr = wr->next;
2023 }
2024 bnxt_qplib_post_send_db(&qp->qplib_qp);
2025 spin_unlock_irqrestore(&qp->sq_lock, flags);
2026
2027 return rc;
2028}
2029
2030static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2031 struct bnxt_re_qp *qp,
2032 struct ib_recv_wr *wr)
2033{
2034 struct bnxt_qplib_swqe wqe;
2035 int rc = 0, payload_sz = 0;
2036
2037 memset(&wqe, 0, sizeof(wqe));
2038 while (wr) {
2039 /* House keeping */
2040 memset(&wqe, 0, sizeof(wqe));
2041
2042 /* Common */
2043 wqe.num_sge = wr->num_sge;
2044 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2045 dev_err(rdev_to_dev(rdev),
2046 "Limit exceeded for Receive SGEs");
2047 rc = -EINVAL;
2048 break;
2049 }
2050 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2051 wr->num_sge);
2052 wqe.wr_id = wr->wr_id;
2053 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2054
2055 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2056 if (rc)
2057 break;
2058
2059 wr = wr->next;
2060 }
2061 if (!rc)
2062 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2063 return rc;
2064}
2065
2066int bnxt_re_post_recv(struct ib_qp *ib_qp, struct ib_recv_wr *wr,
2067 struct ib_recv_wr **bad_wr)
2068{
2069 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2070 struct bnxt_qplib_swqe wqe;
2071 int rc = 0, payload_sz = 0;
2072
2073 while (wr) {
2074 /* House keeping */
2075 memset(&wqe, 0, sizeof(wqe));
2076
2077 /* Common */
2078 wqe.num_sge = wr->num_sge;
2079 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2080 dev_err(rdev_to_dev(qp->rdev),
2081 "Limit exceeded for Receive SGEs");
2082 rc = -EINVAL;
2083 *bad_wr = wr;
2084 break;
2085 }
2086
2087 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2088 wr->num_sge);
2089 wqe.wr_id = wr->wr_id;
2090 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2091
2092 if (ib_qp->qp_type == IB_QPT_GSI)
2093 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2094 payload_sz);
2095 if (!rc)
2096 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2097 if (rc) {
2098 *bad_wr = wr;
2099 break;
2100 }
2101 wr = wr->next;
2102 }
2103 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2104 return rc;
2105}
2106
2107/* Completion Queues */
2108int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
2109{
2110 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2111 struct bnxt_re_dev *rdev = cq->rdev;
2112 int rc;
2113
2114 rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2115 if (rc) {
2116 dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ");
2117 return rc;
2118 }
2119 if (cq->umem && !IS_ERR(cq->umem))
2120 ib_umem_release(cq->umem);
2121
2122 if (cq) {
2123 kfree(cq->cql);
2124 kfree(cq);
2125 }
2126 atomic_dec(&rdev->cq_count);
2127 rdev->nq.budget--;
2128 return 0;
2129}
2130
2131struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
2132 const struct ib_cq_init_attr *attr,
2133 struct ib_ucontext *context,
2134 struct ib_udata *udata)
2135{
2136 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
2137 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
2138 struct bnxt_re_cq *cq = NULL;
2139 int rc, entries;
2140 int cqe = attr->cqe;
2141
2142 /* Validate CQ fields */
2143 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
2144 dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded");
2145 return ERR_PTR(-EINVAL);
2146 }
2147 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
2148 if (!cq)
2149 return ERR_PTR(-ENOMEM);
2150
2151 cq->rdev = rdev;
2152 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2153
2154 entries = roundup_pow_of_two(cqe + 1);
2155 if (entries > dev_attr->max_cq_wqes + 1)
2156 entries = dev_attr->max_cq_wqes + 1;
2157
2158 if (context) {
2159 struct bnxt_re_cq_req req;
2160 struct bnxt_re_ucontext *uctx = container_of
2161 (context,
2162 struct bnxt_re_ucontext,
2163 ib_uctx);
2164 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2165 rc = -EFAULT;
2166 goto fail;
2167 }
2168
2169 cq->umem = ib_umem_get(context, req.cq_va,
2170 entries * sizeof(struct cq_base),
2171 IB_ACCESS_LOCAL_WRITE, 1);
2172 if (IS_ERR(cq->umem)) {
2173 rc = PTR_ERR(cq->umem);
2174 goto fail;
2175 }
2176 cq->qplib_cq.sghead = cq->umem->sg_head.sgl;
2177 cq->qplib_cq.nmap = cq->umem->nmap;
2178 cq->qplib_cq.dpi = uctx->dpi;
2179 } else {
2180 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2181 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2182 GFP_KERNEL);
2183 if (!cq->cql) {
2184 rc = -ENOMEM;
2185 goto fail;
2186 }
2187
2188 cq->qplib_cq.dpi = &rdev->dpi_privileged;
2189 cq->qplib_cq.sghead = NULL;
2190 cq->qplib_cq.nmap = 0;
2191 }
2192 cq->qplib_cq.max_wqe = entries;
2193 cq->qplib_cq.cnq_hw_ring_id = rdev->nq.ring_id;
2194
2195 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2196 if (rc) {
2197 dev_err(rdev_to_dev(rdev), "Failed to create HW CQ");
2198 goto fail;
2199 }
2200
2201 cq->ib_cq.cqe = entries;
2202 cq->cq_period = cq->qplib_cq.period;
2203 rdev->nq.budget++;
2204
2205 atomic_inc(&rdev->cq_count);
2206
2207 if (context) {
2208 struct bnxt_re_cq_resp resp;
2209
2210 resp.cqid = cq->qplib_cq.id;
2211 resp.tail = cq->qplib_cq.hwq.cons;
2212 resp.phase = cq->qplib_cq.period;
2213 resp.rsvd = 0;
2214 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
2215 if (rc) {
2216 dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata");
2217 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2218 goto c2fail;
2219 }
2220 }
2221
2222 return &cq->ib_cq;
2223
2224c2fail:
2225 if (context)
2226 ib_umem_release(cq->umem);
2227fail:
2228 kfree(cq->cql);
2229 kfree(cq);
2230 return ERR_PTR(rc);
2231}
2232
2233static u8 __req_to_ib_wc_status(u8 qstatus)
2234{
2235 switch (qstatus) {
2236 case CQ_REQ_STATUS_OK:
2237 return IB_WC_SUCCESS;
2238 case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
2239 return IB_WC_BAD_RESP_ERR;
2240 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
2241 return IB_WC_LOC_LEN_ERR;
2242 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
2243 return IB_WC_LOC_QP_OP_ERR;
2244 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
2245 return IB_WC_LOC_PROT_ERR;
2246 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
2247 return IB_WC_GENERAL_ERR;
2248 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
2249 return IB_WC_REM_INV_REQ_ERR;
2250 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
2251 return IB_WC_REM_ACCESS_ERR;
2252 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
2253 return IB_WC_REM_OP_ERR;
2254 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
2255 return IB_WC_RNR_RETRY_EXC_ERR;
2256 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
2257 return IB_WC_RETRY_EXC_ERR;
2258 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
2259 return IB_WC_WR_FLUSH_ERR;
2260 default:
2261 return IB_WC_GENERAL_ERR;
2262 }
2263 return 0;
2264}
2265
2266static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
2267{
2268 switch (qstatus) {
2269 case CQ_RES_RAWETH_QP1_STATUS_OK:
2270 return IB_WC_SUCCESS;
2271 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
2272 return IB_WC_LOC_ACCESS_ERR;
2273 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
2274 return IB_WC_LOC_LEN_ERR;
2275 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
2276 return IB_WC_LOC_PROT_ERR;
2277 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
2278 return IB_WC_LOC_QP_OP_ERR;
2279 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
2280 return IB_WC_GENERAL_ERR;
2281 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
2282 return IB_WC_WR_FLUSH_ERR;
2283 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
2284 return IB_WC_WR_FLUSH_ERR;
2285 default:
2286 return IB_WC_GENERAL_ERR;
2287 }
2288}
2289
2290static u8 __rc_to_ib_wc_status(u8 qstatus)
2291{
2292 switch (qstatus) {
2293 case CQ_RES_RC_STATUS_OK:
2294 return IB_WC_SUCCESS;
2295 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
2296 return IB_WC_LOC_ACCESS_ERR;
2297 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
2298 return IB_WC_LOC_LEN_ERR;
2299 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
2300 return IB_WC_LOC_PROT_ERR;
2301 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
2302 return IB_WC_LOC_QP_OP_ERR;
2303 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
2304 return IB_WC_GENERAL_ERR;
2305 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
2306 return IB_WC_REM_INV_REQ_ERR;
2307 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
2308 return IB_WC_WR_FLUSH_ERR;
2309 case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
2310 return IB_WC_WR_FLUSH_ERR;
2311 default:
2312 return IB_WC_GENERAL_ERR;
2313 }
2314}
2315
2316static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
2317{
2318 switch (cqe->type) {
2319 case BNXT_QPLIB_SWQE_TYPE_SEND:
2320 wc->opcode = IB_WC_SEND;
2321 break;
2322 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2323 wc->opcode = IB_WC_SEND;
2324 wc->wc_flags |= IB_WC_WITH_IMM;
2325 break;
2326 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
2327 wc->opcode = IB_WC_SEND;
2328 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2329 break;
2330 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
2331 wc->opcode = IB_WC_RDMA_WRITE;
2332 break;
2333 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
2334 wc->opcode = IB_WC_RDMA_WRITE;
2335 wc->wc_flags |= IB_WC_WITH_IMM;
2336 break;
2337 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
2338 wc->opcode = IB_WC_RDMA_READ;
2339 break;
2340 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
2341 wc->opcode = IB_WC_COMP_SWAP;
2342 break;
2343 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
2344 wc->opcode = IB_WC_FETCH_ADD;
2345 break;
2346 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
2347 wc->opcode = IB_WC_LOCAL_INV;
2348 break;
2349 case BNXT_QPLIB_SWQE_TYPE_REG_MR:
2350 wc->opcode = IB_WC_REG_MR;
2351 break;
2352 default:
2353 wc->opcode = IB_WC_SEND;
2354 break;
2355 }
2356
2357 wc->status = __req_to_ib_wc_status(cqe->status);
2358}
2359
2360static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
2361 u16 raweth_qp1_flags2)
2362{
2363 bool is_udp = false, is_ipv6 = false, is_ipv4 = false;
2364
2365 /* raweth_qp1_flags Bit 9-6 indicates itype */
2366 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
2367 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
2368 return -1;
2369
2370 if (raweth_qp1_flags2 &
2371 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
2372 raweth_qp1_flags2 &
2373 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
2374 is_udp = true;
2375 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
2376 (raweth_qp1_flags2 &
2377 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
2378 (is_ipv6 = true) : (is_ipv4 = true);
2379 return ((is_ipv6) ?
2380 BNXT_RE_ROCEV2_IPV6_PACKET :
2381 BNXT_RE_ROCEV2_IPV4_PACKET);
2382 } else {
2383 return BNXT_RE_ROCE_V1_PACKET;
2384 }
2385}
2386
2387static int bnxt_re_to_ib_nw_type(int nw_type)
2388{
2389 u8 nw_hdr_type = 0xFF;
2390
2391 switch (nw_type) {
2392 case BNXT_RE_ROCE_V1_PACKET:
2393 nw_hdr_type = RDMA_NETWORK_ROCE_V1;
2394 break;
2395 case BNXT_RE_ROCEV2_IPV4_PACKET:
2396 nw_hdr_type = RDMA_NETWORK_IPV4;
2397 break;
2398 case BNXT_RE_ROCEV2_IPV6_PACKET:
2399 nw_hdr_type = RDMA_NETWORK_IPV6;
2400 break;
2401 }
2402 return nw_hdr_type;
2403}
2404
2405static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
2406 void *rq_hdr_buf)
2407{
2408 u8 *tmp_buf = NULL;
2409 struct ethhdr *eth_hdr;
2410 u16 eth_type;
2411 bool rc = false;
2412
2413 tmp_buf = (u8 *)rq_hdr_buf;
2414 /*
2415 * If dest mac is not same as I/F mac, this could be a
2416 * loopback address or multicast address, check whether
2417 * it is a loopback packet
2418 */
2419 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
2420 tmp_buf += 4;
2421 /* Check the ether type */
2422 eth_hdr = (struct ethhdr *)tmp_buf;
2423 eth_type = ntohs(eth_hdr->h_proto);
2424 switch (eth_type) {
2425 case ETH_P_IBOE:
2426 rc = true;
2427 break;
2428 case ETH_P_IP:
2429 case ETH_P_IPV6: {
2430 u32 len;
2431 struct udphdr *udp_hdr;
2432
2433 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
2434 sizeof(struct ipv6hdr));
2435 tmp_buf += sizeof(struct ethhdr) + len;
2436 udp_hdr = (struct udphdr *)tmp_buf;
2437 if (ntohs(udp_hdr->dest) ==
2438 ROCE_V2_UDP_DPORT)
2439 rc = true;
2440 break;
2441 }
2442 default:
2443 break;
2444 }
2445 }
2446
2447 return rc;
2448}
2449
2450static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp,
2451 struct bnxt_qplib_cqe *cqe)
2452{
2453 struct bnxt_re_dev *rdev = qp1_qp->rdev;
2454 struct bnxt_re_sqp_entries *sqp_entry = NULL;
2455 struct bnxt_re_qp *qp = rdev->qp1_sqp;
2456 struct ib_send_wr *swr;
2457 struct ib_ud_wr udwr;
2458 struct ib_recv_wr rwr;
2459 int pkt_type = 0;
2460 u32 tbl_idx;
2461 void *rq_hdr_buf;
2462 dma_addr_t rq_hdr_buf_map;
2463 dma_addr_t shrq_hdr_buf_map;
2464 u32 offset = 0;
2465 u32 skip_bytes = 0;
2466 struct ib_sge s_sge[2];
2467 struct ib_sge r_sge[2];
2468 int rc;
2469
2470 memset(&udwr, 0, sizeof(udwr));
2471 memset(&rwr, 0, sizeof(rwr));
2472 memset(&s_sge, 0, sizeof(s_sge));
2473 memset(&r_sge, 0, sizeof(r_sge));
2474
2475 swr = &udwr.wr;
2476 tbl_idx = cqe->wr_id;
2477
2478 rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf +
2479 (tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size);
2480 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp,
2481 tbl_idx);
2482
2483 /* Shadow QP header buffer */
2484 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp,
2485 tbl_idx);
2486 sqp_entry = &rdev->sqp_tbl[tbl_idx];
2487
2488 /* Store this cqe */
2489 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
2490 sqp_entry->qp1_qp = qp1_qp;
2491
2492 /* Find packet type from the cqe */
2493
2494 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
2495 cqe->raweth_qp1_flags2);
2496 if (pkt_type < 0) {
2497 dev_err(rdev_to_dev(rdev), "Invalid packet\n");
2498 return -EINVAL;
2499 }
2500
2501 /* Adjust the offset for the user buffer and post in the rq */
2502
2503 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
2504 offset = 20;
2505
2506 /*
2507 * QP1 loopback packet has 4 bytes of internal header before
2508 * ether header. Skip these four bytes.
2509 */
2510 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
2511 skip_bytes = 4;
2512
2513 /* First send SGE . Skip the ether header*/
2514 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
2515 + skip_bytes;
2516 s_sge[0].lkey = 0xFFFFFFFF;
2517 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
2518 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
2519
2520 /* Second Send SGE */
2521 s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
2522 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
2523 if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
2524 s_sge[1].addr += 8;
2525 s_sge[1].lkey = 0xFFFFFFFF;
2526 s_sge[1].length = 256;
2527
2528 /* First recv SGE */
2529
2530 r_sge[0].addr = shrq_hdr_buf_map;
2531 r_sge[0].lkey = 0xFFFFFFFF;
2532 r_sge[0].length = 40;
2533
2534 r_sge[1].addr = sqp_entry->sge.addr + offset;
2535 r_sge[1].lkey = sqp_entry->sge.lkey;
2536 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
2537
2538 /* Create receive work request */
2539 rwr.num_sge = 2;
2540 rwr.sg_list = r_sge;
2541 rwr.wr_id = tbl_idx;
2542 rwr.next = NULL;
2543
2544 rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr);
2545 if (rc) {
2546 dev_err(rdev_to_dev(rdev),
2547 "Failed to post Rx buffers to shadow QP");
2548 return -ENOMEM;
2549 }
2550
2551 swr->num_sge = 2;
2552 swr->sg_list = s_sge;
2553 swr->wr_id = tbl_idx;
2554 swr->opcode = IB_WR_SEND;
2555 swr->next = NULL;
2556
2557 udwr.ah = &rdev->sqp_ah->ib_ah;
2558 udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id;
2559 udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey;
2560
2561 /* post data received in the send queue */
2562 rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr);
2563
2564 return 0;
2565}
2566
2567static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
2568 struct bnxt_qplib_cqe *cqe)
2569{
2570 wc->opcode = IB_WC_RECV;
2571 wc->status = __rawqp1_to_ib_wc_status(cqe->status);
2572 wc->wc_flags |= IB_WC_GRH;
2573}
2574
2575static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
2576 struct bnxt_qplib_cqe *cqe)
2577{
2578 wc->opcode = IB_WC_RECV;
2579 wc->status = __rc_to_ib_wc_status(cqe->status);
2580
2581 if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
2582 wc->wc_flags |= IB_WC_WITH_IMM;
2583 if (cqe->flags & CQ_RES_RC_FLAGS_INV)
2584 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2585 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
2586 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
2587 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2588}
2589
2590static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp,
2591 struct ib_wc *wc,
2592 struct bnxt_qplib_cqe *cqe)
2593{
2594 u32 tbl_idx;
2595 struct bnxt_re_dev *rdev = qp->rdev;
2596 struct bnxt_re_qp *qp1_qp = NULL;
2597 struct bnxt_qplib_cqe *orig_cqe = NULL;
2598 struct bnxt_re_sqp_entries *sqp_entry = NULL;
2599 int nw_type;
2600
2601 tbl_idx = cqe->wr_id;
2602
2603 sqp_entry = &rdev->sqp_tbl[tbl_idx];
2604 qp1_qp = sqp_entry->qp1_qp;
2605 orig_cqe = &sqp_entry->cqe;
2606
2607 wc->wr_id = sqp_entry->wrid;
2608 wc->byte_len = orig_cqe->length;
2609 wc->qp = &qp1_qp->ib_qp;
2610
2611 wc->ex.imm_data = orig_cqe->immdata;
2612 wc->src_qp = orig_cqe->src_qp;
2613 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
2614 wc->port_num = 1;
2615 wc->vendor_err = orig_cqe->status;
2616
2617 wc->opcode = IB_WC_RECV;
2618 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
2619 wc->wc_flags |= IB_WC_GRH;
2620
2621 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
2622 orig_cqe->raweth_qp1_flags2);
2623 if (nw_type >= 0) {
2624 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
2625 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
2626 }
2627}
2628
2629static void bnxt_re_process_res_ud_wc(struct ib_wc *wc,
2630 struct bnxt_qplib_cqe *cqe)
2631{
2632 wc->opcode = IB_WC_RECV;
2633 wc->status = __rc_to_ib_wc_status(cqe->status);
2634
2635 if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
2636 wc->wc_flags |= IB_WC_WITH_IMM;
2637 if (cqe->flags & CQ_RES_RC_FLAGS_INV)
2638 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2639 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
2640 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
2641 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2642}
2643
2644int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
2645{
2646 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2647 struct bnxt_re_qp *qp;
2648 struct bnxt_qplib_cqe *cqe;
2649 int i, ncqe, budget;
2650 u32 tbl_idx;
2651 struct bnxt_re_sqp_entries *sqp_entry = NULL;
2652 unsigned long flags;
2653
2654 spin_lock_irqsave(&cq->cq_lock, flags);
2655 budget = min_t(u32, num_entries, cq->max_cql);
2656 if (!cq->cql) {
2657 dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use");
2658 goto exit;
2659 }
2660 cqe = &cq->cql[0];
2661 while (budget) {
2662 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget);
2663 if (!ncqe)
2664 break;
2665
2666 for (i = 0; i < ncqe; i++, cqe++) {
2667 /* Transcribe each qplib_wqe back to ib_wc */
2668 memset(wc, 0, sizeof(*wc));
2669
2670 wc->wr_id = cqe->wr_id;
2671 wc->byte_len = cqe->length;
2672 qp = container_of
2673 ((struct bnxt_qplib_qp *)
2674 (unsigned long)(cqe->qp_handle),
2675 struct bnxt_re_qp, qplib_qp);
2676 if (!qp) {
2677 dev_err(rdev_to_dev(cq->rdev),
2678 "POLL CQ : bad QP handle");
2679 continue;
2680 }
2681 wc->qp = &qp->ib_qp;
2682 wc->ex.imm_data = cqe->immdata;
2683 wc->src_qp = cqe->src_qp;
2684 memcpy(wc->smac, cqe->smac, ETH_ALEN);
2685 wc->port_num = 1;
2686 wc->vendor_err = cqe->status;
2687
2688 switch (cqe->opcode) {
2689 case CQ_BASE_CQE_TYPE_REQ:
2690 if (qp->qplib_qp.id ==
2691 qp->rdev->qp1_sqp->qplib_qp.id) {
2692 /* Handle this completion with
2693 * the stored completion
2694 */
2695 memset(wc, 0, sizeof(*wc));
2696 continue;
2697 }
2698 bnxt_re_process_req_wc(wc, cqe);
2699 break;
2700 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2701 if (!cqe->status) {
2702 int rc = 0;
2703
2704 rc = bnxt_re_process_raw_qp_pkt_rx
2705 (qp, cqe);
2706 if (!rc) {
2707 memset(wc, 0, sizeof(*wc));
2708 continue;
2709 }
2710 cqe->status = -1;
2711 }
2712 /* Errors need not be looped back.
2713 * But change the wr_id to the one
2714 * stored in the table
2715 */
2716 tbl_idx = cqe->wr_id;
2717 sqp_entry = &cq->rdev->sqp_tbl[tbl_idx];
2718 wc->wr_id = sqp_entry->wrid;
2719 bnxt_re_process_res_rawqp1_wc(wc, cqe);
2720 break;
2721 case CQ_BASE_CQE_TYPE_RES_RC:
2722 bnxt_re_process_res_rc_wc(wc, cqe);
2723 break;
2724 case CQ_BASE_CQE_TYPE_RES_UD:
2725 if (qp->qplib_qp.id ==
2726 qp->rdev->qp1_sqp->qplib_qp.id) {
2727 /* Handle this completion with
2728 * the stored completion
2729 */
2730 if (cqe->status) {
2731 continue;
2732 } else {
2733 bnxt_re_process_res_shadow_qp_wc
2734 (qp, wc, cqe);
2735 break;
2736 }
2737 }
2738 bnxt_re_process_res_ud_wc(wc, cqe);
2739 break;
2740 default:
2741 dev_err(rdev_to_dev(cq->rdev),
2742 "POLL CQ : type 0x%x not handled",
2743 cqe->opcode);
2744 continue;
2745 }
2746 wc++;
2747 budget--;
2748 }
2749 }
2750exit:
2751 spin_unlock_irqrestore(&cq->cq_lock, flags);
2752 return num_entries - budget;
2753}
2754
2755int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
2756 enum ib_cq_notify_flags ib_cqn_flags)
2757{
2758 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2759 int type = 0;
2760
2761 /* Trigger on the very next completion */
2762 if (ib_cqn_flags & IB_CQ_NEXT_COMP)
2763 type = DBR_DBR_TYPE_CQ_ARMALL;
2764 /* Trigger on the next solicited completion */
2765 else if (ib_cqn_flags & IB_CQ_SOLICITED)
2766 type = DBR_DBR_TYPE_CQ_ARMSE;
2767
2768 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
2769
2770 return 0;
2771}
2772
2773/* Memory Regions */
2774struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
2775{
2776 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2777 struct bnxt_re_dev *rdev = pd->rdev;
2778 struct bnxt_re_mr *mr;
2779 u64 pbl = 0;
2780 int rc;
2781
2782 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2783 if (!mr)
2784 return ERR_PTR(-ENOMEM);
2785
2786 mr->rdev = rdev;
2787 mr->qplib_mr.pd = &pd->qplib_pd;
2788 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
2789 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2790
2791 /* Allocate and register 0 as the address */
2792 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
2793 if (rc)
2794 goto fail;
2795
2796 mr->qplib_mr.hwq.level = PBL_LVL_MAX;
2797 mr->qplib_mr.total_size = -1; /* Infinte length */
2798 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false);
2799 if (rc)
2800 goto fail_mr;
2801
2802 mr->ib_mr.lkey = mr->qplib_mr.lkey;
2803 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
2804 IB_ACCESS_REMOTE_ATOMIC))
2805 mr->ib_mr.rkey = mr->ib_mr.lkey;
2806 atomic_inc(&rdev->mr_count);
2807
2808 return &mr->ib_mr;
2809
2810fail_mr:
2811 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2812fail:
2813 kfree(mr);
2814 return ERR_PTR(rc);
2815}
2816
2817int bnxt_re_dereg_mr(struct ib_mr *ib_mr)
2818{
2819 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2820 struct bnxt_re_dev *rdev = mr->rdev;
2821 int rc = 0;
2822
2823 if (mr->npages && mr->pages) {
2824 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
2825 &mr->qplib_frpl);
2826 kfree(mr->pages);
2827 mr->npages = 0;
2828 mr->pages = NULL;
2829 }
2830 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2831
2832 if (!IS_ERR(mr->ib_umem) && mr->ib_umem)
2833 ib_umem_release(mr->ib_umem);
2834
2835 kfree(mr);
2836 atomic_dec(&rdev->mr_count);
2837 return rc;
2838}
2839
2840static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
2841{
2842 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2843
2844 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
2845 return -ENOMEM;
2846
2847 mr->pages[mr->npages++] = addr;
2848 return 0;
2849}
2850
2851int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
2852 unsigned int *sg_offset)
2853{
2854 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2855
2856 mr->npages = 0;
2857 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
2858}
2859
2860struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
2861 u32 max_num_sg)
2862{
2863 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2864 struct bnxt_re_dev *rdev = pd->rdev;
2865 struct bnxt_re_mr *mr = NULL;
2866 int rc;
2867
2868 if (type != IB_MR_TYPE_MEM_REG) {
2869 dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type);
2870 return ERR_PTR(-EINVAL);
2871 }
2872 if (max_num_sg > MAX_PBL_LVL_1_PGS)
2873 return ERR_PTR(-EINVAL);
2874
2875 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2876 if (!mr)
2877 return ERR_PTR(-ENOMEM);
2878
2879 mr->rdev = rdev;
2880 mr->qplib_mr.pd = &pd->qplib_pd;
2881 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
2882 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2883
2884 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
2885 if (rc)
2886 goto fail;
2887
2888 mr->ib_mr.lkey = mr->qplib_mr.lkey;
2889 mr->ib_mr.rkey = mr->ib_mr.lkey;
2890
2891 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
2892 if (!mr->pages) {
2893 rc = -ENOMEM;
2894 goto fail;
2895 }
2896 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
2897 &mr->qplib_frpl, max_num_sg);
2898 if (rc) {
2899 dev_err(rdev_to_dev(rdev),
2900 "Failed to allocate HW FR page list");
2901 goto fail_mr;
2902 }
2903
2904 atomic_inc(&rdev->mr_count);
2905 return &mr->ib_mr;
2906
2907fail_mr:
2908 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2909fail:
2910 kfree(mr->pages);
2911 kfree(mr);
2912 return ERR_PTR(rc);
2913}
2914
2915/* Fast Memory Regions */
2916struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *ib_pd, int mr_access_flags,
2917 struct ib_fmr_attr *fmr_attr)
2918{
2919 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2920 struct bnxt_re_dev *rdev = pd->rdev;
2921 struct bnxt_re_fmr *fmr;
2922 int rc;
2923
2924 if (fmr_attr->max_pages > MAX_PBL_LVL_2_PGS ||
2925 fmr_attr->max_maps > rdev->dev_attr.max_map_per_fmr) {
2926 dev_err(rdev_to_dev(rdev), "Allocate FMR exceeded Max limit");
2927 return ERR_PTR(-ENOMEM);
2928 }
2929 fmr = kzalloc(sizeof(*fmr), GFP_KERNEL);
2930 if (!fmr)
2931 return ERR_PTR(-ENOMEM);
2932
2933 fmr->rdev = rdev;
2934 fmr->qplib_fmr.pd = &pd->qplib_pd;
2935 fmr->qplib_fmr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2936
2937 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &fmr->qplib_fmr);
2938 if (rc)
2939 goto fail;
2940
2941 fmr->qplib_fmr.flags = __from_ib_access_flags(mr_access_flags);
2942 fmr->ib_fmr.lkey = fmr->qplib_fmr.lkey;
2943 fmr->ib_fmr.rkey = fmr->ib_fmr.lkey;
2944
2945 atomic_inc(&rdev->mr_count);
2946 return &fmr->ib_fmr;
2947fail:
2948 kfree(fmr);
2949 return ERR_PTR(rc);
2950}
2951
2952int bnxt_re_map_phys_fmr(struct ib_fmr *ib_fmr, u64 *page_list, int list_len,
2953 u64 iova)
2954{
2955 struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr,
2956 ib_fmr);
2957 struct bnxt_re_dev *rdev = fmr->rdev;
2958 int rc;
2959
2960 fmr->qplib_fmr.va = iova;
2961 fmr->qplib_fmr.total_size = list_len * PAGE_SIZE;
2962
2963 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &fmr->qplib_fmr, page_list,
2964 list_len, true);
2965 if (rc)
2966 dev_err(rdev_to_dev(rdev), "Failed to map FMR for lkey = 0x%x!",
2967 fmr->ib_fmr.lkey);
2968 return rc;
2969}
2970
2971int bnxt_re_unmap_fmr(struct list_head *fmr_list)
2972{
2973 struct bnxt_re_dev *rdev;
2974 struct bnxt_re_fmr *fmr;
2975 struct ib_fmr *ib_fmr;
2976 int rc = 0;
2977
2978 /* Validate each FMRs inside the fmr_list */
2979 list_for_each_entry(ib_fmr, fmr_list, list) {
2980 fmr = container_of(ib_fmr, struct bnxt_re_fmr, ib_fmr);
2981 rdev = fmr->rdev;
2982
2983 if (rdev) {
2984 rc = bnxt_qplib_dereg_mrw(&rdev->qplib_res,
2985 &fmr->qplib_fmr, true);
2986 if (rc)
2987 break;
2988 }
2989 }
2990 return rc;
2991}
2992
2993int bnxt_re_dealloc_fmr(struct ib_fmr *ib_fmr)
2994{
2995 struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr,
2996 ib_fmr);
2997 struct bnxt_re_dev *rdev = fmr->rdev;
2998 int rc;
2999
3000 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &fmr->qplib_fmr);
3001 if (rc)
3002 dev_err(rdev_to_dev(rdev), "Failed to free FMR");
3003
3004 kfree(fmr);
3005 atomic_dec(&rdev->mr_count);
3006 return rc;
3007}
3008
3009/* uverbs */
3010struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
3011 u64 virt_addr, int mr_access_flags,
3012 struct ib_udata *udata)
3013{
3014 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3015 struct bnxt_re_dev *rdev = pd->rdev;
3016 struct bnxt_re_mr *mr;
3017 struct ib_umem *umem;
3018 u64 *pbl_tbl, *pbl_tbl_orig;
3019 int i, umem_pgs, pages, page_shift, rc;
3020 struct scatterlist *sg;
3021 int entry;
3022
3023 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3024 if (!mr)
3025 return ERR_PTR(-ENOMEM);
3026
3027 mr->rdev = rdev;
3028 mr->qplib_mr.pd = &pd->qplib_pd;
3029 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3030 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
3031
3032 umem = ib_umem_get(ib_pd->uobject->context, start, length,
3033 mr_access_flags, 0);
3034 if (IS_ERR(umem)) {
3035 dev_err(rdev_to_dev(rdev), "Failed to get umem");
3036 rc = -EFAULT;
3037 goto free_mr;
3038 }
3039 mr->ib_umem = umem;
3040
3041 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3042 if (rc) {
3043 dev_err(rdev_to_dev(rdev), "Failed to allocate MR");
3044 goto release_umem;
3045 }
3046 /* The fixed portion of the rkey is the same as the lkey */
3047 mr->ib_mr.rkey = mr->qplib_mr.rkey;
3048
3049 mr->qplib_mr.va = virt_addr;
3050 umem_pgs = ib_umem_page_count(umem);
3051 if (!umem_pgs) {
3052 dev_err(rdev_to_dev(rdev), "umem is invalid!");
3053 rc = -EINVAL;
3054 goto free_mrw;
3055 }
3056 mr->qplib_mr.total_size = length;
3057
3058 pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL);
3059 if (!pbl_tbl) {
3060 rc = -EINVAL;
3061 goto free_mrw;
3062 }
3063 pbl_tbl_orig = pbl_tbl;
3064
3065 page_shift = ilog2(umem->page_size);
3066 if (umem->hugetlb) {
3067 dev_err(rdev_to_dev(rdev), "umem hugetlb not supported!");
3068 rc = -EFAULT;
3069 goto fail;
3070 }
3071 if (umem->page_size != PAGE_SIZE) {
3072 dev_err(rdev_to_dev(rdev), "umem page size unsupported!");
3073 rc = -EFAULT;
3074 goto fail;
3075 }
3076 /* Map umem buf ptrs to the PBL */
3077 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
3078 pages = sg_dma_len(sg) >> page_shift;
3079 for (i = 0; i < pages; i++, pbl_tbl++)
3080 *pbl_tbl = sg_dma_address(sg) + (i << page_shift);
3081 }
3082 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl_orig,
3083 umem_pgs, false);
3084 if (rc) {
3085 dev_err(rdev_to_dev(rdev), "Failed to register user MR");
3086 goto fail;
3087 }
3088
3089 kfree(pbl_tbl_orig);
3090
3091 mr->ib_mr.lkey = mr->qplib_mr.lkey;
3092 mr->ib_mr.rkey = mr->qplib_mr.lkey;
3093 atomic_inc(&rdev->mr_count);
3094
3095 return &mr->ib_mr;
3096fail:
3097 kfree(pbl_tbl_orig);
3098free_mrw:
3099 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3100release_umem:
3101 ib_umem_release(umem);
3102free_mr:
3103 kfree(mr);
3104 return ERR_PTR(rc);
3105}
3106
3107struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev,
3108 struct ib_udata *udata)
3109{
3110 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
3111 struct bnxt_re_uctx_resp resp;
3112 struct bnxt_re_ucontext *uctx;
3113 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
3114 int rc;
3115
3116 dev_dbg(rdev_to_dev(rdev), "ABI version requested %d",
3117 ibdev->uverbs_abi_ver);
3118
3119 if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
3120 dev_dbg(rdev_to_dev(rdev), " is different from the device %d ",
3121 BNXT_RE_ABI_VERSION);
3122 return ERR_PTR(-EPERM);
3123 }
3124
3125 uctx = kzalloc(sizeof(*uctx), GFP_KERNEL);
3126 if (!uctx)
3127 return ERR_PTR(-ENOMEM);
3128
3129 uctx->rdev = rdev;
3130
3131 uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
3132 if (!uctx->shpg) {
3133 rc = -ENOMEM;
3134 goto fail;
3135 }
3136 spin_lock_init(&uctx->sh_lock);
3137
3138 resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/
3139 resp.max_qp = rdev->qplib_ctx.qpc_count;
3140 resp.pg_size = PAGE_SIZE;
3141 resp.cqe_sz = sizeof(struct cq_base);
3142 resp.max_cqd = dev_attr->max_cq_wqes;
3143 resp.rsvd = 0;
3144
3145 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
3146 if (rc) {
3147 dev_err(rdev_to_dev(rdev), "Failed to copy user context");
3148 rc = -EFAULT;
3149 goto cfail;
3150 }
3151
3152 return &uctx->ib_uctx;
3153cfail:
3154 free_page((unsigned long)uctx->shpg);
3155 uctx->shpg = NULL;
3156fail:
3157 kfree(uctx);
3158 return ERR_PTR(rc);
3159}
3160
3161int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
3162{
3163 struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3164 struct bnxt_re_ucontext,
3165 ib_uctx);
3166 if (uctx->shpg)
3167 free_page((unsigned long)uctx->shpg);
3168 kfree(uctx);
3169 return 0;
3170}
3171
3172/* Helper function to mmap the virtual memory from user app */
3173int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
3174{
3175 struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3176 struct bnxt_re_ucontext,
3177 ib_uctx);
3178 struct bnxt_re_dev *rdev = uctx->rdev;
3179 u64 pfn;
3180
3181 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3182 return -EINVAL;
3183
3184 if (vma->vm_pgoff) {
3185 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3186 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
3187 PAGE_SIZE, vma->vm_page_prot)) {
3188 dev_err(rdev_to_dev(rdev), "Failed to map DPI");
3189 return -EAGAIN;
3190 }
3191 } else {
3192 pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT;
3193 if (remap_pfn_range(vma, vma->vm_start,
3194 pfn, PAGE_SIZE, vma->vm_page_prot)) {
3195 dev_err(rdev_to_dev(rdev),
3196 "Failed to map shared page");
3197 return -EAGAIN;
3198 }
3199 }
3200
3201 return 0;
3202}
drivers/infiniband/hw/bnxt_re/ib_verbs.h+197−0
@@ -0,0 +1,197 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: IB Verbs interpreter (header)
37 */
38
39#ifndef __BNXT_RE_IB_VERBS_H__
40#define __BNXT_RE_IB_VERBS_H__
41
42struct bnxt_re_gid_ctx {
43 u32 idx;
44 u32 refcnt;
45};
46
47struct bnxt_re_pd {
48 struct bnxt_re_dev *rdev;
49 struct ib_pd ib_pd;
50 struct bnxt_qplib_pd qplib_pd;
51 struct bnxt_qplib_dpi dpi;
52};
53
54struct bnxt_re_ah {
55 struct bnxt_re_dev *rdev;
56 struct ib_ah ib_ah;
57 struct bnxt_qplib_ah qplib_ah;
58};
59
60struct bnxt_re_qp {
61 struct list_head list;
62 struct bnxt_re_dev *rdev;
63 struct ib_qp ib_qp;
64 spinlock_t sq_lock; /* protect sq */
65 struct bnxt_qplib_qp qplib_qp;
66 struct ib_umem *sumem;
67 struct ib_umem *rumem;
68 /* QP1 */
69 u32 send_psn;
70 struct ib_ud_header qp1_hdr;
71};
72
73struct bnxt_re_cq {
74 struct bnxt_re_dev *rdev;
75 spinlock_t cq_lock; /* protect cq */
76 u16 cq_count;
77 u16 cq_period;
78 struct ib_cq ib_cq;
79 struct bnxt_qplib_cq qplib_cq;
80 struct bnxt_qplib_cqe *cql;
81#define MAX_CQL_PER_POLL 1024
82 u32 max_cql;
83 struct ib_umem *umem;
84};
85
86struct bnxt_re_mr {
87 struct bnxt_re_dev *rdev;
88 struct ib_mr ib_mr;
89 struct ib_umem *ib_umem;
90 struct bnxt_qplib_mrw qplib_mr;
91 u32 npages;
92 u64 *pages;
93 struct bnxt_qplib_frpl qplib_frpl;
94};
95
96struct bnxt_re_frpl {
97 struct bnxt_re_dev *rdev;
98 struct bnxt_qplib_frpl qplib_frpl;
99 u64 *page_list;
100};
101
102struct bnxt_re_fmr {
103 struct bnxt_re_dev *rdev;
104 struct ib_fmr ib_fmr;
105 struct bnxt_qplib_mrw qplib_fmr;
106};
107
108struct bnxt_re_mw {
109 struct bnxt_re_dev *rdev;
110 struct ib_mw ib_mw;
111 struct bnxt_qplib_mrw qplib_mw;
112};
113
114struct bnxt_re_ucontext {
115 struct bnxt_re_dev *rdev;
116 struct ib_ucontext ib_uctx;
117 struct bnxt_qplib_dpi *dpi;
118 void *shpg;
119 spinlock_t sh_lock; /* protect shpg */
120};
121
122struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num);
123
124int bnxt_re_query_device(struct ib_device *ibdev,
125 struct ib_device_attr *ib_attr,
126 struct ib_udata *udata);
127int bnxt_re_modify_device(struct ib_device *ibdev,
128 int device_modify_mask,
129 struct ib_device_modify *device_modify);
130int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
131 struct ib_port_attr *port_attr);
132int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
133 int port_modify_mask,
134 struct ib_port_modify *port_modify);
135int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
136 struct ib_port_immutable *immutable);
137int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
138 u16 index, u16 *pkey);
139int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
140 unsigned int index, void **context);
141int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
142 unsigned int index, const union ib_gid *gid,
143 const struct ib_gid_attr *attr, void **context);
144int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
145 int index, union ib_gid *gid);
146enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
147 u8 port_num);
148struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev,
149 struct ib_ucontext *context,
150 struct ib_udata *udata);
151int bnxt_re_dealloc_pd(struct ib_pd *pd);
152struct ib_ah *bnxt_re_create_ah(struct ib_pd *pd,
153 struct ib_ah_attr *ah_attr,
154 struct ib_udata *udata);
155int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
156int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
157int bnxt_re_destroy_ah(struct ib_ah *ah);
158struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd,
159 struct ib_qp_init_attr *qp_init_attr,
160 struct ib_udata *udata);
161int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
162 int qp_attr_mask, struct ib_udata *udata);
163int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
164 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
165int bnxt_re_destroy_qp(struct ib_qp *qp);
166int bnxt_re_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr,
167 struct ib_send_wr **bad_send_wr);
168int bnxt_re_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
169 struct ib_recv_wr **bad_recv_wr);
170struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
171 const struct ib_cq_init_attr *attr,
172 struct ib_ucontext *context,
173 struct ib_udata *udata);
174int bnxt_re_destroy_cq(struct ib_cq *cq);
175int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
176int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
177struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
178
179int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
180 unsigned int *sg_offset);
181struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type,
182 u32 max_num_sg);
183int bnxt_re_dereg_mr(struct ib_mr *mr);
184struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
185 struct ib_fmr_attr *fmr_attr);
186int bnxt_re_map_phys_fmr(struct ib_fmr *fmr, u64 *page_list, int list_len,
187 u64 iova);
188int bnxt_re_unmap_fmr(struct list_head *fmr_list);
189int bnxt_re_dealloc_fmr(struct ib_fmr *fmr);
190struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
191 u64 virt_addr, int mr_access_flags,
192 struct ib_udata *udata);
193struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev,
194 struct ib_udata *udata);
195int bnxt_re_dealloc_ucontext(struct ib_ucontext *context);
196int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
197#endif /* __BNXT_RE_IB_VERBS_H__ */
drivers/infiniband/hw/bnxt_re/main.c+1315−0
@@ -0,0 +1,1315 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Main component of the bnxt_re driver
37 */
38
39#include <linux/module.h>
40#include <linux/netdevice.h>
41#include <linux/ethtool.h>
42#include <linux/mutex.h>
43#include <linux/list.h>
44#include <linux/rculist.h>
45#include <linux/spinlock.h>
46#include <linux/pci.h>
47#include <net/dcbnl.h>
48#include <net/ipv6.h>
49#include <net/addrconf.h>
50#include <linux/if_ether.h>
51
52#include <rdma/ib_verbs.h>
53#include <rdma/ib_user_verbs.h>
54#include <rdma/ib_umem.h>
55#include <rdma/ib_addr.h>
56
57#include "bnxt_ulp.h"
58#include "roce_hsi.h"
59#include "qplib_res.h"
60#include "qplib_sp.h"
61#include "qplib_fp.h"
62#include "qplib_rcfw.h"
63#include "bnxt_re.h"
64#include "ib_verbs.h"
65#include <rdma/bnxt_re-abi.h>
66#include "bnxt.h"
67static char version[] =
68 BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n";
69
70MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>");
71MODULE_DESCRIPTION(BNXT_RE_DESC " Driver");
72MODULE_LICENSE("Dual BSD/GPL");
73MODULE_VERSION(ROCE_DRV_MODULE_VERSION);
74
75/* globals */
76static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);
77/* Mutex to protect the list of bnxt_re devices added */
78static DEFINE_MUTEX(bnxt_re_dev_lock);
79static struct workqueue_struct *bnxt_re_wq;
80
81/* for handling bnxt_en callbacks later */
82static void bnxt_re_stop(void *p)
83{
84}
85
86static void bnxt_re_start(void *p)
87{
88}
89
90static void bnxt_re_sriov_config(void *p, int num_vfs)
91{
92}
93
94static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
95 .ulp_async_notifier = NULL,
96 .ulp_stop = bnxt_re_stop,
97 .ulp_start = bnxt_re_start,
98 .ulp_sriov_config = bnxt_re_sriov_config
99};
100
101/* RoCE -> Net driver */
102
103/* Driver registration routines used to let the networking driver (bnxt_en)
104 * to know that the RoCE driver is now installed
105 */
106static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev, bool lock_wait)
107{
108 struct bnxt_en_dev *en_dev;
109 int rc;
110
111 if (!rdev)
112 return -EINVAL;
113
114 en_dev = rdev->en_dev;
115 /* Acquire rtnl lock if it is not invokded from netdev event */
116 if (lock_wait)
117 rtnl_lock();
118
119 rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev,
120 BNXT_ROCE_ULP);
121 if (lock_wait)
122 rtnl_unlock();
123 return rc;
124}
125
126static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
127{
128 struct bnxt_en_dev *en_dev;
129 int rc = 0;
130
131 if (!rdev)
132 return -EINVAL;
133
134 en_dev = rdev->en_dev;
135
136 rtnl_lock();
137 rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP,
138 &bnxt_re_ulp_ops, rdev);
139 rtnl_unlock();
140 return rc;
141}
142
143static int bnxt_re_free_msix(struct bnxt_re_dev *rdev, bool lock_wait)
144{
145 struct bnxt_en_dev *en_dev;
146 int rc;
147
148 if (!rdev)
149 return -EINVAL;
150
151 en_dev = rdev->en_dev;
152
153 if (lock_wait)
154 rtnl_lock();
155
156 rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP);
157
158 if (lock_wait)
159 rtnl_unlock();
160 return rc;
161}
162
163static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
164{
165 int rc = 0, num_msix_want = BNXT_RE_MIN_MSIX, num_msix_got;
166 struct bnxt_en_dev *en_dev;
167
168 if (!rdev)
169 return -EINVAL;
170
171 en_dev = rdev->en_dev;
172
173 rtnl_lock();
174 num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP,
175 rdev->msix_entries,
176 num_msix_want);
177 if (num_msix_got < BNXT_RE_MIN_MSIX) {
178 rc = -EINVAL;
179 goto done;
180 }
181 if (num_msix_got != num_msix_want) {
182 dev_warn(rdev_to_dev(rdev),
183 "Requested %d MSI-X vectors, got %d\n",
184 num_msix_want, num_msix_got);
185 }
186 rdev->num_msix = num_msix_got;
187done:
188 rtnl_unlock();
189 return rc;
190}
191
192static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr,
193 u16 opcd, u16 crid, u16 trid)
194{
195 hdr->req_type = cpu_to_le16(opcd);
196 hdr->cmpl_ring = cpu_to_le16(crid);
197 hdr->target_id = cpu_to_le16(trid);
198}
199
200static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg,
201 int msg_len, void *resp, int resp_max_len,
202 int timeout)
203{
204 fw_msg->msg = msg;
205 fw_msg->msg_len = msg_len;
206 fw_msg->resp = resp;
207 fw_msg->resp_max_len = resp_max_len;
208 fw_msg->timeout = timeout;
209}
210
211static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id,
212 bool lock_wait)
213{
214 struct bnxt_en_dev *en_dev = rdev->en_dev;
215 struct hwrm_ring_free_input req = {0};
216 struct hwrm_ring_free_output resp;
217 struct bnxt_fw_msg fw_msg;
218 bool do_unlock = false;
219 int rc = -EINVAL;
220
221 if (!en_dev)
222 return rc;
223
224 memset(&fw_msg, 0, sizeof(fw_msg));
225 if (lock_wait) {
226 rtnl_lock();
227 do_unlock = true;
228 }
229
230 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1);
231 req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
232 req.ring_id = cpu_to_le16(fw_ring_id);
233 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
234 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
235 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
236 if (rc)
237 dev_err(rdev_to_dev(rdev),
238 "Failed to free HW ring:%d :%#x", req.ring_id, rc);
239 if (do_unlock)
240 rtnl_unlock();
241 return rc;
242}
243
244static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr,
245 int pages, int type, u32 ring_mask,
246 u32 map_index, u16 *fw_ring_id)
247{
248 struct bnxt_en_dev *en_dev = rdev->en_dev;
249 struct hwrm_ring_alloc_input req = {0};
250 struct hwrm_ring_alloc_output resp;
251 struct bnxt_fw_msg fw_msg;
252 int rc = -EINVAL;
253
254 if (!en_dev)
255 return rc;
256
257 memset(&fw_msg, 0, sizeof(fw_msg));
258 rtnl_lock();
259 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1);
260 req.enables = 0;
261 req.page_tbl_addr = cpu_to_le64(dma_arr[0]);
262 if (pages > 1) {
263 /* Page size is in log2 units */
264 req.page_size = BNXT_PAGE_SHIFT;
265 req.page_tbl_depth = 1;
266 }
267 req.fbo = 0;
268 /* Association of ring index with doorbell index and MSIX number */
269 req.logical_id = cpu_to_le16(map_index);
270 req.length = cpu_to_le32(ring_mask + 1);
271 req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
272 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
273 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
274 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
275 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
276 if (!rc)
277 *fw_ring_id = le16_to_cpu(resp.ring_id);
278
279 rtnl_unlock();
280 return rc;
281}
282
283static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
284 u32 fw_stats_ctx_id, bool lock_wait)
285{
286 struct bnxt_en_dev *en_dev = rdev->en_dev;
287 struct hwrm_stat_ctx_free_input req = {0};
288 struct bnxt_fw_msg fw_msg;
289 bool do_unlock = false;
290 int rc = -EINVAL;
291
292 if (!en_dev)
293 return rc;
294
295 memset(&fw_msg, 0, sizeof(fw_msg));
296 if (lock_wait) {
297 rtnl_lock();
298 do_unlock = true;
299 }
300
301 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1);
302 req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);
303 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&req,
304 sizeof(req), DFLT_HWRM_CMD_TIMEOUT);
305 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
306 if (rc)
307 dev_err(rdev_to_dev(rdev),
308 "Failed to free HW stats context %#x", rc);
309
310 if (do_unlock)
311 rtnl_unlock();
312 return rc;
313}
314
315static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev,
316 dma_addr_t dma_map,
317 u32 *fw_stats_ctx_id)
318{
319 struct hwrm_stat_ctx_alloc_output resp = {0};
320 struct hwrm_stat_ctx_alloc_input req = {0};
321 struct bnxt_en_dev *en_dev = rdev->en_dev;
322 struct bnxt_fw_msg fw_msg;
323 int rc = -EINVAL;
324
325 *fw_stats_ctx_id = INVALID_STATS_CTX_ID;
326
327 if (!en_dev)
328 return rc;
329
330 memset(&fw_msg, 0, sizeof(fw_msg));
331 rtnl_lock();
332
333 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1);
334 req.update_period_ms = cpu_to_le32(1000);
335 req.stats_dma_addr = cpu_to_le64(dma_map);
336 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
337 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
338 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
339 if (!rc)
340 *fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id);
341
342 rtnl_unlock();
343 return rc;
344}
345
346/* Device */
347
348static bool is_bnxt_re_dev(struct net_device *netdev)
349{
350 struct ethtool_drvinfo drvinfo;
351
352 if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) {
353 memset(&drvinfo, 0, sizeof(drvinfo));
354 netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo);
355
356 if (strcmp(drvinfo.driver, "bnxt_en"))
357 return false;
358 return true;
359 }
360 return false;
361}
362
363static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
364{
365 struct bnxt_re_dev *rdev;
366
367 rcu_read_lock();
368 list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) {
369 if (rdev->netdev == netdev) {
370 rcu_read_unlock();
371 return rdev;
372 }
373 }
374 rcu_read_unlock();
375 return NULL;
376}
377
378static void bnxt_re_dev_unprobe(struct net_device *netdev,
379 struct bnxt_en_dev *en_dev)
380{
381 dev_put(netdev);
382 module_put(en_dev->pdev->driver->driver.owner);
383}
384
385static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev)
386{
387 struct bnxt *bp = netdev_priv(netdev);
388 struct bnxt_en_dev *en_dev;
389 struct pci_dev *pdev;
390
391 /* Call bnxt_en's RoCE probe via indirect API */
392 if (!bp->ulp_probe)
393 return ERR_PTR(-EINVAL);
394
395 en_dev = bp->ulp_probe(netdev);
396 if (IS_ERR(en_dev))
397 return en_dev;
398
399 pdev = en_dev->pdev;
400 if (!pdev)
401 return ERR_PTR(-EINVAL);
402
403 if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) {
404 dev_dbg(&pdev->dev,
405 "%s: probe error: RoCE is not supported on this device",
406 ROCE_DRV_MODULE_NAME);
407 return ERR_PTR(-ENODEV);
408 }
409
410 /* Bump net device reference count */
411 if (!try_module_get(pdev->driver->driver.owner))
412 return ERR_PTR(-ENODEV);
413
414 dev_hold(netdev);
415
416 return en_dev;
417}
418
419static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev)
420{
421 ib_unregister_device(&rdev->ibdev);
422}
423
424static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
425{
426 struct ib_device *ibdev = &rdev->ibdev;
427
428 /* ib device init */
429 ibdev->owner = THIS_MODULE;
430 ibdev->node_type = RDMA_NODE_IB_CA;
431 strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX);
432 strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA",
433 strlen(BNXT_RE_DESC) + 5);
434 ibdev->phys_port_cnt = 1;
435
436 bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid);
437
438 ibdev->num_comp_vectors = 1;
439 ibdev->dma_device = &rdev->en_dev->pdev->dev;
440 ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
441
442 /* User space */
443 ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION;
444 ibdev->uverbs_cmd_mask =
445 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
446 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
447 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
448 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
449 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
450 (1ull << IB_USER_VERBS_CMD_REG_MR) |
451 (1ull << IB_USER_VERBS_CMD_REREG_MR) |
452 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
453 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
454 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
455 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
456 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
457 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
458 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
459 (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
460 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
461 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
462 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
463 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
464 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
465 (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
466 (1ull << IB_USER_VERBS_CMD_MODIFY_AH) |
467 (1ull << IB_USER_VERBS_CMD_QUERY_AH) |
468 (1ull << IB_USER_VERBS_CMD_DESTROY_AH);
469 /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */
470
471 /* Kernel verbs */
472 ibdev->query_device = bnxt_re_query_device;
473 ibdev->modify_device = bnxt_re_modify_device;
474
475 ibdev->query_port = bnxt_re_query_port;
476 ibdev->modify_port = bnxt_re_modify_port;
477 ibdev->get_port_immutable = bnxt_re_get_port_immutable;
478 ibdev->query_pkey = bnxt_re_query_pkey;
479 ibdev->query_gid = bnxt_re_query_gid;
480 ibdev->get_netdev = bnxt_re_get_netdev;
481 ibdev->add_gid = bnxt_re_add_gid;
482 ibdev->del_gid = bnxt_re_del_gid;
483 ibdev->get_link_layer = bnxt_re_get_link_layer;
484
485 ibdev->alloc_pd = bnxt_re_alloc_pd;
486 ibdev->dealloc_pd = bnxt_re_dealloc_pd;
487
488 ibdev->create_ah = bnxt_re_create_ah;
489 ibdev->modify_ah = bnxt_re_modify_ah;
490 ibdev->query_ah = bnxt_re_query_ah;
491 ibdev->destroy_ah = bnxt_re_destroy_ah;
492
493 ibdev->create_qp = bnxt_re_create_qp;
494 ibdev->modify_qp = bnxt_re_modify_qp;
495 ibdev->query_qp = bnxt_re_query_qp;
496 ibdev->destroy_qp = bnxt_re_destroy_qp;
497
498 ibdev->post_send = bnxt_re_post_send;
499 ibdev->post_recv = bnxt_re_post_recv;
500
501 ibdev->create_cq = bnxt_re_create_cq;
502 ibdev->destroy_cq = bnxt_re_destroy_cq;
503 ibdev->poll_cq = bnxt_re_poll_cq;
504 ibdev->req_notify_cq = bnxt_re_req_notify_cq;
505
506 ibdev->get_dma_mr = bnxt_re_get_dma_mr;
507 ibdev->dereg_mr = bnxt_re_dereg_mr;
508 ibdev->alloc_mr = bnxt_re_alloc_mr;
509 ibdev->map_mr_sg = bnxt_re_map_mr_sg;
510 ibdev->alloc_fmr = bnxt_re_alloc_fmr;
511 ibdev->map_phys_fmr = bnxt_re_map_phys_fmr;
512 ibdev->unmap_fmr = bnxt_re_unmap_fmr;
513 ibdev->dealloc_fmr = bnxt_re_dealloc_fmr;
514
515 ibdev->reg_user_mr = bnxt_re_reg_user_mr;
516 ibdev->alloc_ucontext = bnxt_re_alloc_ucontext;
517 ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext;
518 ibdev->mmap = bnxt_re_mmap;
519
520 return ib_register_device(ibdev, NULL);
521}
522
523static ssize_t show_rev(struct device *device, struct device_attribute *attr,
524 char *buf)
525{
526 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
527
528 return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
529}
530
531static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
532 char *buf)
533{
534 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
535
536 return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->dev_attr.fw_ver);
537}
538
539static ssize_t show_hca(struct device *device, struct device_attribute *attr,
540 char *buf)
541{
542 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
543
544 return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc);
545}
546
547static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL);
548static DEVICE_ATTR(fw_rev, 0444, show_fw_ver, NULL);
549static DEVICE_ATTR(hca_type, 0444, show_hca, NULL);
550
551static struct device_attribute *bnxt_re_attributes[] = {
552 &dev_attr_hw_rev,
553 &dev_attr_fw_rev,
554 &dev_attr_hca_type
555};
556
557static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
558{
559 dev_put(rdev->netdev);
560 rdev->netdev = NULL;
561
562 mutex_lock(&bnxt_re_dev_lock);
563 list_del_rcu(&rdev->list);
564 mutex_unlock(&bnxt_re_dev_lock);
565
566 synchronize_rcu();
567 flush_workqueue(bnxt_re_wq);
568
569 ib_dealloc_device(&rdev->ibdev);
570 /* rdev is gone */
571}
572
573static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
574 struct bnxt_en_dev *en_dev)
575{
576 struct bnxt_re_dev *rdev;
577
578 /* Allocate bnxt_re_dev instance here */
579 rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev));
580 if (!rdev) {
581 dev_err(NULL, "%s: bnxt_re_dev allocation failure!",
582 ROCE_DRV_MODULE_NAME);
583 return NULL;
584 }
585 /* Default values */
586 rdev->netdev = netdev;
587 dev_hold(rdev->netdev);
588 rdev->en_dev = en_dev;
589 rdev->id = rdev->en_dev->pdev->devfn;
590 INIT_LIST_HEAD(&rdev->qp_list);
591 mutex_init(&rdev->qp_lock);
592 atomic_set(&rdev->qp_count, 0);
593 atomic_set(&rdev->cq_count, 0);
594 atomic_set(&rdev->srq_count, 0);
595 atomic_set(&rdev->mr_count, 0);
596 atomic_set(&rdev->mw_count, 0);
597 rdev->cosq[0] = 0xFFFF;
598 rdev->cosq[1] = 0xFFFF;
599
600 mutex_lock(&bnxt_re_dev_lock);
601 list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list);
602 mutex_unlock(&bnxt_re_dev_lock);
603 return rdev;
604}
605
606static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
607 struct creq_func_event *aeqe)
608{
609 switch (aeqe->event) {
610 case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
611 break;
612 case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
613 break;
614 case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
615 break;
616 case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
617 break;
618 case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
619 break;
620 case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
621 break;
622 case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
623 break;
624 case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
625 break;
626 case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
627 break;
628 case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
629 break;
630 case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
631 break;
632 default:
633 return -EINVAL;
634 }
635 return 0;
636}
637
638static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
639 struct bnxt_qplib_cq *handle)
640{
641 struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq,
642 qplib_cq);
643
644 if (!cq) {
645 dev_err(NULL, "%s: CQ is NULL, CQN not handled",
646 ROCE_DRV_MODULE_NAME);
647 return -EINVAL;
648 }
649 if (cq->ib_cq.comp_handler) {
650 /* Lock comp_handler? */
651 (*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context);
652 }
653
654 return 0;
655}
656
657static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
658{
659 if (rdev->nq.hwq.max_elements)
660 bnxt_qplib_disable_nq(&rdev->nq);
661
662 if (rdev->qplib_res.rcfw)
663 bnxt_qplib_cleanup_res(&rdev->qplib_res);
664}
665
666static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
667{
668 int rc = 0;
669
670 bnxt_qplib_init_res(&rdev->qplib_res);
671
672 if (rdev->msix_entries[BNXT_RE_NQ_IDX].vector <= 0)
673 return -EINVAL;
674
675 rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq,
676 rdev->msix_entries[BNXT_RE_NQ_IDX].vector,
677 rdev->msix_entries[BNXT_RE_NQ_IDX].db_offset,
678 &bnxt_re_cqn_handler,
679 NULL);
680
681 if (rc)
682 dev_err(rdev_to_dev(rdev), "Failed to enable NQ: %#x", rc);
683
684 return rc;
685}
686
687static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait)
688{
689 if (rdev->nq.hwq.max_elements) {
690 bnxt_re_net_ring_free(rdev, rdev->nq.ring_id, lock_wait);
691 bnxt_qplib_free_nq(&rdev->nq);
692 }
693 if (rdev->qplib_res.dpi_tbl.max) {
694 bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
695 &rdev->qplib_res.dpi_tbl,
696 &rdev->dpi_privileged);
697 }
698 if (rdev->qplib_res.rcfw) {
699 bnxt_qplib_free_res(&rdev->qplib_res);
700 rdev->qplib_res.rcfw = NULL;
701 }
702}
703
704static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
705{
706 int rc = 0;
707
708 /* Configure and allocate resources for qplib */
709 rdev->qplib_res.rcfw = &rdev->rcfw;
710 rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr);
711 if (rc)
712 goto fail;
713
714 rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev,
715 rdev->netdev, &rdev->dev_attr);
716 if (rc)
717 goto fail;
718
719 rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
720 &rdev->dpi_privileged,
721 rdev);
722 if (rc)
723 goto fail;
724
725 rdev->nq.hwq.max_elements = BNXT_RE_MAX_CQ_COUNT +
726 BNXT_RE_MAX_SRQC_COUNT + 2;
727 rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq);
728 if (rc) {
729 dev_err(rdev_to_dev(rdev),
730 "Failed to allocate NQ memory: %#x", rc);
731 goto fail;
732 }
733 rc = bnxt_re_net_ring_alloc
734 (rdev, rdev->nq.hwq.pbl[PBL_LVL_0].pg_map_arr,
735 rdev->nq.hwq.pbl[rdev->nq.hwq.level].pg_count,
736 HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_NQE_MAX_CNT - 1,
737 rdev->msix_entries[BNXT_RE_NQ_IDX].ring_idx,
738 &rdev->nq.ring_id);
739 if (rc) {
740 dev_err(rdev_to_dev(rdev),
741 "Failed to allocate NQ ring: %#x", rc);
742 goto free_nq;
743 }
744 return 0;
745free_nq:
746 bnxt_qplib_free_nq(&rdev->nq);
747fail:
748 rdev->qplib_res.rcfw = NULL;
749 return rc;
750}
751
752static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
753 u8 port_num, enum ib_event_type event)
754{
755 struct ib_event ib_event;
756
757 ib_event.device = ibdev;
758 if (qp)
759 ib_event.element.qp = qp;
760 else
761 ib_event.element.port_num = port_num;
762 ib_event.event = event;
763 ib_dispatch_event(&ib_event);
764}
765
766#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02
767static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir,
768 u64 *cid_map)
769{
770 struct hwrm_queue_pri2cos_qcfg_input req = {0};
771 struct bnxt *bp = netdev_priv(rdev->netdev);
772 struct hwrm_queue_pri2cos_qcfg_output resp;
773 struct bnxt_en_dev *en_dev = rdev->en_dev;
774 struct bnxt_fw_msg fw_msg;
775 u32 flags = 0;
776 u8 *qcfgmap, *tmp_map;
777 int rc = 0, i;
778
779 if (!cid_map)
780 return -EINVAL;
781
782 memset(&fw_msg, 0, sizeof(fw_msg));
783 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
784 HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
785 flags |= (dir & 0x01);
786 flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN;
787 req.flags = cpu_to_le32(flags);
788 req.port_id = bp->pf.port_id;
789
790 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
791 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
792 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
793 if (rc)
794 return rc;
795
796 if (resp.queue_cfg_info) {
797 dev_warn(rdev_to_dev(rdev),
798 "Asymmetric cos queue configuration detected");
799 dev_warn(rdev_to_dev(rdev),
800 " on device, QoS may not be fully functional\n");
801 }
802 qcfgmap = &resp.pri0_cos_queue_id;
803 tmp_map = (u8 *)cid_map;
804 for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++)
805 tmp_map[i] = qcfgmap[i];
806
807 return rc;
808}
809
810static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
811 struct bnxt_re_qp *qp)
812{
813 return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp);
814}
815
816static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)
817{
818 int mask = IB_QP_STATE;
819 struct ib_qp_attr qp_attr;
820 struct bnxt_re_qp *qp;
821
822 qp_attr.qp_state = IB_QPS_ERR;
823 mutex_lock(&rdev->qp_lock);
824 list_for_each_entry(qp, &rdev->qp_list, list) {
825 /* Modify the state of all QPs except QP1/Shadow QP */
826 if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) {
827 if (qp->qplib_qp.state !=
828 CMDQ_MODIFY_QP_NEW_STATE_RESET &&
829 qp->qplib_qp.state !=
830 CMDQ_MODIFY_QP_NEW_STATE_ERR) {
831 bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp,
832 1, IB_EVENT_QP_FATAL);
833 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask,
834 NULL);
835 }
836 }
837 }
838 mutex_unlock(&rdev->qp_lock);
839}
840
841static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev)
842{
843 u32 prio_map = 0, tmp_map = 0;
844 struct net_device *netdev;
845 struct dcb_app app;
846
847 netdev = rdev->netdev;
848
849 memset(&app, 0, sizeof(app));
850 app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE;
851 app.protocol = ETH_P_IBOE;
852 tmp_map = dcb_ieee_getapp_mask(netdev, &app);
853 prio_map = tmp_map;
854
855 app.selector = IEEE_8021QAZ_APP_SEL_DGRAM;
856 app.protocol = ROCE_V2_UDP_DPORT;
857 tmp_map = dcb_ieee_getapp_mask(netdev, &app);
858 prio_map |= tmp_map;
859
860 if (!prio_map)
861 prio_map = -EFAULT;
862 return prio_map;
863}
864
865static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq)
866{
867 u16 prio;
868 u8 id;
869
870 for (prio = 0, id = 0; prio < 8; prio++) {
871 if (prio_map & (1 << prio)) {
872 cosq[id] = cid_map[prio];
873 id++;
874 if (id == 2) /* Max 2 tcs supported */
875 break;
876 }
877 }
878}
879
880static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
881{
882 u8 prio_map = 0;
883 u64 cid_map;
884 int rc;
885
886 /* Get priority for roce */
887 rc = bnxt_re_get_priority_mask(rdev);
888 if (rc < 0)
889 return rc;
890 prio_map = (u8)rc;
891
892 if (prio_map == rdev->cur_prio_map)
893 return 0;
894 rdev->cur_prio_map = prio_map;
895 /* Get cosq id for this priority */
896 rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map);
897 if (rc) {
898 dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map);
899 return rc;
900 }
901 /* Parse CoS IDs for app priority */
902 bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq);
903
904 /* Config BONO. */
905 rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq);
906 if (rc) {
907 dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n",
908 rdev->cosq[0], rdev->cosq[1]);
909 return rc;
910 }
911
912 return 0;
913}
914
915static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait)
916{
917 int i, rc;
918
919 if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
920 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++)
921 device_remove_file(&rdev->ibdev.dev,
922 bnxt_re_attributes[i]);
923 /* Cleanup ib dev */
924 bnxt_re_unregister_ib(rdev);
925 }
926 if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags))
927 cancel_delayed_work(&rdev->worker);
928
929 bnxt_re_cleanup_res(rdev);
930 bnxt_re_free_res(rdev, lock_wait);
931
932 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) {
933 rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw);
934 if (rc)
935 dev_warn(rdev_to_dev(rdev),
936 "Failed to deinitialize RCFW: %#x", rc);
937 bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id,
938 lock_wait);
939 bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx);
940 bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
941 bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, lock_wait);
942 bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
943 }
944 if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) {
945 rc = bnxt_re_free_msix(rdev, lock_wait);
946 if (rc)
947 dev_warn(rdev_to_dev(rdev),
948 "Failed to free MSI-X vectors: %#x", rc);
949 }
950 if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) {
951 rc = bnxt_re_unregister_netdev(rdev, lock_wait);
952 if (rc)
953 dev_warn(rdev_to_dev(rdev),
954 "Failed to unregister with netdev: %#x", rc);
955 }
956}
957
958static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
959{
960 u32 i;
961
962 rdev->qplib_ctx.qpc_count = BNXT_RE_MAX_QPC_COUNT;
963 rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT;
964 rdev->qplib_ctx.srqc_count = BNXT_RE_MAX_SRQC_COUNT;
965 rdev->qplib_ctx.cq_count = BNXT_RE_MAX_CQ_COUNT;
966 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
967 rdev->qplib_ctx.tqm_count[i] =
968 rdev->dev_attr.tqm_alloc_reqs[i];
969}
970
971/* worker thread for polling periodic events. Now used for QoS programming*/
972static void bnxt_re_worker(struct work_struct *work)
973{
974 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
975 worker.work);
976
977 bnxt_re_setup_qos(rdev);
978 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
979}
980
981static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
982{
983 int i, j, rc;
984
985 /* Registered a new RoCE device instance to netdev */
986 rc = bnxt_re_register_netdev(rdev);
987 if (rc) {
988 pr_err("Failed to register with netedev: %#x\n", rc);
989 return -EINVAL;
990 }
991 set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
992
993 rc = bnxt_re_request_msix(rdev);
994 if (rc) {
995 pr_err("Failed to get MSI-X vectors: %#x\n", rc);
996 rc = -EINVAL;
997 goto fail;
998 }
999 set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags);
1000
1001 /* Establish RCFW Communication Channel to initialize the context
1002 * memory for the function and all child VFs
1003 */
1004 rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw);
1005 if (rc)
1006 goto fail;
1007
1008 rc = bnxt_re_net_ring_alloc
1009 (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr,
1010 rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count,
1011 HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1,
1012 rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx,
1013 &rdev->rcfw.creq_ring_id);
1014 if (rc) {
1015 pr_err("Failed to allocate CREQ: %#x\n", rc);
1016 goto free_rcfw;
1017 }
1018 rc = bnxt_qplib_enable_rcfw_channel
1019 (rdev->en_dev->pdev, &rdev->rcfw,
1020 rdev->msix_entries[BNXT_RE_AEQ_IDX].vector,
1021 rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset,
1022 0, &bnxt_re_aeq_handler);
1023 if (rc) {
1024 pr_err("Failed to enable RCFW channel: %#x\n", rc);
1025 goto free_ring;
1026 }
1027
1028 rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr);
1029 if (rc)
1030 goto disable_rcfw;
1031 bnxt_re_set_resource_limits(rdev);
1032
1033 rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0);
1034 if (rc) {
1035 pr_err("Failed to allocate QPLIB context: %#x\n", rc);
1036 goto disable_rcfw;
1037 }
1038 rc = bnxt_re_net_stats_ctx_alloc(rdev,
1039 rdev->qplib_ctx.stats.dma_map,
1040 &rdev->qplib_ctx.stats.fw_id);
1041 if (rc) {
1042 pr_err("Failed to allocate stats context: %#x\n", rc);
1043 goto free_ctx;
1044 }
1045
1046 rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, 0);
1047 if (rc) {
1048 pr_err("Failed to initialize RCFW: %#x\n", rc);
1049 goto free_sctx;
1050 }
1051 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
1052
1053 /* Resources based on the 'new' device caps */
1054 rc = bnxt_re_alloc_res(rdev);
1055 if (rc) {
1056 pr_err("Failed to allocate resources: %#x\n", rc);
1057 goto fail;
1058 }
1059 rc = bnxt_re_init_res(rdev);
1060 if (rc) {
1061 pr_err("Failed to initialize resources: %#x\n", rc);
1062 goto fail;
1063 }
1064
1065 rc = bnxt_re_setup_qos(rdev);
1066 if (rc)
1067 pr_info("RoCE priority not yet configured\n");
1068
1069 INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
1070 set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags);
1071 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
1072
1073 /* Register ib dev */
1074 rc = bnxt_re_register_ib(rdev);
1075 if (rc) {
1076 pr_err("Failed to register with IB: %#x\n", rc);
1077 goto fail;
1078 }
1079 dev_info(rdev_to_dev(rdev), "Device registered successfully");
1080 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) {
1081 rc = device_create_file(&rdev->ibdev.dev,
1082 bnxt_re_attributes[i]);
1083 if (rc) {
1084 dev_err(rdev_to_dev(rdev),
1085 "Failed to create IB sysfs: %#x", rc);
1086 /* Must clean up all created device files */
1087 for (j = 0; j < i; j++)
1088 device_remove_file(&rdev->ibdev.dev,
1089 bnxt_re_attributes[j]);
1090 bnxt_re_unregister_ib(rdev);
1091 goto fail;
1092 }
1093 }
1094 set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
1095 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);
1096 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE);
1097
1098 return 0;
1099free_sctx:
1100 bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, true);
1101free_ctx:
1102 bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx);
1103disable_rcfw:
1104 bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
1105free_ring:
1106 bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, true);
1107free_rcfw:
1108 bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
1109fail:
1110 bnxt_re_ib_unreg(rdev, true);
1111 return rc;
1112}
1113
1114static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev)
1115{
1116 struct bnxt_en_dev *en_dev = rdev->en_dev;
1117 struct net_device *netdev = rdev->netdev;
1118
1119 bnxt_re_dev_remove(rdev);
1120
1121 if (netdev)
1122 bnxt_re_dev_unprobe(netdev, en_dev);
1123}
1124
1125static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev)
1126{
1127 struct bnxt_en_dev *en_dev;
1128 int rc = 0;
1129
1130 if (!is_bnxt_re_dev(netdev))
1131 return -ENODEV;
1132
1133 en_dev = bnxt_re_dev_probe(netdev);
1134 if (IS_ERR(en_dev)) {
1135 if (en_dev != ERR_PTR(-ENODEV))
1136 pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME);
1137 rc = PTR_ERR(en_dev);
1138 goto exit;
1139 }
1140 *rdev = bnxt_re_dev_add(netdev, en_dev);
1141 if (!*rdev) {
1142 rc = -ENOMEM;
1143 bnxt_re_dev_unprobe(netdev, en_dev);
1144 goto exit;
1145 }
1146exit:
1147 return rc;
1148}
1149
1150static void bnxt_re_remove_one(struct bnxt_re_dev *rdev)
1151{
1152 pci_dev_put(rdev->en_dev->pdev);
1153}
1154
1155/* Handle all deferred netevents tasks */
1156static void bnxt_re_task(struct work_struct *work)
1157{
1158 struct bnxt_re_work *re_work;
1159 struct bnxt_re_dev *rdev;
1160 int rc = 0;
1161
1162 re_work = container_of(work, struct bnxt_re_work, work);
1163 rdev = re_work->rdev;
1164
1165 if (re_work->event != NETDEV_REGISTER &&
1166 !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
1167 return;
1168
1169 switch (re_work->event) {
1170 case NETDEV_REGISTER:
1171 rc = bnxt_re_ib_reg(rdev);
1172 if (rc)
1173 dev_err(rdev_to_dev(rdev),
1174 "Failed to register with IB: %#x", rc);
1175 break;
1176 case NETDEV_UP:
1177 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
1178 IB_EVENT_PORT_ACTIVE);
1179 break;
1180 case NETDEV_DOWN:
1181 bnxt_re_dev_stop(rdev);
1182 break;
1183 case NETDEV_CHANGE:
1184 if (!netif_carrier_ok(rdev->netdev))
1185 bnxt_re_dev_stop(rdev);
1186 else if (netif_carrier_ok(rdev->netdev))
1187 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
1188 IB_EVENT_PORT_ACTIVE);
1189 break;
1190 default:
1191 break;
1192 }
1193 kfree(re_work);
1194}
1195
1196static void bnxt_re_init_one(struct bnxt_re_dev *rdev)
1197{
1198 pci_dev_get(rdev->en_dev->pdev);
1199}
1200
1201/*
1202 * "Notifier chain callback can be invoked for the same chain from
1203 * different CPUs at the same time".
1204 *
1205 * For cases when the netdev is already present, our call to the
1206 * register_netdevice_notifier() will actually get the rtnl_lock()
1207 * before sending NETDEV_REGISTER and (if up) NETDEV_UP
1208 * events.
1209 *
1210 * But for cases when the netdev is not already present, the notifier
1211 * chain is subjected to be invoked from different CPUs simultaneously.
1212 *
1213 * This is protected by the netdev_mutex.
1214 */
1215static int bnxt_re_netdev_event(struct notifier_block *notifier,
1216 unsigned long event, void *ptr)
1217{
1218 struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr);
1219 struct bnxt_re_work *re_work;
1220 struct bnxt_re_dev *rdev;
1221 int rc = 0;
1222 bool sch_work = false;
1223
1224 real_dev = rdma_vlan_dev_real_dev(netdev);
1225 if (!real_dev)
1226 real_dev = netdev;
1227
1228 rdev = bnxt_re_from_netdev(real_dev);
1229 if (!rdev && event != NETDEV_REGISTER)
1230 goto exit;
1231 if (real_dev != netdev)
1232 goto exit;
1233
1234 switch (event) {
1235 case NETDEV_REGISTER:
1236 if (rdev)
1237 break;
1238 rc = bnxt_re_dev_reg(&rdev, real_dev);
1239 if (rc == -ENODEV)
1240 break;
1241 if (rc) {
1242 pr_err("Failed to register with the device %s: %#x\n",
1243 real_dev->name, rc);
1244 break;
1245 }
1246 bnxt_re_init_one(rdev);
1247 sch_work = true;
1248 break;
1249
1250 case NETDEV_UNREGISTER:
1251 bnxt_re_ib_unreg(rdev, false);
1252 bnxt_re_remove_one(rdev);
1253 bnxt_re_dev_unreg(rdev);
1254 break;
1255
1256 default:
1257 sch_work = true;
1258 break;
1259 }
1260 if (sch_work) {
1261 /* Allocate for the deferred task */
1262 re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC);
1263 if (re_work) {
1264 re_work->rdev = rdev;
1265 re_work->event = event;
1266 re_work->vlan_dev = (real_dev == netdev ?
1267 NULL : netdev);
1268 INIT_WORK(&re_work->work, bnxt_re_task);
1269 queue_work(bnxt_re_wq, &re_work->work);
1270 }
1271 }
1272
1273exit:
1274 return NOTIFY_DONE;
1275}
1276
1277static struct notifier_block bnxt_re_netdev_notifier = {
1278 .notifier_call = bnxt_re_netdev_event
1279};
1280
1281static int __init bnxt_re_mod_init(void)
1282{
1283 int rc = 0;
1284
1285 pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);
1286
1287 bnxt_re_wq = create_singlethread_workqueue("bnxt_re");
1288 if (!bnxt_re_wq)
1289 return -ENOMEM;
1290
1291 INIT_LIST_HEAD(&bnxt_re_dev_list);
1292
1293 rc = register_netdevice_notifier(&bnxt_re_netdev_notifier);
1294 if (rc) {
1295 pr_err("%s: Cannot register to netdevice_notifier",
1296 ROCE_DRV_MODULE_NAME);
1297 goto err_netdev;
1298 }
1299 return 0;
1300
1301err_netdev:
1302 destroy_workqueue(bnxt_re_wq);
1303
1304 return rc;
1305}
1306
1307static void __exit bnxt_re_mod_exit(void)
1308{
1309 unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
1310 if (bnxt_re_wq)
1311 destroy_workqueue(bnxt_re_wq);
1312}
1313
1314module_init(bnxt_re_mod_init);
1315module_exit(bnxt_re_mod_exit);
drivers/infiniband/hw/bnxt_re/qplib_fp.c+2167−0
@@ -0,0 +1,2167 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Fast Path Operators
37 */
38
39#include <linux/interrupt.h>
40#include <linux/spinlock.h>
41#include <linux/sched.h>
42#include <linux/slab.h>
43#include <linux/pci.h>
44#include <linux/prefetch.h>
45
46#include "roce_hsi.h"
47
48#include "qplib_res.h"
49#include "qplib_rcfw.h"
50#include "qplib_sp.h"
51#include "qplib_fp.h"
52
53static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
54
55static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
56 struct bnxt_qplib_qp *qp)
57{
58 struct bnxt_qplib_q *rq = &qp->rq;
59 struct bnxt_qplib_q *sq = &qp->sq;
60
61 if (qp->rq_hdr_buf)
62 dma_free_coherent(&res->pdev->dev,
63 rq->hwq.max_elements * qp->rq_hdr_buf_size,
64 qp->rq_hdr_buf, qp->rq_hdr_buf_map);
65 if (qp->sq_hdr_buf)
66 dma_free_coherent(&res->pdev->dev,
67 sq->hwq.max_elements * qp->sq_hdr_buf_size,
68 qp->sq_hdr_buf, qp->sq_hdr_buf_map);
69 qp->rq_hdr_buf = NULL;
70 qp->sq_hdr_buf = NULL;
71 qp->rq_hdr_buf_map = 0;
72 qp->sq_hdr_buf_map = 0;
73 qp->sq_hdr_buf_size = 0;
74 qp->rq_hdr_buf_size = 0;
75}
76
77static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
78 struct bnxt_qplib_qp *qp)
79{
80 struct bnxt_qplib_q *rq = &qp->rq;
81 struct bnxt_qplib_q *sq = &qp->rq;
82 int rc = 0;
83
84 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
85 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
86 sq->hwq.max_elements *
87 qp->sq_hdr_buf_size,
88 &qp->sq_hdr_buf_map, GFP_KERNEL);
89 if (!qp->sq_hdr_buf) {
90 rc = -ENOMEM;
91 dev_err(&res->pdev->dev,
92 "QPLIB: Failed to create sq_hdr_buf");
93 goto fail;
94 }
95 }
96
97 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
98 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
99 rq->hwq.max_elements *
100 qp->rq_hdr_buf_size,
101 &qp->rq_hdr_buf_map,
102 GFP_KERNEL);
103 if (!qp->rq_hdr_buf) {
104 rc = -ENOMEM;
105 dev_err(&res->pdev->dev,
106 "QPLIB: Failed to create rq_hdr_buf");
107 goto fail;
108 }
109 }
110 return 0;
111
112fail:
113 bnxt_qplib_free_qp_hdr_buf(res, qp);
114 return rc;
115}
116
117static void bnxt_qplib_service_nq(unsigned long data)
118{
119 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
120 struct bnxt_qplib_hwq *hwq = &nq->hwq;
121 struct nq_base *nqe, **nq_ptr;
122 int num_cqne_processed = 0;
123 u32 sw_cons, raw_cons;
124 u16 type;
125 int budget = nq->budget;
126 u64 q_handle;
127
128 /* Service the NQ until empty */
129 raw_cons = hwq->cons;
130 while (budget--) {
131 sw_cons = HWQ_CMP(raw_cons, hwq);
132 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
133 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
134 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
135 break;
136
137 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
138 switch (type) {
139 case NQ_BASE_TYPE_CQ_NOTIFICATION:
140 {
141 struct nq_cn *nqcne = (struct nq_cn *)nqe;
142
143 q_handle = le32_to_cpu(nqcne->cq_handle_low);
144 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
145 << 32;
146 bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *)
147 ((unsigned long)q_handle));
148 if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *)
149 ((unsigned long)q_handle)))
150 num_cqne_processed++;
151 else
152 dev_warn(&nq->pdev->dev,
153 "QPLIB: cqn - type 0x%x not handled",
154 type);
155 break;
156 }
157 case NQ_BASE_TYPE_DBQ_EVENT:
158 break;
159 default:
160 dev_warn(&nq->pdev->dev,
161 "QPLIB: nqe with type = 0x%x not handled",
162 type);
163 break;
164 }
165 raw_cons++;
166 }
167 if (hwq->cons != raw_cons) {
168 hwq->cons = raw_cons;
169 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
170 }
171}
172
173static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
174{
175 struct bnxt_qplib_nq *nq = dev_instance;
176 struct bnxt_qplib_hwq *hwq = &nq->hwq;
177 struct nq_base **nq_ptr;
178 u32 sw_cons;
179
180 /* Prefetch the NQ element */
181 sw_cons = HWQ_CMP(hwq->cons, hwq);
182 nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
183 prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
184
185 /* Fan out to CPU affinitized kthreads? */
186 tasklet_schedule(&nq->worker);
187
188 return IRQ_HANDLED;
189}
190
191void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
192{
193 /* Make sure the HW is stopped! */
194 synchronize_irq(nq->vector);
195 tasklet_disable(&nq->worker);
196 tasklet_kill(&nq->worker);
197
198 if (nq->requested) {
199 free_irq(nq->vector, nq);
200 nq->requested = false;
201 }
202 if (nq->bar_reg_iomem)
203 iounmap(nq->bar_reg_iomem);
204 nq->bar_reg_iomem = NULL;
205
206 nq->cqn_handler = NULL;
207 nq->srqn_handler = NULL;
208 nq->vector = 0;
209}
210
211int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
212 int msix_vector, int bar_reg_offset,
213 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
214 struct bnxt_qplib_cq *),
215 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
216 void *, u8 event))
217{
218 resource_size_t nq_base;
219 int rc;
220
221 nq->pdev = pdev;
222 nq->vector = msix_vector;
223
224 nq->cqn_handler = cqn_handler;
225
226 nq->srqn_handler = srqn_handler;
227
228 tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
229
230 nq->requested = false;
231 rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq);
232 if (rc) {
233 dev_err(&nq->pdev->dev,
234 "Failed to request IRQ for NQ: %#x", rc);
235 bnxt_qplib_disable_nq(nq);
236 goto fail;
237 }
238 nq->requested = true;
239 nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
240 nq->bar_reg_off = bar_reg_offset;
241 nq_base = pci_resource_start(pdev, nq->bar_reg);
242 if (!nq_base) {
243 rc = -ENOMEM;
244 goto fail;
245 }
246 nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
247 if (!nq->bar_reg_iomem) {
248 rc = -ENOMEM;
249 goto fail;
250 }
251 NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
252
253 return 0;
254fail:
255 bnxt_qplib_disable_nq(nq);
256 return rc;
257}
258
259void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
260{
261 if (nq->hwq.max_elements)
262 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
263}
264
265int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
266{
267 nq->pdev = pdev;
268 if (!nq->hwq.max_elements ||
269 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
270 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
271
272 if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
273 &nq->hwq.max_elements,
274 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
275 PAGE_SIZE, HWQ_TYPE_L2_CMPL))
276 return -ENOMEM;
277
278 nq->budget = 8;
279 return 0;
280}
281
282/* QP */
283int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
284{
285 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
286 struct cmdq_create_qp1 req;
287 struct creq_create_qp1_resp *resp;
288 struct bnxt_qplib_pbl *pbl;
289 struct bnxt_qplib_q *sq = &qp->sq;
290 struct bnxt_qplib_q *rq = &qp->rq;
291 int rc;
292 u16 cmd_flags = 0;
293 u32 qp_flags = 0;
294
295 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
296
297 /* General */
298 req.type = qp->type;
299 req.dpi = cpu_to_le32(qp->dpi->dpi);
300 req.qp_handle = cpu_to_le64(qp->qp_handle);
301
302 /* SQ */
303 sq->hwq.max_elements = sq->max_wqe;
304 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
305 &sq->hwq.max_elements,
306 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
307 PAGE_SIZE, HWQ_TYPE_QUEUE);
308 if (rc)
309 goto exit;
310
311 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
312 if (!sq->swq) {
313 rc = -ENOMEM;
314 goto fail_sq;
315 }
316 pbl = &sq->hwq.pbl[PBL_LVL_0];
317 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
318 req.sq_pg_size_sq_lvl =
319 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
320 << CMDQ_CREATE_QP1_SQ_LVL_SFT) |
321 (pbl->pg_size == ROCE_PG_SIZE_4K ?
322 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
323 pbl->pg_size == ROCE_PG_SIZE_8K ?
324 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
325 pbl->pg_size == ROCE_PG_SIZE_64K ?
326 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
327 pbl->pg_size == ROCE_PG_SIZE_2M ?
328 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
329 pbl->pg_size == ROCE_PG_SIZE_8M ?
330 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
331 pbl->pg_size == ROCE_PG_SIZE_1G ?
332 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
333 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
334
335 if (qp->scq)
336 req.scq_cid = cpu_to_le32(qp->scq->id);
337
338 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
339
340 /* RQ */
341 if (rq->max_wqe) {
342 rq->hwq.max_elements = qp->rq.max_wqe;
343 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
344 &rq->hwq.max_elements,
345 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
346 PAGE_SIZE, HWQ_TYPE_QUEUE);
347 if (rc)
348 goto fail_sq;
349
350 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
351 GFP_KERNEL);
352 if (!rq->swq) {
353 rc = -ENOMEM;
354 goto fail_rq;
355 }
356 pbl = &rq->hwq.pbl[PBL_LVL_0];
357 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
358 req.rq_pg_size_rq_lvl =
359 ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
360 CMDQ_CREATE_QP1_RQ_LVL_SFT) |
361 (pbl->pg_size == ROCE_PG_SIZE_4K ?
362 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
363 pbl->pg_size == ROCE_PG_SIZE_8K ?
364 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
365 pbl->pg_size == ROCE_PG_SIZE_64K ?
366 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
367 pbl->pg_size == ROCE_PG_SIZE_2M ?
368 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
369 pbl->pg_size == ROCE_PG_SIZE_8M ?
370 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
371 pbl->pg_size == ROCE_PG_SIZE_1G ?
372 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
373 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
374 if (qp->rcq)
375 req.rcq_cid = cpu_to_le32(qp->rcq->id);
376 }
377
378 /* Header buffer - allow hdr_buf pass in */
379 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
380 if (rc) {
381 rc = -ENOMEM;
382 goto fail;
383 }
384 req.qp_flags = cpu_to_le32(qp_flags);
385 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
386 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
387
388 req.sq_fwo_sq_sge =
389 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
390 CMDQ_CREATE_QP1_SQ_SGE_SFT);
391 req.rq_fwo_rq_sge =
392 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
393 CMDQ_CREATE_QP1_RQ_SGE_SFT);
394
395 req.pd_id = cpu_to_le32(qp->pd->id);
396
397 resp = (struct creq_create_qp1_resp *)
398 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
399 NULL, 0);
400 if (!resp) {
401 dev_err(&res->pdev->dev, "QPLIB: FP: CREATE_QP1 send failed");
402 rc = -EINVAL;
403 goto fail;
404 }
405 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
406 /* Cmd timed out */
407 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 timed out");
408 rc = -ETIMEDOUT;
409 goto fail;
410 }
411 if (resp->status ||
412 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
413 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 failed ");
414 dev_err(&rcfw->pdev->dev,
415 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
416 resp->status, le16_to_cpu(req.cookie),
417 le16_to_cpu(resp->cookie));
418 rc = -EINVAL;
419 goto fail;
420 }
421 qp->id = le32_to_cpu(resp->xid);
422 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
423 sq->flush_in_progress = false;
424 rq->flush_in_progress = false;
425
426 return 0;
427
428fail:
429 bnxt_qplib_free_qp_hdr_buf(res, qp);
430fail_rq:
431 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
432 kfree(rq->swq);
433fail_sq:
434 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
435 kfree(sq->swq);
436exit:
437 return rc;
438}
439
440int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
441{
442 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
443 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
444 struct cmdq_create_qp req;
445 struct creq_create_qp_resp *resp;
446 struct bnxt_qplib_pbl *pbl;
447 struct sq_psn_search **psn_search_ptr;
448 unsigned long int psn_search, poff = 0;
449 struct bnxt_qplib_q *sq = &qp->sq;
450 struct bnxt_qplib_q *rq = &qp->rq;
451 struct bnxt_qplib_hwq *xrrq;
452 int i, rc, req_size, psn_sz;
453 u16 cmd_flags = 0, max_ssge;
454 u32 sw_prod, qp_flags = 0;
455
456 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
457
458 /* General */
459 req.type = qp->type;
460 req.dpi = cpu_to_le32(qp->dpi->dpi);
461 req.qp_handle = cpu_to_le64(qp->qp_handle);
462
463 /* SQ */
464 psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
465 sizeof(struct sq_psn_search) : 0;
466 sq->hwq.max_elements = sq->max_wqe;
467 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
468 sq->nmap, &sq->hwq.max_elements,
469 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
470 psn_sz,
471 PAGE_SIZE, HWQ_TYPE_QUEUE);
472 if (rc)
473 goto exit;
474
475 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
476 if (!sq->swq) {
477 rc = -ENOMEM;
478 goto fail_sq;
479 }
480 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
481 if (psn_sz) {
482 psn_search_ptr = (struct sq_psn_search **)
483 &hw_sq_send_ptr[get_sqe_pg
484 (sq->hwq.max_elements)];
485 psn_search = (unsigned long int)
486 &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
487 [get_sqe_idx(sq->hwq.max_elements)];
488 if (psn_search & ~PAGE_MASK) {
489 /* If the psn_search does not start on a page boundary,
490 * then calculate the offset
491 */
492 poff = (psn_search & ~PAGE_MASK) /
493 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
494 }
495 for (i = 0; i < sq->hwq.max_elements; i++)
496 sq->swq[i].psn_search =
497 &psn_search_ptr[get_psne_pg(i + poff)]
498 [get_psne_idx(i + poff)];
499 }
500 pbl = &sq->hwq.pbl[PBL_LVL_0];
501 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
502 req.sq_pg_size_sq_lvl =
503 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
504 << CMDQ_CREATE_QP_SQ_LVL_SFT) |
505 (pbl->pg_size == ROCE_PG_SIZE_4K ?
506 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
507 pbl->pg_size == ROCE_PG_SIZE_8K ?
508 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
509 pbl->pg_size == ROCE_PG_SIZE_64K ?
510 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
511 pbl->pg_size == ROCE_PG_SIZE_2M ?
512 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
513 pbl->pg_size == ROCE_PG_SIZE_8M ?
514 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
515 pbl->pg_size == ROCE_PG_SIZE_1G ?
516 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
517 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
518
519 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
520 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
521 for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
522 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
523 [get_sqe_idx(sw_prod)];
524 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
525 }
526
527 if (qp->scq)
528 req.scq_cid = cpu_to_le32(qp->scq->id);
529
530 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
531 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
532 if (qp->sig_type)
533 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
534
535 /* RQ */
536 if (rq->max_wqe) {
537 rq->hwq.max_elements = rq->max_wqe;
538 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
539 rq->nmap, &rq->hwq.max_elements,
540 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
541 PAGE_SIZE, HWQ_TYPE_QUEUE);
542 if (rc)
543 goto fail_sq;
544
545 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
546 GFP_KERNEL);
547 if (!rq->swq) {
548 rc = -ENOMEM;
549 goto fail_rq;
550 }
551 pbl = &rq->hwq.pbl[PBL_LVL_0];
552 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
553 req.rq_pg_size_rq_lvl =
554 ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
555 CMDQ_CREATE_QP_RQ_LVL_SFT) |
556 (pbl->pg_size == ROCE_PG_SIZE_4K ?
557 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
558 pbl->pg_size == ROCE_PG_SIZE_8K ?
559 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
560 pbl->pg_size == ROCE_PG_SIZE_64K ?
561 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
562 pbl->pg_size == ROCE_PG_SIZE_2M ?
563 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
564 pbl->pg_size == ROCE_PG_SIZE_8M ?
565 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
566 pbl->pg_size == ROCE_PG_SIZE_1G ?
567 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
568 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
569 }
570
571 if (qp->rcq)
572 req.rcq_cid = cpu_to_le32(qp->rcq->id);
573 req.qp_flags = cpu_to_le32(qp_flags);
574 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
575 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
576 qp->sq_hdr_buf = NULL;
577 qp->rq_hdr_buf = NULL;
578
579 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
580 if (rc)
581 goto fail_rq;
582
583 /* CTRL-22434: Irrespective of the requested SGE count on the SQ
584 * always create the QP with max send sges possible if the requested
585 * inline size is greater than 0.
586 */
587 max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
588 req.sq_fwo_sq_sge = cpu_to_le16(
589 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
590 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
591 req.rq_fwo_rq_sge = cpu_to_le16(
592 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
593 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
594 /* ORRQ and IRRQ */
595 if (psn_sz) {
596 xrrq = &qp->orrq;
597 xrrq->max_elements =
598 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
599 req_size = xrrq->max_elements *
600 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
601 req_size &= ~(PAGE_SIZE - 1);
602 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
603 &xrrq->max_elements,
604 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
605 0, req_size, HWQ_TYPE_CTX);
606 if (rc)
607 goto fail_buf_free;
608 pbl = &xrrq->pbl[PBL_LVL_0];
609 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
610
611 xrrq = &qp->irrq;
612 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
613 qp->max_dest_rd_atomic);
614 req_size = xrrq->max_elements *
615 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
616 req_size &= ~(PAGE_SIZE - 1);
617
618 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
619 &xrrq->max_elements,
620 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
621 0, req_size, HWQ_TYPE_CTX);
622 if (rc)
623 goto fail_orrq;
624
625 pbl = &xrrq->pbl[PBL_LVL_0];
626 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
627 }
628 req.pd_id = cpu_to_le32(qp->pd->id);
629
630 resp = (struct creq_create_qp_resp *)
631 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
632 NULL, 0);
633 if (!resp) {
634 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP send failed");
635 rc = -EINVAL;
636 goto fail;
637 }
638 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
639 /* Cmd timed out */
640 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP timed out");
641 rc = -ETIMEDOUT;
642 goto fail;
643 }
644 if (resp->status ||
645 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
646 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP failed ");
647 dev_err(&rcfw->pdev->dev,
648 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
649 resp->status, le16_to_cpu(req.cookie),
650 le16_to_cpu(resp->cookie));
651 rc = -EINVAL;
652 goto fail;
653 }
654 qp->id = le32_to_cpu(resp->xid);
655 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
656 sq->flush_in_progress = false;
657 rq->flush_in_progress = false;
658
659 return 0;
660
661fail:
662 if (qp->irrq.max_elements)
663 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
664fail_orrq:
665 if (qp->orrq.max_elements)
666 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
667fail_buf_free:
668 bnxt_qplib_free_qp_hdr_buf(res, qp);
669fail_rq:
670 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
671 kfree(rq->swq);
672fail_sq:
673 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
674 kfree(sq->swq);
675exit:
676 return rc;
677}
678
679static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
680{
681 switch (qp->state) {
682 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
683 /* INIT->RTR, configure the path_mtu to the default
684 * 2048 if not being requested
685 */
686 if (!(qp->modify_flags &
687 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
688 qp->modify_flags |=
689 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
690 qp->path_mtu =
691 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
692 }
693 qp->modify_flags &=
694 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
695 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
696 if (qp->max_dest_rd_atomic < 1)
697 qp->max_dest_rd_atomic = 1;
698 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
699 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
700 if (!(qp->modify_flags &
701 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
702 qp->modify_flags |=
703 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
704 qp->ah.sgid_index = 0;
705 }
706 break;
707 default:
708 break;
709 }
710}
711
712static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
713{
714 switch (qp->state) {
715 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
716 /* Bono FW requires the max_rd_atomic to be >= 1 */
717 if (qp->max_rd_atomic < 1)
718 qp->max_rd_atomic = 1;
719 /* Bono FW does not allow PKEY_INDEX,
720 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
721 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
722 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
723 * modification
724 */
725 qp->modify_flags &=
726 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
727 CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
728 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
729 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
730 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
731 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
732 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
733 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
734 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
735 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
736 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
737 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
738 break;
739 default:
740 break;
741 }
742}
743
744static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
745{
746 switch (qp->cur_qp_state) {
747 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
748 break;
749 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
750 __modify_flags_from_init_state(qp);
751 break;
752 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
753 __modify_flags_from_rtr_state(qp);
754 break;
755 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
756 break;
757 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
758 break;
759 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
760 break;
761 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
762 break;
763 default:
764 break;
765 }
766}
767
768int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
769{
770 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
771 struct cmdq_modify_qp req;
772 struct creq_modify_qp_resp *resp;
773 u16 cmd_flags = 0, pkey;
774 u32 temp32[4];
775 u32 bmask;
776
777 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
778
779 /* Filter out the qp_attr_mask based on the state->new transition */
780 __filter_modify_flags(qp);
781 bmask = qp->modify_flags;
782 req.modify_mask = cpu_to_le32(qp->modify_flags);
783 req.qp_cid = cpu_to_le32(qp->id);
784 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
785 req.network_type_en_sqd_async_notify_new_state =
786 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
787 (qp->en_sqd_async_notify ?
788 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
789 }
790 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
791
792 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
793 req.access = qp->access;
794
795 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
796 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
797 qp->pkey_index, &pkey))
798 req.pkey = cpu_to_le16(pkey);
799 }
800 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
801 req.qkey = cpu_to_le32(qp->qkey);
802
803 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
804 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
805 req.dgid[0] = cpu_to_le32(temp32[0]);
806 req.dgid[1] = cpu_to_le32(temp32[1]);
807 req.dgid[2] = cpu_to_le32(temp32[2]);
808 req.dgid[3] = cpu_to_le32(temp32[3]);
809 }
810 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
811 req.flow_label = cpu_to_le32(qp->ah.flow_label);
812
813 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
814 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
815 [qp->ah.sgid_index]);
816
817 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
818 req.hop_limit = qp->ah.hop_limit;
819
820 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
821 req.traffic_class = qp->ah.traffic_class;
822
823 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
824 memcpy(req.dest_mac, qp->ah.dmac, 6);
825
826 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
827 req.path_mtu = qp->path_mtu;
828
829 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
830 req.timeout = qp->timeout;
831
832 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
833 req.retry_cnt = qp->retry_cnt;
834
835 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
836 req.rnr_retry = qp->rnr_retry;
837
838 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
839 req.min_rnr_timer = qp->min_rnr_timer;
840
841 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
842 req.rq_psn = cpu_to_le32(qp->rq.psn);
843
844 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
845 req.sq_psn = cpu_to_le32(qp->sq.psn);
846
847 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
848 req.max_rd_atomic =
849 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
850
851 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
852 req.max_dest_rd_atomic =
853 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
854
855 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
856 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
857 req.sq_sge = cpu_to_le16(qp->sq.max_sge);
858 req.rq_sge = cpu_to_le16(qp->rq.max_sge);
859 req.max_inline_data = cpu_to_le32(qp->max_inline_data);
860 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
861 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
862
863 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
864
865 resp = (struct creq_modify_qp_resp *)
866 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
867 NULL, 0);
868 if (!resp) {
869 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP send failed");
870 return -EINVAL;
871 }
872 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
873 /* Cmd timed out */
874 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP timed out");
875 return -ETIMEDOUT;
876 }
877 if (resp->status ||
878 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
879 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP failed ");
880 dev_err(&rcfw->pdev->dev,
881 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
882 resp->status, le16_to_cpu(req.cookie),
883 le16_to_cpu(resp->cookie));
884 return -EINVAL;
885 }
886 qp->cur_qp_state = qp->state;
887 return 0;
888}
889
890int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
891{
892 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
893 struct cmdq_query_qp req;
894 struct creq_query_qp_resp *resp;
895 struct creq_query_qp_resp_sb *sb;
896 u16 cmd_flags = 0;
897 u32 temp32[4];
898 int i;
899
900 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
901
902 req.qp_cid = cpu_to_le32(qp->id);
903 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
904 resp = (struct creq_query_qp_resp *)
905 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
906 (void **)&sb, 0);
907 if (!resp) {
908 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP send failed");
909 return -EINVAL;
910 }
911 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
912 /* Cmd timed out */
913 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP timed out");
914 return -ETIMEDOUT;
915 }
916 if (resp->status ||
917 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
918 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP failed ");
919 dev_err(&rcfw->pdev->dev,
920 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
921 resp->status, le16_to_cpu(req.cookie),
922 le16_to_cpu(resp->cookie));
923 return -EINVAL;
924 }
925 /* Extract the context from the side buffer */
926 qp->state = sb->en_sqd_async_notify_state &
927 CREQ_QUERY_QP_RESP_SB_STATE_MASK;
928 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
929 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
930 true : false;
931 qp->access = sb->access;
932 qp->pkey_index = le16_to_cpu(sb->pkey);
933 qp->qkey = le32_to_cpu(sb->qkey);
934
935 temp32[0] = le32_to_cpu(sb->dgid[0]);
936 temp32[1] = le32_to_cpu(sb->dgid[1]);
937 temp32[2] = le32_to_cpu(sb->dgid[2]);
938 temp32[3] = le32_to_cpu(sb->dgid[3]);
939 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
940
941 qp->ah.flow_label = le32_to_cpu(sb->flow_label);
942
943 qp->ah.sgid_index = 0;
944 for (i = 0; i < res->sgid_tbl.max; i++) {
945 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
946 qp->ah.sgid_index = i;
947 break;
948 }
949 }
950 if (i == res->sgid_tbl.max)
951 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
952
953 qp->ah.hop_limit = sb->hop_limit;
954 qp->ah.traffic_class = sb->traffic_class;
955 memcpy(qp->ah.dmac, sb->dest_mac, 6);
956 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
957 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
958 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
959 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
960 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
961 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
962 qp->timeout = sb->timeout;
963 qp->retry_cnt = sb->retry_cnt;
964 qp->rnr_retry = sb->rnr_retry;
965 qp->min_rnr_timer = sb->min_rnr_timer;
966 qp->rq.psn = le32_to_cpu(sb->rq_psn);
967 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
968 qp->sq.psn = le32_to_cpu(sb->sq_psn);
969 qp->max_dest_rd_atomic =
970 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
971 qp->sq.max_wqe = qp->sq.hwq.max_elements;
972 qp->rq.max_wqe = qp->rq.hwq.max_elements;
973 qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
974 qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
975 qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
976 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
977 memcpy(qp->smac, sb->src_mac, 6);
978 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
979 return 0;
980}
981
982static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
983{
984 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
985 struct cq_base *hw_cqe, **hw_cqe_ptr;
986 int i;
987
988 for (i = 0; i < cq_hwq->max_elements; i++) {
989 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
990 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
991 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
992 continue;
993 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
994 case CQ_BASE_CQE_TYPE_REQ:
995 case CQ_BASE_CQE_TYPE_TERMINAL:
996 {
997 struct cq_req *cqe = (struct cq_req *)hw_cqe;
998
999 if (qp == le64_to_cpu(cqe->qp_handle))
1000 cqe->qp_handle = 0;
1001 break;
1002 }
1003 case CQ_BASE_CQE_TYPE_RES_RC:
1004 case CQ_BASE_CQE_TYPE_RES_UD:
1005 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1006 {
1007 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1008
1009 if (qp == le64_to_cpu(cqe->qp_handle))
1010 cqe->qp_handle = 0;
1011 break;
1012 }
1013 default:
1014 break;
1015 }
1016 }
1017}
1018
1019int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1020 struct bnxt_qplib_qp *qp)
1021{
1022 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1023 struct cmdq_destroy_qp req;
1024 struct creq_destroy_qp_resp *resp;
1025 unsigned long flags;
1026 u16 cmd_flags = 0;
1027
1028 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1029
1030 req.qp_cid = cpu_to_le32(qp->id);
1031 resp = (struct creq_destroy_qp_resp *)
1032 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1033 NULL, 0);
1034 if (!resp) {
1035 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP send failed");
1036 return -EINVAL;
1037 }
1038 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1039 /* Cmd timed out */
1040 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP timed out");
1041 return -ETIMEDOUT;
1042 }
1043 if (resp->status ||
1044 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1045 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP failed ");
1046 dev_err(&rcfw->pdev->dev,
1047 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1048 resp->status, le16_to_cpu(req.cookie),
1049 le16_to_cpu(resp->cookie));
1050 return -EINVAL;
1051 }
1052
1053 /* Must walk the associated CQs to nullified the QP ptr */
1054 spin_lock_irqsave(&qp->scq->hwq.lock, flags);
1055
1056 __clean_cq(qp->scq, (u64)(unsigned long)qp);
1057
1058 if (qp->rcq && qp->rcq != qp->scq) {
1059 spin_lock(&qp->rcq->hwq.lock);
1060 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
1061 spin_unlock(&qp->rcq->hwq.lock);
1062 }
1063
1064 spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
1065
1066 bnxt_qplib_free_qp_hdr_buf(res, qp);
1067 bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
1068 kfree(qp->sq.swq);
1069
1070 bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
1071 kfree(qp->rq.swq);
1072
1073 if (qp->irrq.max_elements)
1074 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1075 if (qp->orrq.max_elements)
1076 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1077
1078 return 0;
1079}
1080
1081void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1082 struct bnxt_qplib_sge *sge)
1083{
1084 struct bnxt_qplib_q *sq = &qp->sq;
1085 u32 sw_prod;
1086
1087 memset(sge, 0, sizeof(*sge));
1088
1089 if (qp->sq_hdr_buf) {
1090 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1091 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1092 sw_prod * qp->sq_hdr_buf_size);
1093 sge->lkey = 0xFFFFFFFF;
1094 sge->size = qp->sq_hdr_buf_size;
1095 return qp->sq_hdr_buf + sw_prod * sge->size;
1096 }
1097 return NULL;
1098}
1099
1100u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1101{
1102 struct bnxt_qplib_q *rq = &qp->rq;
1103
1104 return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1105}
1106
1107dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1108{
1109 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1110}
1111
1112void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1113 struct bnxt_qplib_sge *sge)
1114{
1115 struct bnxt_qplib_q *rq = &qp->rq;
1116 u32 sw_prod;
1117
1118 memset(sge, 0, sizeof(*sge));
1119
1120 if (qp->rq_hdr_buf) {
1121 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1122 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1123 sw_prod * qp->rq_hdr_buf_size);
1124 sge->lkey = 0xFFFFFFFF;
1125 sge->size = qp->rq_hdr_buf_size;
1126 return qp->rq_hdr_buf + sw_prod * sge->size;
1127 }
1128 return NULL;
1129}
1130
1131void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1132{
1133 struct bnxt_qplib_q *sq = &qp->sq;
1134 struct dbr_dbr db_msg = { 0 };
1135 u32 sw_prod;
1136
1137 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1138
1139 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1140 DBR_DBR_INDEX_MASK);
1141 db_msg.type_xid =
1142 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1143 DBR_DBR_TYPE_SQ);
1144 /* Flush all the WQE writes to HW */
1145 wmb();
1146 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1147}
1148
1149int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1150 struct bnxt_qplib_swqe *wqe)
1151{
1152 struct bnxt_qplib_q *sq = &qp->sq;
1153 struct bnxt_qplib_swq *swq;
1154 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1155 struct sq_sge *hw_sge;
1156 u32 sw_prod;
1157 u8 wqe_size16;
1158 int i, rc = 0, data_len = 0, pkt_num = 0;
1159 __le32 temp32;
1160
1161 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1162 rc = -EINVAL;
1163 goto done;
1164 }
1165 if (HWQ_CMP((sq->hwq.prod + 1), &sq->hwq) ==
1166 HWQ_CMP(sq->hwq.cons, &sq->hwq)) {
1167 rc = -ENOMEM;
1168 goto done;
1169 }
1170 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1171 swq = &sq->swq[sw_prod];
1172 swq->wr_id = wqe->wr_id;
1173 swq->type = wqe->type;
1174 swq->flags = wqe->flags;
1175 if (qp->sig_type)
1176 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1177 swq->start_psn = sq->psn & BTH_PSN_MASK;
1178
1179 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1180 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1181 [get_sqe_idx(sw_prod)];
1182
1183 memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1184
1185 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1186 /* Copy the inline data */
1187 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1188 dev_warn(&sq->hwq.pdev->dev,
1189 "QPLIB: Inline data length > 96 detected");
1190 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1191 } else {
1192 data_len = wqe->inline_len;
1193 }
1194 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1195 wqe_size16 = (data_len + 15) >> 4;
1196 } else {
1197 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1198 i < wqe->num_sge; i++, hw_sge++) {
1199 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1200 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1201 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1202 data_len += wqe->sg_list[i].size;
1203 }
1204 /* Each SGE entry = 1 WQE size16 */
1205 wqe_size16 = wqe->num_sge;
1206 }
1207
1208 /* Specifics */
1209 switch (wqe->type) {
1210 case BNXT_QPLIB_SWQE_TYPE_SEND:
1211 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1212 /* Assemble info for Raw Ethertype QPs */
1213 struct sq_send_raweth_qp1 *sqe =
1214 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1215
1216 sqe->wqe_type = wqe->type;
1217 sqe->flags = wqe->flags;
1218 sqe->wqe_size = wqe_size16 +
1219 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1220 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1221 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1222 sqe->length = cpu_to_le32(data_len);
1223 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1224 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1225 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1226
1227 break;
1228 }
1229 /* else, just fall thru */
1230 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1231 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1232 {
1233 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1234
1235 sqe->wqe_type = wqe->type;
1236 sqe->flags = wqe->flags;
1237 sqe->wqe_size = wqe_size16 +
1238 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1239 sqe->inv_key_or_imm_data = cpu_to_le32(
1240 wqe->send.inv_key);
1241 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1242 sqe->q_key = cpu_to_le32(wqe->send.q_key);
1243 sqe->dst_qp = cpu_to_le32(
1244 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1245 sqe->length = cpu_to_le32(data_len);
1246 sqe->avid = cpu_to_le32(wqe->send.avid &
1247 SQ_SEND_AVID_MASK);
1248 sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1249 } else {
1250 sqe->length = cpu_to_le32(data_len);
1251 sqe->dst_qp = 0;
1252 sqe->avid = 0;
1253 if (qp->mtu)
1254 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1255 if (!pkt_num)
1256 pkt_num = 1;
1257 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1258 }
1259 break;
1260 }
1261 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1262 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1263 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1264 {
1265 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1266
1267 sqe->wqe_type = wqe->type;
1268 sqe->flags = wqe->flags;
1269 sqe->wqe_size = wqe_size16 +
1270 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1271 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1272 sqe->length = cpu_to_le32((u32)data_len);
1273 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1274 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1275 if (qp->mtu)
1276 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1277 if (!pkt_num)
1278 pkt_num = 1;
1279 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1280 break;
1281 }
1282 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1283 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1284 {
1285 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1286
1287 sqe->wqe_type = wqe->type;
1288 sqe->flags = wqe->flags;
1289 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1290 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1291 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1292 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1293 if (qp->mtu)
1294 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1295 if (!pkt_num)
1296 pkt_num = 1;
1297 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1298 break;
1299 }
1300 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1301 {
1302 struct sq_localinvalidate *sqe =
1303 (struct sq_localinvalidate *)hw_sq_send_hdr;
1304
1305 sqe->wqe_type = wqe->type;
1306 sqe->flags = wqe->flags;
1307 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1308
1309 break;
1310 }
1311 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1312 {
1313 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1314
1315 sqe->wqe_type = wqe->type;
1316 sqe->flags = wqe->flags;
1317 sqe->access_cntl = wqe->frmr.access_cntl |
1318 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1319 sqe->zero_based_page_size_log =
1320 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1321 SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1322 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1323 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1324 temp32 = cpu_to_le32(wqe->frmr.length);
1325 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1326 sqe->numlevels_pbl_page_size_log =
1327 ((wqe->frmr.pbl_pg_sz_log <<
1328 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1329 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1330 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1331 SQ_FR_PMR_NUMLEVELS_MASK);
1332
1333 for (i = 0; i < wqe->frmr.page_list_len; i++)
1334 wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1335 wqe->frmr.page_list[i] |
1336 PTU_PTE_VALID);
1337 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1338 sqe->va = cpu_to_le64(wqe->frmr.va);
1339
1340 break;
1341 }
1342 case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1343 {
1344 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1345
1346 sqe->wqe_type = wqe->type;
1347 sqe->flags = wqe->flags;
1348 sqe->access_cntl = wqe->bind.access_cntl;
1349 sqe->mw_type_zero_based = wqe->bind.mw_type |
1350 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1351 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1352 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1353 sqe->va = cpu_to_le64(wqe->bind.va);
1354 temp32 = cpu_to_le32(wqe->bind.length);
1355 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1356 break;
1357 }
1358 default:
1359 /* Bad wqe, return error */
1360 rc = -EINVAL;
1361 goto done;
1362 }
1363 swq->next_psn = sq->psn & BTH_PSN_MASK;
1364 if (swq->psn_search) {
1365 swq->psn_search->opcode_start_psn = cpu_to_le32(
1366 ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1367 SQ_PSN_SEARCH_START_PSN_MASK) |
1368 ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1369 SQ_PSN_SEARCH_OPCODE_MASK));
1370 swq->psn_search->flags_next_psn = cpu_to_le32(
1371 ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1372 SQ_PSN_SEARCH_NEXT_PSN_MASK));
1373 }
1374
1375 sq->hwq.prod++;
1376done:
1377 return rc;
1378}
1379
1380void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1381{
1382 struct bnxt_qplib_q *rq = &qp->rq;
1383 struct dbr_dbr db_msg = { 0 };
1384 u32 sw_prod;
1385
1386 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1387 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1388 DBR_DBR_INDEX_MASK);
1389 db_msg.type_xid =
1390 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1391 DBR_DBR_TYPE_RQ);
1392
1393 /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1394 wmb();
1395 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1396}
1397
1398int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1399 struct bnxt_qplib_swqe *wqe)
1400{
1401 struct bnxt_qplib_q *rq = &qp->rq;
1402 struct rq_wqe *rqe, **rqe_ptr;
1403 struct sq_sge *hw_sge;
1404 u32 sw_prod;
1405 int i, rc = 0;
1406
1407 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1408 dev_err(&rq->hwq.pdev->dev,
1409 "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1410 qp->id, qp->state);
1411 rc = -EINVAL;
1412 goto done;
1413 }
1414 if (HWQ_CMP((rq->hwq.prod + 1), &rq->hwq) ==
1415 HWQ_CMP(rq->hwq.cons, &rq->hwq)) {
1416 dev_err(&rq->hwq.pdev->dev,
1417 "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1418 rc = -EINVAL;
1419 goto done;
1420 }
1421 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1422 rq->swq[sw_prod].wr_id = wqe->wr_id;
1423
1424 rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1425 rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1426
1427 memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1428
1429 /* Calculate wqe_size16 and data_len */
1430 for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1431 i < wqe->num_sge; i++, hw_sge++) {
1432 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1433 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1434 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1435 }
1436 rqe->wqe_type = wqe->type;
1437 rqe->flags = wqe->flags;
1438 rqe->wqe_size = wqe->num_sge +
1439 ((offsetof(typeof(*rqe), data) + 15) >> 4);
1440
1441 /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1442 rqe->wr_id[0] = cpu_to_le32(sw_prod);
1443
1444 rq->hwq.prod++;
1445done:
1446 return rc;
1447}
1448
1449/* CQ */
1450
1451/* Spinlock must be held */
1452static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1453{
1454 struct dbr_dbr db_msg = { 0 };
1455
1456 db_msg.type_xid =
1457 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1458 DBR_DBR_TYPE_CQ_ARMENA);
1459 /* Flush memory writes before enabling the CQ */
1460 wmb();
1461 __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1462}
1463
1464static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1465{
1466 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1467 struct dbr_dbr db_msg = { 0 };
1468 u32 sw_cons;
1469
1470 /* Ring DB */
1471 sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1472 db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1473 DBR_DBR_INDEX_MASK);
1474 db_msg.type_xid =
1475 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1476 arm_type);
1477 /* flush memory writes before arming the CQ */
1478 wmb();
1479 __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1480}
1481
1482int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1483{
1484 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1485 struct cmdq_create_cq req;
1486 struct creq_create_cq_resp *resp;
1487 struct bnxt_qplib_pbl *pbl;
1488 u16 cmd_flags = 0;
1489 int rc;
1490
1491 cq->hwq.max_elements = cq->max_wqe;
1492 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1493 cq->nmap, &cq->hwq.max_elements,
1494 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1495 PAGE_SIZE, HWQ_TYPE_QUEUE);
1496 if (rc)
1497 goto exit;
1498
1499 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1500
1501 if (!cq->dpi) {
1502 dev_err(&rcfw->pdev->dev,
1503 "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1504 return -EINVAL;
1505 }
1506 req.dpi = cpu_to_le32(cq->dpi->dpi);
1507 req.cq_handle = cpu_to_le64(cq->cq_handle);
1508
1509 req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1510 pbl = &cq->hwq.pbl[PBL_LVL_0];
1511 req.pg_size_lvl = cpu_to_le32(
1512 ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1513 CMDQ_CREATE_CQ_LVL_SFT) |
1514 (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1515 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1516 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1517 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1518 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1519 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1520 CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1521
1522 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1523
1524 req.cq_fco_cnq_id = cpu_to_le32(
1525 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1526 CMDQ_CREATE_CQ_CNQ_ID_SFT);
1527
1528 resp = (struct creq_create_cq_resp *)
1529 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1530 NULL, 0);
1531 if (!resp) {
1532 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ send failed");
1533 return -EINVAL;
1534 }
1535 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1536 /* Cmd timed out */
1537 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ timed out");
1538 rc = -ETIMEDOUT;
1539 goto fail;
1540 }
1541 if (resp->status ||
1542 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1543 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ failed ");
1544 dev_err(&rcfw->pdev->dev,
1545 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1546 resp->status, le16_to_cpu(req.cookie),
1547 le16_to_cpu(resp->cookie));
1548 rc = -EINVAL;
1549 goto fail;
1550 }
1551 cq->id = le32_to_cpu(resp->xid);
1552 cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1553 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1554 init_waitqueue_head(&cq->waitq);
1555
1556 bnxt_qplib_arm_cq_enable(cq);
1557 return 0;
1558
1559fail:
1560 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1561exit:
1562 return rc;
1563}
1564
1565int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1566{
1567 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1568 struct cmdq_destroy_cq req;
1569 struct creq_destroy_cq_resp *resp;
1570 u16 cmd_flags = 0;
1571
1572 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1573
1574 req.cq_cid = cpu_to_le32(cq->id);
1575 resp = (struct creq_destroy_cq_resp *)
1576 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1577 NULL, 0);
1578 if (!resp) {
1579 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ send failed");
1580 return -EINVAL;
1581 }
1582 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1583 /* Cmd timed out */
1584 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ timed out");
1585 return -ETIMEDOUT;
1586 }
1587 if (resp->status ||
1588 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1589 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ failed ");
1590 dev_err(&rcfw->pdev->dev,
1591 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1592 resp->status, le16_to_cpu(req.cookie),
1593 le16_to_cpu(resp->cookie));
1594 return -EINVAL;
1595 }
1596 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1597 return 0;
1598}
1599
1600static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
1601 struct bnxt_qplib_cqe **pcqe, int *budget)
1602{
1603 u32 sw_prod, sw_cons;
1604 struct bnxt_qplib_cqe *cqe;
1605 int rc = 0;
1606
1607 /* Now complete all outstanding SQEs with FLUSHED_ERR */
1608 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1609 cqe = *pcqe;
1610 while (*budget) {
1611 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1612 if (sw_cons == sw_prod) {
1613 sq->flush_in_progress = false;
1614 break;
1615 }
1616 memset(cqe, 0, sizeof(*cqe));
1617 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
1618 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1619 cqe->qp_handle = (u64)(unsigned long)qp;
1620 cqe->wr_id = sq->swq[sw_cons].wr_id;
1621 cqe->src_qp = qp->id;
1622 cqe->type = sq->swq[sw_cons].type;
1623 cqe++;
1624 (*budget)--;
1625 sq->hwq.cons++;
1626 }
1627 *pcqe = cqe;
1628 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
1629 /* Out of budget */
1630 rc = -EAGAIN;
1631
1632 return rc;
1633}
1634
1635static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
1636 int opcode, struct bnxt_qplib_cqe **pcqe, int *budget)
1637{
1638 struct bnxt_qplib_cqe *cqe;
1639 u32 sw_prod, sw_cons;
1640 int rc = 0;
1641
1642 /* Flush the rest of the RQ */
1643 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1644 cqe = *pcqe;
1645 while (*budget) {
1646 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
1647 if (sw_cons == sw_prod)
1648 break;
1649 memset(cqe, 0, sizeof(*cqe));
1650 cqe->status =
1651 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
1652 cqe->opcode = opcode;
1653 cqe->qp_handle = (unsigned long)qp;
1654 cqe->wr_id = rq->swq[sw_cons].wr_id;
1655 cqe++;
1656 (*budget)--;
1657 rq->hwq.cons++;
1658 }
1659 *pcqe = cqe;
1660 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
1661 /* Out of budget */
1662 rc = -EAGAIN;
1663
1664 return rc;
1665}
1666
1667static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
1668 struct cq_req *hwcqe,
1669 struct bnxt_qplib_cqe **pcqe, int *budget)
1670{
1671 struct bnxt_qplib_qp *qp;
1672 struct bnxt_qplib_q *sq;
1673 struct bnxt_qplib_cqe *cqe;
1674 u32 sw_cons, cqe_cons;
1675 int rc = 0;
1676
1677 qp = (struct bnxt_qplib_qp *)((unsigned long)
1678 le64_to_cpu(hwcqe->qp_handle));
1679 if (!qp) {
1680 dev_err(&cq->hwq.pdev->dev,
1681 "QPLIB: FP: Process Req qp is NULL");
1682 return -EINVAL;
1683 }
1684 sq = &qp->sq;
1685
1686 cqe_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
1687 if (cqe_cons > sq->hwq.max_elements) {
1688 dev_err(&cq->hwq.pdev->dev,
1689 "QPLIB: FP: CQ Process req reported ");
1690 dev_err(&cq->hwq.pdev->dev,
1691 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1692 cqe_cons, sq->hwq.max_elements);
1693 return -EINVAL;
1694 }
1695 /* If we were in the middle of flushing the SQ, continue */
1696 if (sq->flush_in_progress)
1697 goto flush;
1698
1699 /* Require to walk the sq's swq to fabricate CQEs for all previously
1700 * signaled SWQEs due to CQE aggregation from the current sq cons
1701 * to the cqe_cons
1702 */
1703 cqe = *pcqe;
1704 while (*budget) {
1705 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1706 if (sw_cons == cqe_cons)
1707 break;
1708 memset(cqe, 0, sizeof(*cqe));
1709 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1710 cqe->qp_handle = (u64)(unsigned long)qp;
1711 cqe->src_qp = qp->id;
1712 cqe->wr_id = sq->swq[sw_cons].wr_id;
1713 cqe->type = sq->swq[sw_cons].type;
1714
1715 /* For the last CQE, check for status. For errors, regardless
1716 * of the request being signaled or not, it must complete with
1717 * the hwcqe error status
1718 */
1719 if (HWQ_CMP((sw_cons + 1), &sq->hwq) == cqe_cons &&
1720 hwcqe->status != CQ_REQ_STATUS_OK) {
1721 cqe->status = hwcqe->status;
1722 dev_err(&cq->hwq.pdev->dev,
1723 "QPLIB: FP: CQ Processed Req ");
1724 dev_err(&cq->hwq.pdev->dev,
1725 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
1726 sw_cons, cqe->wr_id, cqe->status);
1727 cqe++;
1728 (*budget)--;
1729 sq->flush_in_progress = true;
1730 /* Must block new posting of SQ and RQ */
1731 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1732 } else {
1733 if (sq->swq[sw_cons].flags &
1734 SQ_SEND_FLAGS_SIGNAL_COMP) {
1735 cqe->status = CQ_REQ_STATUS_OK;
1736 cqe++;
1737 (*budget)--;
1738 }
1739 }
1740 sq->hwq.cons++;
1741 }
1742 *pcqe = cqe;
1743 if (!*budget && HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_cons) {
1744 /* Out of budget */
1745 rc = -EAGAIN;
1746 goto done;
1747 }
1748 if (!sq->flush_in_progress)
1749 goto done;
1750flush:
1751 /* Require to walk the sq's swq to fabricate CQEs for all
1752 * previously posted SWQEs due to the error CQE received
1753 */
1754 rc = __flush_sq(sq, qp, pcqe, budget);
1755 if (!rc)
1756 sq->flush_in_progress = false;
1757done:
1758 return rc;
1759}
1760
1761static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
1762 struct cq_res_rc *hwcqe,
1763 struct bnxt_qplib_cqe **pcqe,
1764 int *budget)
1765{
1766 struct bnxt_qplib_qp *qp;
1767 struct bnxt_qplib_q *rq;
1768 struct bnxt_qplib_cqe *cqe;
1769 u32 wr_id_idx;
1770 int rc = 0;
1771
1772 qp = (struct bnxt_qplib_qp *)((unsigned long)
1773 le64_to_cpu(hwcqe->qp_handle));
1774 if (!qp) {
1775 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
1776 return -EINVAL;
1777 }
1778 cqe = *pcqe;
1779 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1780 cqe->length = le32_to_cpu(hwcqe->length);
1781 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
1782 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
1783 cqe->flags = le16_to_cpu(hwcqe->flags);
1784 cqe->status = hwcqe->status;
1785 cqe->qp_handle = (u64)(unsigned long)qp;
1786
1787 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
1788 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
1789 rq = &qp->rq;
1790 if (wr_id_idx > rq->hwq.max_elements) {
1791 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC ");
1792 dev_err(&cq->hwq.pdev->dev,
1793 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
1794 wr_id_idx, rq->hwq.max_elements);
1795 return -EINVAL;
1796 }
1797 if (rq->flush_in_progress)
1798 goto flush_rq;
1799
1800 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1801 cqe++;
1802 (*budget)--;
1803 rq->hwq.cons++;
1804 *pcqe = cqe;
1805
1806 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1807 rq->flush_in_progress = true;
1808flush_rq:
1809 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget);
1810 if (!rc)
1811 rq->flush_in_progress = false;
1812 }
1813 return rc;
1814}
1815
1816static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
1817 struct cq_res_ud *hwcqe,
1818 struct bnxt_qplib_cqe **pcqe,
1819 int *budget)
1820{
1821 struct bnxt_qplib_qp *qp;
1822 struct bnxt_qplib_q *rq;
1823 struct bnxt_qplib_cqe *cqe;
1824 u32 wr_id_idx;
1825 int rc = 0;
1826
1827 qp = (struct bnxt_qplib_qp *)((unsigned long)
1828 le64_to_cpu(hwcqe->qp_handle));
1829 if (!qp) {
1830 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
1831 return -EINVAL;
1832 }
1833 cqe = *pcqe;
1834 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1835 cqe->length = le32_to_cpu(hwcqe->length);
1836 cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
1837 cqe->flags = le16_to_cpu(hwcqe->flags);
1838 cqe->status = hwcqe->status;
1839 cqe->qp_handle = (u64)(unsigned long)qp;
1840 memcpy(cqe->smac, hwcqe->src_mac, 6);
1841 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
1842 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
1843 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
1844 ((le32_to_cpu(
1845 hwcqe->src_qp_high_srq_or_rq_wr_id) &
1846 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
1847
1848 rq = &qp->rq;
1849 if (wr_id_idx > rq->hwq.max_elements) {
1850 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD ");
1851 dev_err(&cq->hwq.pdev->dev,
1852 "QPLIB: wr_id idx %#x exceeded RQ max %#x",
1853 wr_id_idx, rq->hwq.max_elements);
1854 return -EINVAL;
1855 }
1856 if (rq->flush_in_progress)
1857 goto flush_rq;
1858
1859 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1860 cqe++;
1861 (*budget)--;
1862 rq->hwq.cons++;
1863 *pcqe = cqe;
1864
1865 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1866 rq->flush_in_progress = true;
1867flush_rq:
1868 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget);
1869 if (!rc)
1870 rq->flush_in_progress = false;
1871 }
1872 return rc;
1873}
1874
1875static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
1876 struct cq_res_raweth_qp1 *hwcqe,
1877 struct bnxt_qplib_cqe **pcqe,
1878 int *budget)
1879{
1880 struct bnxt_qplib_qp *qp;
1881 struct bnxt_qplib_q *rq;
1882 struct bnxt_qplib_cqe *cqe;
1883 u32 wr_id_idx;
1884 int rc = 0;
1885
1886 qp = (struct bnxt_qplib_qp *)((unsigned long)
1887 le64_to_cpu(hwcqe->qp_handle));
1888 if (!qp) {
1889 dev_err(&cq->hwq.pdev->dev,
1890 "QPLIB: process_cq Raw/QP1 qp is NULL");
1891 return -EINVAL;
1892 }
1893 cqe = *pcqe;
1894 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1895 cqe->flags = le16_to_cpu(hwcqe->flags);
1896 cqe->qp_handle = (u64)(unsigned long)qp;
1897
1898 wr_id_idx =
1899 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
1900 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
1901 cqe->src_qp = qp->id;
1902 if (qp->id == 1 && !cqe->length) {
1903 /* Add workaround for the length misdetection */
1904 cqe->length = 296;
1905 } else {
1906 cqe->length = le16_to_cpu(hwcqe->length);
1907 }
1908 cqe->pkey_index = qp->pkey_index;
1909 memcpy(cqe->smac, qp->smac, 6);
1910
1911 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
1912 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
1913
1914 rq = &qp->rq;
1915 if (wr_id_idx > rq->hwq.max_elements) {
1916 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
1917 dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
1918 wr_id_idx, rq->hwq.max_elements);
1919 return -EINVAL;
1920 }
1921 if (rq->flush_in_progress)
1922 goto flush_rq;
1923
1924 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1925 cqe++;
1926 (*budget)--;
1927 rq->hwq.cons++;
1928 *pcqe = cqe;
1929
1930 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1931 rq->flush_in_progress = true;
1932flush_rq:
1933 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe,
1934 budget);
1935 if (!rc)
1936 rq->flush_in_progress = false;
1937 }
1938 return rc;
1939}
1940
1941static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
1942 struct cq_terminal *hwcqe,
1943 struct bnxt_qplib_cqe **pcqe,
1944 int *budget)
1945{
1946 struct bnxt_qplib_qp *qp;
1947 struct bnxt_qplib_q *sq, *rq;
1948 struct bnxt_qplib_cqe *cqe;
1949 u32 sw_cons = 0, cqe_cons;
1950 int rc = 0;
1951 u8 opcode = 0;
1952
1953 /* Check the Status */
1954 if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
1955 dev_warn(&cq->hwq.pdev->dev,
1956 "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
1957 hwcqe->status);
1958
1959 qp = (struct bnxt_qplib_qp *)((unsigned long)
1960 le64_to_cpu(hwcqe->qp_handle));
1961 if (!qp) {
1962 dev_err(&cq->hwq.pdev->dev,
1963 "QPLIB: FP: CQ Process terminal qp is NULL");
1964 return -EINVAL;
1965 }
1966 /* Must block new posting of SQ and RQ */
1967 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1968
1969 sq = &qp->sq;
1970 rq = &qp->rq;
1971
1972 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
1973 if (cqe_cons == 0xFFFF)
1974 goto do_rq;
1975
1976 if (cqe_cons > sq->hwq.max_elements) {
1977 dev_err(&cq->hwq.pdev->dev,
1978 "QPLIB: FP: CQ Process terminal reported ");
1979 dev_err(&cq->hwq.pdev->dev,
1980 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1981 cqe_cons, sq->hwq.max_elements);
1982 goto do_rq;
1983 }
1984 /* If we were in the middle of flushing, continue */
1985 if (sq->flush_in_progress)
1986 goto flush_sq;
1987
1988 /* Terminal CQE can also include aggregated successful CQEs prior.
1989 * So we must complete all CQEs from the current sq's cons to the
1990 * cq_cons with status OK
1991 */
1992 cqe = *pcqe;
1993 while (*budget) {
1994 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1995 if (sw_cons == cqe_cons)
1996 break;
1997 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
1998 memset(cqe, 0, sizeof(*cqe));
1999 cqe->status = CQ_REQ_STATUS_OK;
2000 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2001 cqe->qp_handle = (u64)(unsigned long)qp;
2002 cqe->src_qp = qp->id;
2003 cqe->wr_id = sq->swq[sw_cons].wr_id;
2004 cqe->type = sq->swq[sw_cons].type;
2005 cqe++;
2006 (*budget)--;
2007 }
2008 sq->hwq.cons++;
2009 }
2010 *pcqe = cqe;
2011 if (!(*budget) && sw_cons != cqe_cons) {
2012 /* Out of budget */
2013 rc = -EAGAIN;
2014 goto sq_done;
2015 }
2016 sq->flush_in_progress = true;
2017flush_sq:
2018 rc = __flush_sq(sq, qp, pcqe, budget);
2019 if (!rc)
2020 sq->flush_in_progress = false;
2021sq_done:
2022 if (rc)
2023 return rc;
2024do_rq:
2025 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2026 if (cqe_cons == 0xFFFF) {
2027 goto done;
2028 } else if (cqe_cons > rq->hwq.max_elements) {
2029 dev_err(&cq->hwq.pdev->dev,
2030 "QPLIB: FP: CQ Processed terminal ");
2031 dev_err(&cq->hwq.pdev->dev,
2032 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2033 cqe_cons, rq->hwq.max_elements);
2034 goto done;
2035 }
2036 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2037 * from the current rq->cons to the rq->prod regardless what the
2038 * rq->cons the terminal CQE indicates
2039 */
2040 rq->flush_in_progress = true;
2041 switch (qp->type) {
2042 case CMDQ_CREATE_QP1_TYPE_GSI:
2043 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2044 break;
2045 case CMDQ_CREATE_QP_TYPE_RC:
2046 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2047 break;
2048 case CMDQ_CREATE_QP_TYPE_UD:
2049 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2050 break;
2051 }
2052
2053 rc = __flush_rq(rq, qp, opcode, pcqe, budget);
2054 if (!rc)
2055 rq->flush_in_progress = false;
2056done:
2057 return rc;
2058}
2059
2060static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2061 struct cq_cutoff *hwcqe)
2062{
2063 /* Check the Status */
2064 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2065 dev_err(&cq->hwq.pdev->dev,
2066 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2067 hwcqe->status);
2068 return -EINVAL;
2069 }
2070 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2071 wake_up_interruptible(&cq->waitq);
2072
2073 return 0;
2074}
2075
2076int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2077 int num_cqes)
2078{
2079 struct cq_base *hw_cqe, **hw_cqe_ptr;
2080 unsigned long flags;
2081 u32 sw_cons, raw_cons;
2082 int budget, rc = 0;
2083
2084 spin_lock_irqsave(&cq->hwq.lock, flags);
2085 raw_cons = cq->hwq.cons;
2086 budget = num_cqes;
2087
2088 while (budget) {
2089 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2090 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2091 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2092
2093 /* Check for Valid bit */
2094 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2095 break;
2096
2097 /* From the device's respective CQE format to qplib_wc*/
2098 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2099 case CQ_BASE_CQE_TYPE_REQ:
2100 rc = bnxt_qplib_cq_process_req(cq,
2101 (struct cq_req *)hw_cqe,
2102 &cqe, &budget);
2103 break;
2104 case CQ_BASE_CQE_TYPE_RES_RC:
2105 rc = bnxt_qplib_cq_process_res_rc(cq,
2106 (struct cq_res_rc *)
2107 hw_cqe, &cqe,
2108 &budget);
2109 break;
2110 case CQ_BASE_CQE_TYPE_RES_UD:
2111 rc = bnxt_qplib_cq_process_res_ud
2112 (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2113 &budget);
2114 break;
2115 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2116 rc = bnxt_qplib_cq_process_res_raweth_qp1
2117 (cq, (struct cq_res_raweth_qp1 *)
2118 hw_cqe, &cqe, &budget);
2119 break;
2120 case CQ_BASE_CQE_TYPE_TERMINAL:
2121 rc = bnxt_qplib_cq_process_terminal
2122 (cq, (struct cq_terminal *)hw_cqe,
2123 &cqe, &budget);
2124 break;
2125 case CQ_BASE_CQE_TYPE_CUT_OFF:
2126 bnxt_qplib_cq_process_cutoff
2127 (cq, (struct cq_cutoff *)hw_cqe);
2128 /* Done processing this CQ */
2129 goto exit;
2130 default:
2131 dev_err(&cq->hwq.pdev->dev,
2132 "QPLIB: process_cq unknown type 0x%lx",
2133 hw_cqe->cqe_type_toggle &
2134 CQ_BASE_CQE_TYPE_MASK);
2135 rc = -EINVAL;
2136 break;
2137 }
2138 if (rc < 0) {
2139 if (rc == -EAGAIN)
2140 break;
2141 /* Error while processing the CQE, just skip to the
2142 * next one
2143 */
2144 dev_err(&cq->hwq.pdev->dev,
2145 "QPLIB: process_cqe error rc = 0x%x", rc);
2146 }
2147 raw_cons++;
2148 }
2149 if (cq->hwq.cons != raw_cons) {
2150 cq->hwq.cons = raw_cons;
2151 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2152 }
2153exit:
2154 spin_unlock_irqrestore(&cq->hwq.lock, flags);
2155 return num_cqes - budget;
2156}
2157
2158void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2159{
2160 unsigned long flags;
2161
2162 spin_lock_irqsave(&cq->hwq.lock, flags);
2163 if (arm_type)
2164 bnxt_qplib_arm_cq(cq, arm_type);
2165
2166 spin_unlock_irqrestore(&cq->hwq.lock, flags);
2167}
drivers/infiniband/hw/bnxt_re/qplib_fp.h+439−0
@@ -0,0 +1,439 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Fast Path Operators (header)
37 */
38
39#ifndef __BNXT_QPLIB_FP_H__
40#define __BNXT_QPLIB_FP_H__
41
42struct bnxt_qplib_sge {
43 u64 addr;
44 u32 lkey;
45 u32 size;
46};
47
48#define BNXT_QPLIB_MAX_SQE_ENTRY_SIZE sizeof(struct sq_send)
49
50#define SQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_SQE_ENTRY_SIZE)
51#define SQE_MAX_IDX_PER_PG (SQE_CNT_PER_PG - 1)
52
53static inline u32 get_sqe_pg(u32 val)
54{
55 return ((val & ~SQE_MAX_IDX_PER_PG) / SQE_CNT_PER_PG);
56}
57
58static inline u32 get_sqe_idx(u32 val)
59{
60 return (val & SQE_MAX_IDX_PER_PG);
61}
62
63#define BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE sizeof(struct sq_psn_search)
64
65#define PSNE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE)
66#define PSNE_MAX_IDX_PER_PG (PSNE_CNT_PER_PG - 1)
67
68static inline u32 get_psne_pg(u32 val)
69{
70 return ((val & ~PSNE_MAX_IDX_PER_PG) / PSNE_CNT_PER_PG);
71}
72
73static inline u32 get_psne_idx(u32 val)
74{
75 return (val & PSNE_MAX_IDX_PER_PG);
76}
77
78#define BNXT_QPLIB_QP_MAX_SGL 6
79
80struct bnxt_qplib_swq {
81 u64 wr_id;
82 u8 type;
83 u8 flags;
84 u32 start_psn;
85 u32 next_psn;
86 struct sq_psn_search *psn_search;
87};
88
89struct bnxt_qplib_swqe {
90 /* General */
91 u64 wr_id;
92 u8 reqs_type;
93 u8 type;
94#define BNXT_QPLIB_SWQE_TYPE_SEND 0
95#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1
96#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2
97#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4
98#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5
99#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6
100#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8
101#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11
102#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12
103#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13
104#define BNXT_QPLIB_SWQE_TYPE_REG_MR 13
105#define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14
106#define BNXT_QPLIB_SWQE_TYPE_RECV 128
107#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129
108 u8 flags;
109#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0)
110#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1)
111#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2)
112#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3)
113#define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4)
114 struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL];
115 int num_sge;
116 /* Max inline data is 96 bytes */
117 u32 inline_len;
118#define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96
119 u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH];
120
121 union {
122 /* Send, with imm, inval key */
123 struct {
124 union {
125 __be32 imm_data;
126 u32 inv_key;
127 };
128 u32 q_key;
129 u32 dst_qp;
130 u16 avid;
131 } send;
132
133 /* Send Raw Ethernet and QP1 */
134 struct {
135 u16 lflags;
136 u16 cfa_action;
137 u32 cfa_meta;
138 } rawqp1;
139
140 /* RDMA write, with imm, read */
141 struct {
142 union {
143 __be32 imm_data;
144 u32 inv_key;
145 };
146 u64 remote_va;
147 u32 r_key;
148 } rdma;
149
150 /* Atomic cmp/swap, fetch/add */
151 struct {
152 u64 remote_va;
153 u32 r_key;
154 u64 swap_data;
155 u64 cmp_data;
156 } atomic;
157
158 /* Local Invalidate */
159 struct {
160 u32 inv_l_key;
161 } local_inv;
162
163 /* FR-PMR */
164 struct {
165 u8 access_cntl;
166 u8 pg_sz_log;
167 bool zero_based;
168 u32 l_key;
169 u32 length;
170 u8 pbl_pg_sz_log;
171#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0
172#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1
173#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4
174#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6
175#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8
176#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9
177#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10
178#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18
179 u8 levels;
180#define PAGE_SHIFT_4K 12
181 __le64 *pbl_ptr;
182 dma_addr_t pbl_dma_ptr;
183 u64 *page_list;
184 u16 page_list_len;
185 u64 va;
186 } frmr;
187
188 /* Bind */
189 struct {
190 u8 access_cntl;
191#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0)
192#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1)
193#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2)
194#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3)
195#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4)
196 bool zero_based;
197 u8 mw_type;
198 u32 parent_l_key;
199 u32 r_key;
200 u64 va;
201 u32 length;
202 } bind;
203 };
204};
205
206#define BNXT_QPLIB_MAX_RQE_ENTRY_SIZE sizeof(struct rq_wqe)
207
208#define RQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_RQE_ENTRY_SIZE)
209#define RQE_MAX_IDX_PER_PG (RQE_CNT_PER_PG - 1)
210#define RQE_PG(x) (((x) & ~RQE_MAX_IDX_PER_PG) / RQE_CNT_PER_PG)
211#define RQE_IDX(x) ((x) & RQE_MAX_IDX_PER_PG)
212
213struct bnxt_qplib_q {
214 struct bnxt_qplib_hwq hwq;
215 struct bnxt_qplib_swq *swq;
216 struct scatterlist *sglist;
217 u32 nmap;
218 u32 max_wqe;
219 u16 max_sge;
220 u32 psn;
221 bool flush_in_progress;
222};
223
224struct bnxt_qplib_qp {
225 struct bnxt_qplib_pd *pd;
226 struct bnxt_qplib_dpi *dpi;
227 u64 qp_handle;
228 u32 id;
229 u8 type;
230 u8 sig_type;
231 u32 modify_flags;
232 u8 state;
233 u8 cur_qp_state;
234 u32 max_inline_data;
235 u32 mtu;
236 u8 path_mtu;
237 bool en_sqd_async_notify;
238 u16 pkey_index;
239 u32 qkey;
240 u32 dest_qp_id;
241 u8 access;
242 u8 timeout;
243 u8 retry_cnt;
244 u8 rnr_retry;
245 u32 min_rnr_timer;
246 u32 max_rd_atomic;
247 u32 max_dest_rd_atomic;
248 u32 dest_qpn;
249 u8 smac[6];
250 u16 vlan_id;
251 u8 nw_type;
252 struct bnxt_qplib_ah ah;
253
254#define BTH_PSN_MASK ((1 << 24) - 1)
255 /* SQ */
256 struct bnxt_qplib_q sq;
257 /* RQ */
258 struct bnxt_qplib_q rq;
259 /* SRQ */
260 struct bnxt_qplib_srq *srq;
261 /* CQ */
262 struct bnxt_qplib_cq *scq;
263 struct bnxt_qplib_cq *rcq;
264 /* IRRQ and ORRQ */
265 struct bnxt_qplib_hwq irrq;
266 struct bnxt_qplib_hwq orrq;
267 /* Header buffer for QP1 */
268 int sq_hdr_buf_size;
269 int rq_hdr_buf_size;
270/*
271 * Buffer space for ETH(14), IP or GRH(40), UDP header(8)
272 * and ib_bth + ib_deth (20).
273 * Max required is 82 when RoCE V2 is enabled
274 */
275#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86
276 /* Ethernet header = 14 */
277 /* ib_grh = 40 (provided by MAD) */
278 /* ib_bth + ib_deth = 20 */
279 /* MAD = 256 (provided by MAD) */
280 /* iCRC = 4 */
281#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14
282#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512
283#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20
284#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40
285#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20
286 void *sq_hdr_buf;
287 dma_addr_t sq_hdr_buf_map;
288 void *rq_hdr_buf;
289 dma_addr_t rq_hdr_buf_map;
290};
291
292#define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base)
293
294#define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE)
295#define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1)
296#define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG)
297#define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG)
298
299#define ROCE_CQE_CMP_V 0
300#define CQE_CMP_VALID(hdr, raw_cons, cp_bit) \
301 (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \
302 !((raw_cons) & (cp_bit)))
303
304struct bnxt_qplib_cqe {
305 u8 status;
306 u8 type;
307 u8 opcode;
308 u32 length;
309 u64 wr_id;
310 union {
311 __be32 immdata;
312 u32 invrkey;
313 };
314 u64 qp_handle;
315 u64 mr_handle;
316 u16 flags;
317 u8 smac[6];
318 u32 src_qp;
319 u16 raweth_qp1_flags;
320 u16 raweth_qp1_errors;
321 u16 raweth_qp1_cfa_code;
322 u32 raweth_qp1_flags2;
323 u32 raweth_qp1_metadata;
324 u8 raweth_qp1_payload_offset;
325 u16 pkey_index;
326};
327
328#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01
329struct bnxt_qplib_cq {
330 struct bnxt_qplib_dpi *dpi;
331 void __iomem *dbr_base;
332 u32 max_wqe;
333 u32 id;
334 u16 count;
335 u16 period;
336 struct bnxt_qplib_hwq hwq;
337 u32 cnq_hw_ring_id;
338 bool resize_in_progress;
339 struct scatterlist *sghead;
340 u32 nmap;
341 u64 cq_handle;
342
343#define CQ_RESIZE_WAIT_TIME_MS 500
344 unsigned long flags;
345#define CQ_FLAGS_RESIZE_IN_PROG 1
346 wait_queue_head_t waitq;
347};
348
349#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq)
350#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq)
351#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2)
352#define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1)
353#define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1)
354#define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1)
355
356#define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base)
357
358#define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE)
359#define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1)
360#define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG)
361#define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG)
362
363#define NQE_CMP_VALID(hdr, raw_cons, cp_bit) \
364 (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \
365 !((raw_cons) & (cp_bit)))
366
367#define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024)
368
369#define NQ_CONS_PCI_BAR_REGION 2
370#define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT)
371#define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID
372#define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK
373#define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \
374 NQ_DB_IDX_VALID)
375#define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \
376 NQ_DB_IDX_VALID | \
377 NQ_DB_IRQ_DIS)
378#define NQ_DB_REARM(db, raw_cons, cp_bit) \
379 writel(NQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db)
380#define NQ_DB(db, raw_cons, cp_bit) \
381 writel(NQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db)
382
383struct bnxt_qplib_nq {
384 struct pci_dev *pdev;
385
386 int vector;
387 int budget;
388 bool requested;
389 struct tasklet_struct worker;
390 struct bnxt_qplib_hwq hwq;
391
392 u16 bar_reg;
393 u16 bar_reg_off;
394 u16 ring_id;
395 void __iomem *bar_reg_iomem;
396
397 int (*cqn_handler)
398 (struct bnxt_qplib_nq *nq,
399 struct bnxt_qplib_cq *cq);
400 int (*srqn_handler)
401 (struct bnxt_qplib_nq *nq,
402 void *srq,
403 u8 event);
404};
405
406void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq);
407int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
408 int msix_vector, int bar_reg_offset,
409 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
410 struct bnxt_qplib_cq *cq),
411 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
412 void *srq,
413 u8 event));
414int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
415int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
416int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
417int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
418int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
419void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
420 struct bnxt_qplib_sge *sge);
421void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
422 struct bnxt_qplib_sge *sge);
423u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp);
424dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp,
425 u32 index);
426void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp);
427int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
428 struct bnxt_qplib_swqe *wqe);
429void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp);
430int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
431 struct bnxt_qplib_swqe *wqe);
432int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
433int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
434int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
435 int num);
436void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
437void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq);
438int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq);
439#endif /* __BNXT_QPLIB_FP_H__ */
drivers/infiniband/hw/bnxt_re/qplib_rcfw.c+694−0
@@ -0,0 +1,694 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: RDMA Controller HW interface
37 */
38#include <linux/interrupt.h>
39#include <linux/spinlock.h>
40#include <linux/pci.h>
41#include <linux/prefetch.h>
42#include "roce_hsi.h"
43#include "qplib_res.h"
44#include "qplib_rcfw.h"
45static void bnxt_qplib_service_creq(unsigned long data);
46
47/* Hardware communication channel */
48int bnxt_qplib_rcfw_wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
49{
50 u16 cbit;
51 int rc;
52
53 cookie &= RCFW_MAX_COOKIE_VALUE;
54 cbit = cookie % RCFW_MAX_OUTSTANDING_CMD;
55 if (!test_bit(cbit, rcfw->cmdq_bitmap))
56 dev_warn(&rcfw->pdev->dev,
57 "QPLIB: CMD bit %d for cookie 0x%x is not set?",
58 cbit, cookie);
59
60 rc = wait_event_timeout(rcfw->waitq,
61 !test_bit(cbit, rcfw->cmdq_bitmap),
62 msecs_to_jiffies(RCFW_CMD_WAIT_TIME_MS));
63 if (!rc) {
64 dev_warn(&rcfw->pdev->dev,
65 "QPLIB: Bono Error: timeout %d msec, msg {0x%x}\n",
66 RCFW_CMD_WAIT_TIME_MS, cookie);
67 }
68
69 return rc;
70};
71
72int bnxt_qplib_rcfw_block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
73{
74 u32 count = -1;
75 u16 cbit;
76
77 cookie &= RCFW_MAX_COOKIE_VALUE;
78 cbit = cookie % RCFW_MAX_OUTSTANDING_CMD;
79 if (!test_bit(cbit, rcfw->cmdq_bitmap))
80 goto done;
81 do {
82 bnxt_qplib_service_creq((unsigned long)rcfw);
83 } while (test_bit(cbit, rcfw->cmdq_bitmap) && --count);
84done:
85 return count;
86};
87
88void *bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
89 struct cmdq_base *req, void **crsbe,
90 u8 is_block)
91{
92 struct bnxt_qplib_crsq *crsq = &rcfw->crsq;
93 struct bnxt_qplib_cmdqe *cmdqe, **cmdq_ptr;
94 struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq;
95 struct bnxt_qplib_hwq *crsb = &rcfw->crsb;
96 struct bnxt_qplib_crsqe *crsqe = NULL;
97 struct bnxt_qplib_crsbe **crsb_ptr;
98 u32 sw_prod, cmdq_prod;
99 u8 retry_cnt = 0xFF;
100 dma_addr_t dma_addr;
101 unsigned long flags;
102 u32 size, opcode;
103 u16 cookie, cbit;
104 int pg, idx;
105 u8 *preq;
106
107retry:
108 opcode = req->opcode;
109 if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) &&
110 (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &&
111 opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW)) {
112 dev_err(&rcfw->pdev->dev,
113 "QPLIB: RCFW not initialized, reject opcode 0x%x",
114 opcode);
115 return NULL;
116 }
117
118 if (test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) &&
119 opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {
120 dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!");
121 return NULL;
122 }
123
124 /* Cmdq are in 16-byte units, each request can consume 1 or more
125 * cmdqe
126 */
127 spin_lock_irqsave(&cmdq->lock, flags);
128 if (req->cmd_size > cmdq->max_elements -
129 ((HWQ_CMP(cmdq->prod, cmdq) - HWQ_CMP(cmdq->cons, cmdq)) &
130 (cmdq->max_elements - 1))) {
131 dev_err(&rcfw->pdev->dev, "QPLIB: RCFW: CMDQ is full!");
132 spin_unlock_irqrestore(&cmdq->lock, flags);
133
134 if (!retry_cnt--)
135 return NULL;
136 goto retry;
137 }
138
139 retry_cnt = 0xFF;
140
141 cookie = atomic_inc_return(&rcfw->seq_num) & RCFW_MAX_COOKIE_VALUE;
142 cbit = cookie % RCFW_MAX_OUTSTANDING_CMD;
143 if (is_block)
144 cookie |= RCFW_CMD_IS_BLOCKING;
145 req->cookie = cpu_to_le16(cookie);
146 if (test_and_set_bit(cbit, rcfw->cmdq_bitmap)) {
147 dev_err(&rcfw->pdev->dev,
148 "QPLIB: RCFW MAX outstanding cmd reached!");
149 atomic_dec(&rcfw->seq_num);
150 spin_unlock_irqrestore(&cmdq->lock, flags);
151
152 if (!retry_cnt--)
153 return NULL;
154 goto retry;
155 }
156 /* Reserve a resp buffer slot if requested */
157 if (req->resp_size && crsbe) {
158 spin_lock(&crsb->lock);
159 sw_prod = HWQ_CMP(crsb->prod, crsb);
160 crsb_ptr = (struct bnxt_qplib_crsbe **)crsb->pbl_ptr;
161 *crsbe = (void *)&crsb_ptr[get_crsb_pg(sw_prod)]
162 [get_crsb_idx(sw_prod)];
163 bnxt_qplib_crsb_dma_next(crsb->pbl_dma_ptr, sw_prod, &dma_addr);
164 req->resp_addr = cpu_to_le64(dma_addr);
165 crsb->prod++;
166 spin_unlock(&crsb->lock);
167
168 req->resp_size = (sizeof(struct bnxt_qplib_crsbe) +
169 BNXT_QPLIB_CMDQE_UNITS - 1) /
170 BNXT_QPLIB_CMDQE_UNITS;
171 }
172 cmdq_ptr = (struct bnxt_qplib_cmdqe **)cmdq->pbl_ptr;
173 preq = (u8 *)req;
174 size = req->cmd_size * BNXT_QPLIB_CMDQE_UNITS;
175 do {
176 pg = 0;
177 idx = 0;
178
179 /* Locate the next cmdq slot */
180 sw_prod = HWQ_CMP(cmdq->prod, cmdq);
181 cmdqe = &cmdq_ptr[get_cmdq_pg(sw_prod)][get_cmdq_idx(sw_prod)];
182 if (!cmdqe) {
183 dev_err(&rcfw->pdev->dev,
184 "QPLIB: RCFW request failed with no cmdqe!");
185 goto done;
186 }
187 /* Copy a segment of the req cmd to the cmdq */
188 memset(cmdqe, 0, sizeof(*cmdqe));
189 memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe)));
190 preq += min_t(u32, size, sizeof(*cmdqe));
191 size -= min_t(u32, size, sizeof(*cmdqe));
192 cmdq->prod++;
193 } while (size > 0);
194
195 cmdq_prod = cmdq->prod;
196 if (rcfw->flags & FIRMWARE_FIRST_FLAG) {
197 /* The very first doorbell write is required to set this flag
198 * which prompts the FW to reset its internal pointers
199 */
200 cmdq_prod |= FIRMWARE_FIRST_FLAG;
201 rcfw->flags &= ~FIRMWARE_FIRST_FLAG;
202 }
203 sw_prod = HWQ_CMP(crsq->prod, crsq);
204 crsqe = &crsq->crsq[sw_prod];
205 memset(crsqe, 0, sizeof(*crsqe));
206 crsq->prod++;
207 crsqe->req_size = req->cmd_size;
208
209 /* ring CMDQ DB */
210 writel(cmdq_prod, rcfw->cmdq_bar_reg_iomem +
211 rcfw->cmdq_bar_reg_prod_off);
212 writel(RCFW_CMDQ_TRIG_VAL, rcfw->cmdq_bar_reg_iomem +
213 rcfw->cmdq_bar_reg_trig_off);
214done:
215 spin_unlock_irqrestore(&cmdq->lock, flags);
216 /* Return the CREQ response pointer */
217 return crsqe ? &crsqe->qp_event : NULL;
218}
219
220/* Completions */
221static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw,
222 struct creq_func_event *func_event)
223{
224 switch (func_event->event) {
225 case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
226 break;
227 case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
228 break;
229 case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
230 break;
231 case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
232 break;
233 case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
234 break;
235 case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
236 break;
237 case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
238 break;
239 case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
240 /* SRQ ctx error, call srq_handler??
241 * But there's no SRQ handle!
242 */
243 break;
244 case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
245 break;
246 case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
247 break;
248 case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
249 break;
250 case CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST:
251 break;
252 case CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED:
253 break;
254 default:
255 return -EINVAL;
256 }
257 return 0;
258}
259
260static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
261 struct creq_qp_event *qp_event)
262{
263 struct bnxt_qplib_crsq *crsq = &rcfw->crsq;
264 struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq;
265 struct bnxt_qplib_crsqe *crsqe;
266 u16 cbit, cookie, blocked = 0;
267 unsigned long flags;
268 u32 sw_cons;
269
270 switch (qp_event->event) {
271 case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
272 dev_dbg(&rcfw->pdev->dev,
273 "QPLIB: Received QP error notification");
274 break;
275 default:
276 /* Command Response */
277 spin_lock_irqsave(&cmdq->lock, flags);
278 sw_cons = HWQ_CMP(crsq->cons, crsq);
279 crsqe = &crsq->crsq[sw_cons];
280 crsq->cons++;
281 memcpy(&crsqe->qp_event, qp_event, sizeof(crsqe->qp_event));
282
283 cookie = le16_to_cpu(crsqe->qp_event.cookie);
284 blocked = cookie & RCFW_CMD_IS_BLOCKING;
285 cookie &= RCFW_MAX_COOKIE_VALUE;
286 cbit = cookie % RCFW_MAX_OUTSTANDING_CMD;
287 if (!test_and_clear_bit(cbit, rcfw->cmdq_bitmap))
288 dev_warn(&rcfw->pdev->dev,
289 "QPLIB: CMD bit %d was not requested", cbit);
290
291 cmdq->cons += crsqe->req_size;
292 spin_unlock_irqrestore(&cmdq->lock, flags);
293 if (!blocked)
294 wake_up(&rcfw->waitq);
295 break;
296 }
297 return 0;
298}
299
300/* SP - CREQ Completion handlers */
301static void bnxt_qplib_service_creq(unsigned long data)
302{
303 struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data;
304 struct bnxt_qplib_hwq *creq = &rcfw->creq;
305 struct creq_base *creqe, **creq_ptr;
306 u32 sw_cons, raw_cons;
307 unsigned long flags;
308 u32 type;
309
310 /* Service the CREQ until empty */
311 spin_lock_irqsave(&creq->lock, flags);
312 raw_cons = creq->cons;
313 while (1) {
314 sw_cons = HWQ_CMP(raw_cons, creq);
315 creq_ptr = (struct creq_base **)creq->pbl_ptr;
316 creqe = &creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)];
317 if (!CREQ_CMP_VALID(creqe, raw_cons, creq->max_elements))
318 break;
319
320 type = creqe->type & CREQ_BASE_TYPE_MASK;
321 switch (type) {
322 case CREQ_BASE_TYPE_QP_EVENT:
323 if (!bnxt_qplib_process_qp_event
324 (rcfw, (struct creq_qp_event *)creqe))
325 rcfw->creq_qp_event_processed++;
326 else {
327 dev_warn(&rcfw->pdev->dev, "QPLIB: crsqe with");
328 dev_warn(&rcfw->pdev->dev,
329 "QPLIB: type = 0x%x not handled",
330 type);
331 }
332 break;
333 case CREQ_BASE_TYPE_FUNC_EVENT:
334 if (!bnxt_qplib_process_func_event
335 (rcfw, (struct creq_func_event *)creqe))
336 rcfw->creq_func_event_processed++;
337 else
338 dev_warn
339 (&rcfw->pdev->dev, "QPLIB:aeqe:%#x Not handled",
340 type);
341 break;
342 default:
343 dev_warn(&rcfw->pdev->dev, "QPLIB: creqe with ");
344 dev_warn(&rcfw->pdev->dev,
345 "QPLIB: op_event = 0x%x not handled", type);
346 break;
347 }
348 raw_cons++;
349 }
350 if (creq->cons != raw_cons) {
351 creq->cons = raw_cons;
352 CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, raw_cons,
353 creq->max_elements);
354 }
355 spin_unlock_irqrestore(&creq->lock, flags);
356}
357
358static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
359{
360 struct bnxt_qplib_rcfw *rcfw = dev_instance;
361 struct bnxt_qplib_hwq *creq = &rcfw->creq;
362 struct creq_base **creq_ptr;
363 u32 sw_cons;
364
365 /* Prefetch the CREQ element */
366 sw_cons = HWQ_CMP(creq->cons, creq);
367 creq_ptr = (struct creq_base **)rcfw->creq.pbl_ptr;
368 prefetch(&creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]);
369
370 tasklet_schedule(&rcfw->worker);
371
372 return IRQ_HANDLED;
373}
374
375/* RCFW */
376int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
377{
378 struct creq_deinitialize_fw_resp *resp;
379 struct cmdq_deinitialize_fw req;
380 u16 cmd_flags = 0;
381
382 RCFW_CMD_PREP(req, DEINITIALIZE_FW, cmd_flags);
383 resp = (struct creq_deinitialize_fw_resp *)
384 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
385 NULL, 0);
386 if (!resp)
387 return -EINVAL;
388
389 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie)))
390 return -ETIMEDOUT;
391
392 if (resp->status ||
393 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie))
394 return -EFAULT;
395
396 clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags);
397 return 0;
398}
399
400static int __get_pbl_pg_idx(struct bnxt_qplib_pbl *pbl)
401{
402 return (pbl->pg_size == ROCE_PG_SIZE_4K ?
403 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K :
404 pbl->pg_size == ROCE_PG_SIZE_8K ?
405 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K :
406 pbl->pg_size == ROCE_PG_SIZE_64K ?
407 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K :
408 pbl->pg_size == ROCE_PG_SIZE_2M ?
409 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M :
410 pbl->pg_size == ROCE_PG_SIZE_8M ?
411 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M :
412 pbl->pg_size == ROCE_PG_SIZE_1G ?
413 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G :
414 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K);
415}
416
417int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
418 struct bnxt_qplib_ctx *ctx, int is_virtfn)
419{
420 struct creq_initialize_fw_resp *resp;
421 struct cmdq_initialize_fw req;
422 u16 cmd_flags = 0, level;
423
424 RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags);
425
426 /*
427 * VFs need not setup the HW context area, PF
428 * shall setup this area for VF. Skipping the
429 * HW programming
430 */
431 if (is_virtfn)
432 goto skip_ctx_setup;
433
434 level = ctx->qpc_tbl.level;
435 req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) |
436 __get_pbl_pg_idx(&ctx->qpc_tbl.pbl[level]);
437 level = ctx->mrw_tbl.level;
438 req.mrw_pg_size_mrw_lvl = (level << CMDQ_INITIALIZE_FW_MRW_LVL_SFT) |
439 __get_pbl_pg_idx(&ctx->mrw_tbl.pbl[level]);
440 level = ctx->srqc_tbl.level;
441 req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) |
442 __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]);
443 level = ctx->cq_tbl.level;
444 req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) |
445 __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]);
446 level = ctx->srqc_tbl.level;
447 req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) |
448 __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]);
449 level = ctx->cq_tbl.level;
450 req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) |
451 __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]);
452 level = ctx->tim_tbl.level;
453 req.tim_pg_size_tim_lvl = (level << CMDQ_INITIALIZE_FW_TIM_LVL_SFT) |
454 __get_pbl_pg_idx(&ctx->tim_tbl.pbl[level]);
455 level = ctx->tqm_pde_level;
456 req.tqm_pg_size_tqm_lvl = (level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) |
457 __get_pbl_pg_idx(&ctx->tqm_pde.pbl[level]);
458
459 req.qpc_page_dir =
460 cpu_to_le64(ctx->qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
461 req.mrw_page_dir =
462 cpu_to_le64(ctx->mrw_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
463 req.srq_page_dir =
464 cpu_to_le64(ctx->srqc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
465 req.cq_page_dir =
466 cpu_to_le64(ctx->cq_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
467 req.tim_page_dir =
468 cpu_to_le64(ctx->tim_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
469 req.tqm_page_dir =
470 cpu_to_le64(ctx->tqm_pde.pbl[PBL_LVL_0].pg_map_arr[0]);
471
472 req.number_of_qp = cpu_to_le32(ctx->qpc_tbl.max_elements);
473 req.number_of_mrw = cpu_to_le32(ctx->mrw_tbl.max_elements);
474 req.number_of_srq = cpu_to_le32(ctx->srqc_tbl.max_elements);
475 req.number_of_cq = cpu_to_le32(ctx->cq_tbl.max_elements);
476
477 req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf);
478 req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf);
479 req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf);
480 req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf);
481 req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf);
482
483skip_ctx_setup:
484 req.stat_ctx_id = cpu_to_le32(ctx->stats.fw_id);
485 resp = (struct creq_initialize_fw_resp *)
486 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
487 NULL, 0);
488 if (!resp) {
489 dev_err(&rcfw->pdev->dev,
490 "QPLIB: RCFW: INITIALIZE_FW send failed");
491 return -EINVAL;
492 }
493 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
494 /* Cmd timed out */
495 dev_err(&rcfw->pdev->dev,
496 "QPLIB: RCFW: INITIALIZE_FW timed out");
497 return -ETIMEDOUT;
498 }
499 if (resp->status ||
500 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
501 dev_err(&rcfw->pdev->dev,
502 "QPLIB: RCFW: INITIALIZE_FW failed");
503 return -EINVAL;
504 }
505 set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags);
506 return 0;
507}
508
509void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
510{
511 bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->crsb);
512 kfree(rcfw->crsq.crsq);
513 bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->cmdq);
514 bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->creq);
515
516 rcfw->pdev = NULL;
517}
518
519int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
520 struct bnxt_qplib_rcfw *rcfw)
521{
522 rcfw->pdev = pdev;
523 rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT;
524 if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL, 0,
525 &rcfw->creq.max_elements,
526 BNXT_QPLIB_CREQE_UNITS, 0, PAGE_SIZE,
527 HWQ_TYPE_L2_CMPL)) {
528 dev_err(&rcfw->pdev->dev,
529 "QPLIB: HW channel CREQ allocation failed");
530 goto fail;
531 }
532 rcfw->cmdq.max_elements = BNXT_QPLIB_CMDQE_MAX_CNT;
533 if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->cmdq, NULL, 0,
534 &rcfw->cmdq.max_elements,
535 BNXT_QPLIB_CMDQE_UNITS, 0, PAGE_SIZE,
536 HWQ_TYPE_CTX)) {
537 dev_err(&rcfw->pdev->dev,
538 "QPLIB: HW channel CMDQ allocation failed");
539 goto fail;
540 }
541
542 rcfw->crsq.max_elements = rcfw->cmdq.max_elements;
543 rcfw->crsq.crsq = kcalloc(rcfw->crsq.max_elements,
544 sizeof(*rcfw->crsq.crsq), GFP_KERNEL);
545 if (!rcfw->crsq.crsq)
546 goto fail;
547
548 rcfw->crsb.max_elements = BNXT_QPLIB_CRSBE_MAX_CNT;
549 if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->crsb, NULL, 0,
550 &rcfw->crsb.max_elements,
551 BNXT_QPLIB_CRSBE_UNITS, 0, PAGE_SIZE,
552 HWQ_TYPE_CTX)) {
553 dev_err(&rcfw->pdev->dev,
554 "QPLIB: HW channel CRSB allocation failed");
555 goto fail;
556 }
557 return 0;
558
559fail:
560 bnxt_qplib_free_rcfw_channel(rcfw);
561 return -ENOMEM;
562}
563
564void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
565{
566 unsigned long indx;
567
568 /* Make sure the HW channel is stopped! */
569 synchronize_irq(rcfw->vector);
570 tasklet_disable(&rcfw->worker);
571 tasklet_kill(&rcfw->worker);
572
573 if (rcfw->requested) {
574 free_irq(rcfw->vector, rcfw);
575 rcfw->requested = false;
576 }
577 if (rcfw->cmdq_bar_reg_iomem)
578 iounmap(rcfw->cmdq_bar_reg_iomem);
579 rcfw->cmdq_bar_reg_iomem = NULL;
580
581 if (rcfw->creq_bar_reg_iomem)
582 iounmap(rcfw->creq_bar_reg_iomem);
583 rcfw->creq_bar_reg_iomem = NULL;
584
585 indx = find_first_bit(rcfw->cmdq_bitmap, rcfw->bmap_size);
586 if (indx != rcfw->bmap_size)
587 dev_err(&rcfw->pdev->dev,
588 "QPLIB: disabling RCFW with pending cmd-bit %lx", indx);
589 kfree(rcfw->cmdq_bitmap);
590 rcfw->bmap_size = 0;
591
592 rcfw->aeq_handler = NULL;
593 rcfw->vector = 0;
594}
595
596int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
597 struct bnxt_qplib_rcfw *rcfw,
598 int msix_vector,
599 int cp_bar_reg_off, int virt_fn,
600 int (*aeq_handler)(struct bnxt_qplib_rcfw *,
601 struct creq_func_event *))
602{
603 resource_size_t res_base;
604 struct cmdq_init init;
605 u16 bmap_size;
606 int rc;
607
608 /* General */
609 atomic_set(&rcfw->seq_num, 0);
610 rcfw->flags = FIRMWARE_FIRST_FLAG;
611 bmap_size = BITS_TO_LONGS(RCFW_MAX_OUTSTANDING_CMD *
612 sizeof(unsigned long));
613 rcfw->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL);
614 if (!rcfw->cmdq_bitmap)
615 return -ENOMEM;
616 rcfw->bmap_size = bmap_size;
617
618 /* CMDQ */
619 rcfw->cmdq_bar_reg = RCFW_COMM_PCI_BAR_REGION;
620 res_base = pci_resource_start(pdev, rcfw->cmdq_bar_reg);
621 if (!res_base)
622 return -ENOMEM;
623
624 rcfw->cmdq_bar_reg_iomem = ioremap_nocache(res_base +
625 RCFW_COMM_BASE_OFFSET,
626 RCFW_COMM_SIZE);
627 if (!rcfw->cmdq_bar_reg_iomem) {
628 dev_err(&rcfw->pdev->dev,
629 "QPLIB: CMDQ BAR region %d mapping failed",
630 rcfw->cmdq_bar_reg);
631 return -ENOMEM;
632 }
633
634 rcfw->cmdq_bar_reg_prod_off = virt_fn ? RCFW_VF_COMM_PROD_OFFSET :
635 RCFW_PF_COMM_PROD_OFFSET;
636
637 rcfw->cmdq_bar_reg_trig_off = RCFW_COMM_TRIG_OFFSET;
638
639 /* CRSQ */
640 rcfw->crsq.prod = 0;
641 rcfw->crsq.cons = 0;
642
643 /* CREQ */
644 rcfw->creq_bar_reg = RCFW_COMM_CONS_PCI_BAR_REGION;
645 res_base = pci_resource_start(pdev, rcfw->creq_bar_reg);
646 if (!res_base)
647 dev_err(&rcfw->pdev->dev,
648 "QPLIB: CREQ BAR region %d resc start is 0!",
649 rcfw->creq_bar_reg);
650 rcfw->creq_bar_reg_iomem = ioremap_nocache(res_base + cp_bar_reg_off,
651 4);
652 if (!rcfw->creq_bar_reg_iomem) {
653 dev_err(&rcfw->pdev->dev,
654 "QPLIB: CREQ BAR region %d mapping failed",
655 rcfw->creq_bar_reg);
656 return -ENOMEM;
657 }
658 rcfw->creq_qp_event_processed = 0;
659 rcfw->creq_func_event_processed = 0;
660
661 rcfw->vector = msix_vector;
662 if (aeq_handler)
663 rcfw->aeq_handler = aeq_handler;
664
665 tasklet_init(&rcfw->worker, bnxt_qplib_service_creq,
666 (unsigned long)rcfw);
667
668 rcfw->requested = false;
669 rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0,
670 "bnxt_qplib_creq", rcfw);
671 if (rc) {
672 dev_err(&rcfw->pdev->dev,
673 "QPLIB: Failed to request IRQ for CREQ rc = 0x%x", rc);
674 bnxt_qplib_disable_rcfw_channel(rcfw);
675 return rc;
676 }
677 rcfw->requested = true;
678
679 init_waitqueue_head(&rcfw->waitq);
680
681 CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, 0, rcfw->creq.max_elements);
682
683 init.cmdq_pbl = cpu_to_le64(rcfw->cmdq.pbl[PBL_LVL_0].pg_map_arr[0]);
684 init.cmdq_size_cmdq_lvl = cpu_to_le16(
685 ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) &
686 CMDQ_INIT_CMDQ_SIZE_MASK) |
687 ((rcfw->cmdq.level << CMDQ_INIT_CMDQ_LVL_SFT) &
688 CMDQ_INIT_CMDQ_LVL_MASK));
689 init.creq_ring_id = cpu_to_le16(rcfw->creq_ring_id);
690
691 /* Write to the Bono mailbox register */
692 __iowrite32_copy(rcfw->cmdq_bar_reg_iomem, &init, sizeof(init) / 4);
693 return 0;
694}
drivers/infiniband/hw/bnxt_re/qplib_rcfw.h+231−0
@@ -0,0 +1,231 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: RDMA Controller HW interface (header)
37 */
38
39#ifndef __BNXT_QPLIB_RCFW_H__
40#define __BNXT_QPLIB_RCFW_H__
41
42#define RCFW_CMDQ_TRIG_VAL 1
43#define RCFW_COMM_PCI_BAR_REGION 0
44#define RCFW_COMM_CONS_PCI_BAR_REGION 2
45#define RCFW_COMM_BASE_OFFSET 0x600
46#define RCFW_PF_COMM_PROD_OFFSET 0xc
47#define RCFW_VF_COMM_PROD_OFFSET 0xc
48#define RCFW_COMM_TRIG_OFFSET 0x100
49#define RCFW_COMM_SIZE 0x104
50
51#define RCFW_DBR_PCI_BAR_REGION 2
52
53#define RCFW_CMD_PREP(req, CMD, cmd_flags) \
54 do { \
55 memset(&(req), 0, sizeof((req))); \
56 (req).opcode = CMDQ_BASE_OPCODE_##CMD; \
57 (req).cmd_size = (sizeof((req)) + \
58 BNXT_QPLIB_CMDQE_UNITS - 1) / \
59 BNXT_QPLIB_CMDQE_UNITS; \
60 (req).flags = cpu_to_le16(cmd_flags); \
61 } while (0)
62
63#define RCFW_CMD_WAIT_TIME_MS 20000 /* 20 Seconds timeout */
64
65/* CMDQ elements */
66#define BNXT_QPLIB_CMDQE_MAX_CNT 256
67#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe)
68#define BNXT_QPLIB_CMDQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CMDQE_UNITS)
69
70#define MAX_CMDQ_IDX (BNXT_QPLIB_CMDQE_MAX_CNT - 1)
71#define MAX_CMDQ_IDX_PER_PG (BNXT_QPLIB_CMDQE_CNT_PER_PG - 1)
72
73#define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT
74#define RCFW_MAX_COOKIE_VALUE 0x7FFF
75#define RCFW_CMD_IS_BLOCKING 0x8000
76
77/* Cmdq contains a fix number of a 16-Byte slots */
78struct bnxt_qplib_cmdqe {
79 u8 data[16];
80};
81
82static inline u32 get_cmdq_pg(u32 val)
83{
84 return (val & ~MAX_CMDQ_IDX_PER_PG) / BNXT_QPLIB_CMDQE_CNT_PER_PG;
85}
86
87static inline u32 get_cmdq_idx(u32 val)
88{
89 return val & MAX_CMDQ_IDX_PER_PG;
90}
91
92/* Crsq buf is 1024-Byte */
93struct bnxt_qplib_crsbe {
94 u8 data[1024];
95};
96
97/* CRSQ SB */
98#define BNXT_QPLIB_CRSBE_MAX_CNT 4
99#define BNXT_QPLIB_CRSBE_UNITS sizeof(struct bnxt_qplib_crsbe)
100#define BNXT_QPLIB_CRSBE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CRSBE_UNITS)
101
102#define MAX_CRSB_IDX (BNXT_QPLIB_CRSBE_MAX_CNT - 1)
103#define MAX_CRSB_IDX_PER_PG (BNXT_QPLIB_CRSBE_CNT_PER_PG - 1)
104
105static inline u32 get_crsb_pg(u32 val)
106{
107 return (val & ~MAX_CRSB_IDX_PER_PG) / BNXT_QPLIB_CRSBE_CNT_PER_PG;
108}
109
110static inline u32 get_crsb_idx(u32 val)
111{
112 return val & MAX_CRSB_IDX_PER_PG;
113}
114
115static inline void bnxt_qplib_crsb_dma_next(dma_addr_t *pg_map_arr,
116 u32 prod, dma_addr_t *dma_addr)
117{
118 *dma_addr = pg_map_arr[(prod) / BNXT_QPLIB_CRSBE_CNT_PER_PG];
119 *dma_addr += ((prod) % BNXT_QPLIB_CRSBE_CNT_PER_PG) *
120 BNXT_QPLIB_CRSBE_UNITS;
121}
122
123/* CREQ */
124/* Allocate 1 per QP for async error notification for now */
125#define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024)
126#define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */
127#define BNXT_QPLIB_CREQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CREQE_UNITS)
128
129#define MAX_CREQ_IDX (BNXT_QPLIB_CREQE_MAX_CNT - 1)
130#define MAX_CREQ_IDX_PER_PG (BNXT_QPLIB_CREQE_CNT_PER_PG - 1)
131
132static inline u32 get_creq_pg(u32 val)
133{
134 return (val & ~MAX_CREQ_IDX_PER_PG) / BNXT_QPLIB_CREQE_CNT_PER_PG;
135}
136
137static inline u32 get_creq_idx(u32 val)
138{
139 return val & MAX_CREQ_IDX_PER_PG;
140}
141
142#define BNXT_QPLIB_CREQE_PER_PG (PAGE_SIZE / sizeof(struct creq_base))
143
144#define CREQ_CMP_VALID(hdr, raw_cons, cp_bit) \
145 (!!((hdr)->v & CREQ_BASE_V) == \
146 !((raw_cons) & (cp_bit)))
147
148#define CREQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT)
149#define CREQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID
150#define CREQ_DB_IRQ_DIS CMPL_DOORBELL_MASK
151#define CREQ_DB_CP_FLAGS_REARM (CREQ_DB_KEY_CP | \
152 CREQ_DB_IDX_VALID)
153#define CREQ_DB_CP_FLAGS (CREQ_DB_KEY_CP | \
154 CREQ_DB_IDX_VALID | \
155 CREQ_DB_IRQ_DIS)
156#define CREQ_DB_REARM(db, raw_cons, cp_bit) \
157 writel(CREQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db)
158#define CREQ_DB(db, raw_cons, cp_bit) \
159 writel(CREQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db)
160
161/* HWQ */
162struct bnxt_qplib_crsqe {
163 struct creq_qp_event qp_event;
164 u32 req_size;
165};
166
167struct bnxt_qplib_crsq {
168 struct bnxt_qplib_crsqe *crsq;
169 u32 prod;
170 u32 cons;
171 u32 max_elements;
172};
173
174/* RCFW Communication Channels */
175struct bnxt_qplib_rcfw {
176 struct pci_dev *pdev;
177 int vector;
178 struct tasklet_struct worker;
179 bool requested;
180 unsigned long *cmdq_bitmap;
181 u32 bmap_size;
182 unsigned long flags;
183#define FIRMWARE_INITIALIZED_FLAG 1
184#define FIRMWARE_FIRST_FLAG BIT(31)
185 wait_queue_head_t waitq;
186 int (*aeq_handler)(struct bnxt_qplib_rcfw *,
187 struct creq_func_event *);
188 atomic_t seq_num;
189
190 /* Bar region info */
191 void __iomem *cmdq_bar_reg_iomem;
192 u16 cmdq_bar_reg;
193 u16 cmdq_bar_reg_prod_off;
194 u16 cmdq_bar_reg_trig_off;
195 u16 creq_ring_id;
196 u16 creq_bar_reg;
197 void __iomem *creq_bar_reg_iomem;
198
199 /* Cmd-Resp and Async Event notification queue */
200 struct bnxt_qplib_hwq creq;
201 u64 creq_qp_event_processed;
202 u64 creq_func_event_processed;
203
204 /* Actual Cmd and Resp Queues */
205 struct bnxt_qplib_hwq cmdq;
206 struct bnxt_qplib_crsq crsq;
207 struct bnxt_qplib_hwq crsb;
208};
209
210void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
211int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
212 struct bnxt_qplib_rcfw *rcfw);
213void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
214int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
215 struct bnxt_qplib_rcfw *rcfw,
216 int msix_vector,
217 int cp_bar_reg_off, int virt_fn,
218 int (*aeq_handler)
219 (struct bnxt_qplib_rcfw *,
220 struct creq_func_event *));
221
222int bnxt_qplib_rcfw_block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie);
223int bnxt_qplib_rcfw_wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie);
224void *bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
225 struct cmdq_base *req, void **crsbe,
226 u8 is_block);
227
228int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw);
229int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
230 struct bnxt_qplib_ctx *ctx, int is_virtfn);
231#endif /* __BNXT_QPLIB_RCFW_H__ */
drivers/infiniband/hw/bnxt_re/qplib_res.c+825−0
@@ -0,0 +1,825 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: QPLib resource manager
37 */
38
39#include <linux/spinlock.h>
40#include <linux/pci.h>
41#include <linux/interrupt.h>
42#include <linux/inetdevice.h>
43#include <linux/dma-mapping.h>
44#include <linux/if_vlan.h>
45#include "roce_hsi.h"
46#include "qplib_res.h"
47#include "qplib_sp.h"
48#include "qplib_rcfw.h"
49
50static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
51 struct bnxt_qplib_stats *stats);
52static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
53 struct bnxt_qplib_stats *stats);
54
55/* PBL */
56static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
57 bool is_umem)
58{
59 int i;
60
61 if (!is_umem) {
62 for (i = 0; i < pbl->pg_count; i++) {
63 if (pbl->pg_arr[i])
64 dma_free_coherent(&pdev->dev, pbl->pg_size,
65 (void *)((unsigned long)
66 pbl->pg_arr[i] &
67 PAGE_MASK),
68 pbl->pg_map_arr[i]);
69 else
70 dev_warn(&pdev->dev,
71 "QPLIB: PBL free pg_arr[%d] empty?!",
72 i);
73 pbl->pg_arr[i] = NULL;
74 }
75 }
76 kfree(pbl->pg_arr);
77 pbl->pg_arr = NULL;
78 kfree(pbl->pg_map_arr);
79 pbl->pg_map_arr = NULL;
80 pbl->pg_count = 0;
81 pbl->pg_size = 0;
82}
83
84static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
85 struct scatterlist *sghead, u32 pages, u32 pg_size)
86{
87 struct scatterlist *sg;
88 bool is_umem = false;
89 int i;
90
91 /* page ptr arrays */
92 pbl->pg_arr = kcalloc(pages, sizeof(void *), GFP_KERNEL);
93 if (!pbl->pg_arr)
94 return -ENOMEM;
95
96 pbl->pg_map_arr = kcalloc(pages, sizeof(dma_addr_t), GFP_KERNEL);
97 if (!pbl->pg_map_arr) {
98 kfree(pbl->pg_arr);
99 pbl->pg_arr = NULL;
100 return -ENOMEM;
101 }
102 pbl->pg_count = 0;
103 pbl->pg_size = pg_size;
104
105 if (!sghead) {
106 for (i = 0; i < pages; i++) {
107 pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
108 pbl->pg_size,
109 &pbl->pg_map_arr[i],
110 GFP_KERNEL);
111 if (!pbl->pg_arr[i])
112 goto fail;
113 memset(pbl->pg_arr[i], 0, pbl->pg_size);
114 pbl->pg_count++;
115 }
116 } else {
117 i = 0;
118 is_umem = true;
119 for_each_sg(sghead, sg, pages, i) {
120 pbl->pg_map_arr[i] = sg_dma_address(sg);
121 pbl->pg_arr[i] = sg_virt(sg);
122 if (!pbl->pg_arr[i])
123 goto fail;
124
125 pbl->pg_count++;
126 }
127 }
128
129 return 0;
130
131fail:
132 __free_pbl(pdev, pbl, is_umem);
133 return -ENOMEM;
134}
135
136/* HWQ */
137void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)
138{
139 int i;
140
141 if (!hwq->max_elements)
142 return;
143 if (hwq->level >= PBL_LVL_MAX)
144 return;
145
146 for (i = 0; i < hwq->level + 1; i++) {
147 if (i == hwq->level)
148 __free_pbl(pdev, &hwq->pbl[i], hwq->is_user);
149 else
150 __free_pbl(pdev, &hwq->pbl[i], false);
151 }
152
153 hwq->level = PBL_LVL_MAX;
154 hwq->max_elements = 0;
155 hwq->element_size = 0;
156 hwq->prod = 0;
157 hwq->cons = 0;
158 hwq->cp_bit = 0;
159}
160
161/* All HWQs are power of 2 in size */
162int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
163 struct scatterlist *sghead, int nmap,
164 u32 *elements, u32 element_size, u32 aux,
165 u32 pg_size, enum bnxt_qplib_hwq_type hwq_type)
166{
167 u32 pages, slots, size, aux_pages = 0, aux_size = 0;
168 dma_addr_t *src_phys_ptr, **dst_virt_ptr;
169 int i, rc;
170
171 hwq->level = PBL_LVL_MAX;
172
173 slots = roundup_pow_of_two(*elements);
174 if (aux) {
175 aux_size = roundup_pow_of_two(aux);
176 aux_pages = (slots * aux_size) / pg_size;
177 if ((slots * aux_size) % pg_size)
178 aux_pages++;
179 }
180 size = roundup_pow_of_two(element_size);
181
182 if (!sghead) {
183 hwq->is_user = false;
184 pages = (slots * size) / pg_size + aux_pages;
185 if ((slots * size) % pg_size)
186 pages++;
187 if (!pages)
188 return -EINVAL;
189 } else {
190 hwq->is_user = true;
191 pages = nmap;
192 }
193
194 /* Alloc the 1st memory block; can be a PDL/PTL/PBL */
195 if (sghead && (pages == MAX_PBL_LVL_0_PGS))
196 rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], sghead,
197 pages, pg_size);
198 else
199 rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], NULL, 1, pg_size);
200 if (rc)
201 goto fail;
202
203 hwq->level = PBL_LVL_0;
204
205 if (pages > MAX_PBL_LVL_0_PGS) {
206 if (pages > MAX_PBL_LVL_1_PGS) {
207 /* 2 levels of indirection */
208 rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], NULL,
209 MAX_PBL_LVL_1_PGS_FOR_LVL_2, pg_size);
210 if (rc)
211 goto fail;
212 /* Fill in lvl0 PBL */
213 dst_virt_ptr =
214 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
215 src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
216 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
217 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
218 src_phys_ptr[i] | PTU_PDE_VALID;
219 hwq->level = PBL_LVL_1;
220
221 rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_2], sghead,
222 pages, pg_size);
223 if (rc)
224 goto fail;
225
226 /* Fill in lvl1 PBL */
227 dst_virt_ptr =
228 (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
229 src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
230 for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
231 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
232 src_phys_ptr[i] | PTU_PTE_VALID;
233 }
234 if (hwq_type == HWQ_TYPE_QUEUE) {
235 /* Find the last pg of the size */
236 i = hwq->pbl[PBL_LVL_2].pg_count;
237 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
238 PTU_PTE_LAST;
239 if (i > 1)
240 dst_virt_ptr[PTR_PG(i - 2)]
241 [PTR_IDX(i - 2)] |=
242 PTU_PTE_NEXT_TO_LAST;
243 }
244 hwq->level = PBL_LVL_2;
245 } else {
246 u32 flag = hwq_type == HWQ_TYPE_L2_CMPL ? 0 :
247 PTU_PTE_VALID;
248
249 /* 1 level of indirection */
250 rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], sghead,
251 pages, pg_size);
252 if (rc)
253 goto fail;
254 /* Fill in lvl0 PBL */
255 dst_virt_ptr =
256 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
257 src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
258 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) {
259 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
260 src_phys_ptr[i] | flag;
261 }
262 if (hwq_type == HWQ_TYPE_QUEUE) {
263 /* Find the last pg of the size */
264 i = hwq->pbl[PBL_LVL_1].pg_count;
265 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
266 PTU_PTE_LAST;
267 if (i > 1)
268 dst_virt_ptr[PTR_PG(i - 2)]
269 [PTR_IDX(i - 2)] |=
270 PTU_PTE_NEXT_TO_LAST;
271 }
272 hwq->level = PBL_LVL_1;
273 }
274 }
275 hwq->pdev = pdev;
276 spin_lock_init(&hwq->lock);
277 hwq->prod = 0;
278 hwq->cons = 0;
279 *elements = hwq->max_elements = slots;
280 hwq->element_size = size;
281
282 /* For direct access to the elements */
283 hwq->pbl_ptr = hwq->pbl[hwq->level].pg_arr;
284 hwq->pbl_dma_ptr = hwq->pbl[hwq->level].pg_map_arr;
285
286 return 0;
287
288fail:
289 bnxt_qplib_free_hwq(pdev, hwq);
290 return -ENOMEM;
291}
292
293/* Context Tables */
294void bnxt_qplib_free_ctx(struct pci_dev *pdev,
295 struct bnxt_qplib_ctx *ctx)
296{
297 int i;
298
299 bnxt_qplib_free_hwq(pdev, &ctx->qpc_tbl);
300 bnxt_qplib_free_hwq(pdev, &ctx->mrw_tbl);
301 bnxt_qplib_free_hwq(pdev, &ctx->srqc_tbl);
302 bnxt_qplib_free_hwq(pdev, &ctx->cq_tbl);
303 bnxt_qplib_free_hwq(pdev, &ctx->tim_tbl);
304 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
305 bnxt_qplib_free_hwq(pdev, &ctx->tqm_tbl[i]);
306 bnxt_qplib_free_hwq(pdev, &ctx->tqm_pde);
307 bnxt_qplib_free_stats_ctx(pdev, &ctx->stats);
308}
309
310/*
311 * Routine: bnxt_qplib_alloc_ctx
312 * Description:
313 * Context tables are memories which are used by the chip fw.
314 * The 6 tables defined are:
315 * QPC ctx - holds QP states
316 * MRW ctx - holds memory region and window
317 * SRQ ctx - holds shared RQ states
318 * CQ ctx - holds completion queue states
319 * TQM ctx - holds Tx Queue Manager context
320 * TIM ctx - holds timer context
321 * Depending on the size of the tbl requested, either a 1 Page Buffer List
322 * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
323 * instead.
324 * Table might be employed as follows:
325 * For 0 < ctx size <= 1 PAGE, 0 level of ind is used
326 * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
327 * For 512 < ctx size <= MAX, 2 levels of ind is used
328 * Returns:
329 * 0 if success, else -ERRORS
330 */
331int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
332 struct bnxt_qplib_ctx *ctx,
333 bool virt_fn)
334{
335 int i, j, k, rc = 0;
336 int fnz_idx = -1;
337 __le64 **pbl_ptr;
338
339 if (virt_fn)
340 goto stats_alloc;
341
342 /* QPC Tables */
343 ctx->qpc_tbl.max_elements = ctx->qpc_count;
344 rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->qpc_tbl, NULL, 0,
345 &ctx->qpc_tbl.max_elements,
346 BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE, 0,
347 PAGE_SIZE, HWQ_TYPE_CTX);
348 if (rc)
349 goto fail;
350
351 /* MRW Tables */
352 ctx->mrw_tbl.max_elements = ctx->mrw_count;
353 rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->mrw_tbl, NULL, 0,
354 &ctx->mrw_tbl.max_elements,
355 BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE, 0,
356 PAGE_SIZE, HWQ_TYPE_CTX);
357 if (rc)
358 goto fail;
359
360 /* SRQ Tables */
361 ctx->srqc_tbl.max_elements = ctx->srqc_count;
362 rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->srqc_tbl, NULL, 0,
363 &ctx->srqc_tbl.max_elements,
364 BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE, 0,
365 PAGE_SIZE, HWQ_TYPE_CTX);
366 if (rc)
367 goto fail;
368
369 /* CQ Tables */
370 ctx->cq_tbl.max_elements = ctx->cq_count;
371 rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->cq_tbl, NULL, 0,
372 &ctx->cq_tbl.max_elements,
373 BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE, 0,
374 PAGE_SIZE, HWQ_TYPE_CTX);
375 if (rc)
376 goto fail;
377
378 /* TQM Buffer */
379 ctx->tqm_pde.max_elements = 512;
380 rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_pde, NULL, 0,
381 &ctx->tqm_pde.max_elements, sizeof(u64),
382 0, PAGE_SIZE, HWQ_TYPE_CTX);
383 if (rc)
384 goto fail;
385
386 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
387 if (!ctx->tqm_count[i])
388 continue;
389 ctx->tqm_tbl[i].max_elements = ctx->qpc_count *
390 ctx->tqm_count[i];
391 rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_tbl[i], NULL, 0,
392 &ctx->tqm_tbl[i].max_elements, 1,
393 0, PAGE_SIZE, HWQ_TYPE_CTX);
394 if (rc)
395 goto fail;
396 }
397 pbl_ptr = (__le64 **)ctx->tqm_pde.pbl_ptr;
398 for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
399 i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
400 if (!ctx->tqm_tbl[i].max_elements)
401 continue;
402 if (fnz_idx == -1)
403 fnz_idx = i;
404 switch (ctx->tqm_tbl[i].level) {
405 case PBL_LVL_2:
406 for (k = 0; k < ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_count;
407 k++)
408 pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)] =
409 cpu_to_le64(
410 ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_map_arr[k]
411 | PTU_PTE_VALID);
412 break;
413 case PBL_LVL_1:
414 case PBL_LVL_0:
415 default:
416 pbl_ptr[PTR_PG(j)][PTR_IDX(j)] = cpu_to_le64(
417 ctx->tqm_tbl[i].pbl[PBL_LVL_0].pg_map_arr[0] |
418 PTU_PTE_VALID);
419 break;
420 }
421 }
422 if (fnz_idx == -1)
423 fnz_idx = 0;
424 ctx->tqm_pde_level = ctx->tqm_tbl[fnz_idx].level == PBL_LVL_2 ?
425 PBL_LVL_2 : ctx->tqm_tbl[fnz_idx].level + 1;
426
427 /* TIM Buffer */
428 ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
429 rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tim_tbl, NULL, 0,
430 &ctx->tim_tbl.max_elements, 1,
431 0, PAGE_SIZE, HWQ_TYPE_CTX);
432 if (rc)
433 goto fail;
434
435stats_alloc:
436 /* Stats */
437 rc = bnxt_qplib_alloc_stats_ctx(pdev, &ctx->stats);
438 if (rc)
439 goto fail;
440
441 return 0;
442
443fail:
444 bnxt_qplib_free_ctx(pdev, ctx);
445 return rc;
446}
447
448/* GUID */
449void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid)
450{
451 u8 mac[ETH_ALEN];
452
453 /* MAC-48 to EUI-64 mapping */
454 memcpy(mac, dev_addr, ETH_ALEN);
455 guid[0] = mac[0] ^ 2;
456 guid[1] = mac[1];
457 guid[2] = mac[2];
458 guid[3] = 0xff;
459 guid[4] = 0xfe;
460 guid[5] = mac[3];
461 guid[6] = mac[4];
462 guid[7] = mac[5];
463}
464
465static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,
466 struct bnxt_qplib_sgid_tbl *sgid_tbl)
467{
468 kfree(sgid_tbl->tbl);
469 kfree(sgid_tbl->hw_id);
470 kfree(sgid_tbl->ctx);
471 sgid_tbl->tbl = NULL;
472 sgid_tbl->hw_id = NULL;
473 sgid_tbl->ctx = NULL;
474 sgid_tbl->max = 0;
475 sgid_tbl->active = 0;
476}
477
478static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,
479 struct bnxt_qplib_sgid_tbl *sgid_tbl,
480 u16 max)
481{
482 sgid_tbl->tbl = kcalloc(max, sizeof(struct bnxt_qplib_gid), GFP_KERNEL);
483 if (!sgid_tbl->tbl)
484 return -ENOMEM;
485
486 sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL);
487 if (!sgid_tbl->hw_id)
488 goto out_free1;
489
490 sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
491 if (!sgid_tbl->ctx)
492 goto out_free2;
493
494 sgid_tbl->max = max;
495 return 0;
496out_free2:
497 kfree(sgid_tbl->hw_id);
498 sgid_tbl->hw_id = NULL;
499out_free1:
500 kfree(sgid_tbl->tbl);
501 sgid_tbl->tbl = NULL;
502 return -ENOMEM;
503};
504
505static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
506 struct bnxt_qplib_sgid_tbl *sgid_tbl)
507{
508 int i;
509
510 for (i = 0; i < sgid_tbl->max; i++) {
511 if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
512 sizeof(bnxt_qplib_gid_zero)))
513 bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i], true);
514 }
515 memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max);
516 memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
517 sgid_tbl->active = 0;
518}
519
520static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
521 struct net_device *netdev)
522{
523 memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max);
524 memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
525}
526
527static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res,
528 struct bnxt_qplib_pkey_tbl *pkey_tbl)
529{
530 if (!pkey_tbl->tbl)
531 dev_dbg(&res->pdev->dev, "QPLIB: PKEY tbl not present");
532 else
533 kfree(pkey_tbl->tbl);
534
535 pkey_tbl->tbl = NULL;
536 pkey_tbl->max = 0;
537 pkey_tbl->active = 0;
538}
539
540static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res,
541 struct bnxt_qplib_pkey_tbl *pkey_tbl,
542 u16 max)
543{
544 pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL);
545 if (!pkey_tbl->tbl)
546 return -ENOMEM;
547
548 pkey_tbl->max = max;
549 return 0;
550};
551
552/* PDs */
553int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd)
554{
555 u32 bit_num;
556
557 bit_num = find_first_bit(pdt->tbl, pdt->max);
558 if (bit_num == pdt->max)
559 return -ENOMEM;
560
561 /* Found unused PD */
562 clear_bit(bit_num, pdt->tbl);
563 pd->id = bit_num;
564 return 0;
565}
566
567int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
568 struct bnxt_qplib_pd_tbl *pdt,
569 struct bnxt_qplib_pd *pd)
570{
571 if (test_and_set_bit(pd->id, pdt->tbl)) {
572 dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d",
573 pd->id);
574 return -EINVAL;
575 }
576 pd->id = 0;
577 return 0;
578}
579
580static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
581{
582 kfree(pdt->tbl);
583 pdt->tbl = NULL;
584 pdt->max = 0;
585}
586
587static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
588 struct bnxt_qplib_pd_tbl *pdt,
589 u32 max)
590{
591 u32 bytes;
592
593 bytes = max >> 3;
594 if (!bytes)
595 bytes = 1;
596 pdt->tbl = kmalloc(bytes, GFP_KERNEL);
597 if (!pdt->tbl)
598 return -ENOMEM;
599
600 pdt->max = max;
601 memset((u8 *)pdt->tbl, 0xFF, bytes);
602
603 return 0;
604}
605
606/* DPIs */
607int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,
608 struct bnxt_qplib_dpi *dpi,
609 void *app)
610{
611 u32 bit_num;
612
613 bit_num = find_first_bit(dpit->tbl, dpit->max);
614 if (bit_num == dpit->max)
615 return -ENOMEM;
616
617 /* Found unused DPI */
618 clear_bit(bit_num, dpit->tbl);
619 dpit->app_tbl[bit_num] = app;
620
621 dpi->dpi = bit_num;
622 dpi->dbr = dpit->dbr_bar_reg_iomem + (bit_num * PAGE_SIZE);
623 dpi->umdbr = dpit->unmapped_dbr + (bit_num * PAGE_SIZE);
624
625 return 0;
626}
627
628int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
629 struct bnxt_qplib_dpi_tbl *dpit,
630 struct bnxt_qplib_dpi *dpi)
631{
632 if (dpi->dpi >= dpit->max) {
633 dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d", dpi->dpi);
634 return -EINVAL;
635 }
636 if (test_and_set_bit(dpi->dpi, dpit->tbl)) {
637 dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d",
638 dpi->dpi);
639 return -EINVAL;
640 }
641 if (dpit->app_tbl)
642 dpit->app_tbl[dpi->dpi] = NULL;
643 memset(dpi, 0, sizeof(*dpi));
644
645 return 0;
646}
647
648static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res *res,
649 struct bnxt_qplib_dpi_tbl *dpit)
650{
651 kfree(dpit->tbl);
652 kfree(dpit->app_tbl);
653 if (dpit->dbr_bar_reg_iomem)
654 pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
655 memset(dpit, 0, sizeof(*dpit));
656}
657
658static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
659 struct bnxt_qplib_dpi_tbl *dpit,
660 u32 dbr_offset)
661{
662 u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION;
663 resource_size_t bar_reg_base;
664 u32 dbr_len, bytes;
665
666 if (dpit->dbr_bar_reg_iomem) {
667 dev_err(&res->pdev->dev,
668 "QPLIB: DBR BAR region %d already mapped", dbr_bar_reg);
669 return -EALREADY;
670 }
671
672 bar_reg_base = pci_resource_start(res->pdev, dbr_bar_reg);
673 if (!bar_reg_base) {
674 dev_err(&res->pdev->dev,
675 "QPLIB: BAR region %d resc start failed", dbr_bar_reg);
676 return -ENOMEM;
677 }
678
679 dbr_len = pci_resource_len(res->pdev, dbr_bar_reg) - dbr_offset;
680 if (!dbr_len || ((dbr_len & (PAGE_SIZE - 1)) != 0)) {
681 dev_err(&res->pdev->dev, "QPLIB: Invalid DBR length %d",
682 dbr_len);
683 return -ENOMEM;
684 }
685
686 dpit->dbr_bar_reg_iomem = ioremap_nocache(bar_reg_base + dbr_offset,
687 dbr_len);
688 if (!dpit->dbr_bar_reg_iomem) {
689 dev_err(&res->pdev->dev,
690 "QPLIB: FP: DBR BAR region %d mapping failed",
691 dbr_bar_reg);
692 return -ENOMEM;
693 }
694
695 dpit->unmapped_dbr = bar_reg_base + dbr_offset;
696 dpit->max = dbr_len / PAGE_SIZE;
697
698 dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL);
699 if (!dpit->app_tbl) {
700 pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
701 dev_err(&res->pdev->dev,
702 "QPLIB: DPI app tbl allocation failed");
703 return -ENOMEM;
704 }
705
706 bytes = dpit->max >> 3;
707 if (!bytes)
708 bytes = 1;
709
710 dpit->tbl = kmalloc(bytes, GFP_KERNEL);
711 if (!dpit->tbl) {
712 pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
713 kfree(dpit->app_tbl);
714 dpit->app_tbl = NULL;
715 dev_err(&res->pdev->dev,
716 "QPLIB: DPI tbl allocation failed for size = %d",
717 bytes);
718 return -ENOMEM;
719 }
720
721 memset((u8 *)dpit->tbl, 0xFF, bytes);
722
723 return 0;
724}
725
726/* PKEYs */
727static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl)
728{
729 memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
730 pkey_tbl->active = 0;
731}
732
733static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res,
734 struct bnxt_qplib_pkey_tbl *pkey_tbl)
735{
736 u16 pkey = 0xFFFF;
737
738 memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
739
740 /* pkey default = 0xFFFF */
741 bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false);
742}
743
744/* Stats */
745static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
746 struct bnxt_qplib_stats *stats)
747{
748 if (stats->dma) {
749 dma_free_coherent(&pdev->dev, stats->size,
750 stats->dma, stats->dma_map);
751 }
752 memset(stats, 0, sizeof(*stats));
753 stats->fw_id = -1;
754}
755
756static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
757 struct bnxt_qplib_stats *stats)
758{
759 memset(stats, 0, sizeof(*stats));
760 stats->fw_id = -1;
761 stats->size = sizeof(struct ctx_hw_stats);
762 stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
763 &stats->dma_map, GFP_KERNEL);
764 if (!stats->dma) {
765 dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed");
766 return -ENOMEM;
767 }
768 return 0;
769}
770
771void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)
772{
773 bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl);
774 bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
775}
776
777int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
778{
779 bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
780 bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl);
781
782 return 0;
783}
784
785void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
786{
787 bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl);
788 bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
789 bnxt_qplib_free_pd_tbl(&res->pd_tbl);
790 bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
791
792 res->netdev = NULL;
793 res->pdev = NULL;
794}
795
796int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,
797 struct net_device *netdev,
798 struct bnxt_qplib_dev_attr *dev_attr)
799{
800 int rc = 0;
801
802 res->pdev = pdev;
803 res->netdev = netdev;
804
805 rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid);
806 if (rc)
807 goto fail;
808
809 rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey);
810 if (rc)
811 goto fail;
812
813 rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);
814 if (rc)
815 goto fail;
816
817 rc = bnxt_qplib_alloc_dpi_tbl(res, &res->dpi_tbl, dev_attr->l2_db_size);
818 if (rc)
819 goto fail;
820
821 return 0;
822fail:
823 bnxt_qplib_free_res(res);
824 return rc;
825}
drivers/infiniband/hw/bnxt_re/qplib_res.h+223−0
@@ -0,0 +1,223 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: QPLib resource manager (header)
37 */
38
39#ifndef __BNXT_QPLIB_RES_H__
40#define __BNXT_QPLIB_RES_H__
41
42extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;
43
44#define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *))
45#define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1)
46#define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG)
47#define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG)
48
49#define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1))
50
51enum bnxt_qplib_hwq_type {
52 HWQ_TYPE_CTX,
53 HWQ_TYPE_QUEUE,
54 HWQ_TYPE_L2_CMPL
55};
56
57#define MAX_PBL_LVL_0_PGS 1
58#define MAX_PBL_LVL_1_PGS 512
59#define MAX_PBL_LVL_1_PGS_SHIFT 9
60#define MAX_PBL_LVL_1_PGS_FOR_LVL_2 256
61#define MAX_PBL_LVL_2_PGS (256 * 512)
62
63enum bnxt_qplib_pbl_lvl {
64 PBL_LVL_0,
65 PBL_LVL_1,
66 PBL_LVL_2,
67 PBL_LVL_MAX
68};
69
70#define ROCE_PG_SIZE_4K (4 * 1024)
71#define ROCE_PG_SIZE_8K (8 * 1024)
72#define ROCE_PG_SIZE_64K (64 * 1024)
73#define ROCE_PG_SIZE_2M (2 * 1024 * 1024)
74#define ROCE_PG_SIZE_8M (8 * 1024 * 1024)
75#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024)
76
77struct bnxt_qplib_pbl {
78 u32 pg_count;
79 u32 pg_size;
80 void **pg_arr;
81 dma_addr_t *pg_map_arr;
82};
83
84struct bnxt_qplib_hwq {
85 struct pci_dev *pdev;
86 /* lock to protect qplib_hwq */
87 spinlock_t lock;
88 struct bnxt_qplib_pbl pbl[PBL_LVL_MAX];
89 enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */
90 /* ptr for easy access to the PBL entries */
91 void **pbl_ptr;
92 /* ptr for easy access to the dma_addr */
93 dma_addr_t *pbl_dma_ptr;
94 u32 max_elements;
95 u16 element_size; /* Size of each entry */
96
97 u32 prod; /* raw */
98 u32 cons; /* raw */
99 u8 cp_bit;
100 u8 is_user;
101};
102
103/* Tables */
104struct bnxt_qplib_pd_tbl {
105 unsigned long *tbl;
106 u32 max;
107};
108
109struct bnxt_qplib_sgid_tbl {
110 struct bnxt_qplib_gid *tbl;
111 u16 *hw_id;
112 u16 max;
113 u16 active;
114 void *ctx;
115};
116
117struct bnxt_qplib_pkey_tbl {
118 u16 *tbl;
119 u16 max;
120 u16 active;
121};
122
123struct bnxt_qplib_dpi {
124 u32 dpi;
125 void __iomem *dbr;
126 u64 umdbr;
127};
128
129struct bnxt_qplib_dpi_tbl {
130 void **app_tbl;
131 unsigned long *tbl;
132 u16 max;
133 void __iomem *dbr_bar_reg_iomem;
134 u64 unmapped_dbr;
135};
136
137struct bnxt_qplib_stats {
138 dma_addr_t dma_map;
139 void *dma;
140 u32 size;
141 u32 fw_id;
142};
143
144struct bnxt_qplib_vf_res {
145 u32 max_qp_per_vf;
146 u32 max_mrw_per_vf;
147 u32 max_srq_per_vf;
148 u32 max_cq_per_vf;
149 u32 max_gid_per_vf;
150};
151
152#define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448
153#define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64
154#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64
155#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128
156
157struct bnxt_qplib_ctx {
158 u32 qpc_count;
159 struct bnxt_qplib_hwq qpc_tbl;
160 u32 mrw_count;
161 struct bnxt_qplib_hwq mrw_tbl;
162 u32 srqc_count;
163 struct bnxt_qplib_hwq srqc_tbl;
164 u32 cq_count;
165 struct bnxt_qplib_hwq cq_tbl;
166 struct bnxt_qplib_hwq tim_tbl;
167#define MAX_TQM_ALLOC_REQ 32
168#define MAX_TQM_ALLOC_BLK_SIZE 8
169 u8 tqm_count[MAX_TQM_ALLOC_REQ];
170 struct bnxt_qplib_hwq tqm_pde;
171 u32 tqm_pde_level;
172 struct bnxt_qplib_hwq tqm_tbl[MAX_TQM_ALLOC_REQ];
173 struct bnxt_qplib_stats stats;
174 struct bnxt_qplib_vf_res vf_res;
175};
176
177struct bnxt_qplib_res {
178 struct pci_dev *pdev;
179 struct net_device *netdev;
180
181 struct bnxt_qplib_rcfw *rcfw;
182
183 struct bnxt_qplib_pd_tbl pd_tbl;
184 struct bnxt_qplib_sgid_tbl sgid_tbl;
185 struct bnxt_qplib_pkey_tbl pkey_tbl;
186 struct bnxt_qplib_dpi_tbl dpi_tbl;
187};
188
189#define to_bnxt_qplib(ptr, type, member) \
190 container_of(ptr, type, member)
191
192struct bnxt_qplib_pd;
193struct bnxt_qplib_dev_attr;
194
195void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq);
196int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
197 struct scatterlist *sl, int nmap, u32 *elements,
198 u32 elements_per_page, u32 aux, u32 pg_size,
199 enum bnxt_qplib_hwq_type hwq_type);
200void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid);
201int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pd_tbl,
202 struct bnxt_qplib_pd *pd);
203int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
204 struct bnxt_qplib_pd_tbl *pd_tbl,
205 struct bnxt_qplib_pd *pd);
206int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,
207 struct bnxt_qplib_dpi *dpi,
208 void *app);
209int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
210 struct bnxt_qplib_dpi_tbl *dpi_tbl,
211 struct bnxt_qplib_dpi *dpi);
212void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res);
213int bnxt_qplib_init_res(struct bnxt_qplib_res *res);
214void bnxt_qplib_free_res(struct bnxt_qplib_res *res);
215int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,
216 struct net_device *netdev,
217 struct bnxt_qplib_dev_attr *dev_attr);
218void bnxt_qplib_free_ctx(struct pci_dev *pdev,
219 struct bnxt_qplib_ctx *ctx);
220int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
221 struct bnxt_qplib_ctx *ctx,
222 bool virt_fn);
223#endif /* __BNXT_QPLIB_RES_H__ */
drivers/infiniband/hw/bnxt_re/qplib_sp.c+838−0
@@ -0,0 +1,838 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Slow Path Operators
37 */
38
39#include <linux/interrupt.h>
40#include <linux/spinlock.h>
41#include <linux/sched.h>
42#include <linux/pci.h>
43
44#include "roce_hsi.h"
45
46#include "qplib_res.h"
47#include "qplib_rcfw.h"
48#include "qplib_sp.h"
49
50const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,
51 0, 0, 0, 0, 0, 0, 0, 0 } };
52
53/* Device */
54int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
55 struct bnxt_qplib_dev_attr *attr)
56{
57 struct cmdq_query_func req;
58 struct creq_query_func_resp *resp;
59 struct creq_query_func_resp_sb *sb;
60 u16 cmd_flags = 0;
61 u32 temp;
62 u8 *tqm_alloc;
63 int i;
64
65 RCFW_CMD_PREP(req, QUERY_FUNC, cmd_flags);
66
67 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
68 resp = (struct creq_query_func_resp *)
69 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void **)&sb,
70 0);
71 if (!resp) {
72 dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC send failed");
73 return -EINVAL;
74 }
75 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
76 /* Cmd timed out */
77 dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC timed out");
78 return -ETIMEDOUT;
79 }
80 if (resp->status ||
81 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
82 dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC failed ");
83 dev_err(&rcfw->pdev->dev,
84 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
85 resp->status, le16_to_cpu(req.cookie),
86 le16_to_cpu(resp->cookie));
87 return -EINVAL;
88 }
89 /* Extract the context from the side buffer */
90 attr->max_qp = le32_to_cpu(sb->max_qp);
91 attr->max_qp_rd_atom =
92 sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
93 BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom;
94 attr->max_qp_init_rd_atom =
95 sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
96 BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom;
97 attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr);
98 attr->max_qp_sges = sb->max_sge;
99 attr->max_cq = le32_to_cpu(sb->max_cq);
100 attr->max_cq_wqes = le32_to_cpu(sb->max_cqe);
101 attr->max_cq_sges = attr->max_qp_sges;
102 attr->max_mr = le32_to_cpu(sb->max_mr);
103 attr->max_mw = le32_to_cpu(sb->max_mw);
104
105 attr->max_mr_size = le64_to_cpu(sb->max_mr_size);
106 attr->max_pd = 64 * 1024;
107 attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp);
108 attr->max_ah = le32_to_cpu(sb->max_ah);
109
110 attr->max_fmr = le32_to_cpu(sb->max_fmr);
111 attr->max_map_per_fmr = sb->max_map_per_fmr;
112
113 attr->max_srq = le16_to_cpu(sb->max_srq);
114 attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
115 attr->max_srq_sges = sb->max_srq_sge;
116 /* Bono only reports 1 PKEY for now, but it can support > 1 */
117 attr->max_pkey = le32_to_cpu(sb->max_pkeys);
118
119 attr->max_inline_data = le32_to_cpu(sb->max_inline_data);
120 attr->l2_db_size = (sb->l2_db_space_size + 1) * PAGE_SIZE;
121 attr->max_sgid = le32_to_cpu(sb->max_gid);
122
123 strlcpy(attr->fw_ver, "20.6.28.0", sizeof(attr->fw_ver));
124
125 for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) {
126 temp = le32_to_cpu(sb->tqm_alloc_reqs[i]);
127 tqm_alloc = (u8 *)&temp;
128 attr->tqm_alloc_reqs[i * 4] = *tqm_alloc;
129 attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc);
130 attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc);
131 attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc);
132 }
133 return 0;
134}
135
136/* SGID */
137int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
138 struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
139 struct bnxt_qplib_gid *gid)
140{
141 if (index > sgid_tbl->max) {
142 dev_err(&res->pdev->dev,
143 "QPLIB: Index %d exceeded SGID table max (%d)",
144 index, sgid_tbl->max);
145 return -EINVAL;
146 }
147 memcpy(gid, &sgid_tbl->tbl[index], sizeof(*gid));
148 return 0;
149}
150
151int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
152 struct bnxt_qplib_gid *gid, bool update)
153{
154 struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
155 struct bnxt_qplib_res,
156 sgid_tbl);
157 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
158 int index;
159
160 if (!sgid_tbl) {
161 dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated");
162 return -EINVAL;
163 }
164 /* Do we need a sgid_lock here? */
165 if (!sgid_tbl->active) {
166 dev_err(&res->pdev->dev,
167 "QPLIB: SGID table has no active entries");
168 return -ENOMEM;
169 }
170 for (index = 0; index < sgid_tbl->max; index++) {
171 if (!memcmp(&sgid_tbl->tbl[index], gid, sizeof(*gid)))
172 break;
173 }
174 if (index == sgid_tbl->max) {
175 dev_warn(&res->pdev->dev, "GID not found in the SGID table");
176 return 0;
177 }
178 /* Remove GID from the SGID table */
179 if (update) {
180 struct cmdq_delete_gid req;
181 struct creq_delete_gid_resp *resp;
182 u16 cmd_flags = 0;
183
184 RCFW_CMD_PREP(req, DELETE_GID, cmd_flags);
185 if (sgid_tbl->hw_id[index] == 0xFFFF) {
186 dev_err(&res->pdev->dev,
187 "QPLIB: GID entry contains an invalid HW id");
188 return -EINVAL;
189 }
190 req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]);
191 resp = (struct creq_delete_gid_resp *)
192 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, NULL,
193 0);
194 if (!resp) {
195 dev_err(&res->pdev->dev,
196 "QPLIB: SP: DELETE_GID send failed");
197 return -EINVAL;
198 }
199 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw,
200 le16_to_cpu(req.cookie))) {
201 /* Cmd timed out */
202 dev_err(&res->pdev->dev,
203 "QPLIB: SP: DELETE_GID timed out");
204 return -ETIMEDOUT;
205 }
206 if (resp->status ||
207 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
208 dev_err(&res->pdev->dev,
209 "QPLIB: SP: DELETE_GID failed ");
210 dev_err(&res->pdev->dev,
211 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
212 resp->status, le16_to_cpu(req.cookie),
213 le16_to_cpu(resp->cookie));
214 return -EINVAL;
215 }
216 }
217 memcpy(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero,
218 sizeof(bnxt_qplib_gid_zero));
219 sgid_tbl->active--;
220 dev_dbg(&res->pdev->dev,
221 "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x",
222 index, sgid_tbl->hw_id[index], sgid_tbl->active);
223 sgid_tbl->hw_id[index] = (u16)-1;
224
225 /* unlock */
226 return 0;
227}
228
229int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
230 struct bnxt_qplib_gid *gid, u8 *smac, u16 vlan_id,
231 bool update, u32 *index)
232{
233 struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
234 struct bnxt_qplib_res,
235 sgid_tbl);
236 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
237 int i, free_idx, rc = 0;
238
239 if (!sgid_tbl) {
240 dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated");
241 return -EINVAL;
242 }
243 /* Do we need a sgid_lock here? */
244 if (sgid_tbl->active == sgid_tbl->max) {
245 dev_err(&res->pdev->dev, "QPLIB: SGID table is full");
246 return -ENOMEM;
247 }
248 free_idx = sgid_tbl->max;
249 for (i = 0; i < sgid_tbl->max; i++) {
250 if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid))) {
251 dev_dbg(&res->pdev->dev,
252 "QPLIB: SGID entry already exist in entry %d!",
253 i);
254 *index = i;
255 return -EALREADY;
256 } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
257 sizeof(bnxt_qplib_gid_zero)) &&
258 free_idx == sgid_tbl->max) {
259 free_idx = i;
260 }
261 }
262 if (free_idx == sgid_tbl->max) {
263 dev_err(&res->pdev->dev,
264 "QPLIB: SGID table is FULL but count is not MAX??");
265 return -ENOMEM;
266 }
267 if (update) {
268 struct cmdq_add_gid req;
269 struct creq_add_gid_resp *resp;
270 u16 cmd_flags = 0;
271 u32 temp32[4];
272 u16 temp16[3];
273
274 RCFW_CMD_PREP(req, ADD_GID, cmd_flags);
275
276 memcpy(temp32, gid->data, sizeof(struct bnxt_qplib_gid));
277 req.gid[0] = cpu_to_be32(temp32[3]);
278 req.gid[1] = cpu_to_be32(temp32[2]);
279 req.gid[2] = cpu_to_be32(temp32[1]);
280 req.gid[3] = cpu_to_be32(temp32[0]);
281 if (vlan_id != 0xFFFF)
282 req.vlan = cpu_to_le16((vlan_id &
283 CMDQ_ADD_GID_VLAN_VLAN_ID_MASK) |
284 CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
285 CMDQ_ADD_GID_VLAN_VLAN_EN);
286
287 /* MAC in network format */
288 memcpy(temp16, smac, 6);
289 req.src_mac[0] = cpu_to_be16(temp16[0]);
290 req.src_mac[1] = cpu_to_be16(temp16[1]);
291 req.src_mac[2] = cpu_to_be16(temp16[2]);
292
293 resp = (struct creq_add_gid_resp *)
294 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
295 NULL, 0);
296 if (!resp) {
297 dev_err(&res->pdev->dev,
298 "QPLIB: SP: ADD_GID send failed");
299 return -EINVAL;
300 }
301 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw,
302 le16_to_cpu(req.cookie))) {
303 /* Cmd timed out */
304 dev_err(&res->pdev->dev,
305 "QPIB: SP: ADD_GID timed out");
306 return -ETIMEDOUT;
307 }
308 if (resp->status ||
309 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
310 dev_err(&res->pdev->dev, "QPLIB: SP: ADD_GID failed ");
311 dev_err(&res->pdev->dev,
312 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
313 resp->status, le16_to_cpu(req.cookie),
314 le16_to_cpu(resp->cookie));
315 return -EINVAL;
316 }
317 sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp->xid);
318 }
319 /* Add GID to the sgid_tbl */
320 memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid));
321 sgid_tbl->active++;
322 dev_dbg(&res->pdev->dev,
323 "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x",
324 free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active);
325
326 *index = free_idx;
327 /* unlock */
328 return rc;
329}
330
331/* pkeys */
332int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,
333 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,
334 u16 *pkey)
335{
336 if (index == 0xFFFF) {
337 *pkey = 0xFFFF;
338 return 0;
339 }
340 if (index > pkey_tbl->max) {
341 dev_err(&res->pdev->dev,
342 "QPLIB: Index %d exceeded PKEY table max (%d)",
343 index, pkey_tbl->max);
344 return -EINVAL;
345 }
346 memcpy(pkey, &pkey_tbl->tbl[index], sizeof(*pkey));
347 return 0;
348}
349
350int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res,
351 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
352 bool update)
353{
354 int i, rc = 0;
355
356 if (!pkey_tbl) {
357 dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated");
358 return -EINVAL;
359 }
360
361 /* Do we need a pkey_lock here? */
362 if (!pkey_tbl->active) {
363 dev_err(&res->pdev->dev,
364 "QPLIB: PKEY table has no active entries");
365 return -ENOMEM;
366 }
367 for (i = 0; i < pkey_tbl->max; i++) {
368 if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey)))
369 break;
370 }
371 if (i == pkey_tbl->max) {
372 dev_err(&res->pdev->dev,
373 "QPLIB: PKEY 0x%04x not found in the pkey table",
374 *pkey);
375 return -ENOMEM;
376 }
377 memset(&pkey_tbl->tbl[i], 0, sizeof(*pkey));
378 pkey_tbl->active--;
379
380 /* unlock */
381 return rc;
382}
383
384int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,
385 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
386 bool update)
387{
388 int i, free_idx, rc = 0;
389
390 if (!pkey_tbl) {
391 dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated");
392 return -EINVAL;
393 }
394
395 /* Do we need a pkey_lock here? */
396 if (pkey_tbl->active == pkey_tbl->max) {
397 dev_err(&res->pdev->dev, "QPLIB: PKEY table is full");
398 return -ENOMEM;
399 }
400 free_idx = pkey_tbl->max;
401 for (i = 0; i < pkey_tbl->max; i++) {
402 if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey)))
403 return -EALREADY;
404 else if (!pkey_tbl->tbl[i] && free_idx == pkey_tbl->max)
405 free_idx = i;
406 }
407 if (free_idx == pkey_tbl->max) {
408 dev_err(&res->pdev->dev,
409 "QPLIB: PKEY table is FULL but count is not MAX??");
410 return -ENOMEM;
411 }
412 /* Add PKEY to the pkey_tbl */
413 memcpy(&pkey_tbl->tbl[free_idx], pkey, sizeof(*pkey));
414 pkey_tbl->active++;
415
416 /* unlock */
417 return rc;
418}
419
420/* AH */
421int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah)
422{
423 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
424 struct cmdq_create_ah req;
425 struct creq_create_ah_resp *resp;
426 u16 cmd_flags = 0;
427 u32 temp32[4];
428 u16 temp16[3];
429
430 RCFW_CMD_PREP(req, CREATE_AH, cmd_flags);
431
432 memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid));
433 req.dgid[0] = cpu_to_le32(temp32[0]);
434 req.dgid[1] = cpu_to_le32(temp32[1]);
435 req.dgid[2] = cpu_to_le32(temp32[2]);
436 req.dgid[3] = cpu_to_le32(temp32[3]);
437
438 req.type = ah->nw_type;
439 req.hop_limit = ah->hop_limit;
440 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]);
441 req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label &
442 CMDQ_CREATE_AH_FLOW_LABEL_MASK) |
443 CMDQ_CREATE_AH_DEST_VLAN_ID_MASK);
444 req.pd_id = cpu_to_le32(ah->pd->id);
445 req.traffic_class = ah->traffic_class;
446
447 /* MAC in network format */
448 memcpy(temp16, ah->dmac, 6);
449 req.dest_mac[0] = cpu_to_le16(temp16[0]);
450 req.dest_mac[1] = cpu_to_le16(temp16[1]);
451 req.dest_mac[2] = cpu_to_le16(temp16[2]);
452
453 resp = (struct creq_create_ah_resp *)
454 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
455 NULL, 1);
456 if (!resp) {
457 dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH send failed");
458 return -EINVAL;
459 }
460 if (!bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie))) {
461 /* Cmd timed out */
462 dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH timed out");
463 return -ETIMEDOUT;
464 }
465 if (resp->status ||
466 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
467 dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH failed ");
468 dev_err(&rcfw->pdev->dev,
469 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
470 resp->status, le16_to_cpu(req.cookie),
471 le16_to_cpu(resp->cookie));
472 return -EINVAL;
473 }
474 ah->id = le32_to_cpu(resp->xid);
475 return 0;
476}
477
478int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah)
479{
480 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
481 struct cmdq_destroy_ah req;
482 struct creq_destroy_ah_resp *resp;
483 u16 cmd_flags = 0;
484
485 /* Clean up the AH table in the device */
486 RCFW_CMD_PREP(req, DESTROY_AH, cmd_flags);
487
488 req.ah_cid = cpu_to_le32(ah->id);
489
490 resp = (struct creq_destroy_ah_resp *)
491 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
492 NULL, 1);
493 if (!resp) {
494 dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH send failed");
495 return -EINVAL;
496 }
497 if (!bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie))) {
498 /* Cmd timed out */
499 dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH timed out");
500 return -ETIMEDOUT;
501 }
502 if (resp->status ||
503 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
504 dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH failed ");
505 dev_err(&rcfw->pdev->dev,
506 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
507 resp->status, le16_to_cpu(req.cookie),
508 le16_to_cpu(resp->cookie));
509 return -EINVAL;
510 }
511 return 0;
512}
513
514/* MRW */
515int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
516{
517 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
518 struct cmdq_deallocate_key req;
519 struct creq_deallocate_key_resp *resp;
520 u16 cmd_flags = 0;
521
522 if (mrw->lkey == 0xFFFFFFFF) {
523 dev_info(&res->pdev->dev,
524 "QPLIB: SP: Free a reserved lkey MRW");
525 return 0;
526 }
527
528 RCFW_CMD_PREP(req, DEALLOCATE_KEY, cmd_flags);
529
530 req.mrw_flags = mrw->type;
531
532 if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) ||
533 (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) ||
534 (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B))
535 req.key = cpu_to_le32(mrw->rkey);
536 else
537 req.key = cpu_to_le32(mrw->lkey);
538
539 resp = (struct creq_deallocate_key_resp *)
540 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
541 NULL, 0);
542 if (!resp) {
543 dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR send failed");
544 return -EINVAL;
545 }
546 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
547 /* Cmd timed out */
548 dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR timed out");
549 return -ETIMEDOUT;
550 }
551 if (resp->status ||
552 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
553 dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR failed ");
554 dev_err(&res->pdev->dev,
555 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
556 resp->status, le16_to_cpu(req.cookie),
557 le16_to_cpu(resp->cookie));
558 return -EINVAL;
559 }
560 /* Free the qplib's MRW memory */
561 if (mrw->hwq.max_elements)
562 bnxt_qplib_free_hwq(res->pdev, &mrw->hwq);
563
564 return 0;
565}
566
567int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
568{
569 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
570 struct cmdq_allocate_mrw req;
571 struct creq_allocate_mrw_resp *resp;
572 u16 cmd_flags = 0;
573 unsigned long tmp;
574
575 RCFW_CMD_PREP(req, ALLOCATE_MRW, cmd_flags);
576
577 req.pd_id = cpu_to_le32(mrw->pd->id);
578 req.mrw_flags = mrw->type;
579 if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR &&
580 mrw->flags & BNXT_QPLIB_FR_PMR) ||
581 mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A ||
582 mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)
583 req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY;
584 tmp = (unsigned long)mrw;
585 req.mrw_handle = cpu_to_le64(tmp);
586
587 resp = (struct creq_allocate_mrw_resp *)
588 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
589 NULL, 0);
590 if (!resp) {
591 dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW send failed");
592 return -EINVAL;
593 }
594 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
595 /* Cmd timed out */
596 dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW timed out");
597 return -ETIMEDOUT;
598 }
599 if (resp->status ||
600 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
601 dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW failed ");
602 dev_err(&rcfw->pdev->dev,
603 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
604 resp->status, le16_to_cpu(req.cookie),
605 le16_to_cpu(resp->cookie));
606 return -EINVAL;
607 }
608 if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) ||
609 (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) ||
610 (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B))
611 mrw->rkey = le32_to_cpu(resp->xid);
612 else
613 mrw->lkey = le32_to_cpu(resp->xid);
614 return 0;
615}
616
617int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
618 bool block)
619{
620 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
621 struct cmdq_deregister_mr req;
622 struct creq_deregister_mr_resp *resp;
623 u16 cmd_flags = 0;
624 int rc;
625
626 RCFW_CMD_PREP(req, DEREGISTER_MR, cmd_flags);
627
628 req.lkey = cpu_to_le32(mrw->lkey);
629 resp = (struct creq_deregister_mr_resp *)
630 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
631 NULL, block);
632 if (!resp) {
633 dev_err(&rcfw->pdev->dev, "QPLIB: SP: DEREG_MR send failed");
634 return -EINVAL;
635 }
636 if (block)
637 rc = bnxt_qplib_rcfw_block_for_resp(rcfw,
638 le16_to_cpu(req.cookie));
639 else
640 rc = bnxt_qplib_rcfw_wait_for_resp(rcfw,
641 le16_to_cpu(req.cookie));
642 if (!rc) {
643 /* Cmd timed out */
644 dev_err(&res->pdev->dev, "QPLIB: SP: DEREG_MR timed out");
645 return -ETIMEDOUT;
646 }
647 if (resp->status ||
648 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
649 dev_err(&rcfw->pdev->dev, "QPLIB: SP: DEREG_MR failed ");
650 dev_err(&rcfw->pdev->dev,
651 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
652 resp->status, le16_to_cpu(req.cookie),
653 le16_to_cpu(resp->cookie));
654 return -EINVAL;
655 }
656
657 /* Free the qplib's MR memory */
658 if (mrw->hwq.max_elements) {
659 mrw->va = 0;
660 mrw->total_size = 0;
661 bnxt_qplib_free_hwq(res->pdev, &mrw->hwq);
662 }
663
664 return 0;
665}
666
667int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
668 u64 *pbl_tbl, int num_pbls, bool block)
669{
670 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
671 struct cmdq_register_mr req;
672 struct creq_register_mr_resp *resp;
673 u16 cmd_flags = 0, level;
674 int pg_ptrs, pages, i, rc;
675 dma_addr_t **pbl_ptr;
676 u32 pg_size;
677
678 if (num_pbls) {
679 pg_ptrs = roundup_pow_of_two(num_pbls);
680 pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT;
681 if (!pages)
682 pages++;
683
684 if (pages > MAX_PBL_LVL_1_PGS) {
685 dev_err(&res->pdev->dev, "QPLIB: SP: Reg MR pages ");
686 dev_err(&res->pdev->dev,
687 "requested (0x%x) exceeded max (0x%x)",
688 pages, MAX_PBL_LVL_1_PGS);
689 return -ENOMEM;
690 }
691 /* Free the hwq if it already exist, must be a rereg */
692 if (mr->hwq.max_elements)
693 bnxt_qplib_free_hwq(res->pdev, &mr->hwq);
694
695 mr->hwq.max_elements = pages;
696 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL, 0,
697 &mr->hwq.max_elements,
698 PAGE_SIZE, 0, PAGE_SIZE,
699 HWQ_TYPE_CTX);
700 if (rc) {
701 dev_err(&res->pdev->dev,
702 "SP: Reg MR memory allocation failed");
703 return -ENOMEM;
704 }
705 /* Write to the hwq */
706 pbl_ptr = (dma_addr_t **)mr->hwq.pbl_ptr;
707 for (i = 0; i < num_pbls; i++)
708 pbl_ptr[PTR_PG(i)][PTR_IDX(i)] =
709 (pbl_tbl[i] & PAGE_MASK) | PTU_PTE_VALID;
710 }
711
712 RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags);
713
714 /* Configure the request */
715 if (mr->hwq.level == PBL_LVL_MAX) {
716 level = 0;
717 req.pbl = 0;
718 pg_size = PAGE_SIZE;
719 } else {
720 level = mr->hwq.level + 1;
721 req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
722 pg_size = mr->hwq.pbl[PBL_LVL_0].pg_size;
723 }
724 req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) |
725 ((ilog2(pg_size) <<
726 CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) &
727 CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK);
728 req.access = (mr->flags & 0xFFFF);
729 req.va = cpu_to_le64(mr->va);
730 req.key = cpu_to_le32(mr->lkey);
731 req.mr_size = cpu_to_le64(mr->total_size);
732
733 resp = (struct creq_register_mr_resp *)
734 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
735 NULL, block);
736 if (!resp) {
737 dev_err(&res->pdev->dev, "SP: REG_MR send failed");
738 rc = -EINVAL;
739 goto fail;
740 }
741 if (block)
742 rc = bnxt_qplib_rcfw_block_for_resp(rcfw,
743 le16_to_cpu(req.cookie));
744 else
745 rc = bnxt_qplib_rcfw_wait_for_resp(rcfw,
746 le16_to_cpu(req.cookie));
747 if (!rc) {
748 /* Cmd timed out */
749 dev_err(&res->pdev->dev, "SP: REG_MR timed out");
750 rc = -ETIMEDOUT;
751 goto fail;
752 }
753 if (resp->status ||
754 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
755 dev_err(&res->pdev->dev, "QPLIB: SP: REG_MR failed ");
756 dev_err(&res->pdev->dev,
757 "QPLIB: SP: with status 0x%x cmdq 0x%x resp 0x%x",
758 resp->status, le16_to_cpu(req.cookie),
759 le16_to_cpu(resp->cookie));
760 rc = -EINVAL;
761 goto fail;
762 }
763 return 0;
764
765fail:
766 if (mr->hwq.max_elements)
767 bnxt_qplib_free_hwq(res->pdev, &mr->hwq);
768 return rc;
769}
770
771int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
772 struct bnxt_qplib_frpl *frpl,
773 int max_pg_ptrs)
774{
775 int pg_ptrs, pages, rc;
776
777 /* Re-calculate the max to fit the HWQ allocation model */
778 pg_ptrs = roundup_pow_of_two(max_pg_ptrs);
779 pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT;
780 if (!pages)
781 pages++;
782
783 if (pages > MAX_PBL_LVL_1_PGS)
784 return -ENOMEM;
785
786 frpl->hwq.max_elements = pages;
787 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &frpl->hwq, NULL, 0,
788 &frpl->hwq.max_elements, PAGE_SIZE, 0,
789 PAGE_SIZE, HWQ_TYPE_CTX);
790 if (!rc)
791 frpl->max_pg_ptrs = pg_ptrs;
792
793 return rc;
794}
795
796int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
797 struct bnxt_qplib_frpl *frpl)
798{
799 bnxt_qplib_free_hwq(res->pdev, &frpl->hwq);
800 return 0;
801}
802
803int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids)
804{
805 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
806 struct cmdq_map_tc_to_cos req;
807 struct creq_map_tc_to_cos_resp *resp;
808 u16 cmd_flags = 0;
809 int tleft;
810
811 RCFW_CMD_PREP(req, MAP_TC_TO_COS, cmd_flags);
812 req.cos0 = cpu_to_le16(cids[0]);
813 req.cos1 = cpu_to_le16(cids[1]);
814
815 resp = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, NULL, 0);
816 if (!resp) {
817 dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS send failed");
818 return -EINVAL;
819 }
820
821 tleft = bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie));
822 if (!tleft) {
823 dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS timed out");
824 return -ETIMEDOUT;
825 }
826
827 if (resp->status ||
828 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
829 dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS failed ");
830 dev_err(&res->pdev->dev,
831 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
832 resp->status, le16_to_cpu(req.cookie),
833 le16_to_cpu(resp->cookie));
834 return -EINVAL;
835 }
836
837 return 0;
838}
drivers/infiniband/hw/bnxt_re/qplib_sp.h+160−0
@@ -0,0 +1,160 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Slow Path Operators (header)
37 *
38 */
39
40#ifndef __BNXT_QPLIB_SP_H__
41#define __BNXT_QPLIB_SP_H__
42
43struct bnxt_qplib_dev_attr {
44 char fw_ver[32];
45 u16 max_sgid;
46 u16 max_mrw;
47 u32 max_qp;
48#define BNXT_QPLIB_MAX_OUT_RD_ATOM 126
49 u32 max_qp_rd_atom;
50 u32 max_qp_init_rd_atom;
51 u32 max_qp_wqes;
52 u32 max_qp_sges;
53 u32 max_cq;
54 u32 max_cq_wqes;
55 u32 max_cq_sges;
56 u32 max_mr;
57 u64 max_mr_size;
58 u32 max_pd;
59 u32 max_mw;
60 u32 max_raw_ethy_qp;
61 u32 max_ah;
62 u32 max_fmr;
63 u32 max_map_per_fmr;
64 u32 max_srq;
65 u32 max_srq_wqes;
66 u32 max_srq_sges;
67 u32 max_pkey;
68 u32 max_inline_data;
69 u32 l2_db_size;
70 u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ];
71};
72
73struct bnxt_qplib_pd {
74 u32 id;
75};
76
77struct bnxt_qplib_gid {
78 u8 data[16];
79};
80
81struct bnxt_qplib_ah {
82 struct bnxt_qplib_gid dgid;
83 struct bnxt_qplib_pd *pd;
84 u32 id;
85 u8 sgid_index;
86 /* For Query AH if the hw table and SW table are differnt */
87 u8 host_sgid_index;
88 u8 traffic_class;
89 u32 flow_label;
90 u8 hop_limit;
91 u8 sl;
92 u8 dmac[6];
93 u16 vlan_id;
94 u8 nw_type;
95};
96
97struct bnxt_qplib_mrw {
98 struct bnxt_qplib_pd *pd;
99 int type;
100 u32 flags;
101#define BNXT_QPLIB_FR_PMR 0x80000000
102 u32 lkey;
103 u32 rkey;
104#define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF
105 u64 va;
106 u64 total_size;
107 u32 npages;
108 u64 mr_handle;
109 struct bnxt_qplib_hwq hwq;
110};
111
112struct bnxt_qplib_frpl {
113 int max_pg_ptrs;
114 struct bnxt_qplib_hwq hwq;
115};
116
117#define BNXT_QPLIB_ACCESS_LOCAL_WRITE BIT(0)
118#define BNXT_QPLIB_ACCESS_REMOTE_READ BIT(1)
119#define BNXT_QPLIB_ACCESS_REMOTE_WRITE BIT(2)
120#define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC BIT(3)
121#define BNXT_QPLIB_ACCESS_MW_BIND BIT(4)
122#define BNXT_QPLIB_ACCESS_ZERO_BASED BIT(5)
123#define BNXT_QPLIB_ACCESS_ON_DEMAND BIT(6)
124
125int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
126 struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
127 struct bnxt_qplib_gid *gid);
128int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
129 struct bnxt_qplib_gid *gid, bool update);
130int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
131 struct bnxt_qplib_gid *gid, u8 *mac, u16 vlan_id,
132 bool update, u32 *index);
133int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,
134 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,
135 u16 *pkey);
136int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res,
137 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
138 bool update);
139int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,
140 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
141 bool update);
142int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
143 struct bnxt_qplib_dev_attr *attr);
144int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah);
145int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah);
146int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res,
147 struct bnxt_qplib_mrw *mrw);
148int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
149 bool block);
150int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
151 u64 *pbl_tbl, int num_pbls, bool block);
152int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr);
153int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res,
154 struct bnxt_qplib_mrw *mr, int max);
155int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
156 struct bnxt_qplib_frpl *frpl, int max);
157int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
158 struct bnxt_qplib_frpl *frpl);
159int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids);
160#endif /* __BNXT_QPLIB_SP_H__*/
drivers/infiniband/hw/bnxt_re/roce_hsi.h+2821−0
@@ -0,0 +1,2821 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: RoCE HSI File - Autogenerated
37 */
38
39#ifndef __BNXT_RE_HSI_H__
40#define __BNXT_RE_HSI_H__
41
42/* include bnxt_hsi.h from bnxt_en driver */
43#include "bnxt_hsi.h"
44
45/* CMP Door Bell Format (4 bytes) */
46struct cmpl_doorbell {
47 __le32 key_mask_valid_idx;
48 #define CMPL_DOORBELL_IDX_MASK 0xffffffUL
49 #define CMPL_DOORBELL_IDX_SFT 0
50 #define CMPL_DOORBELL_RESERVED_MASK 0x3000000UL
51 #define CMPL_DOORBELL_RESERVED_SFT 24
52 #define CMPL_DOORBELL_IDX_VALID 0x4000000UL
53 #define CMPL_DOORBELL_MASK 0x8000000UL
54 #define CMPL_DOORBELL_KEY_MASK 0xf0000000UL
55 #define CMPL_DOORBELL_KEY_SFT 28
56 #define CMPL_DOORBELL_KEY_CMPL (0x2UL << 28)
57};
58
59/* Status Door Bell Format (4 bytes) */
60struct status_doorbell {
61 __le32 key_idx;
62 #define STATUS_DOORBELL_IDX_MASK 0xffffffUL
63 #define STATUS_DOORBELL_IDX_SFT 0
64 #define STATUS_DOORBELL_RESERVED_MASK 0xf000000UL
65 #define STATUS_DOORBELL_RESERVED_SFT 24
66 #define STATUS_DOORBELL_KEY_MASK 0xf0000000UL
67 #define STATUS_DOORBELL_KEY_SFT 28
68 #define STATUS_DOORBELL_KEY_STAT (0x3UL << 28)
69};
70
71/* RoCE Host Structures */
72
73/* Doorbell Structures */
74/* 64b Doorbell Format (8 bytes) */
75struct dbr_dbr {
76 __le32 index;
77 #define DBR_DBR_INDEX_MASK 0xfffffUL
78 #define DBR_DBR_INDEX_SFT 0
79 #define DBR_DBR_RESERVED12_MASK 0xfff00000UL
80 #define DBR_DBR_RESERVED12_SFT 20
81 __le32 type_xid;
82 #define DBR_DBR_XID_MASK 0xfffffUL
83 #define DBR_DBR_XID_SFT 0
84 #define DBR_DBR_RESERVED8_MASK 0xff00000UL
85 #define DBR_DBR_RESERVED8_SFT 20
86 #define DBR_DBR_TYPE_MASK 0xf0000000UL
87 #define DBR_DBR_TYPE_SFT 28
88 #define DBR_DBR_TYPE_SQ (0x0UL << 28)
89 #define DBR_DBR_TYPE_RQ (0x1UL << 28)
90 #define DBR_DBR_TYPE_SRQ (0x2UL << 28)
91 #define DBR_DBR_TYPE_SRQ_ARM (0x3UL << 28)
92 #define DBR_DBR_TYPE_CQ (0x4UL << 28)
93 #define DBR_DBR_TYPE_CQ_ARMSE (0x5UL << 28)
94 #define DBR_DBR_TYPE_CQ_ARMALL (0x6UL << 28)
95 #define DBR_DBR_TYPE_CQ_ARMENA (0x7UL << 28)
96 #define DBR_DBR_TYPE_SRQ_ARMENA (0x8UL << 28)
97 #define DBR_DBR_TYPE_CQ_CUTOFF_ACK (0x9UL << 28)
98 #define DBR_DBR_TYPE_NULL (0xfUL << 28)
99};
100
101/* 32b Doorbell Format (4 bytes) */
102struct dbr_dbr32 {
103 __le32 type_abs_incr_xid;
104 #define DBR_DBR32_XID_MASK 0xfffffUL
105 #define DBR_DBR32_XID_SFT 0
106 #define DBR_DBR32_RESERVED4_MASK 0xf00000UL
107 #define DBR_DBR32_RESERVED4_SFT 20
108 #define DBR_DBR32_INCR_MASK 0xf000000UL
109 #define DBR_DBR32_INCR_SFT 24
110 #define DBR_DBR32_ABS 0x10000000UL
111 #define DBR_DBR32_TYPE_MASK 0xe0000000UL
112 #define DBR_DBR32_TYPE_SFT 29
113 #define DBR_DBR32_TYPE_SQ (0x0UL << 29)
114};
115
116/* SQ WQE Structures */
117/* Base SQ WQE (8 bytes) */
118struct sq_base {
119 u8 wqe_type;
120 #define SQ_BASE_WQE_TYPE_SEND 0x0UL
121 #define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD 0x1UL
122 #define SQ_BASE_WQE_TYPE_SEND_W_INVALID 0x2UL
123 #define SQ_BASE_WQE_TYPE_WRITE_WQE 0x4UL
124 #define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
125 #define SQ_BASE_WQE_TYPE_READ_WQE 0x6UL
126 #define SQ_BASE_WQE_TYPE_ATOMIC_CS 0x8UL
127 #define SQ_BASE_WQE_TYPE_ATOMIC_FA 0xbUL
128 #define SQ_BASE_WQE_TYPE_LOCAL_INVALID 0xcUL
129 #define SQ_BASE_WQE_TYPE_FR_PMR 0xdUL
130 #define SQ_BASE_WQE_TYPE_BIND 0xeUL
131 u8 unused_0[7];
132};
133
134/* WQE SGE (16 bytes) */
135struct sq_sge {
136 __le64 va_or_pa;
137 __le32 l_key;
138 __le32 size;
139};
140
141/* PSN Search Structure (8 bytes) */
142struct sq_psn_search {
143 __le32 opcode_start_psn;
144 #define SQ_PSN_SEARCH_START_PSN_MASK 0xffffffUL
145 #define SQ_PSN_SEARCH_START_PSN_SFT 0
146 #define SQ_PSN_SEARCH_OPCODE_MASK 0xff000000UL
147 #define SQ_PSN_SEARCH_OPCODE_SFT 24
148 __le32 flags_next_psn;
149 #define SQ_PSN_SEARCH_NEXT_PSN_MASK 0xffffffUL
150 #define SQ_PSN_SEARCH_NEXT_PSN_SFT 0
151 #define SQ_PSN_SEARCH_FLAGS_MASK 0xff000000UL
152 #define SQ_PSN_SEARCH_FLAGS_SFT 24
153};
154
155/* Send SQ WQE (40 bytes) */
156struct sq_send {
157 u8 wqe_type;
158 #define SQ_SEND_WQE_TYPE_SEND 0x0UL
159 #define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD 0x1UL
160 #define SQ_SEND_WQE_TYPE_SEND_W_INVALID 0x2UL
161 u8 flags;
162 #define SQ_SEND_FLAGS_SIGNAL_COMP 0x1UL
163 #define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
164 #define SQ_SEND_FLAGS_UC_FENCE 0x4UL
165 #define SQ_SEND_FLAGS_SE 0x8UL
166 #define SQ_SEND_FLAGS_INLINE 0x10UL
167 u8 wqe_size;
168 u8 reserved8_1;
169 __le32 inv_key_or_imm_data;
170 __le32 length;
171 __le32 q_key;
172 __le32 dst_qp;
173 #define SQ_SEND_DST_QP_MASK 0xffffffUL
174 #define SQ_SEND_DST_QP_SFT 0
175 #define SQ_SEND_RESERVED8_2_MASK 0xff000000UL
176 #define SQ_SEND_RESERVED8_2_SFT 24
177 __le32 avid;
178 #define SQ_SEND_AVID_MASK 0xfffffUL
179 #define SQ_SEND_AVID_SFT 0
180 #define SQ_SEND_RESERVED_AVID_MASK 0xfff00000UL
181 #define SQ_SEND_RESERVED_AVID_SFT 20
182 __le64 reserved64;
183 __le32 data[24];
184};
185
186/* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */
187struct sq_send_raweth_qp1 {
188 u8 wqe_type;
189 #define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND 0x0UL
190 u8 flags;
191 #define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP 0x1UL
192 #define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
193 #define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE 0x4UL
194 #define SQ_SEND_RAWETH_QP1_FLAGS_SE 0x8UL
195 #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE 0x10UL
196 u8 wqe_size;
197 u8 reserved8;
198 __le16 lflags;
199 #define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM 0x1UL
200 #define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM 0x2UL
201 #define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC 0x4UL
202 #define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP 0x8UL
203 #define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM 0x10UL
204 #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_1 0x20UL
205 #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_2 0x40UL
206 #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_3 0x80UL
207 #define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC 0x100UL
208 #define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC 0x200UL
209 __le16 cfa_action;
210 __le32 length;
211 __le32 reserved32_1;
212 __le32 cfa_meta;
213 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK 0xfffUL
214 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT 0
215 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE 0x1000UL
216 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK 0xe000UL
217 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT 13
218 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK 0x70000UL
219 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT 16
220 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 (0x0UL << 16)
221 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 (0x1UL << 16)
222 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 (0x2UL << 16)
223 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 (0x3UL << 16)
224 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 (0x4UL << 16)
225 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG (0x5UL << 16)
226 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST \
227 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG
228 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK 0xff80000UL
229 #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19
230 #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK 0xf0000000UL
231 #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT 28
232 #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE (0x0UL << 28)
233 #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG (0x1UL << 28)
234 #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST \
235 SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG
236 __le32 reserved32_2;
237 __le64 reserved64;
238 __le32 data[24];
239};
240
241/* RDMA SQ WQE (40 bytes) */
242struct sq_rdma {
243 u8 wqe_type;
244 #define SQ_RDMA_WQE_TYPE_WRITE_WQE 0x4UL
245 #define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
246 #define SQ_RDMA_WQE_TYPE_READ_WQE 0x6UL
247 u8 flags;
248 #define SQ_RDMA_FLAGS_SIGNAL_COMP 0x1UL
249 #define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
250 #define SQ_RDMA_FLAGS_UC_FENCE 0x4UL
251 #define SQ_RDMA_FLAGS_SE 0x8UL
252 #define SQ_RDMA_FLAGS_INLINE 0x10UL
253 u8 wqe_size;
254 u8 reserved8;
255 __le32 imm_data;
256 __le32 length;
257 __le32 reserved32_1;
258 __le64 remote_va;
259 __le32 remote_key;
260 __le32 reserved32_2;
261 __le32 data[24];
262};
263
264/* Atomic SQ WQE (40 bytes) */
265struct sq_atomic {
266 u8 wqe_type;
267 #define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS 0x8UL
268 #define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA 0xbUL
269 u8 flags;
270 #define SQ_ATOMIC_FLAGS_SIGNAL_COMP 0x1UL
271 #define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
272 #define SQ_ATOMIC_FLAGS_UC_FENCE 0x4UL
273 #define SQ_ATOMIC_FLAGS_SE 0x8UL
274 #define SQ_ATOMIC_FLAGS_INLINE 0x10UL
275 __le16 reserved16;
276 __le32 remote_key;
277 __le64 remote_va;
278 __le64 swap_data;
279 __le64 cmp_data;
280 __le32 data[24];
281};
282
283/* Local Invalidate SQ WQE (40 bytes) */
284struct sq_localinvalidate {
285 u8 wqe_type;
286 #define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID 0xcUL
287 u8 flags;
288 #define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP 0x1UL
289 #define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
290 #define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE 0x4UL
291 #define SQ_LOCALINVALIDATE_FLAGS_SE 0x8UL
292 #define SQ_LOCALINVALIDATE_FLAGS_INLINE 0x10UL
293 __le16 reserved16;
294 __le32 inv_l_key;
295 __le64 reserved64;
296 __le32 reserved128[4];
297 __le32 data[24];
298};
299
300/* FR-PMR SQ WQE (40 bytes) */
301struct sq_fr_pmr {
302 u8 wqe_type;
303 #define SQ_FR_PMR_WQE_TYPE_FR_PMR 0xdUL
304 u8 flags;
305 #define SQ_FR_PMR_FLAGS_SIGNAL_COMP 0x1UL
306 #define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
307 #define SQ_FR_PMR_FLAGS_UC_FENCE 0x4UL
308 #define SQ_FR_PMR_FLAGS_SE 0x8UL
309 #define SQ_FR_PMR_FLAGS_INLINE 0x10UL
310 u8 access_cntl;
311 #define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE 0x1UL
312 #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ 0x2UL
313 #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE 0x4UL
314 #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
315 #define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND 0x10UL
316 u8 zero_based_page_size_log;
317 #define SQ_FR_PMR_PAGE_SIZE_LOG_MASK 0x1fUL
318 #define SQ_FR_PMR_PAGE_SIZE_LOG_SFT 0
319 #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
320 #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
321 #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
322 #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
323 #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
324 #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
325 #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
326 #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
327 #define SQ_FR_PMR_ZERO_BASED 0x20UL
328 #define SQ_FR_PMR_RESERVED2_MASK 0xc0UL
329 #define SQ_FR_PMR_RESERVED2_SFT 6
330 __le32 l_key;
331 u8 length[5];
332 u8 reserved8_1;
333 u8 reserved8_2;
334 u8 numlevels_pbl_page_size_log;
335 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK 0x1fUL
336 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT 0
337 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
338 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
339 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
340 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
341 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
342 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
343 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
344 #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
345 #define SQ_FR_PMR_RESERVED1 0x20UL
346 #define SQ_FR_PMR_NUMLEVELS_MASK 0xc0UL
347 #define SQ_FR_PMR_NUMLEVELS_SFT 6
348 #define SQ_FR_PMR_NUMLEVELS_PHYSICAL (0x0UL << 6)
349 #define SQ_FR_PMR_NUMLEVELS_LAYER1 (0x1UL << 6)
350 #define SQ_FR_PMR_NUMLEVELS_LAYER2 (0x2UL << 6)
351 __le64 pblptr;
352 __le64 va;
353 __le32 data[24];
354};
355
356/* Bind SQ WQE (40 bytes) */
357struct sq_bind {
358 u8 wqe_type;
359 #define SQ_BIND_WQE_TYPE_BIND 0xeUL
360 u8 flags;
361 #define SQ_BIND_FLAGS_SIGNAL_COMP 0x1UL
362 #define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
363 #define SQ_BIND_FLAGS_UC_FENCE 0x4UL
364 #define SQ_BIND_FLAGS_SE 0x8UL
365 #define SQ_BIND_FLAGS_INLINE 0x10UL
366 u8 access_cntl;
367 #define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE 0x1UL
368 #define SQ_BIND_ACCESS_CNTL_REMOTE_READ 0x2UL
369 #define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE 0x4UL
370 #define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
371 #define SQ_BIND_ACCESS_CNTL_WINDOW_BIND 0x10UL
372 u8 reserved8_1;
373 u8 mw_type_zero_based;
374 #define SQ_BIND_ZERO_BASED 0x1UL
375 #define SQ_BIND_MW_TYPE 0x2UL
376 #define SQ_BIND_MW_TYPE_TYPE1 (0x0UL << 1)
377 #define SQ_BIND_MW_TYPE_TYPE2 (0x1UL << 1)
378 #define SQ_BIND_RESERVED6_MASK 0xfcUL
379 #define SQ_BIND_RESERVED6_SFT 2
380 u8 reserved8_2;
381 __le16 reserved16;
382 __le32 parent_l_key;
383 __le32 l_key;
384 __le64 va;
385 u8 length[5];
386 u8 data_reserved24[99];
387 #define SQ_BIND_RESERVED24_MASK 0xffffff00UL
388 #define SQ_BIND_RESERVED24_SFT 8
389 #define SQ_BIND_DATA_MASK 0xffffffffUL
390 #define SQ_BIND_DATA_SFT 0
391};
392
393/* RQ/SRQ WQE Structures */
394/* RQ/SRQ WQE (40 bytes) */
395struct rq_wqe {
396 u8 wqe_type;
397 #define RQ_WQE_WQE_TYPE_RCV 0x80UL
398 u8 flags;
399 u8 wqe_size;
400 u8 reserved8;
401 __le32 reserved32;
402 __le32 wr_id[2];
403 #define RQ_WQE_WR_ID_MASK 0xfffffUL
404 #define RQ_WQE_WR_ID_SFT 0
405 #define RQ_WQE_RESERVED44_MASK 0xfff00000UL
406 #define RQ_WQE_RESERVED44_SFT 20
407 __le32 reserved128[4];
408 __le32 data[24];
409};
410
411/* CQ CQE Structures */
412/* Base CQE (32 bytes) */
413struct cq_base {
414 __le64 reserved64_1;
415 __le64 reserved64_2;
416 __le64 reserved64_3;
417 u8 cqe_type_toggle;
418 #define CQ_BASE_TOGGLE 0x1UL
419 #define CQ_BASE_CQE_TYPE_MASK 0x1eUL
420 #define CQ_BASE_CQE_TYPE_SFT 1
421 #define CQ_BASE_CQE_TYPE_REQ (0x0UL << 1)
422 #define CQ_BASE_CQE_TYPE_RES_RC (0x1UL << 1)
423 #define CQ_BASE_CQE_TYPE_RES_UD (0x2UL << 1)
424 #define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
425 #define CQ_BASE_CQE_TYPE_TERMINAL (0xeUL << 1)
426 #define CQ_BASE_CQE_TYPE_CUT_OFF (0xfUL << 1)
427 #define CQ_BASE_RESERVED3_MASK 0xe0UL
428 #define CQ_BASE_RESERVED3_SFT 5
429 u8 status;
430 __le16 reserved16;
431 __le32 reserved32;
432};
433
434/* Requester CQ CQE (32 bytes) */
435struct cq_req {
436 __le64 qp_handle;
437 __le16 sq_cons_idx;
438 __le16 reserved16_1;
439 __le32 reserved32_2;
440 __le64 reserved64;
441 u8 cqe_type_toggle;
442 #define CQ_REQ_TOGGLE 0x1UL
443 #define CQ_REQ_CQE_TYPE_MASK 0x1eUL
444 #define CQ_REQ_CQE_TYPE_SFT 1
445 #define CQ_REQ_CQE_TYPE_REQ (0x0UL << 1)
446 #define CQ_REQ_RESERVED3_MASK 0xe0UL
447 #define CQ_REQ_RESERVED3_SFT 5
448 u8 status;
449 #define CQ_REQ_STATUS_OK 0x0UL
450 #define CQ_REQ_STATUS_BAD_RESPONSE_ERR 0x1UL
451 #define CQ_REQ_STATUS_LOCAL_LENGTH_ERR 0x2UL
452 #define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR 0x3UL
453 #define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR 0x4UL
454 #define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
455 #define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL
456 #define CQ_REQ_STATUS_REMOTE_ACCESS_ERR 0x7UL
457 #define CQ_REQ_STATUS_REMOTE_OPERATION_ERR 0x8UL
458 #define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR 0x9UL
459 #define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR 0xaUL
460 #define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR 0xbUL
461 __le16 reserved16_2;
462 __le32 reserved32_1;
463};
464
465/* Responder RC CQE (32 bytes) */
466struct cq_res_rc {
467 __le32 length;
468 __le32 imm_data_or_inv_r_key;
469 __le64 qp_handle;
470 __le64 mr_handle;
471 u8 cqe_type_toggle;
472 #define CQ_RES_RC_TOGGLE 0x1UL
473 #define CQ_RES_RC_CQE_TYPE_MASK 0x1eUL
474 #define CQ_RES_RC_CQE_TYPE_SFT 1
475 #define CQ_RES_RC_CQE_TYPE_RES_RC (0x1UL << 1)
476 #define CQ_RES_RC_RESERVED3_MASK 0xe0UL
477 #define CQ_RES_RC_RESERVED3_SFT 5
478 u8 status;
479 #define CQ_RES_RC_STATUS_OK 0x0UL
480 #define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR 0x1UL
481 #define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR 0x2UL
482 #define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR 0x3UL
483 #define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
484 #define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
485 #define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL
486 #define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
487 #define CQ_RES_RC_STATUS_HW_FLUSH_ERR 0x8UL
488 __le16 flags;
489 #define CQ_RES_RC_FLAGS_SRQ 0x1UL
490 #define CQ_RES_RC_FLAGS_SRQ_RQ (0x0UL << 0)
491 #define CQ_RES_RC_FLAGS_SRQ_SRQ (0x1UL << 0)
492 #define CQ_RES_RC_FLAGS_SRQ_LAST CQ_RES_RC_FLAGS_SRQ_SRQ
493 #define CQ_RES_RC_FLAGS_IMM 0x2UL
494 #define CQ_RES_RC_FLAGS_INV 0x4UL
495 #define CQ_RES_RC_FLAGS_RDMA 0x8UL
496 #define CQ_RES_RC_FLAGS_RDMA_SEND (0x0UL << 3)
497 #define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE (0x1UL << 3)
498 #define CQ_RES_RC_FLAGS_RDMA_LAST CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE
499 __le32 srq_or_rq_wr_id;
500 #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
501 #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0
502 #define CQ_RES_RC_RESERVED12_MASK 0xfff00000UL
503 #define CQ_RES_RC_RESERVED12_SFT 20
504};
505
506/* Responder UD CQE (32 bytes) */
507struct cq_res_ud {
508 __le32 length;
509 #define CQ_RES_UD_LENGTH_MASK 0x3fffUL
510 #define CQ_RES_UD_LENGTH_SFT 0
511 #define CQ_RES_UD_RESERVED18_MASK 0xffffc000UL
512 #define CQ_RES_UD_RESERVED18_SFT 14
513 __le32 imm_data;
514 __le64 qp_handle;
515 __le16 src_mac[3];
516 __le16 src_qp_low;
517 u8 cqe_type_toggle;
518 #define CQ_RES_UD_TOGGLE 0x1UL
519 #define CQ_RES_UD_CQE_TYPE_MASK 0x1eUL
520 #define CQ_RES_UD_CQE_TYPE_SFT 1
521 #define CQ_RES_UD_CQE_TYPE_RES_UD (0x2UL << 1)
522 #define CQ_RES_UD_RESERVED3_MASK 0xe0UL
523 #define CQ_RES_UD_RESERVED3_SFT 5
524 u8 status;
525 #define CQ_RES_UD_STATUS_OK 0x0UL
526 #define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR 0x1UL
527 #define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
528 #define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR 0x3UL
529 #define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
530 #define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
531 #define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
532 #define CQ_RES_UD_STATUS_HW_FLUSH_ERR 0x8UL
533 __le16 flags;
534 #define CQ_RES_UD_FLAGS_SRQ 0x1UL
535 #define CQ_RES_UD_FLAGS_SRQ_RQ (0x0UL << 0)
536 #define CQ_RES_UD_FLAGS_SRQ_SRQ (0x1UL << 0)
537 #define CQ_RES_UD_FLAGS_SRQ_LAST CQ_RES_UD_FLAGS_SRQ_SRQ
538 #define CQ_RES_UD_FLAGS_IMM 0x2UL
539 #define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK 0xcUL
540 #define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT 2
541 #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 (0x0UL << 2)
542 #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 2)
543 #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 2)
544 #define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST \
545 CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6
546 __le32 src_qp_high_srq_or_rq_wr_id;
547 #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
548 #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0
549 #define CQ_RES_UD_RESERVED4_MASK 0xf00000UL
550 #define CQ_RES_UD_RESERVED4_SFT 20
551 #define CQ_RES_UD_SRC_QP_HIGH_MASK 0xff000000UL
552 #define CQ_RES_UD_SRC_QP_HIGH_SFT 24
553};
554
555/* Responder RawEth and QP1 CQE (32 bytes) */
556struct cq_res_raweth_qp1 {
557 __le16 length;
558 #define CQ_RES_RAWETH_QP1_LENGTH_MASK 0x3fffUL
559 #define CQ_RES_RAWETH_QP1_LENGTH_SFT 0
560 #define CQ_RES_RAWETH_QP1_RESERVED2_MASK 0xc000UL
561 #define CQ_RES_RAWETH_QP1_RESERVED2_SFT 14
562 __le16 raweth_qp1_flags;
563 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR 0x1UL
564 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_MASK 0x3eUL
565 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_SFT 1
566 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK 0x3c0UL
567 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT 6
568 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL << 6)
569 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP (0x1UL << 6)
570 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP (0x2UL << 6)
571 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP (0x3UL << 6)
572 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE (0x4UL << 6)
573 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE (0x5UL << 6)
574 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP (0x7UL << 6)
575 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP \
576 (0x8UL << 6)
577 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP \
578 (0x9UL << 6)
579 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST \
580 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP
581 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK 0x3ffUL
582 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT 0
583 #define CQ_RES_RAWETH_QP1_RESERVED6_MASK 0xfc00UL
584 #define CQ_RES_RAWETH_QP1_RESERVED6_SFT 10
585 __le16 raweth_qp1_errors;
586 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_MASK 0xfUL
587 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_SFT 0
588 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR 0x10UL
589 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR 0x20UL
590 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL
591 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL
592 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR 0x100UL
593 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL
594 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9
595 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR \
596 (0x0UL << 9)
597 #define \
598 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \
599 (0x1UL << 9)
600 #define \
601 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \
602 (0x2UL << 9)
603 #define \
604 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \
605 (0x3UL << 9)
606 #define \
607 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \
608 (0x4UL << 9)
609 #define \
610 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \
611 (0x5UL << 9)
612 #define \
613 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \
614 (0x6UL << 9)
615 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \
616 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL
617 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL
618 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12
619 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR \
620 (0x0UL << 12)
621 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION \
622 (0x1UL << 12)
623 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \
624 (0x2UL << 12)
625 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL \
626 (0x3UL << 12)
627 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \
628 (0x4UL << 12)
629 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \
630 (0x5UL << 12)
631 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \
632 (0x6UL << 12)
633 #define \
634 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL\
635 (0x7UL << 12)
636 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \
637 (0x8UL << 12)
638 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \
639 CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN
640 __le16 raweth_qp1_cfa_code;
641 __le64 qp_handle;
642 __le32 raweth_qp1_flags2;
643 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC 0x1UL
644 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC 0x2UL
645 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC 0x4UL
646 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC 0x8UL
647 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL
648 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4
649 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE \
650 (0x0UL << 4)
651 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN \
652 (0x1UL << 4)
653 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST\
654 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN
655 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE 0x100UL
656 __le32 raweth_qp1_metadata;
657 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK 0xfffUL
658 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT 0
659 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE 0x1000UL
660 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK 0xe000UL
661 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT 13
662 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK 0xffff0000UL
663 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT 16
664 u8 cqe_type_toggle;
665 #define CQ_RES_RAWETH_QP1_TOGGLE 0x1UL
666 #define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK 0x1eUL
667 #define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT 1
668 #define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
669 #define CQ_RES_RAWETH_QP1_RESERVED3_MASK 0xe0UL
670 #define CQ_RES_RAWETH_QP1_RESERVED3_SFT 5
671 u8 status;
672 #define CQ_RES_RAWETH_QP1_STATUS_OK 0x0UL
673 #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR 0x1UL
674 #define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
675 #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR 0x3UL
676 #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
677 #define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
678 #define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
679 #define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR 0x8UL
680 __le16 flags;
681 #define CQ_RES_RAWETH_QP1_FLAGS_SRQ 0x1UL
682 #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ 0x0UL
683 #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ 0x1UL
684 #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST \
685 CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ
686 __le32 raweth_qp1_payload_offset_srq_or_rq_wr_id;
687 #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
688 #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT 0
689 #define CQ_RES_RAWETH_QP1_RESERVED4_MASK 0xf00000UL
690 #define CQ_RES_RAWETH_QP1_RESERVED4_SFT 20
691 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL
692 #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24
693};
694
695/* Terminal CQE (32 bytes) */
696struct cq_terminal {
697 __le64 qp_handle;
698 __le16 sq_cons_idx;
699 __le16 rq_cons_idx;
700 __le32 reserved32_1;
701 __le64 reserved64_3;
702 u8 cqe_type_toggle;
703 #define CQ_TERMINAL_TOGGLE 0x1UL
704 #define CQ_TERMINAL_CQE_TYPE_MASK 0x1eUL
705 #define CQ_TERMINAL_CQE_TYPE_SFT 1
706 #define CQ_TERMINAL_CQE_TYPE_TERMINAL (0xeUL << 1)
707 #define CQ_TERMINAL_RESERVED3_MASK 0xe0UL
708 #define CQ_TERMINAL_RESERVED3_SFT 5
709 u8 status;
710 #define CQ_TERMINAL_STATUS_OK 0x0UL
711 __le16 reserved16;
712 __le32 reserved32_2;
713};
714
715/* Cutoff CQE (32 bytes) */
716struct cq_cutoff {
717 __le64 reserved64_1;
718 __le64 reserved64_2;
719 __le64 reserved64_3;
720 u8 cqe_type_toggle;
721 #define CQ_CUTOFF_TOGGLE 0x1UL
722 #define CQ_CUTOFF_CQE_TYPE_MASK 0x1eUL
723 #define CQ_CUTOFF_CQE_TYPE_SFT 1
724 #define CQ_CUTOFF_CQE_TYPE_CUT_OFF (0xfUL << 1)
725 #define CQ_CUTOFF_RESERVED3_MASK 0xe0UL
726 #define CQ_CUTOFF_RESERVED3_SFT 5
727 u8 status;
728 #define CQ_CUTOFF_STATUS_OK 0x0UL
729 __le16 reserved16;
730 __le32 reserved32;
731};
732
733/* Notification Queue (NQ) Structures */
734/* Base NQ Record (16 bytes) */
735struct nq_base {
736 __le16 info10_type;
737 #define NQ_BASE_TYPE_MASK 0x3fUL
738 #define NQ_BASE_TYPE_SFT 0
739 #define NQ_BASE_TYPE_CQ_NOTIFICATION 0x30UL
740 #define NQ_BASE_TYPE_SRQ_EVENT 0x32UL
741 #define NQ_BASE_TYPE_DBQ_EVENT 0x34UL
742 #define NQ_BASE_TYPE_QP_EVENT 0x38UL
743 #define NQ_BASE_TYPE_FUNC_EVENT 0x3aUL
744 #define NQ_BASE_INFO10_MASK 0xffc0UL
745 #define NQ_BASE_INFO10_SFT 6
746 __le16 info16;
747 __le32 info32;
748 __le32 info63_v[2];
749 #define NQ_BASE_V 0x1UL
750 #define NQ_BASE_INFO63_MASK 0xfffffffeUL
751 #define NQ_BASE_INFO63_SFT 1
752};
753
754/* Completion Queue Notification (16 bytes) */
755struct nq_cn {
756 __le16 type;
757 #define NQ_CN_TYPE_MASK 0x3fUL
758 #define NQ_CN_TYPE_SFT 0
759 #define NQ_CN_TYPE_CQ_NOTIFICATION 0x30UL
760 #define NQ_CN_RESERVED9_MASK 0xffc0UL
761 #define NQ_CN_RESERVED9_SFT 6
762 __le16 reserved16;
763 __le32 cq_handle_low;
764 __le32 v;
765 #define NQ_CN_V 0x1UL
766 #define NQ_CN_RESERVED31_MASK 0xfffffffeUL
767 #define NQ_CN_RESERVED31_SFT 1
768 __le32 cq_handle_high;
769};
770
771/* SRQ Event Notification (16 bytes) */
772struct nq_srq_event {
773 u8 type;
774 #define NQ_SRQ_EVENT_TYPE_MASK 0x3fUL
775 #define NQ_SRQ_EVENT_TYPE_SFT 0
776 #define NQ_SRQ_EVENT_TYPE_SRQ_EVENT 0x32UL
777 #define NQ_SRQ_EVENT_RESERVED1_MASK 0xc0UL
778 #define NQ_SRQ_EVENT_RESERVED1_SFT 6
779 u8 event;
780 #define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT 0x1UL
781 __le16 reserved16;
782 __le32 srq_handle_low;
783 __le32 v;
784 #define NQ_SRQ_EVENT_V 0x1UL
785 #define NQ_SRQ_EVENT_RESERVED31_MASK 0xfffffffeUL
786 #define NQ_SRQ_EVENT_RESERVED31_SFT 1
787 __le32 srq_handle_high;
788};
789
790/* DBQ Async Event Notification (16 bytes) */
791struct nq_dbq_event {
792 u8 type;
793 #define NQ_DBQ_EVENT_TYPE_MASK 0x3fUL
794 #define NQ_DBQ_EVENT_TYPE_SFT 0
795 #define NQ_DBQ_EVENT_TYPE_DBQ_EVENT 0x34UL
796 #define NQ_DBQ_EVENT_RESERVED1_MASK 0xc0UL
797 #define NQ_DBQ_EVENT_RESERVED1_SFT 6
798 u8 event;
799 #define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT 0x1UL
800 __le16 db_pfid;
801 #define NQ_DBQ_EVENT_DB_PFID_MASK 0xfUL
802 #define NQ_DBQ_EVENT_DB_PFID_SFT 0
803 #define NQ_DBQ_EVENT_RESERVED12_MASK 0xfff0UL
804 #define NQ_DBQ_EVENT_RESERVED12_SFT 4
805 __le32 db_dpi;
806 #define NQ_DBQ_EVENT_DB_DPI_MASK 0xfffffUL
807 #define NQ_DBQ_EVENT_DB_DPI_SFT 0
808 #define NQ_DBQ_EVENT_RESERVED12_2_MASK 0xfff00000UL
809 #define NQ_DBQ_EVENT_RESERVED12_2_SFT 20
810 __le32 v;
811 #define NQ_DBQ_EVENT_V 0x1UL
812 #define NQ_DBQ_EVENT_RESERVED32_MASK 0xfffffffeUL
813 #define NQ_DBQ_EVENT_RESERVED32_SFT 1
814 __le32 db_type_db_xid;
815 #define NQ_DBQ_EVENT_DB_XID_MASK 0xfffffUL
816 #define NQ_DBQ_EVENT_DB_XID_SFT 0
817 #define NQ_DBQ_EVENT_RESERVED8_MASK 0xff00000UL
818 #define NQ_DBQ_EVENT_RESERVED8_SFT 20
819 #define NQ_DBQ_EVENT_DB_TYPE_MASK 0xf0000000UL
820 #define NQ_DBQ_EVENT_DB_TYPE_SFT 28
821};
822
823/* Read Request/Response Queue Structures */
824/* Input Read Request Queue (IRRQ) Message (32 bytes) */
825struct xrrq_irrq {
826 __le16 credits_type;
827 #define XRRQ_IRRQ_TYPE 0x1UL
828 #define XRRQ_IRRQ_TYPE_READ_REQ 0x0UL
829 #define XRRQ_IRRQ_TYPE_ATOMIC_REQ 0x1UL
830 #define XRRQ_IRRQ_RESERVED10_MASK 0x7feUL
831 #define XRRQ_IRRQ_RESERVED10_SFT 1
832 #define XRRQ_IRRQ_CREDITS_MASK 0xf800UL
833 #define XRRQ_IRRQ_CREDITS_SFT 11
834 __le16 reserved16;
835 __le32 reserved32;
836 __le32 psn;
837 #define XRRQ_IRRQ_PSN_MASK 0xffffffUL
838 #define XRRQ_IRRQ_PSN_SFT 0
839 #define XRRQ_IRRQ_RESERVED8_1_MASK 0xff000000UL
840 #define XRRQ_IRRQ_RESERVED8_1_SFT 24
841 __le32 msn;
842 #define XRRQ_IRRQ_MSN_MASK 0xffffffUL
843 #define XRRQ_IRRQ_MSN_SFT 0
844 #define XRRQ_IRRQ_RESERVED8_2_MASK 0xff000000UL
845 #define XRRQ_IRRQ_RESERVED8_2_SFT 24
846 __le64 va_or_atomic_result;
847 __le32 rdma_r_key;
848 __le32 length;
849};
850
851/* Output Read Request Queue (ORRQ) Message (32 bytes) */
852struct xrrq_orrq {
853 __le16 num_sges_type;
854 #define XRRQ_ORRQ_TYPE 0x1UL
855 #define XRRQ_ORRQ_TYPE_READ_REQ 0x0UL
856 #define XRRQ_ORRQ_TYPE_ATOMIC_REQ 0x1UL
857 #define XRRQ_ORRQ_RESERVED10_MASK 0x7feUL
858 #define XRRQ_ORRQ_RESERVED10_SFT 1
859 #define XRRQ_ORRQ_NUM_SGES_MASK 0xf800UL
860 #define XRRQ_ORRQ_NUM_SGES_SFT 11
861 __le16 reserved16;
862 __le32 length;
863 __le32 psn;
864 #define XRRQ_ORRQ_PSN_MASK 0xffffffUL
865 #define XRRQ_ORRQ_PSN_SFT 0
866 #define XRRQ_ORRQ_RESERVED8_1_MASK 0xff000000UL
867 #define XRRQ_ORRQ_RESERVED8_1_SFT 24
868 __le32 end_psn;
869 #define XRRQ_ORRQ_END_PSN_MASK 0xffffffUL
870 #define XRRQ_ORRQ_END_PSN_SFT 0
871 #define XRRQ_ORRQ_RESERVED8_2_MASK 0xff000000UL
872 #define XRRQ_ORRQ_RESERVED8_2_SFT 24
873 __le64 first_sge_phy_or_sing_sge_va;
874 __le32 single_sge_l_key;
875 __le32 single_sge_size;
876};
877
878/* Page Buffer List Memory Structures (PBL) */
879/* Page Table Entry (PTE) (8 bytes) */
880struct ptu_pte {
881 __le32 page_next_to_last_last_valid[2];
882 #define PTU_PTE_VALID 0x1UL
883 #define PTU_PTE_LAST 0x2UL
884 #define PTU_PTE_NEXT_TO_LAST 0x4UL
885 #define PTU_PTE_PAGE_MASK 0xfffff000UL
886 #define PTU_PTE_PAGE_SFT 12
887};
888
889/* Page Directory Entry (PDE) (8 bytes) */
890struct ptu_pde {
891 __le32 page_valid[2];
892 #define PTU_PDE_VALID 0x1UL
893 #define PTU_PDE_PAGE_MASK 0xfffff000UL
894 #define PTU_PDE_PAGE_SFT 12
895};
896
897/* RoCE Fastpath Host Structures */
898/* Command Queue (CMDQ) Interface */
899/* Init CMDQ (16 bytes) */
900struct cmdq_init {
901 __le64 cmdq_pbl;
902 __le16 cmdq_size_cmdq_lvl;
903 #define CMDQ_INIT_CMDQ_LVL_MASK 0x3UL
904 #define CMDQ_INIT_CMDQ_LVL_SFT 0
905 #define CMDQ_INIT_CMDQ_SIZE_MASK 0xfffcUL
906 #define CMDQ_INIT_CMDQ_SIZE_SFT 2
907 __le16 creq_ring_id;
908 __le32 prod_idx;
909};
910
911/* Update CMDQ producer index (16 bytes) */
912struct cmdq_update {
913 __le64 reserved64;
914 __le32 reserved32;
915 __le32 prod_idx;
916};
917
918/* CMDQ common header structure (16 bytes) */
919struct cmdq_base {
920 u8 opcode;
921 #define CMDQ_BASE_OPCODE_CREATE_QP 0x1UL
922 #define CMDQ_BASE_OPCODE_DESTROY_QP 0x2UL
923 #define CMDQ_BASE_OPCODE_MODIFY_QP 0x3UL
924 #define CMDQ_BASE_OPCODE_QUERY_QP 0x4UL
925 #define CMDQ_BASE_OPCODE_CREATE_SRQ 0x5UL
926 #define CMDQ_BASE_OPCODE_DESTROY_SRQ 0x6UL
927 #define CMDQ_BASE_OPCODE_QUERY_SRQ 0x8UL
928 #define CMDQ_BASE_OPCODE_CREATE_CQ 0x9UL
929 #define CMDQ_BASE_OPCODE_DESTROY_CQ 0xaUL
930 #define CMDQ_BASE_OPCODE_RESIZE_CQ 0xcUL
931 #define CMDQ_BASE_OPCODE_ALLOCATE_MRW 0xdUL
932 #define CMDQ_BASE_OPCODE_DEALLOCATE_KEY 0xeUL
933 #define CMDQ_BASE_OPCODE_REGISTER_MR 0xfUL
934 #define CMDQ_BASE_OPCODE_DEREGISTER_MR 0x10UL
935 #define CMDQ_BASE_OPCODE_ADD_GID 0x11UL
936 #define CMDQ_BASE_OPCODE_DELETE_GID 0x12UL
937 #define CMDQ_BASE_OPCODE_MODIFY_GID 0x17UL
938 #define CMDQ_BASE_OPCODE_QUERY_GID 0x18UL
939 #define CMDQ_BASE_OPCODE_CREATE_QP1 0x13UL
940 #define CMDQ_BASE_OPCODE_DESTROY_QP1 0x14UL
941 #define CMDQ_BASE_OPCODE_CREATE_AH 0x15UL
942 #define CMDQ_BASE_OPCODE_DESTROY_AH 0x16UL
943 #define CMDQ_BASE_OPCODE_INITIALIZE_FW 0x80UL
944 #define CMDQ_BASE_OPCODE_DEINITIALIZE_FW 0x81UL
945 #define CMDQ_BASE_OPCODE_STOP_FUNC 0x82UL
946 #define CMDQ_BASE_OPCODE_QUERY_FUNC 0x83UL
947 #define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES 0x84UL
948 #define CMDQ_BASE_OPCODE_READ_CONTEXT 0x85UL
949 #define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST 0x86UL
950 #define CMDQ_BASE_OPCODE_READ_VF_MEMORY 0x87UL
951 #define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST 0x88UL
952 #define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY 0x89UL
953 #define CMDQ_BASE_OPCODE_MAP_TC_TO_COS 0x8aUL
954 #define CMDQ_BASE_OPCODE_QUERY_VERSION 0x8bUL
955 #define CMDQ_BASE_OPCODE_MODIFY_CC 0x8cUL
956 #define CMDQ_BASE_OPCODE_QUERY_CC 0x8dUL
957 u8 cmd_size;
958 __le16 flags;
959 __le16 cookie;
960 u8 resp_size;
961 u8 reserved8;
962 __le64 resp_addr;
963};
964
965/* Create QP command (96 bytes) */
966struct cmdq_create_qp {
967 u8 opcode;
968 #define CMDQ_CREATE_QP_OPCODE_CREATE_QP 0x1UL
969 u8 cmd_size;
970 __le16 flags;
971 __le16 cookie;
972 u8 resp_size;
973 u8 reserved8;
974 __le64 resp_addr;
975 __le64 qp_handle;
976 __le32 qp_flags;
977 #define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED 0x1UL
978 #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL
979 #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
980 #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL
981 u8 type;
982 #define CMDQ_CREATE_QP_TYPE_RC 0x2UL
983 #define CMDQ_CREATE_QP_TYPE_UD 0x4UL
984 #define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE 0x6UL
985 u8 sq_pg_size_sq_lvl;
986 #define CMDQ_CREATE_QP_SQ_LVL_MASK 0xfUL
987 #define CMDQ_CREATE_QP_SQ_LVL_SFT 0
988 #define CMDQ_CREATE_QP_SQ_LVL_LVL_0 0x0UL
989 #define CMDQ_CREATE_QP_SQ_LVL_LVL_1 0x1UL
990 #define CMDQ_CREATE_QP_SQ_LVL_LVL_2 0x2UL
991 #define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK 0xf0UL
992 #define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT 4
993 #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K (0x0UL << 4)
994 #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K (0x1UL << 4)
995 #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K (0x2UL << 4)
996 #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M (0x3UL << 4)
997 #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M (0x4UL << 4)
998 #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G (0x5UL << 4)
999 u8 rq_pg_size_rq_lvl;
1000 #define CMDQ_CREATE_QP_RQ_LVL_MASK 0xfUL
1001 #define CMDQ_CREATE_QP_RQ_LVL_SFT 0
1002 #define CMDQ_CREATE_QP_RQ_LVL_LVL_0 0x0UL
1003 #define CMDQ_CREATE_QP_RQ_LVL_LVL_1 0x1UL
1004 #define CMDQ_CREATE_QP_RQ_LVL_LVL_2 0x2UL
1005 #define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK 0xf0UL
1006 #define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT 4
1007 #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K (0x0UL << 4)
1008 #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K (0x1UL << 4)
1009 #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K (0x2UL << 4)
1010 #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M (0x3UL << 4)
1011 #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M (0x4UL << 4)
1012 #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G (0x5UL << 4)
1013 u8 unused_0;
1014 __le32 dpi;
1015 __le32 sq_size;
1016 __le32 rq_size;
1017 __le16 sq_fwo_sq_sge;
1018 #define CMDQ_CREATE_QP_SQ_SGE_MASK 0xfUL
1019 #define CMDQ_CREATE_QP_SQ_SGE_SFT 0
1020 #define CMDQ_CREATE_QP_SQ_FWO_MASK 0xfff0UL
1021 #define CMDQ_CREATE_QP_SQ_FWO_SFT 4
1022 __le16 rq_fwo_rq_sge;
1023 #define CMDQ_CREATE_QP_RQ_SGE_MASK 0xfUL
1024 #define CMDQ_CREATE_QP_RQ_SGE_SFT 0
1025 #define CMDQ_CREATE_QP_RQ_FWO_MASK 0xfff0UL
1026 #define CMDQ_CREATE_QP_RQ_FWO_SFT 4
1027 __le32 scq_cid;
1028 __le32 rcq_cid;
1029 __le32 srq_cid;
1030 __le32 pd_id;
1031 __le64 sq_pbl;
1032 __le64 rq_pbl;
1033 __le64 irrq_addr;
1034 __le64 orrq_addr;
1035};
1036
1037/* Destroy QP command (24 bytes) */
1038struct cmdq_destroy_qp {
1039 u8 opcode;
1040 #define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP 0x2UL
1041 u8 cmd_size;
1042 __le16 flags;
1043 __le16 cookie;
1044 u8 resp_size;
1045 u8 reserved8;
1046 __le64 resp_addr;
1047 __le32 qp_cid;
1048 __le32 unused_0;
1049};
1050
1051/* Modify QP command (112 bytes) */
1052struct cmdq_modify_qp {
1053 u8 opcode;
1054 #define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP 0x3UL
1055 u8 cmd_size;
1056 __le16 flags;
1057 __le16 cookie;
1058 u8 resp_size;
1059 u8 reserved8;
1060 __le64 resp_addr;
1061 __le32 modify_mask;
1062 #define CMDQ_MODIFY_QP_MODIFY_MASK_STATE 0x1UL
1063 #define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY 0x2UL
1064 #define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS 0x4UL
1065 #define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY 0x8UL
1066 #define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY 0x10UL
1067 #define CMDQ_MODIFY_QP_MODIFY_MASK_DGID 0x20UL
1068 #define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL 0x40UL
1069 #define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX 0x80UL
1070 #define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT 0x100UL
1071 #define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS 0x200UL
1072 #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC 0x400UL
1073 #define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU 0x1000UL
1074 #define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT 0x2000UL
1075 #define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT 0x4000UL
1076 #define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY 0x8000UL
1077 #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN 0x10000UL
1078 #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC 0x20000UL
1079 #define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER 0x40000UL
1080 #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN 0x80000UL
1081 #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC 0x100000UL
1082 #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE 0x200000UL
1083 #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE 0x400000UL
1084 #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE 0x800000UL
1085 #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE 0x1000000UL
1086 #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA 0x2000000UL
1087 #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID 0x4000000UL
1088 #define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC 0x8000000UL
1089 #define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID 0x10000000UL
1090 #define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC 0x20000000UL
1091 #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN 0x40000000UL
1092 #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP 0x80000000UL
1093 __le32 qp_cid;
1094 u8 network_type_en_sqd_async_notify_new_state;
1095 #define CMDQ_MODIFY_QP_NEW_STATE_MASK 0xfUL
1096 #define CMDQ_MODIFY_QP_NEW_STATE_SFT 0
1097 #define CMDQ_MODIFY_QP_NEW_STATE_RESET 0x0UL
1098 #define CMDQ_MODIFY_QP_NEW_STATE_INIT 0x1UL
1099 #define CMDQ_MODIFY_QP_NEW_STATE_RTR 0x2UL
1100 #define CMDQ_MODIFY_QP_NEW_STATE_RTS 0x3UL
1101 #define CMDQ_MODIFY_QP_NEW_STATE_SQD 0x4UL
1102 #define CMDQ_MODIFY_QP_NEW_STATE_SQE 0x5UL
1103 #define CMDQ_MODIFY_QP_NEW_STATE_ERR 0x6UL
1104 #define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY 0x10UL
1105 #define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK 0xc0UL
1106 #define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT 6
1107 #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 (0x0UL << 6)
1108 #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 (0x2UL << 6)
1109 #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 (0x3UL << 6)
1110 u8 access;
1111 #define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE 0x1UL
1112 #define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE 0x2UL
1113 #define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ 0x4UL
1114 #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC 0x8UL
1115 __le16 pkey;
1116 __le32 qkey;
1117 __le32 dgid[4];
1118 __le32 flow_label;
1119 __le16 sgid_index;
1120 u8 hop_limit;
1121 u8 traffic_class;
1122 __le16 dest_mac[3];
1123 u8 tos_dscp_tos_ecn;
1124 #define CMDQ_MODIFY_QP_TOS_ECN_MASK 0x3UL
1125 #define CMDQ_MODIFY_QP_TOS_ECN_SFT 0
1126 #define CMDQ_MODIFY_QP_TOS_DSCP_MASK 0xfcUL
1127 #define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2
1128 u8 path_mtu;
1129 #define CMDQ_MODIFY_QP_PATH_MTU_MASK 0xf0UL
1130 #define CMDQ_MODIFY_QP_PATH_MTU_SFT 4
1131 #define CMDQ_MODIFY_QP_PATH_MTU_MTU_256 (0x0UL << 4)
1132 #define CMDQ_MODIFY_QP_PATH_MTU_MTU_512 (0x1UL << 4)
1133 #define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 (0x2UL << 4)
1134 #define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 (0x3UL << 4)
1135 #define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 (0x4UL << 4)
1136 #define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 (0x5UL << 4)
1137 u8 timeout;
1138 u8 retry_cnt;
1139 u8 rnr_retry;
1140 u8 min_rnr_timer;
1141 __le32 rq_psn;
1142 __le32 sq_psn;
1143 u8 max_rd_atomic;
1144 u8 max_dest_rd_atomic;
1145 __le16 enable_cc;
1146 #define CMDQ_MODIFY_QP_ENABLE_CC 0x1UL
1147 __le32 sq_size;
1148 __le32 rq_size;
1149 __le16 sq_sge;
1150 __le16 rq_sge;
1151 __le32 max_inline_data;
1152 __le32 dest_qp_id;
1153 __le32 unused_3;
1154 __le16 src_mac[3];
1155 __le16 vlan_pcp_vlan_dei_vlan_id;
1156 #define CMDQ_MODIFY_QP_VLAN_ID_MASK 0xfffUL
1157 #define CMDQ_MODIFY_QP_VLAN_ID_SFT 0
1158 #define CMDQ_MODIFY_QP_VLAN_DEI 0x1000UL
1159 #define CMDQ_MODIFY_QP_VLAN_PCP_MASK 0xe000UL
1160 #define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13
1161};
1162
1163/* Query QP command (24 bytes) */
1164struct cmdq_query_qp {
1165 u8 opcode;
1166 #define CMDQ_QUERY_QP_OPCODE_QUERY_QP 0x4UL
1167 u8 cmd_size;
1168 __le16 flags;
1169 __le16 cookie;
1170 u8 resp_size;
1171 u8 reserved8;
1172 __le64 resp_addr;
1173 __le32 qp_cid;
1174 __le32 unused_0;
1175};
1176
1177/* Create SRQ command (48 bytes) */
1178struct cmdq_create_srq {
1179 u8 opcode;
1180 #define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ 0x5UL
1181 u8 cmd_size;
1182 __le16 flags;
1183 __le16 cookie;
1184 u8 resp_size;
1185 u8 reserved8;
1186 __le64 resp_addr;
1187 __le64 srq_handle;
1188 __le16 pg_size_lvl;
1189 #define CMDQ_CREATE_SRQ_LVL_MASK 0x3UL
1190 #define CMDQ_CREATE_SRQ_LVL_SFT 0
1191 #define CMDQ_CREATE_SRQ_LVL_LVL_0 0x0UL
1192 #define CMDQ_CREATE_SRQ_LVL_LVL_1 0x1UL
1193 #define CMDQ_CREATE_SRQ_LVL_LVL_2 0x2UL
1194 #define CMDQ_CREATE_SRQ_PG_SIZE_MASK 0x1cUL
1195 #define CMDQ_CREATE_SRQ_PG_SIZE_SFT 2
1196 #define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K (0x0UL << 2)
1197 #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K (0x1UL << 2)
1198 #define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K (0x2UL << 2)
1199 #define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M (0x3UL << 2)
1200 #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M (0x4UL << 2)
1201 #define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G (0x5UL << 2)
1202 __le16 eventq_id;
1203 #define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK 0xfffUL
1204 #define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0
1205 __le16 srq_size;
1206 __le16 srq_fwo;
1207 __le32 dpi;
1208 __le32 pd_id;
1209 __le64 pbl;
1210};
1211
1212/* Destroy SRQ command (24 bytes) */
1213struct cmdq_destroy_srq {
1214 u8 opcode;
1215 #define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ 0x6UL
1216 u8 cmd_size;
1217 __le16 flags;
1218 __le16 cookie;
1219 u8 resp_size;
1220 u8 reserved8;
1221 __le64 resp_addr;
1222 __le32 srq_cid;
1223 __le32 unused_0;
1224};
1225
1226/* Query SRQ command (24 bytes) */
1227struct cmdq_query_srq {
1228 u8 opcode;
1229 #define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ 0x8UL
1230 u8 cmd_size;
1231 __le16 flags;
1232 __le16 cookie;
1233 u8 resp_size;
1234 u8 reserved8;
1235 __le64 resp_addr;
1236 __le32 srq_cid;
1237 __le32 unused_0;
1238};
1239
1240/* Create CQ command (48 bytes) */
1241struct cmdq_create_cq {
1242 u8 opcode;
1243 #define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ 0x9UL
1244 u8 cmd_size;
1245 __le16 flags;
1246 __le16 cookie;
1247 u8 resp_size;
1248 u8 reserved8;
1249 __le64 resp_addr;
1250 __le64 cq_handle;
1251 __le32 pg_size_lvl;
1252 #define CMDQ_CREATE_CQ_LVL_MASK 0x3UL
1253 #define CMDQ_CREATE_CQ_LVL_SFT 0
1254 #define CMDQ_CREATE_CQ_LVL_LVL_0 0x0UL
1255 #define CMDQ_CREATE_CQ_LVL_LVL_1 0x1UL
1256 #define CMDQ_CREATE_CQ_LVL_LVL_2 0x2UL
1257 #define CMDQ_CREATE_CQ_PG_SIZE_MASK 0x1cUL
1258 #define CMDQ_CREATE_CQ_PG_SIZE_SFT 2
1259 #define CMDQ_CREATE_CQ_PG_SIZE_PG_4K (0x0UL << 2)
1260 #define CMDQ_CREATE_CQ_PG_SIZE_PG_8K (0x1UL << 2)
1261 #define CMDQ_CREATE_CQ_PG_SIZE_PG_64K (0x2UL << 2)
1262 #define CMDQ_CREATE_CQ_PG_SIZE_PG_2M (0x3UL << 2)
1263 #define CMDQ_CREATE_CQ_PG_SIZE_PG_8M (0x4UL << 2)
1264 #define CMDQ_CREATE_CQ_PG_SIZE_PG_1G (0x5UL << 2)
1265 __le32 cq_fco_cnq_id;
1266 #define CMDQ_CREATE_CQ_CNQ_ID_MASK 0xfffUL
1267 #define CMDQ_CREATE_CQ_CNQ_ID_SFT 0
1268 #define CMDQ_CREATE_CQ_CQ_FCO_MASK 0xfffff000UL
1269 #define CMDQ_CREATE_CQ_CQ_FCO_SFT 12
1270 __le32 dpi;
1271 __le32 cq_size;
1272 __le64 pbl;
1273};
1274
1275/* Destroy CQ command (24 bytes) */
1276struct cmdq_destroy_cq {
1277 u8 opcode;
1278 #define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ 0xaUL
1279 u8 cmd_size;
1280 __le16 flags;
1281 __le16 cookie;
1282 u8 resp_size;
1283 u8 reserved8;
1284 __le64 resp_addr;
1285 __le32 cq_cid;
1286 __le32 unused_0;
1287};
1288
1289/* Resize CQ command (40 bytes) */
1290struct cmdq_resize_cq {
1291 u8 opcode;
1292 #define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ 0xcUL
1293 u8 cmd_size;
1294 __le16 flags;
1295 __le16 cookie;
1296 u8 resp_size;
1297 u8 reserved8;
1298 __le64 resp_addr;
1299 __le32 cq_cid;
1300 __le32 new_cq_size_pg_size_lvl;
1301 #define CMDQ_RESIZE_CQ_LVL_MASK 0x3UL
1302 #define CMDQ_RESIZE_CQ_LVL_SFT 0
1303 #define CMDQ_RESIZE_CQ_LVL_LVL_0 0x0UL
1304 #define CMDQ_RESIZE_CQ_LVL_LVL_1 0x1UL
1305 #define CMDQ_RESIZE_CQ_LVL_LVL_2 0x2UL
1306 #define CMDQ_RESIZE_CQ_PG_SIZE_MASK 0x1cUL
1307 #define CMDQ_RESIZE_CQ_PG_SIZE_SFT 2
1308 #define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K (0x0UL << 2)
1309 #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K (0x1UL << 2)
1310 #define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K (0x2UL << 2)
1311 #define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M (0x3UL << 2)
1312 #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M (0x4UL << 2)
1313 #define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G (0x5UL << 2)
1314 #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK 0x1fffe0UL
1315 #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5
1316 __le64 new_pbl;
1317 __le32 new_cq_fco;
1318 __le32 unused_2;
1319};
1320
1321/* Allocate MRW command (32 bytes) */
1322struct cmdq_allocate_mrw {
1323 u8 opcode;
1324 #define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW 0xdUL
1325 u8 cmd_size;
1326 __le16 flags;
1327 __le16 cookie;
1328 u8 resp_size;
1329 u8 reserved8;
1330 __le64 resp_addr;
1331 __le64 mrw_handle;
1332 u8 mrw_flags;
1333 #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK 0xfUL
1334 #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT 0
1335 #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR 0x0UL
1336 #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR 0x1UL
1337 #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 0x2UL
1338 #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A 0x3UL
1339 #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B 0x4UL
1340 u8 access;
1341 #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_MASK 0x1fUL
1342 #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_SFT 0
1343 #define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY 0x20UL
1344 __le16 unused_1;
1345 __le32 pd_id;
1346};
1347
1348/* De-allocate key command (24 bytes) */
1349struct cmdq_deallocate_key {
1350 u8 opcode;
1351 #define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY 0xeUL
1352 u8 cmd_size;
1353 __le16 flags;
1354 __le16 cookie;
1355 u8 resp_size;
1356 u8 reserved8;
1357 __le64 resp_addr;
1358 u8 mrw_flags;
1359 #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK 0xfUL
1360 #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT 0
1361 #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR 0x0UL
1362 #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR 0x1UL
1363 #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 0x2UL
1364 #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A 0x3UL
1365 #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B 0x4UL
1366 u8 unused_1[3];
1367 __le32 key;
1368};
1369
1370/* Register MR command (48 bytes) */
1371struct cmdq_register_mr {
1372 u8 opcode;
1373 #define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR 0xfUL
1374 u8 cmd_size;
1375 __le16 flags;
1376 __le16 cookie;
1377 u8 resp_size;
1378 u8 reserved8;
1379 __le64 resp_addr;
1380 u8 log2_pg_size_lvl;
1381 #define CMDQ_REGISTER_MR_LVL_MASK 0x3UL
1382 #define CMDQ_REGISTER_MR_LVL_SFT 0
1383 #define CMDQ_REGISTER_MR_LVL_LVL_0 0x0UL
1384 #define CMDQ_REGISTER_MR_LVL_LVL_1 0x1UL
1385 #define CMDQ_REGISTER_MR_LVL_LVL_2 0x2UL
1386 #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK 0x7cUL
1387 #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT 2
1388 u8 access;
1389 #define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE 0x1UL
1390 #define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ 0x2UL
1391 #define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE 0x4UL
1392 #define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC 0x8UL
1393 #define CMDQ_REGISTER_MR_ACCESS_MW_BIND 0x10UL
1394 #define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED 0x20UL
1395 __le16 unused_1;
1396 __le32 key;
1397 __le64 pbl;
1398 __le64 va;
1399 __le64 mr_size;
1400};
1401
1402/* Deregister MR command (24 bytes) */
1403struct cmdq_deregister_mr {
1404 u8 opcode;
1405 #define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR 0x10UL
1406 u8 cmd_size;
1407 __le16 flags;
1408 __le16 cookie;
1409 u8 resp_size;
1410 u8 reserved8;
1411 __le64 resp_addr;
1412 __le32 lkey;
1413 __le32 unused_0;
1414};
1415
1416/* Add GID command (48 bytes) */
1417struct cmdq_add_gid {
1418 u8 opcode;
1419 #define CMDQ_ADD_GID_OPCODE_ADD_GID 0x11UL
1420 u8 cmd_size;
1421 __le16 flags;
1422 __le16 cookie;
1423 u8 resp_size;
1424 u8 reserved8;
1425 __le64 resp_addr;
1426 __be32 gid[4];
1427 __be16 src_mac[3];
1428 __le16 vlan;
1429 #define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK 0xfffUL
1430 #define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT 0
1431 #define CMDQ_ADD_GID_VLAN_TPID_MASK 0x7000UL
1432 #define CMDQ_ADD_GID_VLAN_TPID_SFT 12
1433 #define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12)
1434 #define CMDQ_ADD_GID_VLAN_TPID_TPID_8100 (0x1UL << 12)
1435 #define CMDQ_ADD_GID_VLAN_TPID_TPID_9100 (0x2UL << 12)
1436 #define CMDQ_ADD_GID_VLAN_TPID_TPID_9200 (0x3UL << 12)
1437 #define CMDQ_ADD_GID_VLAN_TPID_TPID_9300 (0x4UL << 12)
1438 #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
1439 #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
1440 #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
1441 #define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3
1442 #define CMDQ_ADD_GID_VLAN_VLAN_EN 0x8000UL
1443 __le16 ipid;
1444 __le16 stats_ctx;
1445 #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL
1446 #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT 0
1447 #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL
1448 __le32 unused_0;
1449};
1450
1451/* Delete GID command (24 bytes) */
1452struct cmdq_delete_gid {
1453 u8 opcode;
1454 #define CMDQ_DELETE_GID_OPCODE_DELETE_GID 0x12UL
1455 u8 cmd_size;
1456 __le16 flags;
1457 __le16 cookie;
1458 u8 resp_size;
1459 u8 reserved8;
1460 __le64 resp_addr;
1461 __le16 gid_index;
1462 __le16 unused_0;
1463 __le32 unused_1;
1464};
1465
1466/* Modify GID command (48 bytes) */
1467struct cmdq_modify_gid {
1468 u8 opcode;
1469 #define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID 0x17UL
1470 u8 cmd_size;
1471 __le16 flags;
1472 __le16 cookie;
1473 u8 resp_size;
1474 u8 reserved8;
1475 __le64 resp_addr;
1476 __le32 gid[4];
1477 __le16 src_mac[3];
1478 __le16 vlan;
1479 #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK 0xfffUL
1480 #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT 0
1481 #define CMDQ_MODIFY_GID_VLAN_TPID_MASK 0x7000UL
1482 #define CMDQ_MODIFY_GID_VLAN_TPID_SFT 12
1483 #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12)
1484 #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 (0x1UL << 12)
1485 #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 (0x2UL << 12)
1486 #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 (0x3UL << 12)
1487 #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 (0x4UL << 12)
1488 #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
1489 #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
1490 #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
1491 #define CMDQ_MODIFY_GID_VLAN_TPID_LAST \
1492 CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3
1493 #define CMDQ_MODIFY_GID_VLAN_VLAN_EN 0x8000UL
1494 __le16 ipid;
1495 __le16 gid_index;
1496 __le16 stats_ctx;
1497 #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL
1498 #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT 0
1499 #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL
1500 __le16 unused_0;
1501};
1502
1503/* Query GID command (24 bytes) */
1504struct cmdq_query_gid {
1505 u8 opcode;
1506 #define CMDQ_QUERY_GID_OPCODE_QUERY_GID 0x18UL
1507 u8 cmd_size;
1508 __le16 flags;
1509 __le16 cookie;
1510 u8 resp_size;
1511 u8 reserved8;
1512 __le64 resp_addr;
1513 __le16 gid_index;
1514 __le16 unused_0;
1515 __le32 unused_1;
1516};
1517
1518/* Create QP1 command (80 bytes) */
1519struct cmdq_create_qp1 {
1520 u8 opcode;
1521 #define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 0x13UL
1522 u8 cmd_size;
1523 __le16 flags;
1524 __le16 cookie;
1525 u8 resp_size;
1526 u8 reserved8;
1527 __le64 resp_addr;
1528 __le64 qp_handle;
1529 __le32 qp_flags;
1530 #define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED 0x1UL
1531 #define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION 0x2UL
1532 #define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
1533 u8 type;
1534 #define CMDQ_CREATE_QP1_TYPE_GSI 0x1UL
1535 u8 sq_pg_size_sq_lvl;
1536 #define CMDQ_CREATE_QP1_SQ_LVL_MASK 0xfUL
1537 #define CMDQ_CREATE_QP1_SQ_LVL_SFT 0
1538 #define CMDQ_CREATE_QP1_SQ_LVL_LVL_0 0x0UL
1539 #define CMDQ_CREATE_QP1_SQ_LVL_LVL_1 0x1UL
1540 #define CMDQ_CREATE_QP1_SQ_LVL_LVL_2 0x2UL
1541 #define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK 0xf0UL
1542 #define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT 4
1543 #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K (0x0UL << 4)
1544 #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K (0x1UL << 4)
1545 #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K (0x2UL << 4)
1546 #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M (0x3UL << 4)
1547 #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M (0x4UL << 4)
1548 #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G (0x5UL << 4)
1549 u8 rq_pg_size_rq_lvl;
1550 #define CMDQ_CREATE_QP1_RQ_LVL_MASK 0xfUL
1551 #define CMDQ_CREATE_QP1_RQ_LVL_SFT 0
1552 #define CMDQ_CREATE_QP1_RQ_LVL_LVL_0 0x0UL
1553 #define CMDQ_CREATE_QP1_RQ_LVL_LVL_1 0x1UL
1554 #define CMDQ_CREATE_QP1_RQ_LVL_LVL_2 0x2UL
1555 #define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK 0xf0UL
1556 #define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT 4
1557 #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K (0x0UL << 4)
1558 #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K (0x1UL << 4)
1559 #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K (0x2UL << 4)
1560 #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M (0x3UL << 4)
1561 #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M (0x4UL << 4)
1562 #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G (0x5UL << 4)
1563 u8 unused_0;
1564 __le32 dpi;
1565 __le32 sq_size;
1566 __le32 rq_size;
1567 __le16 sq_fwo_sq_sge;
1568 #define CMDQ_CREATE_QP1_SQ_SGE_MASK 0xfUL
1569 #define CMDQ_CREATE_QP1_SQ_SGE_SFT 0
1570 #define CMDQ_CREATE_QP1_SQ_FWO_MASK 0xfff0UL
1571 #define CMDQ_CREATE_QP1_SQ_FWO_SFT 4
1572 __le16 rq_fwo_rq_sge;
1573 #define CMDQ_CREATE_QP1_RQ_SGE_MASK 0xfUL
1574 #define CMDQ_CREATE_QP1_RQ_SGE_SFT 0
1575 #define CMDQ_CREATE_QP1_RQ_FWO_MASK 0xfff0UL
1576 #define CMDQ_CREATE_QP1_RQ_FWO_SFT 4
1577 __le32 scq_cid;
1578 __le32 rcq_cid;
1579 __le32 srq_cid;
1580 __le32 pd_id;
1581 __le64 sq_pbl;
1582 __le64 rq_pbl;
1583};
1584
1585/* Destroy QP1 command (24 bytes) */
1586struct cmdq_destroy_qp1 {
1587 u8 opcode;
1588 #define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 0x14UL
1589 u8 cmd_size;
1590 __le16 flags;
1591 __le16 cookie;
1592 u8 resp_size;
1593 u8 reserved8;
1594 __le64 resp_addr;
1595 __le32 qp1_cid;
1596 __le32 unused_0;
1597};
1598
1599/* Create AH command (64 bytes) */
1600struct cmdq_create_ah {
1601 u8 opcode;
1602 #define CMDQ_CREATE_AH_OPCODE_CREATE_AH 0x15UL
1603 u8 cmd_size;
1604 __le16 flags;
1605 __le16 cookie;
1606 u8 resp_size;
1607 u8 reserved8;
1608 __le64 resp_addr;
1609 __le64 ah_handle;
1610 __le32 dgid[4];
1611 u8 type;
1612 #define CMDQ_CREATE_AH_TYPE_V1 0x0UL
1613 #define CMDQ_CREATE_AH_TYPE_V2IPV4 0x2UL
1614 #define CMDQ_CREATE_AH_TYPE_V2IPV6 0x3UL
1615 u8 hop_limit;
1616 __le16 sgid_index;
1617 __le32 dest_vlan_id_flow_label;
1618 #define CMDQ_CREATE_AH_FLOW_LABEL_MASK 0xfffffUL
1619 #define CMDQ_CREATE_AH_FLOW_LABEL_SFT 0
1620 #define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK 0xfff00000UL
1621 #define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20
1622 __le32 pd_id;
1623 __le32 unused_0;
1624 __le16 dest_mac[3];
1625 u8 traffic_class;
1626 u8 unused_1;
1627};
1628
1629/* Destroy AH command (24 bytes) */
1630struct cmdq_destroy_ah {
1631 u8 opcode;
1632 #define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH 0x16UL
1633 u8 cmd_size;
1634 __le16 flags;
1635 __le16 cookie;
1636 u8 resp_size;
1637 u8 reserved8;
1638 __le64 resp_addr;
1639 __le32 ah_cid;
1640 __le32 unused_0;
1641};
1642
1643/* Initialize Firmware command (112 bytes) */
1644struct cmdq_initialize_fw {
1645 u8 opcode;
1646 #define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW 0x80UL
1647 u8 cmd_size;
1648 __le16 flags;
1649 __le16 cookie;
1650 u8 resp_size;
1651 u8 reserved8;
1652 __le64 resp_addr;
1653 u8 qpc_pg_size_qpc_lvl;
1654 #define CMDQ_INITIALIZE_FW_QPC_LVL_MASK 0xfUL
1655 #define CMDQ_INITIALIZE_FW_QPC_LVL_SFT 0
1656 #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 0x0UL
1657 #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 0x1UL
1658 #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 0x2UL
1659 #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK 0xf0UL
1660 #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT 4
1661 #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K (0x0UL << 4)
1662 #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K (0x1UL << 4)
1663 #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K (0x2UL << 4)
1664 #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M (0x3UL << 4)
1665 #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M (0x4UL << 4)
1666 #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G (0x5UL << 4)
1667 u8 mrw_pg_size_mrw_lvl;
1668 #define CMDQ_INITIALIZE_FW_MRW_LVL_MASK 0xfUL
1669 #define CMDQ_INITIALIZE_FW_MRW_LVL_SFT 0
1670 #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 0x0UL
1671 #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 0x1UL
1672 #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 0x2UL
1673 #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK 0xf0UL
1674 #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT 4
1675 #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K (0x0UL << 4)
1676 #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K (0x1UL << 4)
1677 #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K (0x2UL << 4)
1678 #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M (0x3UL << 4)
1679 #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M (0x4UL << 4)
1680 #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G (0x5UL << 4)
1681 u8 srq_pg_size_srq_lvl;
1682 #define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK 0xfUL
1683 #define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT 0
1684 #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 0x0UL
1685 #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 0x1UL
1686 #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 0x2UL
1687 #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK 0xf0UL
1688 #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT 4
1689 #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K (0x0UL << 4)
1690 #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K (0x1UL << 4)
1691 #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K (0x2UL << 4)
1692 #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M (0x3UL << 4)
1693 #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M (0x4UL << 4)
1694 #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G (0x5UL << 4)
1695 u8 cq_pg_size_cq_lvl;
1696 #define CMDQ_INITIALIZE_FW_CQ_LVL_MASK 0xfUL
1697 #define CMDQ_INITIALIZE_FW_CQ_LVL_SFT 0
1698 #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 0x0UL
1699 #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 0x1UL
1700 #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 0x2UL
1701 #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK 0xf0UL
1702 #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT 4
1703 #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K (0x0UL << 4)
1704 #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K (0x1UL << 4)
1705 #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K (0x2UL << 4)
1706 #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M (0x3UL << 4)
1707 #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M (0x4UL << 4)
1708 #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G (0x5UL << 4)
1709 u8 tqm_pg_size_tqm_lvl;
1710 #define CMDQ_INITIALIZE_FW_TQM_LVL_MASK 0xfUL
1711 #define CMDQ_INITIALIZE_FW_TQM_LVL_SFT 0
1712 #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 0x0UL
1713 #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 0x1UL
1714 #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 0x2UL
1715 #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK 0xf0UL
1716 #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT 4
1717 #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K (0x0UL << 4)
1718 #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K (0x1UL << 4)
1719 #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K (0x2UL << 4)
1720 #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M (0x3UL << 4)
1721 #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M (0x4UL << 4)
1722 #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G (0x5UL << 4)
1723 u8 tim_pg_size_tim_lvl;
1724 #define CMDQ_INITIALIZE_FW_TIM_LVL_MASK 0xfUL
1725 #define CMDQ_INITIALIZE_FW_TIM_LVL_SFT 0
1726 #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 0x0UL
1727 #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 0x1UL
1728 #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 0x2UL
1729 #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK 0xf0UL
1730 #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT 4
1731 #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K (0x0UL << 4)
1732 #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K (0x1UL << 4)
1733 #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K (0x2UL << 4)
1734 #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M (0x3UL << 4)
1735 #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M (0x4UL << 4)
1736 #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G (0x5UL << 4)
1737 __le16 reserved16;
1738 __le64 qpc_page_dir;
1739 __le64 mrw_page_dir;
1740 __le64 srq_page_dir;
1741 __le64 cq_page_dir;
1742 __le64 tqm_page_dir;
1743 __le64 tim_page_dir;
1744 __le32 number_of_qp;
1745 __le32 number_of_mrw;
1746 __le32 number_of_srq;
1747 __le32 number_of_cq;
1748 __le32 max_qp_per_vf;
1749 __le32 max_mrw_per_vf;
1750 __le32 max_srq_per_vf;
1751 __le32 max_cq_per_vf;
1752 __le32 max_gid_per_vf;
1753 __le32 stat_ctx_id;
1754};
1755
1756/* De-initialize Firmware command (16 bytes) */
1757struct cmdq_deinitialize_fw {
1758 u8 opcode;
1759 #define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW 0x81UL
1760 u8 cmd_size;
1761 __le16 flags;
1762 __le16 cookie;
1763 u8 resp_size;
1764 u8 reserved8;
1765 __le64 resp_addr;
1766};
1767
1768/* Stop function command (16 bytes) */
1769struct cmdq_stop_func {
1770 u8 opcode;
1771 #define CMDQ_STOP_FUNC_OPCODE_STOP_FUNC 0x82UL
1772 u8 cmd_size;
1773 __le16 flags;
1774 __le16 cookie;
1775 u8 resp_size;
1776 u8 reserved8;
1777 __le64 resp_addr;
1778};
1779
1780/* Query function command (16 bytes) */
1781struct cmdq_query_func {
1782 u8 opcode;
1783 #define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC 0x83UL
1784 u8 cmd_size;
1785 __le16 flags;
1786 __le16 cookie;
1787 u8 resp_size;
1788 u8 reserved8;
1789 __le64 resp_addr;
1790};
1791
1792/* Set function resources command (16 bytes) */
1793struct cmdq_set_func_resources {
1794 u8 opcode;
1795 #define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL
1796 u8 cmd_size;
1797 __le16 flags;
1798 __le16 cookie;
1799 u8 resp_size;
1800 u8 reserved8;
1801 __le64 resp_addr;
1802};
1803
1804/* Read hardware resource context command (24 bytes) */
1805struct cmdq_read_context {
1806 u8 opcode;
1807 #define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT 0x85UL
1808 u8 cmd_size;
1809 __le16 flags;
1810 __le16 cookie;
1811 u8 resp_size;
1812 u8 reserved8;
1813 __le64 resp_addr;
1814 __le32 type_xid;
1815 #define CMDQ_READ_CONTEXT_XID_MASK 0xffffffUL
1816 #define CMDQ_READ_CONTEXT_XID_SFT 0
1817 #define CMDQ_READ_CONTEXT_TYPE_MASK 0xff000000UL
1818 #define CMDQ_READ_CONTEXT_TYPE_SFT 24
1819 #define CMDQ_READ_CONTEXT_TYPE_QPC (0x0UL << 24)
1820 #define CMDQ_READ_CONTEXT_TYPE_CQ (0x1UL << 24)
1821 #define CMDQ_READ_CONTEXT_TYPE_MRW (0x2UL << 24)
1822 #define CMDQ_READ_CONTEXT_TYPE_SRQ (0x3UL << 24)
1823 __le32 unused_0;
1824};
1825
1826/* Map TC to COS. Can only be issued from a PF (24 bytes) */
1827struct cmdq_map_tc_to_cos {
1828 u8 opcode;
1829 #define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS 0x8aUL
1830 u8 cmd_size;
1831 __le16 flags;
1832 __le16 cookie;
1833 u8 resp_size;
1834 u8 reserved8;
1835 __le64 resp_addr;
1836 __le16 cos0;
1837 #define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE 0xffffUL
1838 __le16 cos1;
1839 #define CMDQ_MAP_TC_TO_COS_COS1_DISABLE 0x8000UL
1840 #define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE 0xffffUL
1841 __le32 unused_0;
1842};
1843
1844/* Query version command (16 bytes) */
1845struct cmdq_query_version {
1846 u8 opcode;
1847 #define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION 0x8bUL
1848 u8 cmd_size;
1849 __le16 flags;
1850 __le16 cookie;
1851 u8 resp_size;
1852 u8 reserved8;
1853 __le64 resp_addr;
1854};
1855
1856/* Command-Response Event Queue (CREQ) Structures */
1857/* Base CREQ Record (16 bytes) */
1858struct creq_base {
1859 u8 type;
1860 #define CREQ_BASE_TYPE_MASK 0x3fUL
1861 #define CREQ_BASE_TYPE_SFT 0
1862 #define CREQ_BASE_TYPE_QP_EVENT 0x38UL
1863 #define CREQ_BASE_TYPE_FUNC_EVENT 0x3aUL
1864 #define CREQ_BASE_RESERVED2_MASK 0xc0UL
1865 #define CREQ_BASE_RESERVED2_SFT 6
1866 u8 reserved56[7];
1867 u8 v;
1868 #define CREQ_BASE_V 0x1UL
1869 #define CREQ_BASE_RESERVED7_MASK 0xfeUL
1870 #define CREQ_BASE_RESERVED7_SFT 1
1871 u8 event;
1872 __le16 reserved48[3];
1873};
1874
1875/* RoCE Function Async Event Notification (16 bytes) */
1876struct creq_func_event {
1877 u8 type;
1878 #define CREQ_FUNC_EVENT_TYPE_MASK 0x3fUL
1879 #define CREQ_FUNC_EVENT_TYPE_SFT 0
1880 #define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT 0x3aUL
1881 #define CREQ_FUNC_EVENT_RESERVED2_MASK 0xc0UL
1882 #define CREQ_FUNC_EVENT_RESERVED2_SFT 6
1883 u8 reserved56[7];
1884 u8 v;
1885 #define CREQ_FUNC_EVENT_V 0x1UL
1886 #define CREQ_FUNC_EVENT_RESERVED7_MASK 0xfeUL
1887 #define CREQ_FUNC_EVENT_RESERVED7_SFT 1
1888 u8 event;
1889 #define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR 0x1UL
1890 #define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR 0x2UL
1891 #define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR 0x3UL
1892 #define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR 0x4UL
1893 #define CREQ_FUNC_EVENT_EVENT_CQ_ERROR 0x5UL
1894 #define CREQ_FUNC_EVENT_EVENT_TQM_ERROR 0x6UL
1895 #define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR 0x7UL
1896 #define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR 0x8UL
1897 #define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR 0x9UL
1898 #define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR 0xaUL
1899 #define CREQ_FUNC_EVENT_EVENT_TIM_ERROR 0xbUL
1900 #define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST 0x80UL
1901 #define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED 0x81UL
1902 __le16 reserved48[3];
1903};
1904
1905/* RoCE Slowpath Command Completion (16 bytes) */
1906struct creq_qp_event {
1907 u8 type;
1908 #define CREQ_QP_EVENT_TYPE_MASK 0x3fUL
1909 #define CREQ_QP_EVENT_TYPE_SFT 0
1910 #define CREQ_QP_EVENT_TYPE_QP_EVENT 0x38UL
1911 #define CREQ_QP_EVENT_RESERVED2_MASK 0xc0UL
1912 #define CREQ_QP_EVENT_RESERVED2_SFT 6
1913 u8 status;
1914 __le16 cookie;
1915 __le32 reserved32;
1916 u8 v;
1917 #define CREQ_QP_EVENT_V 0x1UL
1918 #define CREQ_QP_EVENT_RESERVED7_MASK 0xfeUL
1919 #define CREQ_QP_EVENT_RESERVED7_SFT 1
1920 u8 event;
1921 #define CREQ_QP_EVENT_EVENT_CREATE_QP 0x1UL
1922 #define CREQ_QP_EVENT_EVENT_DESTROY_QP 0x2UL
1923 #define CREQ_QP_EVENT_EVENT_MODIFY_QP 0x3UL
1924 #define CREQ_QP_EVENT_EVENT_QUERY_QP 0x4UL
1925 #define CREQ_QP_EVENT_EVENT_CREATE_SRQ 0x5UL
1926 #define CREQ_QP_EVENT_EVENT_DESTROY_SRQ 0x6UL
1927 #define CREQ_QP_EVENT_EVENT_QUERY_SRQ 0x8UL
1928 #define CREQ_QP_EVENT_EVENT_CREATE_CQ 0x9UL
1929 #define CREQ_QP_EVENT_EVENT_DESTROY_CQ 0xaUL
1930 #define CREQ_QP_EVENT_EVENT_RESIZE_CQ 0xcUL
1931 #define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW 0xdUL
1932 #define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY 0xeUL
1933 #define CREQ_QP_EVENT_EVENT_REGISTER_MR 0xfUL
1934 #define CREQ_QP_EVENT_EVENT_DEREGISTER_MR 0x10UL
1935 #define CREQ_QP_EVENT_EVENT_ADD_GID 0x11UL
1936 #define CREQ_QP_EVENT_EVENT_DELETE_GID 0x12UL
1937 #define CREQ_QP_EVENT_EVENT_MODIFY_GID 0x17UL
1938 #define CREQ_QP_EVENT_EVENT_QUERY_GID 0x18UL
1939 #define CREQ_QP_EVENT_EVENT_CREATE_QP1 0x13UL
1940 #define CREQ_QP_EVENT_EVENT_DESTROY_QP1 0x14UL
1941 #define CREQ_QP_EVENT_EVENT_CREATE_AH 0x15UL
1942 #define CREQ_QP_EVENT_EVENT_DESTROY_AH 0x16UL
1943 #define CREQ_QP_EVENT_EVENT_INITIALIZE_FW 0x80UL
1944 #define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW 0x81UL
1945 #define CREQ_QP_EVENT_EVENT_STOP_FUNC 0x82UL
1946 #define CREQ_QP_EVENT_EVENT_QUERY_FUNC 0x83UL
1947 #define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES 0x84UL
1948 #define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS 0x8aUL
1949 #define CREQ_QP_EVENT_EVENT_QUERY_VERSION 0x8bUL
1950 #define CREQ_QP_EVENT_EVENT_MODIFY_CC 0x8cUL
1951 #define CREQ_QP_EVENT_EVENT_QUERY_CC 0x8dUL
1952 #define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION 0xc0UL
1953 __le16 reserved48[3];
1954};
1955
1956/* Create QP command response (16 bytes) */
1957struct creq_create_qp_resp {
1958 u8 type;
1959 #define CREQ_CREATE_QP_RESP_TYPE_MASK 0x3fUL
1960 #define CREQ_CREATE_QP_RESP_TYPE_SFT 0
1961 #define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT 0x38UL
1962 #define CREQ_CREATE_QP_RESP_RESERVED2_MASK 0xc0UL
1963 #define CREQ_CREATE_QP_RESP_RESERVED2_SFT 6
1964 u8 status;
1965 __le16 cookie;
1966 __le32 xid;
1967 u8 v;
1968 #define CREQ_CREATE_QP_RESP_V 0x1UL
1969 #define CREQ_CREATE_QP_RESP_RESERVED7_MASK 0xfeUL
1970 #define CREQ_CREATE_QP_RESP_RESERVED7_SFT 1
1971 u8 event;
1972 #define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP 0x1UL
1973 __le16 reserved48[3];
1974};
1975
1976/* Destroy QP command response (16 bytes) */
1977struct creq_destroy_qp_resp {
1978 u8 type;
1979 #define CREQ_DESTROY_QP_RESP_TYPE_MASK 0x3fUL
1980 #define CREQ_DESTROY_QP_RESP_TYPE_SFT 0
1981 #define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT 0x38UL
1982 #define CREQ_DESTROY_QP_RESP_RESERVED2_MASK 0xc0UL
1983 #define CREQ_DESTROY_QP_RESP_RESERVED2_SFT 6
1984 u8 status;
1985 __le16 cookie;
1986 __le32 xid;
1987 u8 v;
1988 #define CREQ_DESTROY_QP_RESP_V 0x1UL
1989 #define CREQ_DESTROY_QP_RESP_RESERVED7_MASK 0xfeUL
1990 #define CREQ_DESTROY_QP_RESP_RESERVED7_SFT 1
1991 u8 event;
1992 #define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP 0x2UL
1993 __le16 reserved48[3];
1994};
1995
1996/* Modify QP command response (16 bytes) */
1997struct creq_modify_qp_resp {
1998 u8 type;
1999 #define CREQ_MODIFY_QP_RESP_TYPE_MASK 0x3fUL
2000 #define CREQ_MODIFY_QP_RESP_TYPE_SFT 0
2001 #define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT 0x38UL
2002 #define CREQ_MODIFY_QP_RESP_RESERVED2_MASK 0xc0UL
2003 #define CREQ_MODIFY_QP_RESP_RESERVED2_SFT 6
2004 u8 status;
2005 __le16 cookie;
2006 __le32 xid;
2007 u8 v;
2008 #define CREQ_MODIFY_QP_RESP_V 0x1UL
2009 #define CREQ_MODIFY_QP_RESP_RESERVED7_MASK 0xfeUL
2010 #define CREQ_MODIFY_QP_RESP_RESERVED7_SFT 1
2011 u8 event;
2012 #define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP 0x3UL
2013 __le16 reserved48[3];
2014};
2015
2016/* Query QP command response (16 bytes) */
2017struct creq_query_qp_resp {
2018 u8 type;
2019 #define CREQ_QUERY_QP_RESP_TYPE_MASK 0x3fUL
2020 #define CREQ_QUERY_QP_RESP_TYPE_SFT 0
2021 #define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT 0x38UL
2022 #define CREQ_QUERY_QP_RESP_RESERVED2_MASK 0xc0UL
2023 #define CREQ_QUERY_QP_RESP_RESERVED2_SFT 6
2024 u8 status;
2025 __le16 cookie;
2026 __le32 size;
2027 u8 v;
2028 #define CREQ_QUERY_QP_RESP_V 0x1UL
2029 #define CREQ_QUERY_QP_RESP_RESERVED7_MASK 0xfeUL
2030 #define CREQ_QUERY_QP_RESP_RESERVED7_SFT 1
2031 u8 event;
2032 #define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP 0x4UL
2033 __le16 reserved48[3];
2034};
2035
2036/* Query QP command response side buffer structure (104 bytes) */
2037struct creq_query_qp_resp_sb {
2038 u8 opcode;
2039 #define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP 0x4UL
2040 u8 status;
2041 __le16 cookie;
2042 __le16 flags;
2043 u8 resp_size;
2044 u8 reserved8;
2045 __le32 xid;
2046 u8 en_sqd_async_notify_state;
2047 #define CREQ_QUERY_QP_RESP_SB_STATE_MASK 0xfUL
2048 #define CREQ_QUERY_QP_RESP_SB_STATE_SFT 0
2049 #define CREQ_QUERY_QP_RESP_SB_STATE_RESET 0x0UL
2050 #define CREQ_QUERY_QP_RESP_SB_STATE_INIT 0x1UL
2051 #define CREQ_QUERY_QP_RESP_SB_STATE_RTR 0x2UL
2052 #define CREQ_QUERY_QP_RESP_SB_STATE_RTS 0x3UL
2053 #define CREQ_QUERY_QP_RESP_SB_STATE_SQD 0x4UL
2054 #define CREQ_QUERY_QP_RESP_SB_STATE_SQE 0x5UL
2055 #define CREQ_QUERY_QP_RESP_SB_STATE_ERR 0x6UL
2056 #define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY 0x10UL
2057 u8 access;
2058 #define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE 0x1UL
2059 #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE 0x2UL
2060 #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ 0x4UL
2061 #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC 0x8UL
2062 __le16 pkey;
2063 __le32 qkey;
2064 __le32 reserved32;
2065 __le32 dgid[4];
2066 __le32 flow_label;
2067 __le16 sgid_index;
2068 u8 hop_limit;
2069 u8 traffic_class;
2070 __le16 dest_mac[3];
2071 __le16 path_mtu_dest_vlan_id;
2072 #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK 0xfffUL
2073 #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0
2074 #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK 0xf000UL
2075 #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT 12
2076 #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 (0x0UL << 12)
2077 #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 (0x1UL << 12)
2078 #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 (0x2UL << 12)
2079 #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 (0x3UL << 12)
2080 #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 (0x4UL << 12)
2081 #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 (0x5UL << 12)
2082 u8 timeout;
2083 u8 retry_cnt;
2084 u8 rnr_retry;
2085 u8 min_rnr_timer;
2086 __le32 rq_psn;
2087 __le32 sq_psn;
2088 u8 max_rd_atomic;
2089 u8 max_dest_rd_atomic;
2090 u8 tos_dscp_tos_ecn;
2091 #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK 0x3UL
2092 #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT 0
2093 #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK 0xfcUL
2094 #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2
2095 u8 enable_cc;
2096 #define CREQ_QUERY_QP_RESP_SB_ENABLE_CC 0x1UL
2097 #define CREQ_QUERY_QP_RESP_SB_RESERVED7_MASK 0xfeUL
2098 #define CREQ_QUERY_QP_RESP_SB_RESERVED7_SFT 1
2099 __le32 sq_size;
2100 __le32 rq_size;
2101 __le16 sq_sge;
2102 __le16 rq_sge;
2103 __le32 max_inline_data;
2104 __le32 dest_qp_id;
2105 __le32 unused_1;
2106 __le16 src_mac[3];
2107 __le16 vlan_pcp_vlan_dei_vlan_id;
2108 #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK 0xfffUL
2109 #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT 0
2110 #define CREQ_QUERY_QP_RESP_SB_VLAN_DEI 0x1000UL
2111 #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK 0xe000UL
2112 #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13
2113};
2114
2115/* Create SRQ command response (16 bytes) */
2116struct creq_create_srq_resp {
2117 u8 type;
2118 #define CREQ_CREATE_SRQ_RESP_TYPE_MASK 0x3fUL
2119 #define CREQ_CREATE_SRQ_RESP_TYPE_SFT 0
2120 #define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT 0x38UL
2121 #define CREQ_CREATE_SRQ_RESP_RESERVED2_MASK 0xc0UL
2122 #define CREQ_CREATE_SRQ_RESP_RESERVED2_SFT 6
2123 u8 status;
2124 __le16 cookie;
2125 __le32 xid;
2126 u8 v;
2127 #define CREQ_CREATE_SRQ_RESP_V 0x1UL
2128 #define CREQ_CREATE_SRQ_RESP_RESERVED7_MASK 0xfeUL
2129 #define CREQ_CREATE_SRQ_RESP_RESERVED7_SFT 1
2130 u8 event;
2131 #define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ 0x5UL
2132 __le16 reserved48[3];
2133};
2134
2135/* Destroy SRQ command response (16 bytes) */
2136struct creq_destroy_srq_resp {
2137 u8 type;
2138 #define CREQ_DESTROY_SRQ_RESP_TYPE_MASK 0x3fUL
2139 #define CREQ_DESTROY_SRQ_RESP_TYPE_SFT 0
2140 #define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT 0x38UL
2141 #define CREQ_DESTROY_SRQ_RESP_RESERVED2_MASK 0xc0UL
2142 #define CREQ_DESTROY_SRQ_RESP_RESERVED2_SFT 6
2143 u8 status;
2144 __le16 cookie;
2145 __le32 xid;
2146 u8 v;
2147 #define CREQ_DESTROY_SRQ_RESP_V 0x1UL
2148 #define CREQ_DESTROY_SRQ_RESP_RESERVED7_MASK 0xfeUL
2149 #define CREQ_DESTROY_SRQ_RESP_RESERVED7_SFT 1
2150 u8 event;
2151 #define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ 0x6UL
2152 __le16 enable_for_arm[3];
2153 #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK 0x30000UL
2154 #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16
2155 #define CREQ_DESTROY_SRQ_RESP_RESERVED46_MASK 0xfffc0000UL
2156 #define CREQ_DESTROY_SRQ_RESP_RESERVED46_SFT 18
2157};
2158
2159/* Query SRQ command response (16 bytes) */
2160struct creq_query_srq_resp {
2161 u8 type;
2162 #define CREQ_QUERY_SRQ_RESP_TYPE_MASK 0x3fUL
2163 #define CREQ_QUERY_SRQ_RESP_TYPE_SFT 0
2164 #define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT 0x38UL
2165 #define CREQ_QUERY_SRQ_RESP_RESERVED2_MASK 0xc0UL
2166 #define CREQ_QUERY_SRQ_RESP_RESERVED2_SFT 6
2167 u8 status;
2168 __le16 cookie;
2169 __le32 size;
2170 u8 v;
2171 #define CREQ_QUERY_SRQ_RESP_V 0x1UL
2172 #define CREQ_QUERY_SRQ_RESP_RESERVED7_MASK 0xfeUL
2173 #define CREQ_QUERY_SRQ_RESP_RESERVED7_SFT 1
2174 u8 event;
2175 #define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ 0x8UL
2176 __le16 reserved48[3];
2177};
2178
2179/* Query SRQ command response side buffer structure (24 bytes) */
2180struct creq_query_srq_resp_sb {
2181 u8 opcode;
2182 #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ 0x8UL
2183 u8 status;
2184 __le16 cookie;
2185 __le16 flags;
2186 u8 resp_size;
2187 u8 reserved8;
2188 __le32 xid;
2189 __le16 srq_limit;
2190 __le16 reserved16;
2191 __le32 data[4];
2192};
2193
2194/* Create CQ command Response (16 bytes) */
2195struct creq_create_cq_resp {
2196 u8 type;
2197 #define CREQ_CREATE_CQ_RESP_TYPE_MASK 0x3fUL
2198 #define CREQ_CREATE_CQ_RESP_TYPE_SFT 0
2199 #define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT 0x38UL
2200 #define CREQ_CREATE_CQ_RESP_RESERVED2_MASK 0xc0UL
2201 #define CREQ_CREATE_CQ_RESP_RESERVED2_SFT 6
2202 u8 status;
2203 __le16 cookie;
2204 __le32 xid;
2205 u8 v;
2206 #define CREQ_CREATE_CQ_RESP_V 0x1UL
2207 #define CREQ_CREATE_CQ_RESP_RESERVED7_MASK 0xfeUL
2208 #define CREQ_CREATE_CQ_RESP_RESERVED7_SFT 1
2209 u8 event;
2210 #define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ 0x9UL
2211 __le16 reserved48[3];
2212};
2213
2214/* Destroy CQ command response (16 bytes) */
2215struct creq_destroy_cq_resp {
2216 u8 type;
2217 #define CREQ_DESTROY_CQ_RESP_TYPE_MASK 0x3fUL
2218 #define CREQ_DESTROY_CQ_RESP_TYPE_SFT 0
2219 #define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT 0x38UL
2220 #define CREQ_DESTROY_CQ_RESP_RESERVED2_MASK 0xc0UL
2221 #define CREQ_DESTROY_CQ_RESP_RESERVED2_SFT 6
2222 u8 status;
2223 __le16 cookie;
2224 __le32 xid;
2225 u8 v;
2226 #define CREQ_DESTROY_CQ_RESP_V 0x1UL
2227 #define CREQ_DESTROY_CQ_RESP_RESERVED7_MASK 0xfeUL
2228 #define CREQ_DESTROY_CQ_RESP_RESERVED7_SFT 1
2229 u8 event;
2230 #define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ 0xaUL
2231 __le16 cq_arm_lvl;
2232 #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK 0x3UL
2233 #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0
2234 #define CREQ_DESTROY_CQ_RESP_RESERVED14_MASK 0xfffcUL
2235 #define CREQ_DESTROY_CQ_RESP_RESERVED14_SFT 2
2236 __le16 total_cnq_events;
2237 __le16 reserved16;
2238};
2239
2240/* Resize CQ command response (16 bytes) */
2241struct creq_resize_cq_resp {
2242 u8 type;
2243 #define CREQ_RESIZE_CQ_RESP_TYPE_MASK 0x3fUL
2244 #define CREQ_RESIZE_CQ_RESP_TYPE_SFT 0
2245 #define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT 0x38UL
2246 #define CREQ_RESIZE_CQ_RESP_RESERVED2_MASK 0xc0UL
2247 #define CREQ_RESIZE_CQ_RESP_RESERVED2_SFT 6
2248 u8 status;
2249 __le16 cookie;
2250 __le32 xid;
2251 u8 v;
2252 #define CREQ_RESIZE_CQ_RESP_V 0x1UL
2253 #define CREQ_RESIZE_CQ_RESP_RESERVED7_MASK 0xfeUL
2254 #define CREQ_RESIZE_CQ_RESP_RESERVED7_SFT 1
2255 u8 event;
2256 #define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ 0xcUL
2257 __le16 reserved48[3];
2258};
2259
2260/* Allocate MRW command response (16 bytes) */
2261struct creq_allocate_mrw_resp {
2262 u8 type;
2263 #define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK 0x3fUL
2264 #define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT 0
2265 #define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT 0x38UL
2266 #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_MASK 0xc0UL
2267 #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_SFT 6
2268 u8 status;
2269 __le16 cookie;
2270 __le32 xid;
2271 u8 v;
2272 #define CREQ_ALLOCATE_MRW_RESP_V 0x1UL
2273 #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_MASK 0xfeUL
2274 #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_SFT 1
2275 u8 event;
2276 #define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW 0xdUL
2277 __le16 reserved48[3];
2278};
2279
2280/* De-allocate key command response (16 bytes) */
2281struct creq_deallocate_key_resp {
2282 u8 type;
2283 #define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK 0x3fUL
2284 #define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT 0
2285 #define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT 0x38UL
2286 #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_MASK 0xc0UL
2287 #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_SFT 6
2288 u8 status;
2289 __le16 cookie;
2290 __le32 xid;
2291 u8 v;
2292 #define CREQ_DEALLOCATE_KEY_RESP_V 0x1UL
2293 #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_MASK 0xfeUL
2294 #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_SFT 1
2295 u8 event;
2296 #define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY 0xeUL
2297 __le16 reserved16;
2298 __le32 bound_window_info;
2299};
2300
2301/* Register MR command response (16 bytes) */
2302struct creq_register_mr_resp {
2303 u8 type;
2304 #define CREQ_REGISTER_MR_RESP_TYPE_MASK 0x3fUL
2305 #define CREQ_REGISTER_MR_RESP_TYPE_SFT 0
2306 #define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL
2307 #define CREQ_REGISTER_MR_RESP_RESERVED2_MASK 0xc0UL
2308 #define CREQ_REGISTER_MR_RESP_RESERVED2_SFT 6
2309 u8 status;
2310 __le16 cookie;
2311 __le32 xid;
2312 u8 v;
2313 #define CREQ_REGISTER_MR_RESP_V 0x1UL
2314 #define CREQ_REGISTER_MR_RESP_RESERVED7_MASK 0xfeUL
2315 #define CREQ_REGISTER_MR_RESP_RESERVED7_SFT 1
2316 u8 event;
2317 #define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR 0xfUL
2318 __le16 reserved48[3];
2319};
2320
2321/* Deregister MR command response (16 bytes) */
2322struct creq_deregister_mr_resp {
2323 u8 type;
2324 #define CREQ_DEREGISTER_MR_RESP_TYPE_MASK 0x3fUL
2325 #define CREQ_DEREGISTER_MR_RESP_TYPE_SFT 0
2326 #define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL
2327 #define CREQ_DEREGISTER_MR_RESP_RESERVED2_MASK 0xc0UL
2328 #define CREQ_DEREGISTER_MR_RESP_RESERVED2_SFT 6
2329 u8 status;
2330 __le16 cookie;
2331 __le32 xid;
2332 u8 v;
2333 #define CREQ_DEREGISTER_MR_RESP_V 0x1UL
2334 #define CREQ_DEREGISTER_MR_RESP_RESERVED7_MASK 0xfeUL
2335 #define CREQ_DEREGISTER_MR_RESP_RESERVED7_SFT 1
2336 u8 event;
2337 #define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR 0x10UL
2338 __le16 reserved16;
2339 __le32 bound_windows;
2340};
2341
2342/* Add GID command response (16 bytes) */
2343struct creq_add_gid_resp {
2344 u8 type;
2345 #define CREQ_ADD_GID_RESP_TYPE_MASK 0x3fUL
2346 #define CREQ_ADD_GID_RESP_TYPE_SFT 0
2347 #define CREQ_ADD_GID_RESP_TYPE_QP_EVENT 0x38UL
2348 #define CREQ_ADD_GID_RESP_RESERVED2_MASK 0xc0UL
2349 #define CREQ_ADD_GID_RESP_RESERVED2_SFT 6
2350 u8 status;
2351 __le16 cookie;
2352 __le32 xid;
2353 u8 v;
2354 #define CREQ_ADD_GID_RESP_V 0x1UL
2355 #define CREQ_ADD_GID_RESP_RESERVED7_MASK 0xfeUL
2356 #define CREQ_ADD_GID_RESP_RESERVED7_SFT 1
2357 u8 event;
2358 #define CREQ_ADD_GID_RESP_EVENT_ADD_GID 0x11UL
2359 __le16 reserved48[3];
2360};
2361
2362/* Delete GID command response (16 bytes) */
2363struct creq_delete_gid_resp {
2364 u8 type;
2365 #define CREQ_DELETE_GID_RESP_TYPE_MASK 0x3fUL
2366 #define CREQ_DELETE_GID_RESP_TYPE_SFT 0
2367 #define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT 0x38UL
2368 #define CREQ_DELETE_GID_RESP_RESERVED2_MASK 0xc0UL
2369 #define CREQ_DELETE_GID_RESP_RESERVED2_SFT 6
2370 u8 status;
2371 __le16 cookie;
2372 __le32 xid;
2373 u8 v;
2374 #define CREQ_DELETE_GID_RESP_V 0x1UL
2375 #define CREQ_DELETE_GID_RESP_RESERVED7_MASK 0xfeUL
2376 #define CREQ_DELETE_GID_RESP_RESERVED7_SFT 1
2377 u8 event;
2378 #define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID 0x12UL
2379 __le16 reserved48[3];
2380};
2381
2382/* Modify GID command response (16 bytes) */
2383struct creq_modify_gid_resp {
2384 u8 type;
2385 #define CREQ_MODIFY_GID_RESP_TYPE_MASK 0x3fUL
2386 #define CREQ_MODIFY_GID_RESP_TYPE_SFT 0
2387 #define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT 0x38UL
2388 #define CREQ_MODIFY_GID_RESP_RESERVED2_MASK 0xc0UL
2389 #define CREQ_MODIFY_GID_RESP_RESERVED2_SFT 6
2390 u8 status;
2391 __le16 cookie;
2392 __le32 xid;
2393 u8 v;
2394 #define CREQ_MODIFY_GID_RESP_V 0x1UL
2395 #define CREQ_MODIFY_GID_RESP_RESERVED7_MASK 0xfeUL
2396 #define CREQ_MODIFY_GID_RESP_RESERVED7_SFT 1
2397 u8 event;
2398 #define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID 0x11UL
2399 __le16 reserved48[3];
2400};
2401
2402/* Query GID command response (16 bytes) */
2403struct creq_query_gid_resp {
2404 u8 type;
2405 #define CREQ_QUERY_GID_RESP_TYPE_MASK 0x3fUL
2406 #define CREQ_QUERY_GID_RESP_TYPE_SFT 0
2407 #define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT 0x38UL
2408 #define CREQ_QUERY_GID_RESP_RESERVED2_MASK 0xc0UL
2409 #define CREQ_QUERY_GID_RESP_RESERVED2_SFT 6
2410 u8 status;
2411 __le16 cookie;
2412 __le32 size;
2413 u8 v;
2414 #define CREQ_QUERY_GID_RESP_V 0x1UL
2415 #define CREQ_QUERY_GID_RESP_RESERVED7_MASK 0xfeUL
2416 #define CREQ_QUERY_GID_RESP_RESERVED7_SFT 1
2417 u8 event;
2418 #define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID 0x18UL
2419 __le16 reserved48[3];
2420};
2421
2422/* Query GID command response side buffer structure (40 bytes) */
2423struct creq_query_gid_resp_sb {
2424 u8 opcode;
2425 #define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID 0x18UL
2426 u8 status;
2427 __le16 cookie;
2428 __le16 flags;
2429 u8 resp_size;
2430 u8 reserved8;
2431 __le32 gid[4];
2432 __le16 src_mac[3];
2433 __le16 vlan;
2434 #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK 0xfffUL
2435 #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT 0
2436 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK 0x7000UL
2437 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT 12
2438 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 (0x0UL << 12)
2439 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 (0x1UL << 12)
2440 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 (0x2UL << 12)
2441 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 (0x3UL << 12)
2442 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 (0x4UL << 12)
2443 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
2444 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
2445 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
2446 #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST \
2447 CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3
2448 #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN 0x8000UL
2449 __le16 ipid;
2450 __le16 gid_index;
2451 __le32 unused_0;
2452};
2453
2454/* Create QP1 command response (16 bytes) */
2455struct creq_create_qp1_resp {
2456 u8 type;
2457 #define CREQ_CREATE_QP1_RESP_TYPE_MASK 0x3fUL
2458 #define CREQ_CREATE_QP1_RESP_TYPE_SFT 0
2459 #define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT 0x38UL
2460 #define CREQ_CREATE_QP1_RESP_RESERVED2_MASK 0xc0UL
2461 #define CREQ_CREATE_QP1_RESP_RESERVED2_SFT 6
2462 u8 status;
2463 __le16 cookie;
2464 __le32 xid;
2465 u8 v;
2466 #define CREQ_CREATE_QP1_RESP_V 0x1UL
2467 #define CREQ_CREATE_QP1_RESP_RESERVED7_MASK 0xfeUL
2468 #define CREQ_CREATE_QP1_RESP_RESERVED7_SFT 1
2469 u8 event;
2470 #define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 0x13UL
2471 __le16 reserved48[3];
2472};
2473
2474/* Destroy QP1 command response (16 bytes) */
2475struct creq_destroy_qp1_resp {
2476 u8 type;
2477 #define CREQ_DESTROY_QP1_RESP_TYPE_MASK 0x3fUL
2478 #define CREQ_DESTROY_QP1_RESP_TYPE_SFT 0
2479 #define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT 0x38UL
2480 #define CREQ_DESTROY_QP1_RESP_RESERVED2_MASK 0xc0UL
2481 #define CREQ_DESTROY_QP1_RESP_RESERVED2_SFT 6
2482 u8 status;
2483 __le16 cookie;
2484 __le32 xid;
2485 u8 v;
2486 #define CREQ_DESTROY_QP1_RESP_V 0x1UL
2487 #define CREQ_DESTROY_QP1_RESP_RESERVED7_MASK 0xfeUL
2488 #define CREQ_DESTROY_QP1_RESP_RESERVED7_SFT 1
2489 u8 event;
2490 #define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 0x14UL
2491 __le16 reserved48[3];
2492};
2493
2494/* Create AH command response (16 bytes) */
2495struct creq_create_ah_resp {
2496 u8 type;
2497 #define CREQ_CREATE_AH_RESP_TYPE_MASK 0x3fUL
2498 #define CREQ_CREATE_AH_RESP_TYPE_SFT 0
2499 #define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT 0x38UL
2500 #define CREQ_CREATE_AH_RESP_RESERVED2_MASK 0xc0UL
2501 #define CREQ_CREATE_AH_RESP_RESERVED2_SFT 6
2502 u8 status;
2503 __le16 cookie;
2504 __le32 xid;
2505 u8 v;
2506 #define CREQ_CREATE_AH_RESP_V 0x1UL
2507 #define CREQ_CREATE_AH_RESP_RESERVED7_MASK 0xfeUL
2508 #define CREQ_CREATE_AH_RESP_RESERVED7_SFT 1
2509 u8 event;
2510 #define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH 0x15UL
2511 __le16 reserved48[3];
2512};
2513
2514/* Destroy AH command response (16 bytes) */
2515struct creq_destroy_ah_resp {
2516 u8 type;
2517 #define CREQ_DESTROY_AH_RESP_TYPE_MASK 0x3fUL
2518 #define CREQ_DESTROY_AH_RESP_TYPE_SFT 0
2519 #define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT 0x38UL
2520 #define CREQ_DESTROY_AH_RESP_RESERVED2_MASK 0xc0UL
2521 #define CREQ_DESTROY_AH_RESP_RESERVED2_SFT 6
2522 u8 status;
2523 __le16 cookie;
2524 __le32 xid;
2525 u8 v;
2526 #define CREQ_DESTROY_AH_RESP_V 0x1UL
2527 #define CREQ_DESTROY_AH_RESP_RESERVED7_MASK 0xfeUL
2528 #define CREQ_DESTROY_AH_RESP_RESERVED7_SFT 1
2529 u8 event;
2530 #define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH 0x16UL
2531 __le16 reserved48[3];
2532};
2533
2534/* Initialize Firmware command response (16 bytes) */
2535struct creq_initialize_fw_resp {
2536 u8 type;
2537 #define CREQ_INITIALIZE_FW_RESP_TYPE_MASK 0x3fUL
2538 #define CREQ_INITIALIZE_FW_RESP_TYPE_SFT 0
2539 #define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL
2540 #define CREQ_INITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL
2541 #define CREQ_INITIALIZE_FW_RESP_RESERVED2_SFT 6
2542 u8 status;
2543 __le16 cookie;
2544 __le32 reserved32;
2545 u8 v;
2546 #define CREQ_INITIALIZE_FW_RESP_V 0x1UL
2547 #define CREQ_INITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL
2548 #define CREQ_INITIALIZE_FW_RESP_RESERVED7_SFT 1
2549 u8 event;
2550 #define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW 0x80UL
2551 __le16 reserved48[3];
2552};
2553
2554/* De-initialize Firmware command response (16 bytes) */
2555struct creq_deinitialize_fw_resp {
2556 u8 type;
2557 #define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK 0x3fUL
2558 #define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT 0
2559 #define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL
2560 #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL
2561 #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_SFT 6
2562 u8 status;
2563 __le16 cookie;
2564 __le32 reserved32;
2565 u8 v;
2566 #define CREQ_DEINITIALIZE_FW_RESP_V 0x1UL
2567 #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL
2568 #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_SFT 1
2569 u8 event;
2570 #define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW 0x81UL
2571 __le16 reserved48[3];
2572};
2573
2574/* Stop function command response (16 bytes) */
2575struct creq_stop_func_resp {
2576 u8 type;
2577 #define CREQ_STOP_FUNC_RESP_TYPE_MASK 0x3fUL
2578 #define CREQ_STOP_FUNC_RESP_TYPE_SFT 0
2579 #define CREQ_STOP_FUNC_RESP_TYPE_QP_EVENT 0x38UL
2580 #define CREQ_STOP_FUNC_RESP_RESERVED2_MASK 0xc0UL
2581 #define CREQ_STOP_FUNC_RESP_RESERVED2_SFT 6
2582 u8 status;
2583 __le16 cookie;
2584 __le32 reserved32;
2585 u8 v;
2586 #define CREQ_STOP_FUNC_RESP_V 0x1UL
2587 #define CREQ_STOP_FUNC_RESP_RESERVED7_MASK 0xfeUL
2588 #define CREQ_STOP_FUNC_RESP_RESERVED7_SFT 1
2589 u8 event;
2590 #define CREQ_STOP_FUNC_RESP_EVENT_STOP_FUNC 0x82UL
2591 __le16 reserved48[3];
2592};
2593
2594/* Query function command response (16 bytes) */
2595struct creq_query_func_resp {
2596 u8 type;
2597 #define CREQ_QUERY_FUNC_RESP_TYPE_MASK 0x3fUL
2598 #define CREQ_QUERY_FUNC_RESP_TYPE_SFT 0
2599 #define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT 0x38UL
2600 #define CREQ_QUERY_FUNC_RESP_RESERVED2_MASK 0xc0UL
2601 #define CREQ_QUERY_FUNC_RESP_RESERVED2_SFT 6
2602 u8 status;
2603 __le16 cookie;
2604 __le32 size;
2605 u8 v;
2606 #define CREQ_QUERY_FUNC_RESP_V 0x1UL
2607 #define CREQ_QUERY_FUNC_RESP_RESERVED7_MASK 0xfeUL
2608 #define CREQ_QUERY_FUNC_RESP_RESERVED7_SFT 1
2609 u8 event;
2610 #define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC 0x83UL
2611 __le16 reserved48[3];
2612};
2613
2614/* Query function command response side buffer structure (88 bytes) */
2615struct creq_query_func_resp_sb {
2616 u8 opcode;
2617 #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC 0x83UL
2618 u8 status;
2619 __le16 cookie;
2620 __le16 flags;
2621 u8 resp_size;
2622 u8 reserved8;
2623 __le64 max_mr_size;
2624 __le32 max_qp;
2625 __le16 max_qp_wr;
2626 __le16 dev_cap_flags;
2627 #define CREQ_QUERY_FUNC_RESP_SB_DEV_CAP_FLAGS_RESIZE_QP 0x1UL
2628 __le32 max_cq;
2629 __le32 max_cqe;
2630 __le32 max_pd;
2631 u8 max_sge;
2632 u8 max_srq_sge;
2633 u8 max_qp_rd_atom;
2634 u8 max_qp_init_rd_atom;
2635 __le32 max_mr;
2636 __le32 max_mw;
2637 __le32 max_raw_eth_qp;
2638 __le32 max_ah;
2639 __le32 max_fmr;
2640 __le32 max_srq_wr;
2641 __le32 max_pkeys;
2642 __le32 max_inline_data;
2643 u8 max_map_per_fmr;
2644 u8 l2_db_space_size;
2645 __le16 max_srq;
2646 __le32 max_gid;
2647 __le32 tqm_alloc_reqs[8];
2648};
2649
2650/* Set resources command response (16 bytes) */
2651struct creq_set_func_resources_resp {
2652 u8 type;
2653 #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK 0x3fUL
2654 #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT 0
2655 #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT 0x38UL
2656 #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_MASK 0xc0UL
2657 #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_SFT 6
2658 u8 status;
2659 __le16 cookie;
2660 __le32 reserved32;
2661 u8 v;
2662 #define CREQ_SET_FUNC_RESOURCES_RESP_V 0x1UL
2663 #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_MASK 0xfeUL
2664 #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_SFT 1
2665 u8 event;
2666 #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL
2667 __le16 reserved48[3];
2668};
2669
2670/* Map TC to COS response (16 bytes) */
2671struct creq_map_tc_to_cos_resp {
2672 u8 type;
2673 #define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK 0x3fUL
2674 #define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT 0
2675 #define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT 0x38UL
2676 #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_MASK 0xc0UL
2677 #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_SFT 6
2678 u8 status;
2679 __le16 cookie;
2680 __le32 reserved32;
2681 u8 v;
2682 #define CREQ_MAP_TC_TO_COS_RESP_V 0x1UL
2683 #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_MASK 0xfeUL
2684 #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_SFT 1
2685 u8 event;
2686 #define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS 0x8aUL
2687 __le16 reserved48[3];
2688};
2689
2690/* Query version response (16 bytes) */
2691struct creq_query_version_resp {
2692 u8 type;
2693 #define CREQ_QUERY_VERSION_RESP_TYPE_MASK 0x3fUL
2694 #define CREQ_QUERY_VERSION_RESP_TYPE_SFT 0
2695 #define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT 0x38UL
2696 #define CREQ_QUERY_VERSION_RESP_RESERVED2_MASK 0xc0UL
2697 #define CREQ_QUERY_VERSION_RESP_RESERVED2_SFT 6
2698 u8 status;
2699 __le16 cookie;
2700 u8 fw_maj;
2701 u8 fw_minor;
2702 u8 fw_bld;
2703 u8 fw_rsvd;
2704 u8 v;
2705 #define CREQ_QUERY_VERSION_RESP_V 0x1UL
2706 #define CREQ_QUERY_VERSION_RESP_RESERVED7_MASK 0xfeUL
2707 #define CREQ_QUERY_VERSION_RESP_RESERVED7_SFT 1
2708 u8 event;
2709 #define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION 0x8bUL
2710 __le16 reserved16;
2711 u8 intf_maj;
2712 u8 intf_minor;
2713 u8 intf_bld;
2714 u8 intf_rsvd;
2715};
2716
2717/* Modify congestion control command response (16 bytes) */
2718struct creq_modify_cc_resp {
2719 u8 type;
2720 #define CREQ_MODIFY_CC_RESP_TYPE_MASK 0x3fUL
2721 #define CREQ_MODIFY_CC_RESP_TYPE_SFT 0
2722 #define CREQ_MODIFY_CC_RESP_TYPE_QP_EVENT 0x38UL
2723 #define CREQ_MODIFY_CC_RESP_RESERVED2_MASK 0xc0UL
2724 #define CREQ_MODIFY_CC_RESP_RESERVED2_SFT 6
2725 u8 status;
2726 __le16 cookie;
2727 __le32 reserved32;
2728 u8 v;
2729 #define CREQ_MODIFY_CC_RESP_V 0x1UL
2730 #define CREQ_MODIFY_CC_RESP_RESERVED7_MASK 0xfeUL
2731 #define CREQ_MODIFY_CC_RESP_RESERVED7_SFT 1
2732 u8 event;
2733 #define CREQ_MODIFY_CC_RESP_EVENT_MODIFY_CC 0x8cUL
2734 __le16 reserved48[3];
2735};
2736
2737/* Query congestion control command response (16 bytes) */
2738struct creq_query_cc_resp {
2739 u8 type;
2740 #define CREQ_QUERY_CC_RESP_TYPE_MASK 0x3fUL
2741 #define CREQ_QUERY_CC_RESP_TYPE_SFT 0
2742 #define CREQ_QUERY_CC_RESP_TYPE_QP_EVENT 0x38UL
2743 #define CREQ_QUERY_CC_RESP_RESERVED2_MASK 0xc0UL
2744 #define CREQ_QUERY_CC_RESP_RESERVED2_SFT 6
2745 u8 status;
2746 __le16 cookie;
2747 __le32 size;
2748 u8 v;
2749 #define CREQ_QUERY_CC_RESP_V 0x1UL
2750 #define CREQ_QUERY_CC_RESP_RESERVED7_MASK 0xfeUL
2751 #define CREQ_QUERY_CC_RESP_RESERVED7_SFT 1
2752 u8 event;
2753 #define CREQ_QUERY_CC_RESP_EVENT_QUERY_CC 0x8dUL
2754 __le16 reserved48[3];
2755};
2756
2757/* Query congestion control command response side buffer structure (32 bytes) */
2758struct creq_query_cc_resp_sb {
2759 u8 opcode;
2760 #define CREQ_QUERY_CC_RESP_SB_OPCODE_QUERY_CC 0x8dUL
2761 u8 status;
2762 __le16 cookie;
2763 __le16 flags;
2764 u8 resp_size;
2765 u8 reserved8;
2766 u8 enable_cc;
2767 #define CREQ_QUERY_CC_RESP_SB_ENABLE_CC 0x1UL
2768 u8 g;
2769 #define CREQ_QUERY_CC_RESP_SB_G_MASK 0x7UL
2770 #define CREQ_QUERY_CC_RESP_SB_G_SFT 0
2771 u8 num_phases_per_state;
2772 __le16 init_cr;
2773 u8 unused_2;
2774 __le16 unused_3;
2775 u8 unused_4;
2776 __le16 init_tr;
2777 u8 tos_dscp_tos_ecn;
2778 #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_MASK 0x3UL
2779 #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_SFT 0
2780 #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_MASK 0xfcUL
2781 #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_SFT 2
2782 __le64 reserved64;
2783 __le64 reserved64_1;
2784};
2785
2786/* QP error notification event (16 bytes) */
2787struct creq_qp_error_notification {
2788 u8 type;
2789 #define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK 0x3fUL
2790 #define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT 0
2791 #define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT 0x38UL
2792 #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_MASK 0xc0UL
2793 #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_SFT 6
2794 u8 status;
2795 u8 req_slow_path_state;
2796 u8 req_err_state_reason;
2797 __le32 xid;
2798 u8 v;
2799 #define CREQ_QP_ERROR_NOTIFICATION_V 0x1UL
2800 #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_MASK 0xfeUL
2801 #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_SFT 1
2802 u8 event;
2803 #define CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION 0xc0UL
2804 u8 res_slow_path_state;
2805 u8 res_err_state_reason;
2806 __le16 sq_cons_idx;
2807 __le16 rq_cons_idx;
2808};
2809
2810/* RoCE Slowpath HSI Specification 1.6.0 */
2811#define ROCE_SP_HSI_VERSION_MAJOR 1
2812#define ROCE_SP_HSI_VERSION_MINOR 6
2813#define ROCE_SP_HSI_VERSION_UPDATE 0
2814
2815#define ROCE_SP_HSI_VERSION_STR "1.6.0"
2816/*
2817 * Following is the signature for ROCE_SP_HSI message field that indicates not
2818 * applicable (All F's). Need to cast it the size of the field if needed.
2819 */
2820#define ROCE_SP_HSI_NA_SIGNATURE ((__le32)(-1))
2821#endif /* __BNXT_RE_HSI_H__ */
include/uapi/rdma/bnxt_re-abi.h+89−0
@@ -0,0 +1,89 @@
1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Uverbs ABI header file
37 */
38
39#ifndef __BNXT_RE_UVERBS_ABI_H__
40#define __BNXT_RE_UVERBS_ABI_H__
41
42#define BNXT_RE_ABI_VERSION 1
43
44struct bnxt_re_uctx_resp {
45 __u32 dev_id;
46 __u32 max_qp;
47 __u32 pg_size;
48 __u32 cqe_sz;
49 __u32 max_cqd;
50 __u32 rsvd;
51};
52
53struct bnxt_re_pd_resp {
54 __u32 pdid;
55 __u32 dpi;
56 __u64 dbr;
57};
58
59struct bnxt_re_cq_req {
60 __u64 cq_va;
61 __u64 cq_handle;
62};
63
64struct bnxt_re_cq_resp {
65 __u32 cqid;
66 __u32 tail;
67 __u32 phase;
68 __u32 rsvd;
69};
70
71struct bnxt_re_qp_req {
72 __u64 qpsva;
73 __u64 qprva;
74 __u64 qp_handle;
75};
76
77struct bnxt_re_qp_resp {
78 __u32 qpid;
79 __u32 rsvd;
80};
81
82enum bnxt_re_shpg_offt {
83 BNXT_RE_BEG_RESV_OFFT = 0x00,
84 BNXT_RE_AVID_OFFT = 0x10,
85 BNXT_RE_AVID_SIZE = 0x04,
86 BNXT_RE_END_RESV_OFFT = 0xFF0
87};
88
89#endif /* __BNXT_RE_UVERBS_ABI_H__*/

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t