This patch introduces the RoCE driver for the Broadcom NetXtreme-E 10/25/40/50G RoCE HCAs. The RoCE driver is a two part driver that relies on the parent bnxt_en NIC driver to operate. The changes needed in the bnxt_en driver have already been incorporated via Dave Miller's net tree into the mainline kernel. The vendor official git repository for this driver is available on github as: https://github.com/Broadcom/linux-rdma-nxt/
Unified · Split
| @@ -0,0 +1,146 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: Slow Path Operators (header) | |
| 37 | + * | |
| 38 | + */ | |
| 39 | + | |
| 40 | +#ifndef __BNXT_RE_H__ | |
| 41 | +#define __BNXT_RE_H__ | |
| 42 | +#define ROCE_DRV_MODULE_NAME "bnxt_re" | |
| 43 | +#define ROCE_DRV_MODULE_VERSION "1.0.0" | |
| 44 | + | |
| 45 | +#define BNXT_RE_DESC "Broadcom NetXtreme-C/E RoCE Driver" | |
| 46 | + | |
| 47 | +#define BNXT_RE_PAGE_SIZE_4K BIT(12) | |
| 48 | +#define BNXT_RE_PAGE_SIZE_8K BIT(13) | |
| 49 | +#define BNXT_RE_PAGE_SIZE_64K BIT(16) | |
| 50 | +#define BNXT_RE_PAGE_SIZE_2M BIT(21) | |
| 51 | +#define BNXT_RE_PAGE_SIZE_8M BIT(23) | |
| 52 | +#define BNXT_RE_PAGE_SIZE_1G BIT(30) | |
| 53 | + | |
| 54 | +#define BNXT_RE_MAX_QPC_COUNT (64 * 1024) | |
| 55 | +#define BNXT_RE_MAX_MRW_COUNT (64 * 1024) | |
| 56 | +#define BNXT_RE_MAX_SRQC_COUNT (64 * 1024) | |
| 57 | +#define BNXT_RE_MAX_CQ_COUNT (64 * 1024) | |
| 58 | + | |
| 59 | +struct bnxt_re_work { | |
| 60 | + struct work_struct work; | |
| 61 | + unsigned long event; | |
| 62 | + struct bnxt_re_dev *rdev; | |
| 63 | + struct net_device *vlan_dev; | |
| 64 | +}; | |
| 65 | + | |
| 66 | +struct bnxt_re_sqp_entries { | |
| 67 | + struct bnxt_qplib_sge sge; | |
| 68 | + u64 wrid; | |
| 69 | + /* For storing the actual qp1 cqe */ | |
| 70 | + struct bnxt_qplib_cqe cqe; | |
| 71 | + struct bnxt_re_qp *qp1_qp; | |
| 72 | +}; | |
| 73 | + | |
| 74 | +#define BNXT_RE_MIN_MSIX 2 | |
| 75 | +#define BNXT_RE_MAX_MSIX 16 | |
| 76 | +#define BNXT_RE_AEQ_IDX 0 | |
| 77 | +#define BNXT_RE_NQ_IDX 1 | |
| 78 | + | |
| 79 | +struct bnxt_re_dev { | |
| 80 | + struct ib_device ibdev; | |
| 81 | + struct list_head list; | |
| 82 | + unsigned long flags; | |
| 83 | +#define BNXT_RE_FLAG_NETDEV_REGISTERED 0 | |
| 84 | +#define BNXT_RE_FLAG_IBDEV_REGISTERED 1 | |
| 85 | +#define BNXT_RE_FLAG_GOT_MSIX 2 | |
| 86 | +#define BNXT_RE_FLAG_RCFW_CHANNEL_EN 8 | |
| 87 | +#define BNXT_RE_FLAG_QOS_WORK_REG 16 | |
| 88 | + struct net_device *netdev; | |
| 89 | + unsigned int version, major, minor; | |
| 90 | + struct bnxt_en_dev *en_dev; | |
| 91 | + struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX]; | |
| 92 | + int num_msix; | |
| 93 | + | |
| 94 | + int id; | |
| 95 | + | |
| 96 | + struct delayed_work worker; | |
| 97 | + u8 cur_prio_map; | |
| 98 | + | |
| 99 | + /* FP Notification Queue (CQ & SRQ) */ | |
| 100 | + struct tasklet_struct nq_task; | |
| 101 | + | |
| 102 | + /* RCFW Channel */ | |
| 103 | + struct bnxt_qplib_rcfw rcfw; | |
| 104 | + | |
| 105 | + /* NQ */ | |
| 106 | + struct bnxt_qplib_nq nq; | |
| 107 | + | |
| 108 | + /* Device Resources */ | |
| 109 | + struct bnxt_qplib_dev_attr dev_attr; | |
| 110 | + struct bnxt_qplib_ctx qplib_ctx; | |
| 111 | + struct bnxt_qplib_res qplib_res; | |
| 112 | + struct bnxt_qplib_dpi dpi_privileged; | |
| 113 | + | |
| 114 | + atomic_t qp_count; | |
| 115 | + struct mutex qp_lock; /* protect qp list */ | |
| 116 | + struct list_head qp_list; | |
| 117 | + | |
| 118 | + atomic_t cq_count; | |
| 119 | + atomic_t srq_count; | |
| 120 | + atomic_t mr_count; | |
| 121 | + atomic_t mw_count; | |
| 122 | + /* Max of 2 lossless traffic class supported per port */ | |
| 123 | + u16 cosq[2]; | |
| 124 | + | |
| 125 | + /* QP for for handling QP1 packets */ | |
| 126 | + u32 sqp_id; | |
| 127 | + struct bnxt_re_qp *qp1_sqp; | |
| 128 | + struct bnxt_re_ah *sqp_ah; | |
| 129 | + struct bnxt_re_sqp_entries sqp_tbl[1024]; | |
| 130 | +}; | |
| 131 | + | |
| 132 | +#define to_bnxt_re_dev(ptr, member) \ | |
| 133 | + container_of((ptr), struct bnxt_re_dev, member) | |
| 134 | + | |
| 135 | +#define BNXT_RE_ROCE_V1_PACKET 0 | |
| 136 | +#define BNXT_RE_ROCEV2_IPV4_PACKET 2 | |
| 137 | +#define BNXT_RE_ROCEV2_IPV6_PACKET 3 | |
| 138 | + | |
| 139 | +static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev) | |
| 140 | +{ | |
| 141 | + if (rdev) | |
| 142 | + return &rdev->ibdev.dev; | |
| 143 | + return NULL; | |
| 144 | +} | |
| 145 | + | |
| 146 | +#endif | |
| @@ -0,0 +1,3202 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: IB Verbs interpreter | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#include <linux/interrupt.h> | |
| 40 | +#include <linux/types.h> | |
| 41 | +#include <linux/pci.h> | |
| 42 | +#include <linux/netdevice.h> | |
| 43 | +#include <linux/if_ether.h> | |
| 44 | + | |
| 45 | +#include <rdma/ib_verbs.h> | |
| 46 | +#include <rdma/ib_user_verbs.h> | |
| 47 | +#include <rdma/ib_umem.h> | |
| 48 | +#include <rdma/ib_addr.h> | |
| 49 | +#include <rdma/ib_mad.h> | |
| 50 | +#include <rdma/ib_cache.h> | |
| 51 | + | |
| 52 | +#include "bnxt_ulp.h" | |
| 53 | + | |
| 54 | +#include "roce_hsi.h" | |
| 55 | +#include "qplib_res.h" | |
| 56 | +#include "qplib_sp.h" | |
| 57 | +#include "qplib_fp.h" | |
| 58 | +#include "qplib_rcfw.h" | |
| 59 | + | |
| 60 | +#include "bnxt_re.h" | |
| 61 | +#include "ib_verbs.h" | |
| 62 | +#include <rdma/bnxt_re-abi.h> | |
| 63 | + | |
| 64 | +static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, | |
| 65 | + struct bnxt_qplib_sge *sg_list, int num) | |
| 66 | +{ | |
| 67 | + int i, total = 0; | |
| 68 | + | |
| 69 | + for (i = 0; i < num; i++) { | |
| 70 | + sg_list[i].addr = ib_sg_list[i].addr; | |
| 71 | + sg_list[i].lkey = ib_sg_list[i].lkey; | |
| 72 | + sg_list[i].size = ib_sg_list[i].length; | |
| 73 | + total += sg_list[i].size; | |
| 74 | + } | |
| 75 | + return total; | |
| 76 | +} | |
| 77 | + | |
| 78 | +/* Device */ | |
| 79 | +struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num) | |
| 80 | +{ | |
| 81 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 82 | + struct net_device *netdev = NULL; | |
| 83 | + | |
| 84 | + rcu_read_lock(); | |
| 85 | + if (rdev) | |
| 86 | + netdev = rdev->netdev; | |
| 87 | + if (netdev) | |
| 88 | + dev_hold(netdev); | |
| 89 | + | |
| 90 | + rcu_read_unlock(); | |
| 91 | + return netdev; | |
| 92 | +} | |
| 93 | + | |
| 94 | +int bnxt_re_query_device(struct ib_device *ibdev, | |
| 95 | + struct ib_device_attr *ib_attr, | |
| 96 | + struct ib_udata *udata) | |
| 97 | +{ | |
| 98 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 99 | + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | |
| 100 | + | |
| 101 | + memset(ib_attr, 0, sizeof(*ib_attr)); | |
| 102 | + | |
| 103 | + ib_attr->fw_ver = (u64)(unsigned long)(dev_attr->fw_ver); | |
| 104 | + bnxt_qplib_get_guid(rdev->netdev->dev_addr, | |
| 105 | + (u8 *)&ib_attr->sys_image_guid); | |
| 106 | + ib_attr->max_mr_size = ~0ull; | |
| 107 | + ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_8K | | |
| 108 | + BNXT_RE_PAGE_SIZE_64K | BNXT_RE_PAGE_SIZE_2M | | |
| 109 | + BNXT_RE_PAGE_SIZE_8M | BNXT_RE_PAGE_SIZE_1G; | |
| 110 | + | |
| 111 | + ib_attr->vendor_id = rdev->en_dev->pdev->vendor; | |
| 112 | + ib_attr->vendor_part_id = rdev->en_dev->pdev->device; | |
| 113 | + ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; | |
| 114 | + ib_attr->max_qp = dev_attr->max_qp; | |
| 115 | + ib_attr->max_qp_wr = dev_attr->max_qp_wqes; | |
| 116 | + ib_attr->device_cap_flags = | |
| 117 | + IB_DEVICE_CURR_QP_STATE_MOD | |
| 118 | + | IB_DEVICE_RC_RNR_NAK_GEN | |
| 119 | + | IB_DEVICE_SHUTDOWN_PORT | |
| 120 | + | IB_DEVICE_SYS_IMAGE_GUID | |
| 121 | + | IB_DEVICE_LOCAL_DMA_LKEY | |
| 122 | + | IB_DEVICE_RESIZE_MAX_WR | |
| 123 | + | IB_DEVICE_PORT_ACTIVE_EVENT | |
| 124 | + | IB_DEVICE_N_NOTIFY_CQ | |
| 125 | + | IB_DEVICE_MEM_WINDOW | |
| 126 | + | IB_DEVICE_MEM_WINDOW_TYPE_2B | |
| 127 | + | IB_DEVICE_MEM_MGT_EXTENSIONS; | |
| 128 | + ib_attr->max_sge = dev_attr->max_qp_sges; | |
| 129 | + ib_attr->max_sge_rd = dev_attr->max_qp_sges; | |
| 130 | + ib_attr->max_cq = dev_attr->max_cq; | |
| 131 | + ib_attr->max_cqe = dev_attr->max_cq_wqes; | |
| 132 | + ib_attr->max_mr = dev_attr->max_mr; | |
| 133 | + ib_attr->max_pd = dev_attr->max_pd; | |
| 134 | + ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; | |
| 135 | + ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_rd_atom; | |
| 136 | + ib_attr->atomic_cap = IB_ATOMIC_HCA; | |
| 137 | + ib_attr->masked_atomic_cap = IB_ATOMIC_HCA; | |
| 138 | + | |
| 139 | + ib_attr->max_ee_rd_atom = 0; | |
| 140 | + ib_attr->max_res_rd_atom = 0; | |
| 141 | + ib_attr->max_ee_init_rd_atom = 0; | |
| 142 | + ib_attr->max_ee = 0; | |
| 143 | + ib_attr->max_rdd = 0; | |
| 144 | + ib_attr->max_mw = dev_attr->max_mw; | |
| 145 | + ib_attr->max_raw_ipv6_qp = 0; | |
| 146 | + ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; | |
| 147 | + ib_attr->max_mcast_grp = 0; | |
| 148 | + ib_attr->max_mcast_qp_attach = 0; | |
| 149 | + ib_attr->max_total_mcast_qp_attach = 0; | |
| 150 | + ib_attr->max_ah = dev_attr->max_ah; | |
| 151 | + | |
| 152 | + ib_attr->max_fmr = dev_attr->max_fmr; | |
| 153 | + ib_attr->max_map_per_fmr = 1; /* ? */ | |
| 154 | + | |
| 155 | + ib_attr->max_srq = dev_attr->max_srq; | |
| 156 | + ib_attr->max_srq_wr = dev_attr->max_srq_wqes; | |
| 157 | + ib_attr->max_srq_sge = dev_attr->max_srq_sges; | |
| 158 | + | |
| 159 | + ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; | |
| 160 | + | |
| 161 | + ib_attr->max_pkeys = 1; | |
| 162 | + ib_attr->local_ca_ack_delay = 0; | |
| 163 | + return 0; | |
| 164 | +} | |
| 165 | + | |
| 166 | +int bnxt_re_modify_device(struct ib_device *ibdev, | |
| 167 | + int device_modify_mask, | |
| 168 | + struct ib_device_modify *device_modify) | |
| 169 | +{ | |
| 170 | + switch (device_modify_mask) { | |
| 171 | + case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: | |
| 172 | + /* Modify the GUID requires the modification of the GID table */ | |
| 173 | + /* GUID should be made as READ-ONLY */ | |
| 174 | + break; | |
| 175 | + case IB_DEVICE_MODIFY_NODE_DESC: | |
| 176 | + /* Node Desc should be made as READ-ONLY */ | |
| 177 | + break; | |
| 178 | + default: | |
| 179 | + break; | |
| 180 | + } | |
| 181 | + return 0; | |
| 182 | +} | |
| 183 | + | |
| 184 | +static void __to_ib_speed_width(struct net_device *netdev, u8 *speed, u8 *width) | |
| 185 | +{ | |
| 186 | + struct ethtool_link_ksettings lksettings; | |
| 187 | + u32 espeed; | |
| 188 | + | |
| 189 | + if (netdev->ethtool_ops && netdev->ethtool_ops->get_link_ksettings) { | |
| 190 | + memset(&lksettings, 0, sizeof(lksettings)); | |
| 191 | + rtnl_lock(); | |
| 192 | + netdev->ethtool_ops->get_link_ksettings(netdev, &lksettings); | |
| 193 | + rtnl_unlock(); | |
| 194 | + espeed = lksettings.base.speed; | |
| 195 | + } else { | |
| 196 | + espeed = SPEED_UNKNOWN; | |
| 197 | + } | |
| 198 | + switch (espeed) { | |
| 199 | + case SPEED_1000: | |
| 200 | + *speed = IB_SPEED_SDR; | |
| 201 | + *width = IB_WIDTH_1X; | |
| 202 | + break; | |
| 203 | + case SPEED_10000: | |
| 204 | + *speed = IB_SPEED_QDR; | |
| 205 | + *width = IB_WIDTH_1X; | |
| 206 | + break; | |
| 207 | + case SPEED_20000: | |
| 208 | + *speed = IB_SPEED_DDR; | |
| 209 | + *width = IB_WIDTH_4X; | |
| 210 | + break; | |
| 211 | + case SPEED_25000: | |
| 212 | + *speed = IB_SPEED_EDR; | |
| 213 | + *width = IB_WIDTH_1X; | |
| 214 | + break; | |
| 215 | + case SPEED_40000: | |
| 216 | + *speed = IB_SPEED_QDR; | |
| 217 | + *width = IB_WIDTH_4X; | |
| 218 | + break; | |
| 219 | + case SPEED_50000: | |
| 220 | + break; | |
| 221 | + default: | |
| 222 | + *speed = IB_SPEED_SDR; | |
| 223 | + *width = IB_WIDTH_1X; | |
| 224 | + break; | |
| 225 | + } | |
| 226 | +} | |
| 227 | + | |
| 228 | +/* Port */ | |
| 229 | +int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, | |
| 230 | + struct ib_port_attr *port_attr) | |
| 231 | +{ | |
| 232 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 233 | + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | |
| 234 | + | |
| 235 | + memset(port_attr, 0, sizeof(*port_attr)); | |
| 236 | + | |
| 237 | + if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { | |
| 238 | + port_attr->state = IB_PORT_ACTIVE; | |
| 239 | + port_attr->phys_state = 5; | |
| 240 | + } else { | |
| 241 | + port_attr->state = IB_PORT_DOWN; | |
| 242 | + port_attr->phys_state = 3; | |
| 243 | + } | |
| 244 | + port_attr->max_mtu = IB_MTU_4096; | |
| 245 | + port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); | |
| 246 | + port_attr->gid_tbl_len = dev_attr->max_sgid; | |
| 247 | + port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | | |
| 248 | + IB_PORT_DEVICE_MGMT_SUP | | |
| 249 | + IB_PORT_VENDOR_CLASS_SUP | | |
| 250 | + IB_PORT_IP_BASED_GIDS; | |
| 251 | + | |
| 252 | + /* Max MSG size set to 2G for now */ | |
| 253 | + port_attr->max_msg_sz = 0x80000000; | |
| 254 | + port_attr->bad_pkey_cntr = 0; | |
| 255 | + port_attr->qkey_viol_cntr = 0; | |
| 256 | + port_attr->pkey_tbl_len = dev_attr->max_pkey; | |
| 257 | + port_attr->lid = 0; | |
| 258 | + port_attr->sm_lid = 0; | |
| 259 | + port_attr->lmc = 0; | |
| 260 | + port_attr->max_vl_num = 4; | |
| 261 | + port_attr->sm_sl = 0; | |
| 262 | + port_attr->subnet_timeout = 0; | |
| 263 | + port_attr->init_type_reply = 0; | |
| 264 | + /* call the underlying netdev's ethtool hooks to query speed settings | |
| 265 | + * for which we acquire rtnl_lock _only_ if it's registered with | |
| 266 | + * IB stack to avoid race in the NETDEV_UNREG path | |
| 267 | + */ | |
| 268 | + if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) | |
| 269 | + __to_ib_speed_width(rdev->netdev, &port_attr->active_speed, | |
| 270 | + &port_attr->active_width); | |
| 271 | + return 0; | |
| 272 | +} | |
| 273 | + | |
| 274 | +int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, | |
| 275 | + int port_modify_mask, | |
| 276 | + struct ib_port_modify *port_modify) | |
| 277 | +{ | |
| 278 | + switch (port_modify_mask) { | |
| 279 | + case IB_PORT_SHUTDOWN: | |
| 280 | + break; | |
| 281 | + case IB_PORT_INIT_TYPE: | |
| 282 | + break; | |
| 283 | + case IB_PORT_RESET_QKEY_CNTR: | |
| 284 | + break; | |
| 285 | + default: | |
| 286 | + break; | |
| 287 | + } | |
| 288 | + return 0; | |
| 289 | +} | |
| 290 | + | |
| 291 | +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, | |
| 292 | + struct ib_port_immutable *immutable) | |
| 293 | +{ | |
| 294 | + struct ib_port_attr port_attr; | |
| 295 | + | |
| 296 | + if (bnxt_re_query_port(ibdev, port_num, &port_attr)) | |
| 297 | + return -EINVAL; | |
| 298 | + | |
| 299 | + immutable->pkey_tbl_len = port_attr.pkey_tbl_len; | |
| 300 | + immutable->gid_tbl_len = port_attr.gid_tbl_len; | |
| 301 | + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; | |
| 302 | + immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; | |
| 303 | + immutable->max_mad_size = IB_MGMT_MAD_SIZE; | |
| 304 | + return 0; | |
| 305 | +} | |
| 306 | + | |
| 307 | +int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, | |
| 308 | + u16 index, u16 *pkey) | |
| 309 | +{ | |
| 310 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 311 | + | |
| 312 | + /* Ignore port_num */ | |
| 313 | + | |
| 314 | + memset(pkey, 0, sizeof(*pkey)); | |
| 315 | + return bnxt_qplib_get_pkey(&rdev->qplib_res, | |
| 316 | + &rdev->qplib_res.pkey_tbl, index, pkey); | |
| 317 | +} | |
| 318 | + | |
| 319 | +int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, | |
| 320 | + int index, union ib_gid *gid) | |
| 321 | +{ | |
| 322 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 323 | + int rc = 0; | |
| 324 | + | |
| 325 | + /* Ignore port_num */ | |
| 326 | + memset(gid, 0, sizeof(*gid)); | |
| 327 | + rc = bnxt_qplib_get_sgid(&rdev->qplib_res, | |
| 328 | + &rdev->qplib_res.sgid_tbl, index, | |
| 329 | + (struct bnxt_qplib_gid *)gid); | |
| 330 | + return rc; | |
| 331 | +} | |
| 332 | + | |
| 333 | +int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, | |
| 334 | + unsigned int index, void **context) | |
| 335 | +{ | |
| 336 | + int rc = 0; | |
| 337 | + struct bnxt_re_gid_ctx *ctx, **ctx_tbl; | |
| 338 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 339 | + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; | |
| 340 | + | |
| 341 | + /* Delete the entry from the hardware */ | |
| 342 | + ctx = *context; | |
| 343 | + if (!ctx) | |
| 344 | + return -EINVAL; | |
| 345 | + | |
| 346 | + if (sgid_tbl && sgid_tbl->active) { | |
| 347 | + if (ctx->idx >= sgid_tbl->max) | |
| 348 | + return -EINVAL; | |
| 349 | + ctx->refcnt--; | |
| 350 | + if (!ctx->refcnt) { | |
| 351 | + rc = bnxt_qplib_del_sgid | |
| 352 | + (sgid_tbl, | |
| 353 | + &sgid_tbl->tbl[ctx->idx], true); | |
| 354 | + if (rc) | |
| 355 | + dev_err(rdev_to_dev(rdev), | |
| 356 | + "Failed to remove GID: %#x", rc); | |
| 357 | + ctx_tbl = sgid_tbl->ctx; | |
| 358 | + ctx_tbl[ctx->idx] = NULL; | |
| 359 | + kfree(ctx); | |
| 360 | + } | |
| 361 | + } else { | |
| 362 | + return -EINVAL; | |
| 363 | + } | |
| 364 | + return rc; | |
| 365 | +} | |
| 366 | + | |
| 367 | +int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, | |
| 368 | + unsigned int index, const union ib_gid *gid, | |
| 369 | + const struct ib_gid_attr *attr, void **context) | |
| 370 | +{ | |
| 371 | + int rc; | |
| 372 | + u32 tbl_idx = 0; | |
| 373 | + u16 vlan_id = 0xFFFF; | |
| 374 | + struct bnxt_re_gid_ctx *ctx, **ctx_tbl; | |
| 375 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 376 | + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; | |
| 377 | + | |
| 378 | + if ((attr->ndev) && is_vlan_dev(attr->ndev)) | |
| 379 | + vlan_id = vlan_dev_vlan_id(attr->ndev); | |
| 380 | + | |
| 381 | + rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)gid, | |
| 382 | + rdev->qplib_res.netdev->dev_addr, | |
| 383 | + vlan_id, true, &tbl_idx); | |
| 384 | + if (rc == -EALREADY) { | |
| 385 | + ctx_tbl = sgid_tbl->ctx; | |
| 386 | + ctx_tbl[tbl_idx]->refcnt++; | |
| 387 | + *context = ctx_tbl[tbl_idx]; | |
| 388 | + return 0; | |
| 389 | + } | |
| 390 | + | |
| 391 | + if (rc < 0) { | |
| 392 | + dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc); | |
| 393 | + return rc; | |
| 394 | + } | |
| 395 | + | |
| 396 | + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); | |
| 397 | + if (!ctx) | |
| 398 | + return -ENOMEM; | |
| 399 | + ctx_tbl = sgid_tbl->ctx; | |
| 400 | + ctx->idx = tbl_idx; | |
| 401 | + ctx->refcnt = 1; | |
| 402 | + ctx_tbl[tbl_idx] = ctx; | |
| 403 | + | |
| 404 | + return rc; | |
| 405 | +} | |
| 406 | + | |
| 407 | +enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, | |
| 408 | + u8 port_num) | |
| 409 | +{ | |
| 410 | + return IB_LINK_LAYER_ETHERNET; | |
| 411 | +} | |
| 412 | + | |
| 413 | +/* Protection Domains */ | |
| 414 | +int bnxt_re_dealloc_pd(struct ib_pd *ib_pd) | |
| 415 | +{ | |
| 416 | + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | |
| 417 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 418 | + int rc; | |
| 419 | + | |
| 420 | + if (ib_pd->uobject && pd->dpi.dbr) { | |
| 421 | + struct ib_ucontext *ib_uctx = ib_pd->uobject->context; | |
| 422 | + struct bnxt_re_ucontext *ucntx; | |
| 423 | + | |
| 424 | + /* Free DPI only if this is the first PD allocated by the | |
| 425 | + * application and mark the context dpi as NULL | |
| 426 | + */ | |
| 427 | + ucntx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); | |
| 428 | + | |
| 429 | + rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, | |
| 430 | + &rdev->qplib_res.dpi_tbl, | |
| 431 | + &pd->dpi); | |
| 432 | + if (rc) | |
| 433 | + dev_err(rdev_to_dev(rdev), "Failed to deallocate HW DPI"); | |
| 434 | + /* Don't fail, continue*/ | |
| 435 | + ucntx->dpi = NULL; | |
| 436 | + } | |
| 437 | + | |
| 438 | + rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, | |
| 439 | + &rdev->qplib_res.pd_tbl, | |
| 440 | + &pd->qplib_pd); | |
| 441 | + if (rc) { | |
| 442 | + dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD"); | |
| 443 | + return rc; | |
| 444 | + } | |
| 445 | + | |
| 446 | + kfree(pd); | |
| 447 | + return 0; | |
| 448 | +} | |
| 449 | + | |
| 450 | +struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev, | |
| 451 | + struct ib_ucontext *ucontext, | |
| 452 | + struct ib_udata *udata) | |
| 453 | +{ | |
| 454 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 455 | + struct bnxt_re_ucontext *ucntx = container_of(ucontext, | |
| 456 | + struct bnxt_re_ucontext, | |
| 457 | + ib_uctx); | |
| 458 | + struct bnxt_re_pd *pd; | |
| 459 | + int rc; | |
| 460 | + | |
| 461 | + pd = kzalloc(sizeof(*pd), GFP_KERNEL); | |
| 462 | + if (!pd) | |
| 463 | + return ERR_PTR(-ENOMEM); | |
| 464 | + | |
| 465 | + pd->rdev = rdev; | |
| 466 | + if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { | |
| 467 | + dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD"); | |
| 468 | + rc = -ENOMEM; | |
| 469 | + goto fail; | |
| 470 | + } | |
| 471 | + | |
| 472 | + if (udata) { | |
| 473 | + struct bnxt_re_pd_resp resp; | |
| 474 | + | |
| 475 | + if (!ucntx->dpi) { | |
| 476 | + /* Allocate DPI in alloc_pd to avoid failing of | |
| 477 | + * ibv_devinfo and family of application when DPIs | |
| 478 | + * are depleted. | |
| 479 | + */ | |
| 480 | + if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, | |
| 481 | + &pd->dpi, ucntx)) { | |
| 482 | + rc = -ENOMEM; | |
| 483 | + goto dbfail; | |
| 484 | + } | |
| 485 | + ucntx->dpi = &pd->dpi; | |
| 486 | + } | |
| 487 | + | |
| 488 | + resp.pdid = pd->qplib_pd.id; | |
| 489 | + /* Still allow mapping this DBR to the new user PD. */ | |
| 490 | + resp.dpi = ucntx->dpi->dpi; | |
| 491 | + resp.dbr = (u64)ucntx->dpi->umdbr; | |
| 492 | + | |
| 493 | + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); | |
| 494 | + if (rc) { | |
| 495 | + dev_err(rdev_to_dev(rdev), | |
| 496 | + "Failed to copy user response\n"); | |
| 497 | + goto dbfail; | |
| 498 | + } | |
| 499 | + } | |
| 500 | + | |
| 501 | + return &pd->ib_pd; | |
| 502 | +dbfail: | |
| 503 | + (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, | |
| 504 | + &pd->qplib_pd); | |
| 505 | +fail: | |
| 506 | + kfree(pd); | |
| 507 | + return ERR_PTR(rc); | |
| 508 | +} | |
| 509 | + | |
| 510 | +/* Address Handles */ | |
| 511 | +int bnxt_re_destroy_ah(struct ib_ah *ib_ah) | |
| 512 | +{ | |
| 513 | + struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); | |
| 514 | + struct bnxt_re_dev *rdev = ah->rdev; | |
| 515 | + int rc; | |
| 516 | + | |
| 517 | + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah); | |
| 518 | + if (rc) { | |
| 519 | + dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH"); | |
| 520 | + return rc; | |
| 521 | + } | |
| 522 | + kfree(ah); | |
| 523 | + return 0; | |
| 524 | +} | |
| 525 | + | |
| 526 | +struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd, | |
| 527 | + struct ib_ah_attr *ah_attr, | |
| 528 | + struct ib_udata *udata) | |
| 529 | +{ | |
| 530 | + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | |
| 531 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 532 | + struct bnxt_re_ah *ah; | |
| 533 | + int rc; | |
| 534 | + u16 vlan_tag; | |
| 535 | + u8 nw_type; | |
| 536 | + | |
| 537 | + struct ib_gid_attr sgid_attr; | |
| 538 | + | |
| 539 | + if (!(ah_attr->ah_flags & IB_AH_GRH)) { | |
| 540 | + dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set"); | |
| 541 | + return ERR_PTR(-EINVAL); | |
| 542 | + } | |
| 543 | + ah = kzalloc(sizeof(*ah), GFP_ATOMIC); | |
| 544 | + if (!ah) | |
| 545 | + return ERR_PTR(-ENOMEM); | |
| 546 | + | |
| 547 | + ah->rdev = rdev; | |
| 548 | + ah->qplib_ah.pd = &pd->qplib_pd; | |
| 549 | + | |
| 550 | + /* Supply the configuration for the HW */ | |
| 551 | + memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw, | |
| 552 | + sizeof(union ib_gid)); | |
| 553 | + /* | |
| 554 | + * If RoCE V2 is enabled, stack will have two entries for | |
| 555 | + * each GID entry. Avoiding this duplicte entry in HW. Dividing | |
| 556 | + * the GID index by 2 for RoCE V2 | |
| 557 | + */ | |
| 558 | + ah->qplib_ah.sgid_index = ah_attr->grh.sgid_index / 2; | |
| 559 | + ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index; | |
| 560 | + ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class; | |
| 561 | + ah->qplib_ah.flow_label = ah_attr->grh.flow_label; | |
| 562 | + ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit; | |
| 563 | + ah->qplib_ah.sl = ah_attr->sl; | |
| 564 | + if (ib_pd->uobject && | |
| 565 | + !rdma_is_multicast_addr((struct in6_addr *) | |
| 566 | + ah_attr->grh.dgid.raw) && | |
| 567 | + !rdma_link_local_addr((struct in6_addr *) | |
| 568 | + ah_attr->grh.dgid.raw)) { | |
| 569 | + union ib_gid sgid; | |
| 570 | + | |
| 571 | + rc = ib_get_cached_gid(&rdev->ibdev, 1, | |
| 572 | + ah_attr->grh.sgid_index, &sgid, | |
| 573 | + &sgid_attr); | |
| 574 | + if (rc) { | |
| 575 | + dev_err(rdev_to_dev(rdev), | |
| 576 | + "Failed to query gid at index %d", | |
| 577 | + ah_attr->grh.sgid_index); | |
| 578 | + goto fail; | |
| 579 | + } | |
| 580 | + if (sgid_attr.ndev) { | |
| 581 | + if (is_vlan_dev(sgid_attr.ndev)) | |
| 582 | + vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev); | |
| 583 | + dev_put(sgid_attr.ndev); | |
| 584 | + } | |
| 585 | + /* Get network header type for this GID */ | |
| 586 | + nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); | |
| 587 | + switch (nw_type) { | |
| 588 | + case RDMA_NETWORK_IPV4: | |
| 589 | + ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; | |
| 590 | + break; | |
| 591 | + case RDMA_NETWORK_IPV6: | |
| 592 | + ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; | |
| 593 | + break; | |
| 594 | + default: | |
| 595 | + ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1; | |
| 596 | + break; | |
| 597 | + } | |
| 598 | + rc = rdma_addr_find_l2_eth_by_grh(&sgid, &ah_attr->grh.dgid, | |
| 599 | + ah_attr->dmac, &vlan_tag, | |
| 600 | + &sgid_attr.ndev->ifindex, | |
| 601 | + NULL); | |
| 602 | + if (rc) { | |
| 603 | + dev_err(rdev_to_dev(rdev), "Failed to get dmac\n"); | |
| 604 | + goto fail; | |
| 605 | + } | |
| 606 | + } | |
| 607 | + | |
| 608 | + memcpy(ah->qplib_ah.dmac, ah_attr->dmac, ETH_ALEN); | |
| 609 | + rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah); | |
| 610 | + if (rc) { | |
| 611 | + dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH"); | |
| 612 | + goto fail; | |
| 613 | + } | |
| 614 | + | |
| 615 | + /* Write AVID to shared page. */ | |
| 616 | + if (ib_pd->uobject) { | |
| 617 | + struct ib_ucontext *ib_uctx = ib_pd->uobject->context; | |
| 618 | + struct bnxt_re_ucontext *uctx; | |
| 619 | + unsigned long flag; | |
| 620 | + u32 *wrptr; | |
| 621 | + | |
| 622 | + uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); | |
| 623 | + spin_lock_irqsave(&uctx->sh_lock, flag); | |
| 624 | + wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); | |
| 625 | + *wrptr = ah->qplib_ah.id; | |
| 626 | + wmb(); /* make sure cache is updated. */ | |
| 627 | + spin_unlock_irqrestore(&uctx->sh_lock, flag); | |
| 628 | + } | |
| 629 | + | |
| 630 | + return &ah->ib_ah; | |
| 631 | + | |
| 632 | +fail: | |
| 633 | + kfree(ah); | |
| 634 | + return ERR_PTR(rc); | |
| 635 | +} | |
| 636 | + | |
| 637 | +int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) | |
| 638 | +{ | |
| 639 | + return 0; | |
| 640 | +} | |
| 641 | + | |
| 642 | +int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) | |
| 643 | +{ | |
| 644 | + struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); | |
| 645 | + | |
| 646 | + memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data, | |
| 647 | + sizeof(union ib_gid)); | |
| 648 | + ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index; | |
| 649 | + ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class; | |
| 650 | + ah_attr->sl = ah->qplib_ah.sl; | |
| 651 | + memcpy(ah_attr->dmac, ah->qplib_ah.dmac, ETH_ALEN); | |
| 652 | + ah_attr->ah_flags = IB_AH_GRH; | |
| 653 | + ah_attr->port_num = 1; | |
| 654 | + ah_attr->static_rate = 0; | |
| 655 | + return 0; | |
| 656 | +} | |
| 657 | + | |
| 658 | +/* Queue Pairs */ | |
| 659 | +int bnxt_re_destroy_qp(struct ib_qp *ib_qp) | |
| 660 | +{ | |
| 661 | + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | |
| 662 | + struct bnxt_re_dev *rdev = qp->rdev; | |
| 663 | + int rc; | |
| 664 | + | |
| 665 | + rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); | |
| 666 | + if (rc) { | |
| 667 | + dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP"); | |
| 668 | + return rc; | |
| 669 | + } | |
| 670 | + if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) { | |
| 671 | + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, | |
| 672 | + &rdev->sqp_ah->qplib_ah); | |
| 673 | + if (rc) { | |
| 674 | + dev_err(rdev_to_dev(rdev), | |
| 675 | + "Failed to destroy HW AH for shadow QP"); | |
| 676 | + return rc; | |
| 677 | + } | |
| 678 | + | |
| 679 | + rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, | |
| 680 | + &rdev->qp1_sqp->qplib_qp); | |
| 681 | + if (rc) { | |
| 682 | + dev_err(rdev_to_dev(rdev), | |
| 683 | + "Failed to destroy Shadow QP"); | |
| 684 | + return rc; | |
| 685 | + } | |
| 686 | + mutex_lock(&rdev->qp_lock); | |
| 687 | + list_del(&rdev->qp1_sqp->list); | |
| 688 | + atomic_dec(&rdev->qp_count); | |
| 689 | + mutex_unlock(&rdev->qp_lock); | |
| 690 | + | |
| 691 | + kfree(rdev->sqp_ah); | |
| 692 | + kfree(rdev->qp1_sqp); | |
| 693 | + } | |
| 694 | + | |
| 695 | + if (qp->rumem && !IS_ERR(qp->rumem)) | |
| 696 | + ib_umem_release(qp->rumem); | |
| 697 | + if (qp->sumem && !IS_ERR(qp->sumem)) | |
| 698 | + ib_umem_release(qp->sumem); | |
| 699 | + | |
| 700 | + mutex_lock(&rdev->qp_lock); | |
| 701 | + list_del(&qp->list); | |
| 702 | + atomic_dec(&rdev->qp_count); | |
| 703 | + mutex_unlock(&rdev->qp_lock); | |
| 704 | + kfree(qp); | |
| 705 | + return 0; | |
| 706 | +} | |
| 707 | + | |
| 708 | +static u8 __from_ib_qp_type(enum ib_qp_type type) | |
| 709 | +{ | |
| 710 | + switch (type) { | |
| 711 | + case IB_QPT_GSI: | |
| 712 | + return CMDQ_CREATE_QP1_TYPE_GSI; | |
| 713 | + case IB_QPT_RC: | |
| 714 | + return CMDQ_CREATE_QP_TYPE_RC; | |
| 715 | + case IB_QPT_UD: | |
| 716 | + return CMDQ_CREATE_QP_TYPE_UD; | |
| 717 | + default: | |
| 718 | + return IB_QPT_MAX; | |
| 719 | + } | |
| 720 | +} | |
| 721 | + | |
| 722 | +static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, | |
| 723 | + struct bnxt_re_qp *qp, struct ib_udata *udata) | |
| 724 | +{ | |
| 725 | + struct bnxt_re_qp_req ureq; | |
| 726 | + struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; | |
| 727 | + struct ib_umem *umem; | |
| 728 | + int bytes = 0; | |
| 729 | + struct ib_ucontext *context = pd->ib_pd.uobject->context; | |
| 730 | + struct bnxt_re_ucontext *cntx = container_of(context, | |
| 731 | + struct bnxt_re_ucontext, | |
| 732 | + ib_uctx); | |
| 733 | + if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) | |
| 734 | + return -EFAULT; | |
| 735 | + | |
| 736 | + bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); | |
| 737 | + /* Consider mapping PSN search memory only for RC QPs. */ | |
| 738 | + if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) | |
| 739 | + bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search)); | |
| 740 | + bytes = PAGE_ALIGN(bytes); | |
| 741 | + umem = ib_umem_get(context, ureq.qpsva, bytes, | |
| 742 | + IB_ACCESS_LOCAL_WRITE, 1); | |
| 743 | + if (IS_ERR(umem)) | |
| 744 | + return PTR_ERR(umem); | |
| 745 | + | |
| 746 | + qp->sumem = umem; | |
| 747 | + qplib_qp->sq.sglist = umem->sg_head.sgl; | |
| 748 | + qplib_qp->sq.nmap = umem->nmap; | |
| 749 | + qplib_qp->qp_handle = ureq.qp_handle; | |
| 750 | + | |
| 751 | + if (!qp->qplib_qp.srq) { | |
| 752 | + bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); | |
| 753 | + bytes = PAGE_ALIGN(bytes); | |
| 754 | + umem = ib_umem_get(context, ureq.qprva, bytes, | |
| 755 | + IB_ACCESS_LOCAL_WRITE, 1); | |
| 756 | + if (IS_ERR(umem)) | |
| 757 | + goto rqfail; | |
| 758 | + qp->rumem = umem; | |
| 759 | + qplib_qp->rq.sglist = umem->sg_head.sgl; | |
| 760 | + qplib_qp->rq.nmap = umem->nmap; | |
| 761 | + } | |
| 762 | + | |
| 763 | + qplib_qp->dpi = cntx->dpi; | |
| 764 | + return 0; | |
| 765 | +rqfail: | |
| 766 | + ib_umem_release(qp->sumem); | |
| 767 | + qp->sumem = NULL; | |
| 768 | + qplib_qp->sq.sglist = NULL; | |
| 769 | + qplib_qp->sq.nmap = 0; | |
| 770 | + | |
| 771 | + return PTR_ERR(umem); | |
| 772 | +} | |
| 773 | + | |
| 774 | +static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah | |
| 775 | + (struct bnxt_re_pd *pd, | |
| 776 | + struct bnxt_qplib_res *qp1_res, | |
| 777 | + struct bnxt_qplib_qp *qp1_qp) | |
| 778 | +{ | |
| 779 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 780 | + struct bnxt_re_ah *ah; | |
| 781 | + union ib_gid sgid; | |
| 782 | + int rc; | |
| 783 | + | |
| 784 | + ah = kzalloc(sizeof(*ah), GFP_KERNEL); | |
| 785 | + if (!ah) | |
| 786 | + return NULL; | |
| 787 | + | |
| 788 | + memset(ah, 0, sizeof(*ah)); | |
| 789 | + ah->rdev = rdev; | |
| 790 | + ah->qplib_ah.pd = &pd->qplib_pd; | |
| 791 | + | |
| 792 | + rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); | |
| 793 | + if (rc) | |
| 794 | + goto fail; | |
| 795 | + | |
| 796 | + /* supply the dgid data same as sgid */ | |
| 797 | + memcpy(ah->qplib_ah.dgid.data, &sgid.raw, | |
| 798 | + sizeof(union ib_gid)); | |
| 799 | + ah->qplib_ah.sgid_index = 0; | |
| 800 | + | |
| 801 | + ah->qplib_ah.traffic_class = 0; | |
| 802 | + ah->qplib_ah.flow_label = 0; | |
| 803 | + ah->qplib_ah.hop_limit = 1; | |
| 804 | + ah->qplib_ah.sl = 0; | |
| 805 | + /* Have DMAC same as SMAC */ | |
| 806 | + ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); | |
| 807 | + | |
| 808 | + rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah); | |
| 809 | + if (rc) { | |
| 810 | + dev_err(rdev_to_dev(rdev), | |
| 811 | + "Failed to allocate HW AH for Shadow QP"); | |
| 812 | + goto fail; | |
| 813 | + } | |
| 814 | + | |
| 815 | + return ah; | |
| 816 | + | |
| 817 | +fail: | |
| 818 | + kfree(ah); | |
| 819 | + return NULL; | |
| 820 | +} | |
| 821 | + | |
| 822 | +static struct bnxt_re_qp *bnxt_re_create_shadow_qp | |
| 823 | + (struct bnxt_re_pd *pd, | |
| 824 | + struct bnxt_qplib_res *qp1_res, | |
| 825 | + struct bnxt_qplib_qp *qp1_qp) | |
| 826 | +{ | |
| 827 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 828 | + struct bnxt_re_qp *qp; | |
| 829 | + int rc; | |
| 830 | + | |
| 831 | + qp = kzalloc(sizeof(*qp), GFP_KERNEL); | |
| 832 | + if (!qp) | |
| 833 | + return NULL; | |
| 834 | + | |
| 835 | + memset(qp, 0, sizeof(*qp)); | |
| 836 | + qp->rdev = rdev; | |
| 837 | + | |
| 838 | + /* Initialize the shadow QP structure from the QP1 values */ | |
| 839 | + ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); | |
| 840 | + | |
| 841 | + qp->qplib_qp.pd = &pd->qplib_pd; | |
| 842 | + qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); | |
| 843 | + qp->qplib_qp.type = IB_QPT_UD; | |
| 844 | + | |
| 845 | + qp->qplib_qp.max_inline_data = 0; | |
| 846 | + qp->qplib_qp.sig_type = true; | |
| 847 | + | |
| 848 | + /* Shadow QP SQ depth should be same as QP1 RQ depth */ | |
| 849 | + qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; | |
| 850 | + qp->qplib_qp.sq.max_sge = 2; | |
| 851 | + | |
| 852 | + qp->qplib_qp.scq = qp1_qp->scq; | |
| 853 | + qp->qplib_qp.rcq = qp1_qp->rcq; | |
| 854 | + | |
| 855 | + qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; | |
| 856 | + qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; | |
| 857 | + | |
| 858 | + qp->qplib_qp.mtu = qp1_qp->mtu; | |
| 859 | + | |
| 860 | + qp->qplib_qp.sq_hdr_buf_size = 0; | |
| 861 | + qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; | |
| 862 | + qp->qplib_qp.dpi = &rdev->dpi_privileged; | |
| 863 | + | |
| 864 | + rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); | |
| 865 | + if (rc) | |
| 866 | + goto fail; | |
| 867 | + | |
| 868 | + rdev->sqp_id = qp->qplib_qp.id; | |
| 869 | + | |
| 870 | + spin_lock_init(&qp->sq_lock); | |
| 871 | + INIT_LIST_HEAD(&qp->list); | |
| 872 | + mutex_lock(&rdev->qp_lock); | |
| 873 | + list_add_tail(&qp->list, &rdev->qp_list); | |
| 874 | + atomic_inc(&rdev->qp_count); | |
| 875 | + mutex_unlock(&rdev->qp_lock); | |
| 876 | + return qp; | |
| 877 | +fail: | |
| 878 | + kfree(qp); | |
| 879 | + return NULL; | |
| 880 | +} | |
| 881 | + | |
| 882 | +struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, | |
| 883 | + struct ib_qp_init_attr *qp_init_attr, | |
| 884 | + struct ib_udata *udata) | |
| 885 | +{ | |
| 886 | + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | |
| 887 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 888 | + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | |
| 889 | + struct bnxt_re_qp *qp; | |
| 890 | + struct bnxt_re_cq *cq; | |
| 891 | + int rc, entries; | |
| 892 | + | |
| 893 | + if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || | |
| 894 | + (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || | |
| 895 | + (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || | |
| 896 | + (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || | |
| 897 | + (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data)) | |
| 898 | + return ERR_PTR(-EINVAL); | |
| 899 | + | |
| 900 | + qp = kzalloc(sizeof(*qp), GFP_KERNEL); | |
| 901 | + if (!qp) | |
| 902 | + return ERR_PTR(-ENOMEM); | |
| 903 | + | |
| 904 | + qp->rdev = rdev; | |
| 905 | + ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); | |
| 906 | + qp->qplib_qp.pd = &pd->qplib_pd; | |
| 907 | + qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); | |
| 908 | + qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type); | |
| 909 | + if (qp->qplib_qp.type == IB_QPT_MAX) { | |
| 910 | + dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported", | |
| 911 | + qp->qplib_qp.type); | |
| 912 | + rc = -EINVAL; | |
| 913 | + goto fail; | |
| 914 | + } | |
| 915 | + qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data; | |
| 916 | + qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type == | |
| 917 | + IB_SIGNAL_ALL_WR) ? true : false); | |
| 918 | + | |
| 919 | + entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1); | |
| 920 | + qp->qplib_qp.sq.max_wqe = min_t(u32, entries, | |
| 921 | + dev_attr->max_qp_wqes + 1); | |
| 922 | + | |
| 923 | + qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge; | |
| 924 | + if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) | |
| 925 | + qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; | |
| 926 | + | |
| 927 | + if (qp_init_attr->send_cq) { | |
| 928 | + cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq, | |
| 929 | + ib_cq); | |
| 930 | + if (!cq) { | |
| 931 | + dev_err(rdev_to_dev(rdev), "Send CQ not found"); | |
| 932 | + rc = -EINVAL; | |
| 933 | + goto fail; | |
| 934 | + } | |
| 935 | + qp->qplib_qp.scq = &cq->qplib_cq; | |
| 936 | + } | |
| 937 | + | |
| 938 | + if (qp_init_attr->recv_cq) { | |
| 939 | + cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq, | |
| 940 | + ib_cq); | |
| 941 | + if (!cq) { | |
| 942 | + dev_err(rdev_to_dev(rdev), "Receive CQ not found"); | |
| 943 | + rc = -EINVAL; | |
| 944 | + goto fail; | |
| 945 | + } | |
| 946 | + qp->qplib_qp.rcq = &cq->qplib_cq; | |
| 947 | + } | |
| 948 | + | |
| 949 | + if (qp_init_attr->srq) { | |
| 950 | + dev_err(rdev_to_dev(rdev), "SRQ not supported"); | |
| 951 | + rc = -ENOTSUPP; | |
| 952 | + goto fail; | |
| 953 | + } else { | |
| 954 | + /* Allocate 1 more than what's provided so posting max doesn't | |
| 955 | + * mean empty | |
| 956 | + */ | |
| 957 | + entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1); | |
| 958 | + qp->qplib_qp.rq.max_wqe = min_t(u32, entries, | |
| 959 | + dev_attr->max_qp_wqes + 1); | |
| 960 | + | |
| 961 | + qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge; | |
| 962 | + if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) | |
| 963 | + qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; | |
| 964 | + } | |
| 965 | + | |
| 966 | + qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); | |
| 967 | + | |
| 968 | + if (qp_init_attr->qp_type == IB_QPT_GSI) { | |
| 969 | + qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; | |
| 970 | + if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) | |
| 971 | + qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; | |
| 972 | + qp->qplib_qp.sq.max_sge++; | |
| 973 | + if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) | |
| 974 | + qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; | |
| 975 | + | |
| 976 | + qp->qplib_qp.rq_hdr_buf_size = | |
| 977 | + BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; | |
| 978 | + | |
| 979 | + qp->qplib_qp.sq_hdr_buf_size = | |
| 980 | + BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; | |
| 981 | + qp->qplib_qp.dpi = &rdev->dpi_privileged; | |
| 982 | + rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp); | |
| 983 | + if (rc) { | |
| 984 | + dev_err(rdev_to_dev(rdev), "Failed to create HW QP1"); | |
| 985 | + goto fail; | |
| 986 | + } | |
| 987 | + /* Create a shadow QP to handle the QP1 traffic */ | |
| 988 | + rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, | |
| 989 | + &qp->qplib_qp); | |
| 990 | + if (!rdev->qp1_sqp) { | |
| 991 | + rc = -EINVAL; | |
| 992 | + dev_err(rdev_to_dev(rdev), | |
| 993 | + "Failed to create Shadow QP for QP1"); | |
| 994 | + goto qp_destroy; | |
| 995 | + } | |
| 996 | + rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, | |
| 997 | + &qp->qplib_qp); | |
| 998 | + if (!rdev->sqp_ah) { | |
| 999 | + bnxt_qplib_destroy_qp(&rdev->qplib_res, | |
| 1000 | + &rdev->qp1_sqp->qplib_qp); | |
| 1001 | + rc = -EINVAL; | |
| 1002 | + dev_err(rdev_to_dev(rdev), | |
| 1003 | + "Failed to create AH entry for ShadowQP"); | |
| 1004 | + goto qp_destroy; | |
| 1005 | + } | |
| 1006 | + | |
| 1007 | + } else { | |
| 1008 | + qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom; | |
| 1009 | + qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; | |
| 1010 | + if (udata) { | |
| 1011 | + rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); | |
| 1012 | + if (rc) | |
| 1013 | + goto fail; | |
| 1014 | + } else { | |
| 1015 | + qp->qplib_qp.dpi = &rdev->dpi_privileged; | |
| 1016 | + } | |
| 1017 | + | |
| 1018 | + rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); | |
| 1019 | + if (rc) { | |
| 1020 | + dev_err(rdev_to_dev(rdev), "Failed to create HW QP"); | |
| 1021 | + goto fail; | |
| 1022 | + } | |
| 1023 | + } | |
| 1024 | + | |
| 1025 | + qp->ib_qp.qp_num = qp->qplib_qp.id; | |
| 1026 | + spin_lock_init(&qp->sq_lock); | |
| 1027 | + | |
| 1028 | + if (udata) { | |
| 1029 | + struct bnxt_re_qp_resp resp; | |
| 1030 | + | |
| 1031 | + resp.qpid = qp->ib_qp.qp_num; | |
| 1032 | + resp.rsvd = 0; | |
| 1033 | + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); | |
| 1034 | + if (rc) { | |
| 1035 | + dev_err(rdev_to_dev(rdev), "Failed to copy QP udata"); | |
| 1036 | + goto qp_destroy; | |
| 1037 | + } | |
| 1038 | + } | |
| 1039 | + INIT_LIST_HEAD(&qp->list); | |
| 1040 | + mutex_lock(&rdev->qp_lock); | |
| 1041 | + list_add_tail(&qp->list, &rdev->qp_list); | |
| 1042 | + atomic_inc(&rdev->qp_count); | |
| 1043 | + mutex_unlock(&rdev->qp_lock); | |
| 1044 | + | |
| 1045 | + return &qp->ib_qp; | |
| 1046 | +qp_destroy: | |
| 1047 | + bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); | |
| 1048 | +fail: | |
| 1049 | + kfree(qp); | |
| 1050 | + return ERR_PTR(rc); | |
| 1051 | +} | |
| 1052 | + | |
| 1053 | +static u8 __from_ib_qp_state(enum ib_qp_state state) | |
| 1054 | +{ | |
| 1055 | + switch (state) { | |
| 1056 | + case IB_QPS_RESET: | |
| 1057 | + return CMDQ_MODIFY_QP_NEW_STATE_RESET; | |
| 1058 | + case IB_QPS_INIT: | |
| 1059 | + return CMDQ_MODIFY_QP_NEW_STATE_INIT; | |
| 1060 | + case IB_QPS_RTR: | |
| 1061 | + return CMDQ_MODIFY_QP_NEW_STATE_RTR; | |
| 1062 | + case IB_QPS_RTS: | |
| 1063 | + return CMDQ_MODIFY_QP_NEW_STATE_RTS; | |
| 1064 | + case IB_QPS_SQD: | |
| 1065 | + return CMDQ_MODIFY_QP_NEW_STATE_SQD; | |
| 1066 | + case IB_QPS_SQE: | |
| 1067 | + return CMDQ_MODIFY_QP_NEW_STATE_SQE; | |
| 1068 | + case IB_QPS_ERR: | |
| 1069 | + default: | |
| 1070 | + return CMDQ_MODIFY_QP_NEW_STATE_ERR; | |
| 1071 | + } | |
| 1072 | +} | |
| 1073 | + | |
| 1074 | +static enum ib_qp_state __to_ib_qp_state(u8 state) | |
| 1075 | +{ | |
| 1076 | + switch (state) { | |
| 1077 | + case CMDQ_MODIFY_QP_NEW_STATE_RESET: | |
| 1078 | + return IB_QPS_RESET; | |
| 1079 | + case CMDQ_MODIFY_QP_NEW_STATE_INIT: | |
| 1080 | + return IB_QPS_INIT; | |
| 1081 | + case CMDQ_MODIFY_QP_NEW_STATE_RTR: | |
| 1082 | + return IB_QPS_RTR; | |
| 1083 | + case CMDQ_MODIFY_QP_NEW_STATE_RTS: | |
| 1084 | + return IB_QPS_RTS; | |
| 1085 | + case CMDQ_MODIFY_QP_NEW_STATE_SQD: | |
| 1086 | + return IB_QPS_SQD; | |
| 1087 | + case CMDQ_MODIFY_QP_NEW_STATE_SQE: | |
| 1088 | + return IB_QPS_SQE; | |
| 1089 | + case CMDQ_MODIFY_QP_NEW_STATE_ERR: | |
| 1090 | + default: | |
| 1091 | + return IB_QPS_ERR; | |
| 1092 | + } | |
| 1093 | +} | |
| 1094 | + | |
| 1095 | +static u32 __from_ib_mtu(enum ib_mtu mtu) | |
| 1096 | +{ | |
| 1097 | + switch (mtu) { | |
| 1098 | + case IB_MTU_256: | |
| 1099 | + return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; | |
| 1100 | + case IB_MTU_512: | |
| 1101 | + return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; | |
| 1102 | + case IB_MTU_1024: | |
| 1103 | + return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; | |
| 1104 | + case IB_MTU_2048: | |
| 1105 | + return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; | |
| 1106 | + case IB_MTU_4096: | |
| 1107 | + return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; | |
| 1108 | + default: | |
| 1109 | + return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; | |
| 1110 | + } | |
| 1111 | +} | |
| 1112 | + | |
| 1113 | +static enum ib_mtu __to_ib_mtu(u32 mtu) | |
| 1114 | +{ | |
| 1115 | + switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { | |
| 1116 | + case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: | |
| 1117 | + return IB_MTU_256; | |
| 1118 | + case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: | |
| 1119 | + return IB_MTU_512; | |
| 1120 | + case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: | |
| 1121 | + return IB_MTU_1024; | |
| 1122 | + case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: | |
| 1123 | + return IB_MTU_2048; | |
| 1124 | + case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: | |
| 1125 | + return IB_MTU_4096; | |
| 1126 | + default: | |
| 1127 | + return IB_MTU_2048; | |
| 1128 | + } | |
| 1129 | +} | |
| 1130 | + | |
| 1131 | +static int __from_ib_access_flags(int iflags) | |
| 1132 | +{ | |
| 1133 | + int qflags = 0; | |
| 1134 | + | |
| 1135 | + if (iflags & IB_ACCESS_LOCAL_WRITE) | |
| 1136 | + qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; | |
| 1137 | + if (iflags & IB_ACCESS_REMOTE_READ) | |
| 1138 | + qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; | |
| 1139 | + if (iflags & IB_ACCESS_REMOTE_WRITE) | |
| 1140 | + qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; | |
| 1141 | + if (iflags & IB_ACCESS_REMOTE_ATOMIC) | |
| 1142 | + qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; | |
| 1143 | + if (iflags & IB_ACCESS_MW_BIND) | |
| 1144 | + qflags |= BNXT_QPLIB_ACCESS_MW_BIND; | |
| 1145 | + if (iflags & IB_ZERO_BASED) | |
| 1146 | + qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; | |
| 1147 | + if (iflags & IB_ACCESS_ON_DEMAND) | |
| 1148 | + qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; | |
| 1149 | + return qflags; | |
| 1150 | +}; | |
| 1151 | + | |
| 1152 | +static enum ib_access_flags __to_ib_access_flags(int qflags) | |
| 1153 | +{ | |
| 1154 | + enum ib_access_flags iflags = 0; | |
| 1155 | + | |
| 1156 | + if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) | |
| 1157 | + iflags |= IB_ACCESS_LOCAL_WRITE; | |
| 1158 | + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) | |
| 1159 | + iflags |= IB_ACCESS_REMOTE_WRITE; | |
| 1160 | + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) | |
| 1161 | + iflags |= IB_ACCESS_REMOTE_READ; | |
| 1162 | + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) | |
| 1163 | + iflags |= IB_ACCESS_REMOTE_ATOMIC; | |
| 1164 | + if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) | |
| 1165 | + iflags |= IB_ACCESS_MW_BIND; | |
| 1166 | + if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) | |
| 1167 | + iflags |= IB_ZERO_BASED; | |
| 1168 | + if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) | |
| 1169 | + iflags |= IB_ACCESS_ON_DEMAND; | |
| 1170 | + return iflags; | |
| 1171 | +}; | |
| 1172 | + | |
| 1173 | +static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, | |
| 1174 | + struct bnxt_re_qp *qp1_qp, | |
| 1175 | + int qp_attr_mask) | |
| 1176 | +{ | |
| 1177 | + struct bnxt_re_qp *qp = rdev->qp1_sqp; | |
| 1178 | + int rc = 0; | |
| 1179 | + | |
| 1180 | + if (qp_attr_mask & IB_QP_STATE) { | |
| 1181 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; | |
| 1182 | + qp->qplib_qp.state = qp1_qp->qplib_qp.state; | |
| 1183 | + } | |
| 1184 | + if (qp_attr_mask & IB_QP_PKEY_INDEX) { | |
| 1185 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; | |
| 1186 | + qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; | |
| 1187 | + } | |
| 1188 | + | |
| 1189 | + if (qp_attr_mask & IB_QP_QKEY) { | |
| 1190 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; | |
| 1191 | + /* Using a Random QKEY */ | |
| 1192 | + qp->qplib_qp.qkey = 0x81818181; | |
| 1193 | + } | |
| 1194 | + if (qp_attr_mask & IB_QP_SQ_PSN) { | |
| 1195 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; | |
| 1196 | + qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; | |
| 1197 | + } | |
| 1198 | + | |
| 1199 | + rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); | |
| 1200 | + if (rc) | |
| 1201 | + dev_err(rdev_to_dev(rdev), | |
| 1202 | + "Failed to modify Shadow QP for QP1"); | |
| 1203 | + return rc; | |
| 1204 | +} | |
| 1205 | + | |
| 1206 | +int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, | |
| 1207 | + int qp_attr_mask, struct ib_udata *udata) | |
| 1208 | +{ | |
| 1209 | + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | |
| 1210 | + struct bnxt_re_dev *rdev = qp->rdev; | |
| 1211 | + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | |
| 1212 | + enum ib_qp_state curr_qp_state, new_qp_state; | |
| 1213 | + int rc, entries; | |
| 1214 | + int status; | |
| 1215 | + union ib_gid sgid; | |
| 1216 | + struct ib_gid_attr sgid_attr; | |
| 1217 | + u8 nw_type; | |
| 1218 | + | |
| 1219 | + qp->qplib_qp.modify_flags = 0; | |
| 1220 | + if (qp_attr_mask & IB_QP_STATE) { | |
| 1221 | + curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); | |
| 1222 | + new_qp_state = qp_attr->qp_state; | |
| 1223 | + if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, | |
| 1224 | + ib_qp->qp_type, qp_attr_mask, | |
| 1225 | + IB_LINK_LAYER_ETHERNET)) { | |
| 1226 | + dev_err(rdev_to_dev(rdev), | |
| 1227 | + "Invalid attribute mask: %#x specified ", | |
| 1228 | + qp_attr_mask); | |
| 1229 | + dev_err(rdev_to_dev(rdev), | |
| 1230 | + "for qpn: %#x type: %#x", | |
| 1231 | + ib_qp->qp_num, ib_qp->qp_type); | |
| 1232 | + dev_err(rdev_to_dev(rdev), | |
| 1233 | + "curr_qp_state=0x%x, new_qp_state=0x%x\n", | |
| 1234 | + curr_qp_state, new_qp_state); | |
| 1235 | + return -EINVAL; | |
| 1236 | + } | |
| 1237 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; | |
| 1238 | + qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); | |
| 1239 | + } | |
| 1240 | + if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { | |
| 1241 | + qp->qplib_qp.modify_flags |= | |
| 1242 | + CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; | |
| 1243 | + qp->qplib_qp.en_sqd_async_notify = true; | |
| 1244 | + } | |
| 1245 | + if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { | |
| 1246 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; | |
| 1247 | + qp->qplib_qp.access = | |
| 1248 | + __from_ib_access_flags(qp_attr->qp_access_flags); | |
| 1249 | + /* LOCAL_WRITE access must be set to allow RC receive */ | |
| 1250 | + qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; | |
| 1251 | + } | |
| 1252 | + if (qp_attr_mask & IB_QP_PKEY_INDEX) { | |
| 1253 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; | |
| 1254 | + qp->qplib_qp.pkey_index = qp_attr->pkey_index; | |
| 1255 | + } | |
| 1256 | + if (qp_attr_mask & IB_QP_QKEY) { | |
| 1257 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; | |
| 1258 | + qp->qplib_qp.qkey = qp_attr->qkey; | |
| 1259 | + } | |
| 1260 | + if (qp_attr_mask & IB_QP_AV) { | |
| 1261 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | | |
| 1262 | + CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | | |
| 1263 | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | | |
| 1264 | + CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | | |
| 1265 | + CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | | |
| 1266 | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | | |
| 1267 | + CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; | |
| 1268 | + memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw, | |
| 1269 | + sizeof(qp->qplib_qp.ah.dgid.data)); | |
| 1270 | + qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label; | |
| 1271 | + /* If RoCE V2 is enabled, stack will have two entries for | |
| 1272 | + * each GID entry. Avoiding this duplicte entry in HW. Dividing | |
| 1273 | + * the GID index by 2 for RoCE V2 | |
| 1274 | + */ | |
| 1275 | + qp->qplib_qp.ah.sgid_index = | |
| 1276 | + qp_attr->ah_attr.grh.sgid_index / 2; | |
| 1277 | + qp->qplib_qp.ah.host_sgid_index = | |
| 1278 | + qp_attr->ah_attr.grh.sgid_index; | |
| 1279 | + qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit; | |
| 1280 | + qp->qplib_qp.ah.traffic_class = | |
| 1281 | + qp_attr->ah_attr.grh.traffic_class; | |
| 1282 | + qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl; | |
| 1283 | + ether_addr_copy(qp->qplib_qp.ah.dmac, qp_attr->ah_attr.dmac); | |
| 1284 | + | |
| 1285 | + status = ib_get_cached_gid(&rdev->ibdev, 1, | |
| 1286 | + qp_attr->ah_attr.grh.sgid_index, | |
| 1287 | + &sgid, &sgid_attr); | |
| 1288 | + if (!status && sgid_attr.ndev) { | |
| 1289 | + memcpy(qp->qplib_qp.smac, sgid_attr.ndev->dev_addr, | |
| 1290 | + ETH_ALEN); | |
| 1291 | + dev_put(sgid_attr.ndev); | |
| 1292 | + nw_type = ib_gid_to_network_type(sgid_attr.gid_type, | |
| 1293 | + &sgid); | |
| 1294 | + switch (nw_type) { | |
| 1295 | + case RDMA_NETWORK_IPV4: | |
| 1296 | + qp->qplib_qp.nw_type = | |
| 1297 | + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; | |
| 1298 | + break; | |
| 1299 | + case RDMA_NETWORK_IPV6: | |
| 1300 | + qp->qplib_qp.nw_type = | |
| 1301 | + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; | |
| 1302 | + break; | |
| 1303 | + default: | |
| 1304 | + qp->qplib_qp.nw_type = | |
| 1305 | + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; | |
| 1306 | + break; | |
| 1307 | + } | |
| 1308 | + } | |
| 1309 | + } | |
| 1310 | + | |
| 1311 | + if (qp_attr_mask & IB_QP_PATH_MTU) { | |
| 1312 | + qp->qplib_qp.modify_flags |= | |
| 1313 | + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; | |
| 1314 | + qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); | |
| 1315 | + } else if (qp_attr->qp_state == IB_QPS_RTR) { | |
| 1316 | + qp->qplib_qp.modify_flags |= | |
| 1317 | + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; | |
| 1318 | + qp->qplib_qp.path_mtu = | |
| 1319 | + __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); | |
| 1320 | + } | |
| 1321 | + | |
| 1322 | + if (qp_attr_mask & IB_QP_TIMEOUT) { | |
| 1323 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; | |
| 1324 | + qp->qplib_qp.timeout = qp_attr->timeout; | |
| 1325 | + } | |
| 1326 | + if (qp_attr_mask & IB_QP_RETRY_CNT) { | |
| 1327 | + qp->qplib_qp.modify_flags |= | |
| 1328 | + CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; | |
| 1329 | + qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; | |
| 1330 | + } | |
| 1331 | + if (qp_attr_mask & IB_QP_RNR_RETRY) { | |
| 1332 | + qp->qplib_qp.modify_flags |= | |
| 1333 | + CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; | |
| 1334 | + qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; | |
| 1335 | + } | |
| 1336 | + if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { | |
| 1337 | + qp->qplib_qp.modify_flags |= | |
| 1338 | + CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; | |
| 1339 | + qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; | |
| 1340 | + } | |
| 1341 | + if (qp_attr_mask & IB_QP_RQ_PSN) { | |
| 1342 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; | |
| 1343 | + qp->qplib_qp.rq.psn = qp_attr->rq_psn; | |
| 1344 | + } | |
| 1345 | + if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { | |
| 1346 | + qp->qplib_qp.modify_flags |= | |
| 1347 | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; | |
| 1348 | + qp->qplib_qp.max_rd_atomic = qp_attr->max_rd_atomic; | |
| 1349 | + } | |
| 1350 | + if (qp_attr_mask & IB_QP_SQ_PSN) { | |
| 1351 | + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; | |
| 1352 | + qp->qplib_qp.sq.psn = qp_attr->sq_psn; | |
| 1353 | + } | |
| 1354 | + if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { | |
| 1355 | + qp->qplib_qp.modify_flags |= | |
| 1356 | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; | |
| 1357 | + qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; | |
| 1358 | + } | |
| 1359 | + if (qp_attr_mask & IB_QP_CAP) { | |
| 1360 | + qp->qplib_qp.modify_flags |= | |
| 1361 | + CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | | |
| 1362 | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | | |
| 1363 | + CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | | |
| 1364 | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | | |
| 1365 | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; | |
| 1366 | + if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || | |
| 1367 | + (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || | |
| 1368 | + (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || | |
| 1369 | + (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || | |
| 1370 | + (qp_attr->cap.max_inline_data >= | |
| 1371 | + dev_attr->max_inline_data)) { | |
| 1372 | + dev_err(rdev_to_dev(rdev), | |
| 1373 | + "Create QP failed - max exceeded"); | |
| 1374 | + return -EINVAL; | |
| 1375 | + } | |
| 1376 | + entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); | |
| 1377 | + qp->qplib_qp.sq.max_wqe = min_t(u32, entries, | |
| 1378 | + dev_attr->max_qp_wqes + 1); | |
| 1379 | + qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; | |
| 1380 | + if (qp->qplib_qp.rq.max_wqe) { | |
| 1381 | + entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); | |
| 1382 | + qp->qplib_qp.rq.max_wqe = | |
| 1383 | + min_t(u32, entries, dev_attr->max_qp_wqes + 1); | |
| 1384 | + qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; | |
| 1385 | + } else { | |
| 1386 | + /* SRQ was used prior, just ignore the RQ caps */ | |
| 1387 | + } | |
| 1388 | + } | |
| 1389 | + if (qp_attr_mask & IB_QP_DEST_QPN) { | |
| 1390 | + qp->qplib_qp.modify_flags |= | |
| 1391 | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; | |
| 1392 | + qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; | |
| 1393 | + } | |
| 1394 | + rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); | |
| 1395 | + if (rc) { | |
| 1396 | + dev_err(rdev_to_dev(rdev), "Failed to modify HW QP"); | |
| 1397 | + return rc; | |
| 1398 | + } | |
| 1399 | + if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) | |
| 1400 | + rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); | |
| 1401 | + return rc; | |
| 1402 | +} | |
| 1403 | + | |
| 1404 | +int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, | |
| 1405 | + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) | |
| 1406 | +{ | |
| 1407 | + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | |
| 1408 | + struct bnxt_re_dev *rdev = qp->rdev; | |
| 1409 | + struct bnxt_qplib_qp qplib_qp; | |
| 1410 | + int rc; | |
| 1411 | + | |
| 1412 | + memset(&qplib_qp, 0, sizeof(struct bnxt_qplib_qp)); | |
| 1413 | + qplib_qp.id = qp->qplib_qp.id; | |
| 1414 | + qplib_qp.ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; | |
| 1415 | + | |
| 1416 | + rc = bnxt_qplib_query_qp(&rdev->qplib_res, &qplib_qp); | |
| 1417 | + if (rc) { | |
| 1418 | + dev_err(rdev_to_dev(rdev), "Failed to query HW QP"); | |
| 1419 | + return rc; | |
| 1420 | + } | |
| 1421 | + qp_attr->qp_state = __to_ib_qp_state(qplib_qp.state); | |
| 1422 | + qp_attr->en_sqd_async_notify = qplib_qp.en_sqd_async_notify ? 1 : 0; | |
| 1423 | + qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp.access); | |
| 1424 | + qp_attr->pkey_index = qplib_qp.pkey_index; | |
| 1425 | + qp_attr->qkey = qplib_qp.qkey; | |
| 1426 | + memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp.ah.dgid.data, | |
| 1427 | + sizeof(qplib_qp.ah.dgid.data)); | |
| 1428 | + qp_attr->ah_attr.grh.flow_label = qplib_qp.ah.flow_label; | |
| 1429 | + qp_attr->ah_attr.grh.sgid_index = qplib_qp.ah.host_sgid_index; | |
| 1430 | + qp_attr->ah_attr.grh.hop_limit = qplib_qp.ah.hop_limit; | |
| 1431 | + qp_attr->ah_attr.grh.traffic_class = qplib_qp.ah.traffic_class; | |
| 1432 | + qp_attr->ah_attr.sl = qplib_qp.ah.sl; | |
| 1433 | + ether_addr_copy(qp_attr->ah_attr.dmac, qplib_qp.ah.dmac); | |
| 1434 | + qp_attr->path_mtu = __to_ib_mtu(qplib_qp.path_mtu); | |
| 1435 | + qp_attr->timeout = qplib_qp.timeout; | |
| 1436 | + qp_attr->retry_cnt = qplib_qp.retry_cnt; | |
| 1437 | + qp_attr->rnr_retry = qplib_qp.rnr_retry; | |
| 1438 | + qp_attr->min_rnr_timer = qplib_qp.min_rnr_timer; | |
| 1439 | + qp_attr->rq_psn = qplib_qp.rq.psn; | |
| 1440 | + qp_attr->max_rd_atomic = qplib_qp.max_rd_atomic; | |
| 1441 | + qp_attr->sq_psn = qplib_qp.sq.psn; | |
| 1442 | + qp_attr->max_dest_rd_atomic = qplib_qp.max_dest_rd_atomic; | |
| 1443 | + qp_init_attr->sq_sig_type = qplib_qp.sig_type ? IB_SIGNAL_ALL_WR : | |
| 1444 | + IB_SIGNAL_REQ_WR; | |
| 1445 | + qp_attr->dest_qp_num = qplib_qp.dest_qpn; | |
| 1446 | + | |
| 1447 | + qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; | |
| 1448 | + qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; | |
| 1449 | + qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; | |
| 1450 | + qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; | |
| 1451 | + qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; | |
| 1452 | + qp_init_attr->cap = qp_attr->cap; | |
| 1453 | + | |
| 1454 | + return 0; | |
| 1455 | +} | |
| 1456 | + | |
| 1457 | +/* Routine for sending QP1 packets for RoCE V1 an V2 | |
| 1458 | + */ | |
| 1459 | +static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, | |
| 1460 | + struct ib_send_wr *wr, | |
| 1461 | + struct bnxt_qplib_swqe *wqe, | |
| 1462 | + int payload_size) | |
| 1463 | +{ | |
| 1464 | + struct ib_device *ibdev = &qp->rdev->ibdev; | |
| 1465 | + struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, | |
| 1466 | + ib_ah); | |
| 1467 | + struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; | |
| 1468 | + struct bnxt_qplib_sge sge; | |
| 1469 | + union ib_gid sgid; | |
| 1470 | + u8 nw_type; | |
| 1471 | + u16 ether_type; | |
| 1472 | + struct ib_gid_attr sgid_attr; | |
| 1473 | + union ib_gid dgid; | |
| 1474 | + bool is_eth = false; | |
| 1475 | + bool is_vlan = false; | |
| 1476 | + bool is_grh = false; | |
| 1477 | + bool is_udp = false; | |
| 1478 | + u8 ip_version = 0; | |
| 1479 | + u16 vlan_id = 0xFFFF; | |
| 1480 | + void *buf; | |
| 1481 | + int i, rc = 0, size; | |
| 1482 | + | |
| 1483 | + memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); | |
| 1484 | + | |
| 1485 | + rc = ib_get_cached_gid(ibdev, 1, | |
| 1486 | + qplib_ah->host_sgid_index, &sgid, | |
| 1487 | + &sgid_attr); | |
| 1488 | + if (rc) { | |
| 1489 | + dev_err(rdev_to_dev(qp->rdev), | |
| 1490 | + "Failed to query gid at index %d", | |
| 1491 | + qplib_ah->host_sgid_index); | |
| 1492 | + return rc; | |
| 1493 | + } | |
| 1494 | + if (sgid_attr.ndev) { | |
| 1495 | + if (is_vlan_dev(sgid_attr.ndev)) | |
| 1496 | + vlan_id = vlan_dev_vlan_id(sgid_attr.ndev); | |
| 1497 | + dev_put(sgid_attr.ndev); | |
| 1498 | + } | |
| 1499 | + /* Get network header type for this GID */ | |
| 1500 | + nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); | |
| 1501 | + switch (nw_type) { | |
| 1502 | + case RDMA_NETWORK_IPV4: | |
| 1503 | + nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; | |
| 1504 | + break; | |
| 1505 | + case RDMA_NETWORK_IPV6: | |
| 1506 | + nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; | |
| 1507 | + break; | |
| 1508 | + default: | |
| 1509 | + nw_type = BNXT_RE_ROCE_V1_PACKET; | |
| 1510 | + break; | |
| 1511 | + } | |
| 1512 | + memcpy(&dgid.raw, &qplib_ah->dgid, 16); | |
| 1513 | + is_udp = sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; | |
| 1514 | + if (is_udp) { | |
| 1515 | + if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) { | |
| 1516 | + ip_version = 4; | |
| 1517 | + ether_type = ETH_P_IP; | |
| 1518 | + } else { | |
| 1519 | + ip_version = 6; | |
| 1520 | + ether_type = ETH_P_IPV6; | |
| 1521 | + } | |
| 1522 | + is_grh = false; | |
| 1523 | + } else { | |
| 1524 | + ether_type = ETH_P_IBOE; | |
| 1525 | + is_grh = true; | |
| 1526 | + } | |
| 1527 | + | |
| 1528 | + is_eth = true; | |
| 1529 | + is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; | |
| 1530 | + | |
| 1531 | + ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, | |
| 1532 | + ip_version, is_udp, 0, &qp->qp1_hdr); | |
| 1533 | + | |
| 1534 | + /* ETH */ | |
| 1535 | + ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); | |
| 1536 | + ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); | |
| 1537 | + | |
| 1538 | + /* For vlan, check the sgid for vlan existence */ | |
| 1539 | + | |
| 1540 | + if (!is_vlan) { | |
| 1541 | + qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); | |
| 1542 | + } else { | |
| 1543 | + qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); | |
| 1544 | + qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); | |
| 1545 | + } | |
| 1546 | + | |
| 1547 | + if (is_grh || (ip_version == 6)) { | |
| 1548 | + memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); | |
| 1549 | + memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, | |
| 1550 | + sizeof(sgid)); | |
| 1551 | + qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; | |
| 1552 | + } | |
| 1553 | + | |
| 1554 | + if (ip_version == 4) { | |
| 1555 | + qp->qp1_hdr.ip4.tos = 0; | |
| 1556 | + qp->qp1_hdr.ip4.id = 0; | |
| 1557 | + qp->qp1_hdr.ip4.frag_off = htons(IP_DF); | |
| 1558 | + qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; | |
| 1559 | + | |
| 1560 | + memcpy(&qp->qp1_hdr.ip4.saddr, sgid.raw + 12, 4); | |
| 1561 | + memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); | |
| 1562 | + qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); | |
| 1563 | + } | |
| 1564 | + | |
| 1565 | + if (is_udp) { | |
| 1566 | + qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); | |
| 1567 | + qp->qp1_hdr.udp.sport = htons(0x8CD1); | |
| 1568 | + qp->qp1_hdr.udp.csum = 0; | |
| 1569 | + } | |
| 1570 | + | |
| 1571 | + /* BTH */ | |
| 1572 | + if (wr->opcode == IB_WR_SEND_WITH_IMM) { | |
| 1573 | + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; | |
| 1574 | + qp->qp1_hdr.immediate_present = 1; | |
| 1575 | + } else { | |
| 1576 | + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; | |
| 1577 | + } | |
| 1578 | + if (wr->send_flags & IB_SEND_SOLICITED) | |
| 1579 | + qp->qp1_hdr.bth.solicited_event = 1; | |
| 1580 | + /* pad_count */ | |
| 1581 | + qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; | |
| 1582 | + | |
| 1583 | + /* P_key for QP1 is for all members */ | |
| 1584 | + qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); | |
| 1585 | + qp->qp1_hdr.bth.destination_qpn = IB_QP1; | |
| 1586 | + qp->qp1_hdr.bth.ack_req = 0; | |
| 1587 | + qp->send_psn++; | |
| 1588 | + qp->send_psn &= BTH_PSN_MASK; | |
| 1589 | + qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); | |
| 1590 | + /* DETH */ | |
| 1591 | + /* Use the priviledged Q_Key for QP1 */ | |
| 1592 | + qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); | |
| 1593 | + qp->qp1_hdr.deth.source_qpn = IB_QP1; | |
| 1594 | + | |
| 1595 | + /* Pack the QP1 to the transmit buffer */ | |
| 1596 | + buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); | |
| 1597 | + if (buf) { | |
| 1598 | + size = ib_ud_header_pack(&qp->qp1_hdr, buf); | |
| 1599 | + for (i = wqe->num_sge; i; i--) { | |
| 1600 | + wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; | |
| 1601 | + wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; | |
| 1602 | + wqe->sg_list[i].size = wqe->sg_list[i - 1].size; | |
| 1603 | + } | |
| 1604 | + | |
| 1605 | + /* | |
| 1606 | + * Max Header buf size for IPV6 RoCE V2 is 86, | |
| 1607 | + * which is same as the QP1 SQ header buffer. | |
| 1608 | + * Header buf size for IPV4 RoCE V2 can be 66. | |
| 1609 | + * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). | |
| 1610 | + * Subtract 20 bytes from QP1 SQ header buf size | |
| 1611 | + */ | |
| 1612 | + if (is_udp && ip_version == 4) | |
| 1613 | + sge.size -= 20; | |
| 1614 | + /* | |
| 1615 | + * Max Header buf size for RoCE V1 is 78. | |
| 1616 | + * ETH(14) + VLAN(4) + GRH(40) + BTH(20). | |
| 1617 | + * Subtract 8 bytes from QP1 SQ header buf size | |
| 1618 | + */ | |
| 1619 | + if (!is_udp) | |
| 1620 | + sge.size -= 8; | |
| 1621 | + | |
| 1622 | + /* Subtract 4 bytes for non vlan packets */ | |
| 1623 | + if (!is_vlan) | |
| 1624 | + sge.size -= 4; | |
| 1625 | + | |
| 1626 | + wqe->sg_list[0].addr = sge.addr; | |
| 1627 | + wqe->sg_list[0].lkey = sge.lkey; | |
| 1628 | + wqe->sg_list[0].size = sge.size; | |
| 1629 | + wqe->num_sge++; | |
| 1630 | + | |
| 1631 | + } else { | |
| 1632 | + dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!"); | |
| 1633 | + rc = -ENOMEM; | |
| 1634 | + } | |
| 1635 | + return rc; | |
| 1636 | +} | |
| 1637 | + | |
| 1638 | +/* For the MAD layer, it only provides the recv SGE the size of | |
| 1639 | + * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, | |
| 1640 | + * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire | |
| 1641 | + * receive packet (334 bytes) with no VLAN and then copy the GRH | |
| 1642 | + * and the MAD datagram out to the provided SGE. | |
| 1643 | + */ | |
| 1644 | +static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, | |
| 1645 | + struct ib_recv_wr *wr, | |
| 1646 | + struct bnxt_qplib_swqe *wqe, | |
| 1647 | + int payload_size) | |
| 1648 | +{ | |
| 1649 | + struct bnxt_qplib_sge ref, sge; | |
| 1650 | + u32 rq_prod_index; | |
| 1651 | + struct bnxt_re_sqp_entries *sqp_entry; | |
| 1652 | + | |
| 1653 | + rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); | |
| 1654 | + | |
| 1655 | + if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) | |
| 1656 | + return -ENOMEM; | |
| 1657 | + | |
| 1658 | + /* Create 1 SGE to receive the entire | |
| 1659 | + * ethernet packet | |
| 1660 | + */ | |
| 1661 | + /* Save the reference from ULP */ | |
| 1662 | + ref.addr = wqe->sg_list[0].addr; | |
| 1663 | + ref.lkey = wqe->sg_list[0].lkey; | |
| 1664 | + ref.size = wqe->sg_list[0].size; | |
| 1665 | + | |
| 1666 | + sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index]; | |
| 1667 | + | |
| 1668 | + /* SGE 1 */ | |
| 1669 | + wqe->sg_list[0].addr = sge.addr; | |
| 1670 | + wqe->sg_list[0].lkey = sge.lkey; | |
| 1671 | + wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; | |
| 1672 | + sge.size -= wqe->sg_list[0].size; | |
| 1673 | + | |
| 1674 | + sqp_entry->sge.addr = ref.addr; | |
| 1675 | + sqp_entry->sge.lkey = ref.lkey; | |
| 1676 | + sqp_entry->sge.size = ref.size; | |
| 1677 | + /* Store the wrid for reporting completion */ | |
| 1678 | + sqp_entry->wrid = wqe->wr_id; | |
| 1679 | + /* change the wqe->wrid to table index */ | |
| 1680 | + wqe->wr_id = rq_prod_index; | |
| 1681 | + return 0; | |
| 1682 | +} | |
| 1683 | + | |
| 1684 | +static int is_ud_qp(struct bnxt_re_qp *qp) | |
| 1685 | +{ | |
| 1686 | + return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD; | |
| 1687 | +} | |
| 1688 | + | |
| 1689 | +static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, | |
| 1690 | + struct ib_send_wr *wr, | |
| 1691 | + struct bnxt_qplib_swqe *wqe) | |
| 1692 | +{ | |
| 1693 | + struct bnxt_re_ah *ah = NULL; | |
| 1694 | + | |
| 1695 | + if (is_ud_qp(qp)) { | |
| 1696 | + ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); | |
| 1697 | + wqe->send.q_key = ud_wr(wr)->remote_qkey; | |
| 1698 | + wqe->send.dst_qp = ud_wr(wr)->remote_qpn; | |
| 1699 | + wqe->send.avid = ah->qplib_ah.id; | |
| 1700 | + } | |
| 1701 | + switch (wr->opcode) { | |
| 1702 | + case IB_WR_SEND: | |
| 1703 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; | |
| 1704 | + break; | |
| 1705 | + case IB_WR_SEND_WITH_IMM: | |
| 1706 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; | |
| 1707 | + wqe->send.imm_data = wr->ex.imm_data; | |
| 1708 | + break; | |
| 1709 | + case IB_WR_SEND_WITH_INV: | |
| 1710 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; | |
| 1711 | + wqe->send.inv_key = wr->ex.invalidate_rkey; | |
| 1712 | + break; | |
| 1713 | + default: | |
| 1714 | + return -EINVAL; | |
| 1715 | + } | |
| 1716 | + if (wr->send_flags & IB_SEND_SIGNALED) | |
| 1717 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | |
| 1718 | + if (wr->send_flags & IB_SEND_FENCE) | |
| 1719 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | |
| 1720 | + if (wr->send_flags & IB_SEND_SOLICITED) | |
| 1721 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; | |
| 1722 | + if (wr->send_flags & IB_SEND_INLINE) | |
| 1723 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; | |
| 1724 | + | |
| 1725 | + return 0; | |
| 1726 | +} | |
| 1727 | + | |
| 1728 | +static int bnxt_re_build_rdma_wqe(struct ib_send_wr *wr, | |
| 1729 | + struct bnxt_qplib_swqe *wqe) | |
| 1730 | +{ | |
| 1731 | + switch (wr->opcode) { | |
| 1732 | + case IB_WR_RDMA_WRITE: | |
| 1733 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; | |
| 1734 | + break; | |
| 1735 | + case IB_WR_RDMA_WRITE_WITH_IMM: | |
| 1736 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; | |
| 1737 | + wqe->rdma.imm_data = wr->ex.imm_data; | |
| 1738 | + break; | |
| 1739 | + case IB_WR_RDMA_READ: | |
| 1740 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; | |
| 1741 | + wqe->rdma.inv_key = wr->ex.invalidate_rkey; | |
| 1742 | + break; | |
| 1743 | + default: | |
| 1744 | + return -EINVAL; | |
| 1745 | + } | |
| 1746 | + wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; | |
| 1747 | + wqe->rdma.r_key = rdma_wr(wr)->rkey; | |
| 1748 | + if (wr->send_flags & IB_SEND_SIGNALED) | |
| 1749 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | |
| 1750 | + if (wr->send_flags & IB_SEND_FENCE) | |
| 1751 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | |
| 1752 | + if (wr->send_flags & IB_SEND_SOLICITED) | |
| 1753 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; | |
| 1754 | + if (wr->send_flags & IB_SEND_INLINE) | |
| 1755 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; | |
| 1756 | + | |
| 1757 | + return 0; | |
| 1758 | +} | |
| 1759 | + | |
| 1760 | +static int bnxt_re_build_atomic_wqe(struct ib_send_wr *wr, | |
| 1761 | + struct bnxt_qplib_swqe *wqe) | |
| 1762 | +{ | |
| 1763 | + switch (wr->opcode) { | |
| 1764 | + case IB_WR_ATOMIC_CMP_AND_SWP: | |
| 1765 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; | |
| 1766 | + wqe->atomic.swap_data = atomic_wr(wr)->swap; | |
| 1767 | + break; | |
| 1768 | + case IB_WR_ATOMIC_FETCH_AND_ADD: | |
| 1769 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; | |
| 1770 | + wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; | |
| 1771 | + break; | |
| 1772 | + default: | |
| 1773 | + return -EINVAL; | |
| 1774 | + } | |
| 1775 | + wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; | |
| 1776 | + wqe->atomic.r_key = atomic_wr(wr)->rkey; | |
| 1777 | + if (wr->send_flags & IB_SEND_SIGNALED) | |
| 1778 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | |
| 1779 | + if (wr->send_flags & IB_SEND_FENCE) | |
| 1780 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | |
| 1781 | + if (wr->send_flags & IB_SEND_SOLICITED) | |
| 1782 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; | |
| 1783 | + return 0; | |
| 1784 | +} | |
| 1785 | + | |
| 1786 | +static int bnxt_re_build_inv_wqe(struct ib_send_wr *wr, | |
| 1787 | + struct bnxt_qplib_swqe *wqe) | |
| 1788 | +{ | |
| 1789 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; | |
| 1790 | + wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; | |
| 1791 | + | |
| 1792 | + if (wr->send_flags & IB_SEND_SIGNALED) | |
| 1793 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | |
| 1794 | + if (wr->send_flags & IB_SEND_FENCE) | |
| 1795 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | |
| 1796 | + if (wr->send_flags & IB_SEND_SOLICITED) | |
| 1797 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; | |
| 1798 | + | |
| 1799 | + return 0; | |
| 1800 | +} | |
| 1801 | + | |
| 1802 | +static int bnxt_re_build_reg_wqe(struct ib_reg_wr *wr, | |
| 1803 | + struct bnxt_qplib_swqe *wqe) | |
| 1804 | +{ | |
| 1805 | + struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); | |
| 1806 | + struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; | |
| 1807 | + int access = wr->access; | |
| 1808 | + | |
| 1809 | + wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; | |
| 1810 | + wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; | |
| 1811 | + wqe->frmr.page_list = mr->pages; | |
| 1812 | + wqe->frmr.page_list_len = mr->npages; | |
| 1813 | + wqe->frmr.levels = qplib_frpl->hwq.level + 1; | |
| 1814 | + wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; | |
| 1815 | + | |
| 1816 | + if (wr->wr.send_flags & IB_SEND_FENCE) | |
| 1817 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | |
| 1818 | + if (wr->wr.send_flags & IB_SEND_SIGNALED) | |
| 1819 | + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | |
| 1820 | + | |
| 1821 | + if (access & IB_ACCESS_LOCAL_WRITE) | |
| 1822 | + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; | |
| 1823 | + if (access & IB_ACCESS_REMOTE_READ) | |
| 1824 | + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; | |
| 1825 | + if (access & IB_ACCESS_REMOTE_WRITE) | |
| 1826 | + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; | |
| 1827 | + if (access & IB_ACCESS_REMOTE_ATOMIC) | |
| 1828 | + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; | |
| 1829 | + if (access & IB_ACCESS_MW_BIND) | |
| 1830 | + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; | |
| 1831 | + | |
| 1832 | + wqe->frmr.l_key = wr->key; | |
| 1833 | + wqe->frmr.length = wr->mr->length; | |
| 1834 | + wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1; | |
| 1835 | + wqe->frmr.va = wr->mr->iova; | |
| 1836 | + return 0; | |
| 1837 | +} | |
| 1838 | + | |
| 1839 | +static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, | |
| 1840 | + struct ib_send_wr *wr, | |
| 1841 | + struct bnxt_qplib_swqe *wqe) | |
| 1842 | +{ | |
| 1843 | + /* Copy the inline data to the data field */ | |
| 1844 | + u8 *in_data; | |
| 1845 | + u32 i, sge_len; | |
| 1846 | + void *sge_addr; | |
| 1847 | + | |
| 1848 | + in_data = wqe->inline_data; | |
| 1849 | + for (i = 0; i < wr->num_sge; i++) { | |
| 1850 | + sge_addr = (void *)(unsigned long) | |
| 1851 | + wr->sg_list[i].addr; | |
| 1852 | + sge_len = wr->sg_list[i].length; | |
| 1853 | + | |
| 1854 | + if ((sge_len + wqe->inline_len) > | |
| 1855 | + BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { | |
| 1856 | + dev_err(rdev_to_dev(rdev), | |
| 1857 | + "Inline data size requested > supported value"); | |
| 1858 | + return -EINVAL; | |
| 1859 | + } | |
| 1860 | + sge_len = wr->sg_list[i].length; | |
| 1861 | + | |
| 1862 | + memcpy(in_data, sge_addr, sge_len); | |
| 1863 | + in_data += wr->sg_list[i].length; | |
| 1864 | + wqe->inline_len += wr->sg_list[i].length; | |
| 1865 | + } | |
| 1866 | + return wqe->inline_len; | |
| 1867 | +} | |
| 1868 | + | |
| 1869 | +static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, | |
| 1870 | + struct ib_send_wr *wr, | |
| 1871 | + struct bnxt_qplib_swqe *wqe) | |
| 1872 | +{ | |
| 1873 | + int payload_sz = 0; | |
| 1874 | + | |
| 1875 | + if (wr->send_flags & IB_SEND_INLINE) | |
| 1876 | + payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); | |
| 1877 | + else | |
| 1878 | + payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, | |
| 1879 | + wqe->num_sge); | |
| 1880 | + | |
| 1881 | + return payload_sz; | |
| 1882 | +} | |
| 1883 | + | |
| 1884 | +static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, | |
| 1885 | + struct bnxt_re_qp *qp, | |
| 1886 | + struct ib_send_wr *wr) | |
| 1887 | +{ | |
| 1888 | + struct bnxt_qplib_swqe wqe; | |
| 1889 | + int rc = 0, payload_sz = 0; | |
| 1890 | + unsigned long flags; | |
| 1891 | + | |
| 1892 | + spin_lock_irqsave(&qp->sq_lock, flags); | |
| 1893 | + memset(&wqe, 0, sizeof(wqe)); | |
| 1894 | + while (wr) { | |
| 1895 | + /* House keeping */ | |
| 1896 | + memset(&wqe, 0, sizeof(wqe)); | |
| 1897 | + | |
| 1898 | + /* Common */ | |
| 1899 | + wqe.num_sge = wr->num_sge; | |
| 1900 | + if (wr->num_sge > qp->qplib_qp.sq.max_sge) { | |
| 1901 | + dev_err(rdev_to_dev(rdev), | |
| 1902 | + "Limit exceeded for Send SGEs"); | |
| 1903 | + rc = -EINVAL; | |
| 1904 | + goto bad; | |
| 1905 | + } | |
| 1906 | + | |
| 1907 | + payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); | |
| 1908 | + if (payload_sz < 0) { | |
| 1909 | + rc = -EINVAL; | |
| 1910 | + goto bad; | |
| 1911 | + } | |
| 1912 | + wqe.wr_id = wr->wr_id; | |
| 1913 | + | |
| 1914 | + wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; | |
| 1915 | + | |
| 1916 | + rc = bnxt_re_build_send_wqe(qp, wr, &wqe); | |
| 1917 | + if (!rc) | |
| 1918 | + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); | |
| 1919 | +bad: | |
| 1920 | + if (rc) { | |
| 1921 | + dev_err(rdev_to_dev(rdev), | |
| 1922 | + "Post send failed opcode = %#x rc = %d", | |
| 1923 | + wr->opcode, rc); | |
| 1924 | + break; | |
| 1925 | + } | |
| 1926 | + wr = wr->next; | |
| 1927 | + } | |
| 1928 | + bnxt_qplib_post_send_db(&qp->qplib_qp); | |
| 1929 | + spin_unlock_irqrestore(&qp->sq_lock, flags); | |
| 1930 | + return rc; | |
| 1931 | +} | |
| 1932 | + | |
| 1933 | +int bnxt_re_post_send(struct ib_qp *ib_qp, struct ib_send_wr *wr, | |
| 1934 | + struct ib_send_wr **bad_wr) | |
| 1935 | +{ | |
| 1936 | + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | |
| 1937 | + struct bnxt_qplib_swqe wqe; | |
| 1938 | + int rc = 0, payload_sz = 0; | |
| 1939 | + unsigned long flags; | |
| 1940 | + | |
| 1941 | + spin_lock_irqsave(&qp->sq_lock, flags); | |
| 1942 | + while (wr) { | |
| 1943 | + /* House keeping */ | |
| 1944 | + memset(&wqe, 0, sizeof(wqe)); | |
| 1945 | + | |
| 1946 | + /* Common */ | |
| 1947 | + wqe.num_sge = wr->num_sge; | |
| 1948 | + if (wr->num_sge > qp->qplib_qp.sq.max_sge) { | |
| 1949 | + dev_err(rdev_to_dev(qp->rdev), | |
| 1950 | + "Limit exceeded for Send SGEs"); | |
| 1951 | + rc = -EINVAL; | |
| 1952 | + goto bad; | |
| 1953 | + } | |
| 1954 | + | |
| 1955 | + payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); | |
| 1956 | + if (payload_sz < 0) { | |
| 1957 | + rc = -EINVAL; | |
| 1958 | + goto bad; | |
| 1959 | + } | |
| 1960 | + wqe.wr_id = wr->wr_id; | |
| 1961 | + | |
| 1962 | + switch (wr->opcode) { | |
| 1963 | + case IB_WR_SEND: | |
| 1964 | + case IB_WR_SEND_WITH_IMM: | |
| 1965 | + if (ib_qp->qp_type == IB_QPT_GSI) { | |
| 1966 | + rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, | |
| 1967 | + payload_sz); | |
| 1968 | + if (rc) | |
| 1969 | + goto bad; | |
| 1970 | + wqe.rawqp1.lflags |= | |
| 1971 | + SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; | |
| 1972 | + } | |
| 1973 | + switch (wr->send_flags) { | |
| 1974 | + case IB_SEND_IP_CSUM: | |
| 1975 | + wqe.rawqp1.lflags |= | |
| 1976 | + SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; | |
| 1977 | + break; | |
| 1978 | + default: | |
| 1979 | + break; | |
| 1980 | + } | |
| 1981 | + /* Fall thru to build the wqe */ | |
| 1982 | + case IB_WR_SEND_WITH_INV: | |
| 1983 | + rc = bnxt_re_build_send_wqe(qp, wr, &wqe); | |
| 1984 | + break; | |
| 1985 | + case IB_WR_RDMA_WRITE: | |
| 1986 | + case IB_WR_RDMA_WRITE_WITH_IMM: | |
| 1987 | + case IB_WR_RDMA_READ: | |
| 1988 | + rc = bnxt_re_build_rdma_wqe(wr, &wqe); | |
| 1989 | + break; | |
| 1990 | + case IB_WR_ATOMIC_CMP_AND_SWP: | |
| 1991 | + case IB_WR_ATOMIC_FETCH_AND_ADD: | |
| 1992 | + rc = bnxt_re_build_atomic_wqe(wr, &wqe); | |
| 1993 | + break; | |
| 1994 | + case IB_WR_RDMA_READ_WITH_INV: | |
| 1995 | + dev_err(rdev_to_dev(qp->rdev), | |
| 1996 | + "RDMA Read with Invalidate is not supported"); | |
| 1997 | + rc = -EINVAL; | |
| 1998 | + goto bad; | |
| 1999 | + case IB_WR_LOCAL_INV: | |
| 2000 | + rc = bnxt_re_build_inv_wqe(wr, &wqe); | |
| 2001 | + break; | |
| 2002 | + case IB_WR_REG_MR: | |
| 2003 | + rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); | |
| 2004 | + break; | |
| 2005 | + default: | |
| 2006 | + /* Unsupported WRs */ | |
| 2007 | + dev_err(rdev_to_dev(qp->rdev), | |
| 2008 | + "WR (%#x) is not supported", wr->opcode); | |
| 2009 | + rc = -EINVAL; | |
| 2010 | + goto bad; | |
| 2011 | + } | |
| 2012 | + if (!rc) | |
| 2013 | + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); | |
| 2014 | +bad: | |
| 2015 | + if (rc) { | |
| 2016 | + dev_err(rdev_to_dev(qp->rdev), | |
| 2017 | + "post_send failed op:%#x qps = %#x rc = %d\n", | |
| 2018 | + wr->opcode, qp->qplib_qp.state, rc); | |
| 2019 | + *bad_wr = wr; | |
| 2020 | + break; | |
| 2021 | + } | |
| 2022 | + wr = wr->next; | |
| 2023 | + } | |
| 2024 | + bnxt_qplib_post_send_db(&qp->qplib_qp); | |
| 2025 | + spin_unlock_irqrestore(&qp->sq_lock, flags); | |
| 2026 | + | |
| 2027 | + return rc; | |
| 2028 | +} | |
| 2029 | + | |
| 2030 | +static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, | |
| 2031 | + struct bnxt_re_qp *qp, | |
| 2032 | + struct ib_recv_wr *wr) | |
| 2033 | +{ | |
| 2034 | + struct bnxt_qplib_swqe wqe; | |
| 2035 | + int rc = 0, payload_sz = 0; | |
| 2036 | + | |
| 2037 | + memset(&wqe, 0, sizeof(wqe)); | |
| 2038 | + while (wr) { | |
| 2039 | + /* House keeping */ | |
| 2040 | + memset(&wqe, 0, sizeof(wqe)); | |
| 2041 | + | |
| 2042 | + /* Common */ | |
| 2043 | + wqe.num_sge = wr->num_sge; | |
| 2044 | + if (wr->num_sge > qp->qplib_qp.rq.max_sge) { | |
| 2045 | + dev_err(rdev_to_dev(rdev), | |
| 2046 | + "Limit exceeded for Receive SGEs"); | |
| 2047 | + rc = -EINVAL; | |
| 2048 | + break; | |
| 2049 | + } | |
| 2050 | + payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, | |
| 2051 | + wr->num_sge); | |
| 2052 | + wqe.wr_id = wr->wr_id; | |
| 2053 | + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; | |
| 2054 | + | |
| 2055 | + rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); | |
| 2056 | + if (rc) | |
| 2057 | + break; | |
| 2058 | + | |
| 2059 | + wr = wr->next; | |
| 2060 | + } | |
| 2061 | + if (!rc) | |
| 2062 | + bnxt_qplib_post_recv_db(&qp->qplib_qp); | |
| 2063 | + return rc; | |
| 2064 | +} | |
| 2065 | + | |
| 2066 | +int bnxt_re_post_recv(struct ib_qp *ib_qp, struct ib_recv_wr *wr, | |
| 2067 | + struct ib_recv_wr **bad_wr) | |
| 2068 | +{ | |
| 2069 | + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | |
| 2070 | + struct bnxt_qplib_swqe wqe; | |
| 2071 | + int rc = 0, payload_sz = 0; | |
| 2072 | + | |
| 2073 | + while (wr) { | |
| 2074 | + /* House keeping */ | |
| 2075 | + memset(&wqe, 0, sizeof(wqe)); | |
| 2076 | + | |
| 2077 | + /* Common */ | |
| 2078 | + wqe.num_sge = wr->num_sge; | |
| 2079 | + if (wr->num_sge > qp->qplib_qp.rq.max_sge) { | |
| 2080 | + dev_err(rdev_to_dev(qp->rdev), | |
| 2081 | + "Limit exceeded for Receive SGEs"); | |
| 2082 | + rc = -EINVAL; | |
| 2083 | + *bad_wr = wr; | |
| 2084 | + break; | |
| 2085 | + } | |
| 2086 | + | |
| 2087 | + payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, | |
| 2088 | + wr->num_sge); | |
| 2089 | + wqe.wr_id = wr->wr_id; | |
| 2090 | + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; | |
| 2091 | + | |
| 2092 | + if (ib_qp->qp_type == IB_QPT_GSI) | |
| 2093 | + rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, | |
| 2094 | + payload_sz); | |
| 2095 | + if (!rc) | |
| 2096 | + rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); | |
| 2097 | + if (rc) { | |
| 2098 | + *bad_wr = wr; | |
| 2099 | + break; | |
| 2100 | + } | |
| 2101 | + wr = wr->next; | |
| 2102 | + } | |
| 2103 | + bnxt_qplib_post_recv_db(&qp->qplib_qp); | |
| 2104 | + return rc; | |
| 2105 | +} | |
| 2106 | + | |
| 2107 | +/* Completion Queues */ | |
| 2108 | +int bnxt_re_destroy_cq(struct ib_cq *ib_cq) | |
| 2109 | +{ | |
| 2110 | + struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); | |
| 2111 | + struct bnxt_re_dev *rdev = cq->rdev; | |
| 2112 | + int rc; | |
| 2113 | + | |
| 2114 | + rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); | |
| 2115 | + if (rc) { | |
| 2116 | + dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ"); | |
| 2117 | + return rc; | |
| 2118 | + } | |
| 2119 | + if (cq->umem && !IS_ERR(cq->umem)) | |
| 2120 | + ib_umem_release(cq->umem); | |
| 2121 | + | |
| 2122 | + if (cq) { | |
| 2123 | + kfree(cq->cql); | |
| 2124 | + kfree(cq); | |
| 2125 | + } | |
| 2126 | + atomic_dec(&rdev->cq_count); | |
| 2127 | + rdev->nq.budget--; | |
| 2128 | + return 0; | |
| 2129 | +} | |
| 2130 | + | |
| 2131 | +struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, | |
| 2132 | + const struct ib_cq_init_attr *attr, | |
| 2133 | + struct ib_ucontext *context, | |
| 2134 | + struct ib_udata *udata) | |
| 2135 | +{ | |
| 2136 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 2137 | + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | |
| 2138 | + struct bnxt_re_cq *cq = NULL; | |
| 2139 | + int rc, entries; | |
| 2140 | + int cqe = attr->cqe; | |
| 2141 | + | |
| 2142 | + /* Validate CQ fields */ | |
| 2143 | + if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { | |
| 2144 | + dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded"); | |
| 2145 | + return ERR_PTR(-EINVAL); | |
| 2146 | + } | |
| 2147 | + cq = kzalloc(sizeof(*cq), GFP_KERNEL); | |
| 2148 | + if (!cq) | |
| 2149 | + return ERR_PTR(-ENOMEM); | |
| 2150 | + | |
| 2151 | + cq->rdev = rdev; | |
| 2152 | + cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); | |
| 2153 | + | |
| 2154 | + entries = roundup_pow_of_two(cqe + 1); | |
| 2155 | + if (entries > dev_attr->max_cq_wqes + 1) | |
| 2156 | + entries = dev_attr->max_cq_wqes + 1; | |
| 2157 | + | |
| 2158 | + if (context) { | |
| 2159 | + struct bnxt_re_cq_req req; | |
| 2160 | + struct bnxt_re_ucontext *uctx = container_of | |
| 2161 | + (context, | |
| 2162 | + struct bnxt_re_ucontext, | |
| 2163 | + ib_uctx); | |
| 2164 | + if (ib_copy_from_udata(&req, udata, sizeof(req))) { | |
| 2165 | + rc = -EFAULT; | |
| 2166 | + goto fail; | |
| 2167 | + } | |
| 2168 | + | |
| 2169 | + cq->umem = ib_umem_get(context, req.cq_va, | |
| 2170 | + entries * sizeof(struct cq_base), | |
| 2171 | + IB_ACCESS_LOCAL_WRITE, 1); | |
| 2172 | + if (IS_ERR(cq->umem)) { | |
| 2173 | + rc = PTR_ERR(cq->umem); | |
| 2174 | + goto fail; | |
| 2175 | + } | |
| 2176 | + cq->qplib_cq.sghead = cq->umem->sg_head.sgl; | |
| 2177 | + cq->qplib_cq.nmap = cq->umem->nmap; | |
| 2178 | + cq->qplib_cq.dpi = uctx->dpi; | |
| 2179 | + } else { | |
| 2180 | + cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); | |
| 2181 | + cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), | |
| 2182 | + GFP_KERNEL); | |
| 2183 | + if (!cq->cql) { | |
| 2184 | + rc = -ENOMEM; | |
| 2185 | + goto fail; | |
| 2186 | + } | |
| 2187 | + | |
| 2188 | + cq->qplib_cq.dpi = &rdev->dpi_privileged; | |
| 2189 | + cq->qplib_cq.sghead = NULL; | |
| 2190 | + cq->qplib_cq.nmap = 0; | |
| 2191 | + } | |
| 2192 | + cq->qplib_cq.max_wqe = entries; | |
| 2193 | + cq->qplib_cq.cnq_hw_ring_id = rdev->nq.ring_id; | |
| 2194 | + | |
| 2195 | + rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); | |
| 2196 | + if (rc) { | |
| 2197 | + dev_err(rdev_to_dev(rdev), "Failed to create HW CQ"); | |
| 2198 | + goto fail; | |
| 2199 | + } | |
| 2200 | + | |
| 2201 | + cq->ib_cq.cqe = entries; | |
| 2202 | + cq->cq_period = cq->qplib_cq.period; | |
| 2203 | + rdev->nq.budget++; | |
| 2204 | + | |
| 2205 | + atomic_inc(&rdev->cq_count); | |
| 2206 | + | |
| 2207 | + if (context) { | |
| 2208 | + struct bnxt_re_cq_resp resp; | |
| 2209 | + | |
| 2210 | + resp.cqid = cq->qplib_cq.id; | |
| 2211 | + resp.tail = cq->qplib_cq.hwq.cons; | |
| 2212 | + resp.phase = cq->qplib_cq.period; | |
| 2213 | + resp.rsvd = 0; | |
| 2214 | + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); | |
| 2215 | + if (rc) { | |
| 2216 | + dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata"); | |
| 2217 | + bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); | |
| 2218 | + goto c2fail; | |
| 2219 | + } | |
| 2220 | + } | |
| 2221 | + | |
| 2222 | + return &cq->ib_cq; | |
| 2223 | + | |
| 2224 | +c2fail: | |
| 2225 | + if (context) | |
| 2226 | + ib_umem_release(cq->umem); | |
| 2227 | +fail: | |
| 2228 | + kfree(cq->cql); | |
| 2229 | + kfree(cq); | |
| 2230 | + return ERR_PTR(rc); | |
| 2231 | +} | |
| 2232 | + | |
| 2233 | +static u8 __req_to_ib_wc_status(u8 qstatus) | |
| 2234 | +{ | |
| 2235 | + switch (qstatus) { | |
| 2236 | + case CQ_REQ_STATUS_OK: | |
| 2237 | + return IB_WC_SUCCESS; | |
| 2238 | + case CQ_REQ_STATUS_BAD_RESPONSE_ERR: | |
| 2239 | + return IB_WC_BAD_RESP_ERR; | |
| 2240 | + case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: | |
| 2241 | + return IB_WC_LOC_LEN_ERR; | |
| 2242 | + case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: | |
| 2243 | + return IB_WC_LOC_QP_OP_ERR; | |
| 2244 | + case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: | |
| 2245 | + return IB_WC_LOC_PROT_ERR; | |
| 2246 | + case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: | |
| 2247 | + return IB_WC_GENERAL_ERR; | |
| 2248 | + case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: | |
| 2249 | + return IB_WC_REM_INV_REQ_ERR; | |
| 2250 | + case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: | |
| 2251 | + return IB_WC_REM_ACCESS_ERR; | |
| 2252 | + case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: | |
| 2253 | + return IB_WC_REM_OP_ERR; | |
| 2254 | + case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: | |
| 2255 | + return IB_WC_RNR_RETRY_EXC_ERR; | |
| 2256 | + case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: | |
| 2257 | + return IB_WC_RETRY_EXC_ERR; | |
| 2258 | + case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: | |
| 2259 | + return IB_WC_WR_FLUSH_ERR; | |
| 2260 | + default: | |
| 2261 | + return IB_WC_GENERAL_ERR; | |
| 2262 | + } | |
| 2263 | + return 0; | |
| 2264 | +} | |
| 2265 | + | |
| 2266 | +static u8 __rawqp1_to_ib_wc_status(u8 qstatus) | |
| 2267 | +{ | |
| 2268 | + switch (qstatus) { | |
| 2269 | + case CQ_RES_RAWETH_QP1_STATUS_OK: | |
| 2270 | + return IB_WC_SUCCESS; | |
| 2271 | + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: | |
| 2272 | + return IB_WC_LOC_ACCESS_ERR; | |
| 2273 | + case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: | |
| 2274 | + return IB_WC_LOC_LEN_ERR; | |
| 2275 | + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: | |
| 2276 | + return IB_WC_LOC_PROT_ERR; | |
| 2277 | + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: | |
| 2278 | + return IB_WC_LOC_QP_OP_ERR; | |
| 2279 | + case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: | |
| 2280 | + return IB_WC_GENERAL_ERR; | |
| 2281 | + case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: | |
| 2282 | + return IB_WC_WR_FLUSH_ERR; | |
| 2283 | + case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: | |
| 2284 | + return IB_WC_WR_FLUSH_ERR; | |
| 2285 | + default: | |
| 2286 | + return IB_WC_GENERAL_ERR; | |
| 2287 | + } | |
| 2288 | +} | |
| 2289 | + | |
| 2290 | +static u8 __rc_to_ib_wc_status(u8 qstatus) | |
| 2291 | +{ | |
| 2292 | + switch (qstatus) { | |
| 2293 | + case CQ_RES_RC_STATUS_OK: | |
| 2294 | + return IB_WC_SUCCESS; | |
| 2295 | + case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: | |
| 2296 | + return IB_WC_LOC_ACCESS_ERR; | |
| 2297 | + case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: | |
| 2298 | + return IB_WC_LOC_LEN_ERR; | |
| 2299 | + case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: | |
| 2300 | + return IB_WC_LOC_PROT_ERR; | |
| 2301 | + case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: | |
| 2302 | + return IB_WC_LOC_QP_OP_ERR; | |
| 2303 | + case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: | |
| 2304 | + return IB_WC_GENERAL_ERR; | |
| 2305 | + case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: | |
| 2306 | + return IB_WC_REM_INV_REQ_ERR; | |
| 2307 | + case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: | |
| 2308 | + return IB_WC_WR_FLUSH_ERR; | |
| 2309 | + case CQ_RES_RC_STATUS_HW_FLUSH_ERR: | |
| 2310 | + return IB_WC_WR_FLUSH_ERR; | |
| 2311 | + default: | |
| 2312 | + return IB_WC_GENERAL_ERR; | |
| 2313 | + } | |
| 2314 | +} | |
| 2315 | + | |
| 2316 | +static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) | |
| 2317 | +{ | |
| 2318 | + switch (cqe->type) { | |
| 2319 | + case BNXT_QPLIB_SWQE_TYPE_SEND: | |
| 2320 | + wc->opcode = IB_WC_SEND; | |
| 2321 | + break; | |
| 2322 | + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: | |
| 2323 | + wc->opcode = IB_WC_SEND; | |
| 2324 | + wc->wc_flags |= IB_WC_WITH_IMM; | |
| 2325 | + break; | |
| 2326 | + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: | |
| 2327 | + wc->opcode = IB_WC_SEND; | |
| 2328 | + wc->wc_flags |= IB_WC_WITH_INVALIDATE; | |
| 2329 | + break; | |
| 2330 | + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: | |
| 2331 | + wc->opcode = IB_WC_RDMA_WRITE; | |
| 2332 | + break; | |
| 2333 | + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: | |
| 2334 | + wc->opcode = IB_WC_RDMA_WRITE; | |
| 2335 | + wc->wc_flags |= IB_WC_WITH_IMM; | |
| 2336 | + break; | |
| 2337 | + case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: | |
| 2338 | + wc->opcode = IB_WC_RDMA_READ; | |
| 2339 | + break; | |
| 2340 | + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: | |
| 2341 | + wc->opcode = IB_WC_COMP_SWAP; | |
| 2342 | + break; | |
| 2343 | + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: | |
| 2344 | + wc->opcode = IB_WC_FETCH_ADD; | |
| 2345 | + break; | |
| 2346 | + case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: | |
| 2347 | + wc->opcode = IB_WC_LOCAL_INV; | |
| 2348 | + break; | |
| 2349 | + case BNXT_QPLIB_SWQE_TYPE_REG_MR: | |
| 2350 | + wc->opcode = IB_WC_REG_MR; | |
| 2351 | + break; | |
| 2352 | + default: | |
| 2353 | + wc->opcode = IB_WC_SEND; | |
| 2354 | + break; | |
| 2355 | + } | |
| 2356 | + | |
| 2357 | + wc->status = __req_to_ib_wc_status(cqe->status); | |
| 2358 | +} | |
| 2359 | + | |
| 2360 | +static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, | |
| 2361 | + u16 raweth_qp1_flags2) | |
| 2362 | +{ | |
| 2363 | + bool is_udp = false, is_ipv6 = false, is_ipv4 = false; | |
| 2364 | + | |
| 2365 | + /* raweth_qp1_flags Bit 9-6 indicates itype */ | |
| 2366 | + if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) | |
| 2367 | + != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) | |
| 2368 | + return -1; | |
| 2369 | + | |
| 2370 | + if (raweth_qp1_flags2 & | |
| 2371 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && | |
| 2372 | + raweth_qp1_flags2 & | |
| 2373 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { | |
| 2374 | + is_udp = true; | |
| 2375 | + /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ | |
| 2376 | + (raweth_qp1_flags2 & | |
| 2377 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? | |
| 2378 | + (is_ipv6 = true) : (is_ipv4 = true); | |
| 2379 | + return ((is_ipv6) ? | |
| 2380 | + BNXT_RE_ROCEV2_IPV6_PACKET : | |
| 2381 | + BNXT_RE_ROCEV2_IPV4_PACKET); | |
| 2382 | + } else { | |
| 2383 | + return BNXT_RE_ROCE_V1_PACKET; | |
| 2384 | + } | |
| 2385 | +} | |
| 2386 | + | |
| 2387 | +static int bnxt_re_to_ib_nw_type(int nw_type) | |
| 2388 | +{ | |
| 2389 | + u8 nw_hdr_type = 0xFF; | |
| 2390 | + | |
| 2391 | + switch (nw_type) { | |
| 2392 | + case BNXT_RE_ROCE_V1_PACKET: | |
| 2393 | + nw_hdr_type = RDMA_NETWORK_ROCE_V1; | |
| 2394 | + break; | |
| 2395 | + case BNXT_RE_ROCEV2_IPV4_PACKET: | |
| 2396 | + nw_hdr_type = RDMA_NETWORK_IPV4; | |
| 2397 | + break; | |
| 2398 | + case BNXT_RE_ROCEV2_IPV6_PACKET: | |
| 2399 | + nw_hdr_type = RDMA_NETWORK_IPV6; | |
| 2400 | + break; | |
| 2401 | + } | |
| 2402 | + return nw_hdr_type; | |
| 2403 | +} | |
| 2404 | + | |
| 2405 | +static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, | |
| 2406 | + void *rq_hdr_buf) | |
| 2407 | +{ | |
| 2408 | + u8 *tmp_buf = NULL; | |
| 2409 | + struct ethhdr *eth_hdr; | |
| 2410 | + u16 eth_type; | |
| 2411 | + bool rc = false; | |
| 2412 | + | |
| 2413 | + tmp_buf = (u8 *)rq_hdr_buf; | |
| 2414 | + /* | |
| 2415 | + * If dest mac is not same as I/F mac, this could be a | |
| 2416 | + * loopback address or multicast address, check whether | |
| 2417 | + * it is a loopback packet | |
| 2418 | + */ | |
| 2419 | + if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { | |
| 2420 | + tmp_buf += 4; | |
| 2421 | + /* Check the ether type */ | |
| 2422 | + eth_hdr = (struct ethhdr *)tmp_buf; | |
| 2423 | + eth_type = ntohs(eth_hdr->h_proto); | |
| 2424 | + switch (eth_type) { | |
| 2425 | + case ETH_P_IBOE: | |
| 2426 | + rc = true; | |
| 2427 | + break; | |
| 2428 | + case ETH_P_IP: | |
| 2429 | + case ETH_P_IPV6: { | |
| 2430 | + u32 len; | |
| 2431 | + struct udphdr *udp_hdr; | |
| 2432 | + | |
| 2433 | + len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : | |
| 2434 | + sizeof(struct ipv6hdr)); | |
| 2435 | + tmp_buf += sizeof(struct ethhdr) + len; | |
| 2436 | + udp_hdr = (struct udphdr *)tmp_buf; | |
| 2437 | + if (ntohs(udp_hdr->dest) == | |
| 2438 | + ROCE_V2_UDP_DPORT) | |
| 2439 | + rc = true; | |
| 2440 | + break; | |
| 2441 | + } | |
| 2442 | + default: | |
| 2443 | + break; | |
| 2444 | + } | |
| 2445 | + } | |
| 2446 | + | |
| 2447 | + return rc; | |
| 2448 | +} | |
| 2449 | + | |
| 2450 | +static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp, | |
| 2451 | + struct bnxt_qplib_cqe *cqe) | |
| 2452 | +{ | |
| 2453 | + struct bnxt_re_dev *rdev = qp1_qp->rdev; | |
| 2454 | + struct bnxt_re_sqp_entries *sqp_entry = NULL; | |
| 2455 | + struct bnxt_re_qp *qp = rdev->qp1_sqp; | |
| 2456 | + struct ib_send_wr *swr; | |
| 2457 | + struct ib_ud_wr udwr; | |
| 2458 | + struct ib_recv_wr rwr; | |
| 2459 | + int pkt_type = 0; | |
| 2460 | + u32 tbl_idx; | |
| 2461 | + void *rq_hdr_buf; | |
| 2462 | + dma_addr_t rq_hdr_buf_map; | |
| 2463 | + dma_addr_t shrq_hdr_buf_map; | |
| 2464 | + u32 offset = 0; | |
| 2465 | + u32 skip_bytes = 0; | |
| 2466 | + struct ib_sge s_sge[2]; | |
| 2467 | + struct ib_sge r_sge[2]; | |
| 2468 | + int rc; | |
| 2469 | + | |
| 2470 | + memset(&udwr, 0, sizeof(udwr)); | |
| 2471 | + memset(&rwr, 0, sizeof(rwr)); | |
| 2472 | + memset(&s_sge, 0, sizeof(s_sge)); | |
| 2473 | + memset(&r_sge, 0, sizeof(r_sge)); | |
| 2474 | + | |
| 2475 | + swr = &udwr.wr; | |
| 2476 | + tbl_idx = cqe->wr_id; | |
| 2477 | + | |
| 2478 | + rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf + | |
| 2479 | + (tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size); | |
| 2480 | + rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp, | |
| 2481 | + tbl_idx); | |
| 2482 | + | |
| 2483 | + /* Shadow QP header buffer */ | |
| 2484 | + shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp, | |
| 2485 | + tbl_idx); | |
| 2486 | + sqp_entry = &rdev->sqp_tbl[tbl_idx]; | |
| 2487 | + | |
| 2488 | + /* Store this cqe */ | |
| 2489 | + memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); | |
| 2490 | + sqp_entry->qp1_qp = qp1_qp; | |
| 2491 | + | |
| 2492 | + /* Find packet type from the cqe */ | |
| 2493 | + | |
| 2494 | + pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, | |
| 2495 | + cqe->raweth_qp1_flags2); | |
| 2496 | + if (pkt_type < 0) { | |
| 2497 | + dev_err(rdev_to_dev(rdev), "Invalid packet\n"); | |
| 2498 | + return -EINVAL; | |
| 2499 | + } | |
| 2500 | + | |
| 2501 | + /* Adjust the offset for the user buffer and post in the rq */ | |
| 2502 | + | |
| 2503 | + if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) | |
| 2504 | + offset = 20; | |
| 2505 | + | |
| 2506 | + /* | |
| 2507 | + * QP1 loopback packet has 4 bytes of internal header before | |
| 2508 | + * ether header. Skip these four bytes. | |
| 2509 | + */ | |
| 2510 | + if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) | |
| 2511 | + skip_bytes = 4; | |
| 2512 | + | |
| 2513 | + /* First send SGE . Skip the ether header*/ | |
| 2514 | + s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE | |
| 2515 | + + skip_bytes; | |
| 2516 | + s_sge[0].lkey = 0xFFFFFFFF; | |
| 2517 | + s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : | |
| 2518 | + BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; | |
| 2519 | + | |
| 2520 | + /* Second Send SGE */ | |
| 2521 | + s_sge[1].addr = s_sge[0].addr + s_sge[0].length + | |
| 2522 | + BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; | |
| 2523 | + if (pkt_type != BNXT_RE_ROCE_V1_PACKET) | |
| 2524 | + s_sge[1].addr += 8; | |
| 2525 | + s_sge[1].lkey = 0xFFFFFFFF; | |
| 2526 | + s_sge[1].length = 256; | |
| 2527 | + | |
| 2528 | + /* First recv SGE */ | |
| 2529 | + | |
| 2530 | + r_sge[0].addr = shrq_hdr_buf_map; | |
| 2531 | + r_sge[0].lkey = 0xFFFFFFFF; | |
| 2532 | + r_sge[0].length = 40; | |
| 2533 | + | |
| 2534 | + r_sge[1].addr = sqp_entry->sge.addr + offset; | |
| 2535 | + r_sge[1].lkey = sqp_entry->sge.lkey; | |
| 2536 | + r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; | |
| 2537 | + | |
| 2538 | + /* Create receive work request */ | |
| 2539 | + rwr.num_sge = 2; | |
| 2540 | + rwr.sg_list = r_sge; | |
| 2541 | + rwr.wr_id = tbl_idx; | |
| 2542 | + rwr.next = NULL; | |
| 2543 | + | |
| 2544 | + rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr); | |
| 2545 | + if (rc) { | |
| 2546 | + dev_err(rdev_to_dev(rdev), | |
| 2547 | + "Failed to post Rx buffers to shadow QP"); | |
| 2548 | + return -ENOMEM; | |
| 2549 | + } | |
| 2550 | + | |
| 2551 | + swr->num_sge = 2; | |
| 2552 | + swr->sg_list = s_sge; | |
| 2553 | + swr->wr_id = tbl_idx; | |
| 2554 | + swr->opcode = IB_WR_SEND; | |
| 2555 | + swr->next = NULL; | |
| 2556 | + | |
| 2557 | + udwr.ah = &rdev->sqp_ah->ib_ah; | |
| 2558 | + udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id; | |
| 2559 | + udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey; | |
| 2560 | + | |
| 2561 | + /* post data received in the send queue */ | |
| 2562 | + rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr); | |
| 2563 | + | |
| 2564 | + return 0; | |
| 2565 | +} | |
| 2566 | + | |
| 2567 | +static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, | |
| 2568 | + struct bnxt_qplib_cqe *cqe) | |
| 2569 | +{ | |
| 2570 | + wc->opcode = IB_WC_RECV; | |
| 2571 | + wc->status = __rawqp1_to_ib_wc_status(cqe->status); | |
| 2572 | + wc->wc_flags |= IB_WC_GRH; | |
| 2573 | +} | |
| 2574 | + | |
| 2575 | +static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, | |
| 2576 | + struct bnxt_qplib_cqe *cqe) | |
| 2577 | +{ | |
| 2578 | + wc->opcode = IB_WC_RECV; | |
| 2579 | + wc->status = __rc_to_ib_wc_status(cqe->status); | |
| 2580 | + | |
| 2581 | + if (cqe->flags & CQ_RES_RC_FLAGS_IMM) | |
| 2582 | + wc->wc_flags |= IB_WC_WITH_IMM; | |
| 2583 | + if (cqe->flags & CQ_RES_RC_FLAGS_INV) | |
| 2584 | + wc->wc_flags |= IB_WC_WITH_INVALIDATE; | |
| 2585 | + if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == | |
| 2586 | + (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) | |
| 2587 | + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; | |
| 2588 | +} | |
| 2589 | + | |
| 2590 | +static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp, | |
| 2591 | + struct ib_wc *wc, | |
| 2592 | + struct bnxt_qplib_cqe *cqe) | |
| 2593 | +{ | |
| 2594 | + u32 tbl_idx; | |
| 2595 | + struct bnxt_re_dev *rdev = qp->rdev; | |
| 2596 | + struct bnxt_re_qp *qp1_qp = NULL; | |
| 2597 | + struct bnxt_qplib_cqe *orig_cqe = NULL; | |
| 2598 | + struct bnxt_re_sqp_entries *sqp_entry = NULL; | |
| 2599 | + int nw_type; | |
| 2600 | + | |
| 2601 | + tbl_idx = cqe->wr_id; | |
| 2602 | + | |
| 2603 | + sqp_entry = &rdev->sqp_tbl[tbl_idx]; | |
| 2604 | + qp1_qp = sqp_entry->qp1_qp; | |
| 2605 | + orig_cqe = &sqp_entry->cqe; | |
| 2606 | + | |
| 2607 | + wc->wr_id = sqp_entry->wrid; | |
| 2608 | + wc->byte_len = orig_cqe->length; | |
| 2609 | + wc->qp = &qp1_qp->ib_qp; | |
| 2610 | + | |
| 2611 | + wc->ex.imm_data = orig_cqe->immdata; | |
| 2612 | + wc->src_qp = orig_cqe->src_qp; | |
| 2613 | + memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); | |
| 2614 | + wc->port_num = 1; | |
| 2615 | + wc->vendor_err = orig_cqe->status; | |
| 2616 | + | |
| 2617 | + wc->opcode = IB_WC_RECV; | |
| 2618 | + wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); | |
| 2619 | + wc->wc_flags |= IB_WC_GRH; | |
| 2620 | + | |
| 2621 | + nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, | |
| 2622 | + orig_cqe->raweth_qp1_flags2); | |
| 2623 | + if (nw_type >= 0) { | |
| 2624 | + wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); | |
| 2625 | + wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; | |
| 2626 | + } | |
| 2627 | +} | |
| 2628 | + | |
| 2629 | +static void bnxt_re_process_res_ud_wc(struct ib_wc *wc, | |
| 2630 | + struct bnxt_qplib_cqe *cqe) | |
| 2631 | +{ | |
| 2632 | + wc->opcode = IB_WC_RECV; | |
| 2633 | + wc->status = __rc_to_ib_wc_status(cqe->status); | |
| 2634 | + | |
| 2635 | + if (cqe->flags & CQ_RES_RC_FLAGS_IMM) | |
| 2636 | + wc->wc_flags |= IB_WC_WITH_IMM; | |
| 2637 | + if (cqe->flags & CQ_RES_RC_FLAGS_INV) | |
| 2638 | + wc->wc_flags |= IB_WC_WITH_INVALIDATE; | |
| 2639 | + if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == | |
| 2640 | + (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) | |
| 2641 | + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; | |
| 2642 | +} | |
| 2643 | + | |
| 2644 | +int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) | |
| 2645 | +{ | |
| 2646 | + struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); | |
| 2647 | + struct bnxt_re_qp *qp; | |
| 2648 | + struct bnxt_qplib_cqe *cqe; | |
| 2649 | + int i, ncqe, budget; | |
| 2650 | + u32 tbl_idx; | |
| 2651 | + struct bnxt_re_sqp_entries *sqp_entry = NULL; | |
| 2652 | + unsigned long flags; | |
| 2653 | + | |
| 2654 | + spin_lock_irqsave(&cq->cq_lock, flags); | |
| 2655 | + budget = min_t(u32, num_entries, cq->max_cql); | |
| 2656 | + if (!cq->cql) { | |
| 2657 | + dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use"); | |
| 2658 | + goto exit; | |
| 2659 | + } | |
| 2660 | + cqe = &cq->cql[0]; | |
| 2661 | + while (budget) { | |
| 2662 | + ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget); | |
| 2663 | + if (!ncqe) | |
| 2664 | + break; | |
| 2665 | + | |
| 2666 | + for (i = 0; i < ncqe; i++, cqe++) { | |
| 2667 | + /* Transcribe each qplib_wqe back to ib_wc */ | |
| 2668 | + memset(wc, 0, sizeof(*wc)); | |
| 2669 | + | |
| 2670 | + wc->wr_id = cqe->wr_id; | |
| 2671 | + wc->byte_len = cqe->length; | |
| 2672 | + qp = container_of | |
| 2673 | + ((struct bnxt_qplib_qp *) | |
| 2674 | + (unsigned long)(cqe->qp_handle), | |
| 2675 | + struct bnxt_re_qp, qplib_qp); | |
| 2676 | + if (!qp) { | |
| 2677 | + dev_err(rdev_to_dev(cq->rdev), | |
| 2678 | + "POLL CQ : bad QP handle"); | |
| 2679 | + continue; | |
| 2680 | + } | |
| 2681 | + wc->qp = &qp->ib_qp; | |
| 2682 | + wc->ex.imm_data = cqe->immdata; | |
| 2683 | + wc->src_qp = cqe->src_qp; | |
| 2684 | + memcpy(wc->smac, cqe->smac, ETH_ALEN); | |
| 2685 | + wc->port_num = 1; | |
| 2686 | + wc->vendor_err = cqe->status; | |
| 2687 | + | |
| 2688 | + switch (cqe->opcode) { | |
| 2689 | + case CQ_BASE_CQE_TYPE_REQ: | |
| 2690 | + if (qp->qplib_qp.id == | |
| 2691 | + qp->rdev->qp1_sqp->qplib_qp.id) { | |
| 2692 | + /* Handle this completion with | |
| 2693 | + * the stored completion | |
| 2694 | + */ | |
| 2695 | + memset(wc, 0, sizeof(*wc)); | |
| 2696 | + continue; | |
| 2697 | + } | |
| 2698 | + bnxt_re_process_req_wc(wc, cqe); | |
| 2699 | + break; | |
| 2700 | + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: | |
| 2701 | + if (!cqe->status) { | |
| 2702 | + int rc = 0; | |
| 2703 | + | |
| 2704 | + rc = bnxt_re_process_raw_qp_pkt_rx | |
| 2705 | + (qp, cqe); | |
| 2706 | + if (!rc) { | |
| 2707 | + memset(wc, 0, sizeof(*wc)); | |
| 2708 | + continue; | |
| 2709 | + } | |
| 2710 | + cqe->status = -1; | |
| 2711 | + } | |
| 2712 | + /* Errors need not be looped back. | |
| 2713 | + * But change the wr_id to the one | |
| 2714 | + * stored in the table | |
| 2715 | + */ | |
| 2716 | + tbl_idx = cqe->wr_id; | |
| 2717 | + sqp_entry = &cq->rdev->sqp_tbl[tbl_idx]; | |
| 2718 | + wc->wr_id = sqp_entry->wrid; | |
| 2719 | + bnxt_re_process_res_rawqp1_wc(wc, cqe); | |
| 2720 | + break; | |
| 2721 | + case CQ_BASE_CQE_TYPE_RES_RC: | |
| 2722 | + bnxt_re_process_res_rc_wc(wc, cqe); | |
| 2723 | + break; | |
| 2724 | + case CQ_BASE_CQE_TYPE_RES_UD: | |
| 2725 | + if (qp->qplib_qp.id == | |
| 2726 | + qp->rdev->qp1_sqp->qplib_qp.id) { | |
| 2727 | + /* Handle this completion with | |
| 2728 | + * the stored completion | |
| 2729 | + */ | |
| 2730 | + if (cqe->status) { | |
| 2731 | + continue; | |
| 2732 | + } else { | |
| 2733 | + bnxt_re_process_res_shadow_qp_wc | |
| 2734 | + (qp, wc, cqe); | |
| 2735 | + break; | |
| 2736 | + } | |
| 2737 | + } | |
| 2738 | + bnxt_re_process_res_ud_wc(wc, cqe); | |
| 2739 | + break; | |
| 2740 | + default: | |
| 2741 | + dev_err(rdev_to_dev(cq->rdev), | |
| 2742 | + "POLL CQ : type 0x%x not handled", | |
| 2743 | + cqe->opcode); | |
| 2744 | + continue; | |
| 2745 | + } | |
| 2746 | + wc++; | |
| 2747 | + budget--; | |
| 2748 | + } | |
| 2749 | + } | |
| 2750 | +exit: | |
| 2751 | + spin_unlock_irqrestore(&cq->cq_lock, flags); | |
| 2752 | + return num_entries - budget; | |
| 2753 | +} | |
| 2754 | + | |
| 2755 | +int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, | |
| 2756 | + enum ib_cq_notify_flags ib_cqn_flags) | |
| 2757 | +{ | |
| 2758 | + struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); | |
| 2759 | + int type = 0; | |
| 2760 | + | |
| 2761 | + /* Trigger on the very next completion */ | |
| 2762 | + if (ib_cqn_flags & IB_CQ_NEXT_COMP) | |
| 2763 | + type = DBR_DBR_TYPE_CQ_ARMALL; | |
| 2764 | + /* Trigger on the next solicited completion */ | |
| 2765 | + else if (ib_cqn_flags & IB_CQ_SOLICITED) | |
| 2766 | + type = DBR_DBR_TYPE_CQ_ARMSE; | |
| 2767 | + | |
| 2768 | + bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); | |
| 2769 | + | |
| 2770 | + return 0; | |
| 2771 | +} | |
| 2772 | + | |
| 2773 | +/* Memory Regions */ | |
| 2774 | +struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) | |
| 2775 | +{ | |
| 2776 | + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | |
| 2777 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 2778 | + struct bnxt_re_mr *mr; | |
| 2779 | + u64 pbl = 0; | |
| 2780 | + int rc; | |
| 2781 | + | |
| 2782 | + mr = kzalloc(sizeof(*mr), GFP_KERNEL); | |
| 2783 | + if (!mr) | |
| 2784 | + return ERR_PTR(-ENOMEM); | |
| 2785 | + | |
| 2786 | + mr->rdev = rdev; | |
| 2787 | + mr->qplib_mr.pd = &pd->qplib_pd; | |
| 2788 | + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); | |
| 2789 | + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; | |
| 2790 | + | |
| 2791 | + /* Allocate and register 0 as the address */ | |
| 2792 | + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); | |
| 2793 | + if (rc) | |
| 2794 | + goto fail; | |
| 2795 | + | |
| 2796 | + mr->qplib_mr.hwq.level = PBL_LVL_MAX; | |
| 2797 | + mr->qplib_mr.total_size = -1; /* Infinte length */ | |
| 2798 | + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false); | |
| 2799 | + if (rc) | |
| 2800 | + goto fail_mr; | |
| 2801 | + | |
| 2802 | + mr->ib_mr.lkey = mr->qplib_mr.lkey; | |
| 2803 | + if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | | |
| 2804 | + IB_ACCESS_REMOTE_ATOMIC)) | |
| 2805 | + mr->ib_mr.rkey = mr->ib_mr.lkey; | |
| 2806 | + atomic_inc(&rdev->mr_count); | |
| 2807 | + | |
| 2808 | + return &mr->ib_mr; | |
| 2809 | + | |
| 2810 | +fail_mr: | |
| 2811 | + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); | |
| 2812 | +fail: | |
| 2813 | + kfree(mr); | |
| 2814 | + return ERR_PTR(rc); | |
| 2815 | +} | |
| 2816 | + | |
| 2817 | +int bnxt_re_dereg_mr(struct ib_mr *ib_mr) | |
| 2818 | +{ | |
| 2819 | + struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); | |
| 2820 | + struct bnxt_re_dev *rdev = mr->rdev; | |
| 2821 | + int rc = 0; | |
| 2822 | + | |
| 2823 | + if (mr->npages && mr->pages) { | |
| 2824 | + rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, | |
| 2825 | + &mr->qplib_frpl); | |
| 2826 | + kfree(mr->pages); | |
| 2827 | + mr->npages = 0; | |
| 2828 | + mr->pages = NULL; | |
| 2829 | + } | |
| 2830 | + rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); | |
| 2831 | + | |
| 2832 | + if (!IS_ERR(mr->ib_umem) && mr->ib_umem) | |
| 2833 | + ib_umem_release(mr->ib_umem); | |
| 2834 | + | |
| 2835 | + kfree(mr); | |
| 2836 | + atomic_dec(&rdev->mr_count); | |
| 2837 | + return rc; | |
| 2838 | +} | |
| 2839 | + | |
| 2840 | +static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) | |
| 2841 | +{ | |
| 2842 | + struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); | |
| 2843 | + | |
| 2844 | + if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) | |
| 2845 | + return -ENOMEM; | |
| 2846 | + | |
| 2847 | + mr->pages[mr->npages++] = addr; | |
| 2848 | + return 0; | |
| 2849 | +} | |
| 2850 | + | |
| 2851 | +int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, | |
| 2852 | + unsigned int *sg_offset) | |
| 2853 | +{ | |
| 2854 | + struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); | |
| 2855 | + | |
| 2856 | + mr->npages = 0; | |
| 2857 | + return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); | |
| 2858 | +} | |
| 2859 | + | |
| 2860 | +struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, | |
| 2861 | + u32 max_num_sg) | |
| 2862 | +{ | |
| 2863 | + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | |
| 2864 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 2865 | + struct bnxt_re_mr *mr = NULL; | |
| 2866 | + int rc; | |
| 2867 | + | |
| 2868 | + if (type != IB_MR_TYPE_MEM_REG) { | |
| 2869 | + dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type); | |
| 2870 | + return ERR_PTR(-EINVAL); | |
| 2871 | + } | |
| 2872 | + if (max_num_sg > MAX_PBL_LVL_1_PGS) | |
| 2873 | + return ERR_PTR(-EINVAL); | |
| 2874 | + | |
| 2875 | + mr = kzalloc(sizeof(*mr), GFP_KERNEL); | |
| 2876 | + if (!mr) | |
| 2877 | + return ERR_PTR(-ENOMEM); | |
| 2878 | + | |
| 2879 | + mr->rdev = rdev; | |
| 2880 | + mr->qplib_mr.pd = &pd->qplib_pd; | |
| 2881 | + mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; | |
| 2882 | + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; | |
| 2883 | + | |
| 2884 | + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); | |
| 2885 | + if (rc) | |
| 2886 | + goto fail; | |
| 2887 | + | |
| 2888 | + mr->ib_mr.lkey = mr->qplib_mr.lkey; | |
| 2889 | + mr->ib_mr.rkey = mr->ib_mr.lkey; | |
| 2890 | + | |
| 2891 | + mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); | |
| 2892 | + if (!mr->pages) { | |
| 2893 | + rc = -ENOMEM; | |
| 2894 | + goto fail; | |
| 2895 | + } | |
| 2896 | + rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, | |
| 2897 | + &mr->qplib_frpl, max_num_sg); | |
| 2898 | + if (rc) { | |
| 2899 | + dev_err(rdev_to_dev(rdev), | |
| 2900 | + "Failed to allocate HW FR page list"); | |
| 2901 | + goto fail_mr; | |
| 2902 | + } | |
| 2903 | + | |
| 2904 | + atomic_inc(&rdev->mr_count); | |
| 2905 | + return &mr->ib_mr; | |
| 2906 | + | |
| 2907 | +fail_mr: | |
| 2908 | + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); | |
| 2909 | +fail: | |
| 2910 | + kfree(mr->pages); | |
| 2911 | + kfree(mr); | |
| 2912 | + return ERR_PTR(rc); | |
| 2913 | +} | |
| 2914 | + | |
| 2915 | +/* Fast Memory Regions */ | |
| 2916 | +struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *ib_pd, int mr_access_flags, | |
| 2917 | + struct ib_fmr_attr *fmr_attr) | |
| 2918 | +{ | |
| 2919 | + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | |
| 2920 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 2921 | + struct bnxt_re_fmr *fmr; | |
| 2922 | + int rc; | |
| 2923 | + | |
| 2924 | + if (fmr_attr->max_pages > MAX_PBL_LVL_2_PGS || | |
| 2925 | + fmr_attr->max_maps > rdev->dev_attr.max_map_per_fmr) { | |
| 2926 | + dev_err(rdev_to_dev(rdev), "Allocate FMR exceeded Max limit"); | |
| 2927 | + return ERR_PTR(-ENOMEM); | |
| 2928 | + } | |
| 2929 | + fmr = kzalloc(sizeof(*fmr), GFP_KERNEL); | |
| 2930 | + if (!fmr) | |
| 2931 | + return ERR_PTR(-ENOMEM); | |
| 2932 | + | |
| 2933 | + fmr->rdev = rdev; | |
| 2934 | + fmr->qplib_fmr.pd = &pd->qplib_pd; | |
| 2935 | + fmr->qplib_fmr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; | |
| 2936 | + | |
| 2937 | + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &fmr->qplib_fmr); | |
| 2938 | + if (rc) | |
| 2939 | + goto fail; | |
| 2940 | + | |
| 2941 | + fmr->qplib_fmr.flags = __from_ib_access_flags(mr_access_flags); | |
| 2942 | + fmr->ib_fmr.lkey = fmr->qplib_fmr.lkey; | |
| 2943 | + fmr->ib_fmr.rkey = fmr->ib_fmr.lkey; | |
| 2944 | + | |
| 2945 | + atomic_inc(&rdev->mr_count); | |
| 2946 | + return &fmr->ib_fmr; | |
| 2947 | +fail: | |
| 2948 | + kfree(fmr); | |
| 2949 | + return ERR_PTR(rc); | |
| 2950 | +} | |
| 2951 | + | |
| 2952 | +int bnxt_re_map_phys_fmr(struct ib_fmr *ib_fmr, u64 *page_list, int list_len, | |
| 2953 | + u64 iova) | |
| 2954 | +{ | |
| 2955 | + struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr, | |
| 2956 | + ib_fmr); | |
| 2957 | + struct bnxt_re_dev *rdev = fmr->rdev; | |
| 2958 | + int rc; | |
| 2959 | + | |
| 2960 | + fmr->qplib_fmr.va = iova; | |
| 2961 | + fmr->qplib_fmr.total_size = list_len * PAGE_SIZE; | |
| 2962 | + | |
| 2963 | + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &fmr->qplib_fmr, page_list, | |
| 2964 | + list_len, true); | |
| 2965 | + if (rc) | |
| 2966 | + dev_err(rdev_to_dev(rdev), "Failed to map FMR for lkey = 0x%x!", | |
| 2967 | + fmr->ib_fmr.lkey); | |
| 2968 | + return rc; | |
| 2969 | +} | |
| 2970 | + | |
| 2971 | +int bnxt_re_unmap_fmr(struct list_head *fmr_list) | |
| 2972 | +{ | |
| 2973 | + struct bnxt_re_dev *rdev; | |
| 2974 | + struct bnxt_re_fmr *fmr; | |
| 2975 | + struct ib_fmr *ib_fmr; | |
| 2976 | + int rc = 0; | |
| 2977 | + | |
| 2978 | + /* Validate each FMRs inside the fmr_list */ | |
| 2979 | + list_for_each_entry(ib_fmr, fmr_list, list) { | |
| 2980 | + fmr = container_of(ib_fmr, struct bnxt_re_fmr, ib_fmr); | |
| 2981 | + rdev = fmr->rdev; | |
| 2982 | + | |
| 2983 | + if (rdev) { | |
| 2984 | + rc = bnxt_qplib_dereg_mrw(&rdev->qplib_res, | |
| 2985 | + &fmr->qplib_fmr, true); | |
| 2986 | + if (rc) | |
| 2987 | + break; | |
| 2988 | + } | |
| 2989 | + } | |
| 2990 | + return rc; | |
| 2991 | +} | |
| 2992 | + | |
| 2993 | +int bnxt_re_dealloc_fmr(struct ib_fmr *ib_fmr) | |
| 2994 | +{ | |
| 2995 | + struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr, | |
| 2996 | + ib_fmr); | |
| 2997 | + struct bnxt_re_dev *rdev = fmr->rdev; | |
| 2998 | + int rc; | |
| 2999 | + | |
| 3000 | + rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &fmr->qplib_fmr); | |
| 3001 | + if (rc) | |
| 3002 | + dev_err(rdev_to_dev(rdev), "Failed to free FMR"); | |
| 3003 | + | |
| 3004 | + kfree(fmr); | |
| 3005 | + atomic_dec(&rdev->mr_count); | |
| 3006 | + return rc; | |
| 3007 | +} | |
| 3008 | + | |
| 3009 | +/* uverbs */ | |
| 3010 | +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, | |
| 3011 | + u64 virt_addr, int mr_access_flags, | |
| 3012 | + struct ib_udata *udata) | |
| 3013 | +{ | |
| 3014 | + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | |
| 3015 | + struct bnxt_re_dev *rdev = pd->rdev; | |
| 3016 | + struct bnxt_re_mr *mr; | |
| 3017 | + struct ib_umem *umem; | |
| 3018 | + u64 *pbl_tbl, *pbl_tbl_orig; | |
| 3019 | + int i, umem_pgs, pages, page_shift, rc; | |
| 3020 | + struct scatterlist *sg; | |
| 3021 | + int entry; | |
| 3022 | + | |
| 3023 | + mr = kzalloc(sizeof(*mr), GFP_KERNEL); | |
| 3024 | + if (!mr) | |
| 3025 | + return ERR_PTR(-ENOMEM); | |
| 3026 | + | |
| 3027 | + mr->rdev = rdev; | |
| 3028 | + mr->qplib_mr.pd = &pd->qplib_pd; | |
| 3029 | + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); | |
| 3030 | + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; | |
| 3031 | + | |
| 3032 | + umem = ib_umem_get(ib_pd->uobject->context, start, length, | |
| 3033 | + mr_access_flags, 0); | |
| 3034 | + if (IS_ERR(umem)) { | |
| 3035 | + dev_err(rdev_to_dev(rdev), "Failed to get umem"); | |
| 3036 | + rc = -EFAULT; | |
| 3037 | + goto free_mr; | |
| 3038 | + } | |
| 3039 | + mr->ib_umem = umem; | |
| 3040 | + | |
| 3041 | + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); | |
| 3042 | + if (rc) { | |
| 3043 | + dev_err(rdev_to_dev(rdev), "Failed to allocate MR"); | |
| 3044 | + goto release_umem; | |
| 3045 | + } | |
| 3046 | + /* The fixed portion of the rkey is the same as the lkey */ | |
| 3047 | + mr->ib_mr.rkey = mr->qplib_mr.rkey; | |
| 3048 | + | |
| 3049 | + mr->qplib_mr.va = virt_addr; | |
| 3050 | + umem_pgs = ib_umem_page_count(umem); | |
| 3051 | + if (!umem_pgs) { | |
| 3052 | + dev_err(rdev_to_dev(rdev), "umem is invalid!"); | |
| 3053 | + rc = -EINVAL; | |
| 3054 | + goto free_mrw; | |
| 3055 | + } | |
| 3056 | + mr->qplib_mr.total_size = length; | |
| 3057 | + | |
| 3058 | + pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL); | |
| 3059 | + if (!pbl_tbl) { | |
| 3060 | + rc = -EINVAL; | |
| 3061 | + goto free_mrw; | |
| 3062 | + } | |
| 3063 | + pbl_tbl_orig = pbl_tbl; | |
| 3064 | + | |
| 3065 | + page_shift = ilog2(umem->page_size); | |
| 3066 | + if (umem->hugetlb) { | |
| 3067 | + dev_err(rdev_to_dev(rdev), "umem hugetlb not supported!"); | |
| 3068 | + rc = -EFAULT; | |
| 3069 | + goto fail; | |
| 3070 | + } | |
| 3071 | + if (umem->page_size != PAGE_SIZE) { | |
| 3072 | + dev_err(rdev_to_dev(rdev), "umem page size unsupported!"); | |
| 3073 | + rc = -EFAULT; | |
| 3074 | + goto fail; | |
| 3075 | + } | |
| 3076 | + /* Map umem buf ptrs to the PBL */ | |
| 3077 | + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { | |
| 3078 | + pages = sg_dma_len(sg) >> page_shift; | |
| 3079 | + for (i = 0; i < pages; i++, pbl_tbl++) | |
| 3080 | + *pbl_tbl = sg_dma_address(sg) + (i << page_shift); | |
| 3081 | + } | |
| 3082 | + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl_orig, | |
| 3083 | + umem_pgs, false); | |
| 3084 | + if (rc) { | |
| 3085 | + dev_err(rdev_to_dev(rdev), "Failed to register user MR"); | |
| 3086 | + goto fail; | |
| 3087 | + } | |
| 3088 | + | |
| 3089 | + kfree(pbl_tbl_orig); | |
| 3090 | + | |
| 3091 | + mr->ib_mr.lkey = mr->qplib_mr.lkey; | |
| 3092 | + mr->ib_mr.rkey = mr->qplib_mr.lkey; | |
| 3093 | + atomic_inc(&rdev->mr_count); | |
| 3094 | + | |
| 3095 | + return &mr->ib_mr; | |
| 3096 | +fail: | |
| 3097 | + kfree(pbl_tbl_orig); | |
| 3098 | +free_mrw: | |
| 3099 | + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); | |
| 3100 | +release_umem: | |
| 3101 | + ib_umem_release(umem); | |
| 3102 | +free_mr: | |
| 3103 | + kfree(mr); | |
| 3104 | + return ERR_PTR(rc); | |
| 3105 | +} | |
| 3106 | + | |
| 3107 | +struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev, | |
| 3108 | + struct ib_udata *udata) | |
| 3109 | +{ | |
| 3110 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | |
| 3111 | + struct bnxt_re_uctx_resp resp; | |
| 3112 | + struct bnxt_re_ucontext *uctx; | |
| 3113 | + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | |
| 3114 | + int rc; | |
| 3115 | + | |
| 3116 | + dev_dbg(rdev_to_dev(rdev), "ABI version requested %d", | |
| 3117 | + ibdev->uverbs_abi_ver); | |
| 3118 | + | |
| 3119 | + if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) { | |
| 3120 | + dev_dbg(rdev_to_dev(rdev), " is different from the device %d ", | |
| 3121 | + BNXT_RE_ABI_VERSION); | |
| 3122 | + return ERR_PTR(-EPERM); | |
| 3123 | + } | |
| 3124 | + | |
| 3125 | + uctx = kzalloc(sizeof(*uctx), GFP_KERNEL); | |
| 3126 | + if (!uctx) | |
| 3127 | + return ERR_PTR(-ENOMEM); | |
| 3128 | + | |
| 3129 | + uctx->rdev = rdev; | |
| 3130 | + | |
| 3131 | + uctx->shpg = (void *)__get_free_page(GFP_KERNEL); | |
| 3132 | + if (!uctx->shpg) { | |
| 3133 | + rc = -ENOMEM; | |
| 3134 | + goto fail; | |
| 3135 | + } | |
| 3136 | + spin_lock_init(&uctx->sh_lock); | |
| 3137 | + | |
| 3138 | + resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/ | |
| 3139 | + resp.max_qp = rdev->qplib_ctx.qpc_count; | |
| 3140 | + resp.pg_size = PAGE_SIZE; | |
| 3141 | + resp.cqe_sz = sizeof(struct cq_base); | |
| 3142 | + resp.max_cqd = dev_attr->max_cq_wqes; | |
| 3143 | + resp.rsvd = 0; | |
| 3144 | + | |
| 3145 | + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); | |
| 3146 | + if (rc) { | |
| 3147 | + dev_err(rdev_to_dev(rdev), "Failed to copy user context"); | |
| 3148 | + rc = -EFAULT; | |
| 3149 | + goto cfail; | |
| 3150 | + } | |
| 3151 | + | |
| 3152 | + return &uctx->ib_uctx; | |
| 3153 | +cfail: | |
| 3154 | + free_page((unsigned long)uctx->shpg); | |
| 3155 | + uctx->shpg = NULL; | |
| 3156 | +fail: | |
| 3157 | + kfree(uctx); | |
| 3158 | + return ERR_PTR(rc); | |
| 3159 | +} | |
| 3160 | + | |
| 3161 | +int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) | |
| 3162 | +{ | |
| 3163 | + struct bnxt_re_ucontext *uctx = container_of(ib_uctx, | |
| 3164 | + struct bnxt_re_ucontext, | |
| 3165 | + ib_uctx); | |
| 3166 | + if (uctx->shpg) | |
| 3167 | + free_page((unsigned long)uctx->shpg); | |
| 3168 | + kfree(uctx); | |
| 3169 | + return 0; | |
| 3170 | +} | |
| 3171 | + | |
| 3172 | +/* Helper function to mmap the virtual memory from user app */ | |
| 3173 | +int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) | |
| 3174 | +{ | |
| 3175 | + struct bnxt_re_ucontext *uctx = container_of(ib_uctx, | |
| 3176 | + struct bnxt_re_ucontext, | |
| 3177 | + ib_uctx); | |
| 3178 | + struct bnxt_re_dev *rdev = uctx->rdev; | |
| 3179 | + u64 pfn; | |
| 3180 | + | |
| 3181 | + if (vma->vm_end - vma->vm_start != PAGE_SIZE) | |
| 3182 | + return -EINVAL; | |
| 3183 | + | |
| 3184 | + if (vma->vm_pgoff) { | |
| 3185 | + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); | |
| 3186 | + if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, | |
| 3187 | + PAGE_SIZE, vma->vm_page_prot)) { | |
| 3188 | + dev_err(rdev_to_dev(rdev), "Failed to map DPI"); | |
| 3189 | + return -EAGAIN; | |
| 3190 | + } | |
| 3191 | + } else { | |
| 3192 | + pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT; | |
| 3193 | + if (remap_pfn_range(vma, vma->vm_start, | |
| 3194 | + pfn, PAGE_SIZE, vma->vm_page_prot)) { | |
| 3195 | + dev_err(rdev_to_dev(rdev), | |
| 3196 | + "Failed to map shared page"); | |
| 3197 | + return -EAGAIN; | |
| 3198 | + } | |
| 3199 | + } | |
| 3200 | + | |
| 3201 | + return 0; | |
| 3202 | +} | |
| @@ -0,0 +1,197 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: IB Verbs interpreter (header) | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#ifndef __BNXT_RE_IB_VERBS_H__ | |
| 40 | +#define __BNXT_RE_IB_VERBS_H__ | |
| 41 | + | |
| 42 | +struct bnxt_re_gid_ctx { | |
| 43 | + u32 idx; | |
| 44 | + u32 refcnt; | |
| 45 | +}; | |
| 46 | + | |
| 47 | +struct bnxt_re_pd { | |
| 48 | + struct bnxt_re_dev *rdev; | |
| 49 | + struct ib_pd ib_pd; | |
| 50 | + struct bnxt_qplib_pd qplib_pd; | |
| 51 | + struct bnxt_qplib_dpi dpi; | |
| 52 | +}; | |
| 53 | + | |
| 54 | +struct bnxt_re_ah { | |
| 55 | + struct bnxt_re_dev *rdev; | |
| 56 | + struct ib_ah ib_ah; | |
| 57 | + struct bnxt_qplib_ah qplib_ah; | |
| 58 | +}; | |
| 59 | + | |
| 60 | +struct bnxt_re_qp { | |
| 61 | + struct list_head list; | |
| 62 | + struct bnxt_re_dev *rdev; | |
| 63 | + struct ib_qp ib_qp; | |
| 64 | + spinlock_t sq_lock; /* protect sq */ | |
| 65 | + struct bnxt_qplib_qp qplib_qp; | |
| 66 | + struct ib_umem *sumem; | |
| 67 | + struct ib_umem *rumem; | |
| 68 | + /* QP1 */ | |
| 69 | + u32 send_psn; | |
| 70 | + struct ib_ud_header qp1_hdr; | |
| 71 | +}; | |
| 72 | + | |
| 73 | +struct bnxt_re_cq { | |
| 74 | + struct bnxt_re_dev *rdev; | |
| 75 | + spinlock_t cq_lock; /* protect cq */ | |
| 76 | + u16 cq_count; | |
| 77 | + u16 cq_period; | |
| 78 | + struct ib_cq ib_cq; | |
| 79 | + struct bnxt_qplib_cq qplib_cq; | |
| 80 | + struct bnxt_qplib_cqe *cql; | |
| 81 | +#define MAX_CQL_PER_POLL 1024 | |
| 82 | + u32 max_cql; | |
| 83 | + struct ib_umem *umem; | |
| 84 | +}; | |
| 85 | + | |
| 86 | +struct bnxt_re_mr { | |
| 87 | + struct bnxt_re_dev *rdev; | |
| 88 | + struct ib_mr ib_mr; | |
| 89 | + struct ib_umem *ib_umem; | |
| 90 | + struct bnxt_qplib_mrw qplib_mr; | |
| 91 | + u32 npages; | |
| 92 | + u64 *pages; | |
| 93 | + struct bnxt_qplib_frpl qplib_frpl; | |
| 94 | +}; | |
| 95 | + | |
| 96 | +struct bnxt_re_frpl { | |
| 97 | + struct bnxt_re_dev *rdev; | |
| 98 | + struct bnxt_qplib_frpl qplib_frpl; | |
| 99 | + u64 *page_list; | |
| 100 | +}; | |
| 101 | + | |
| 102 | +struct bnxt_re_fmr { | |
| 103 | + struct bnxt_re_dev *rdev; | |
| 104 | + struct ib_fmr ib_fmr; | |
| 105 | + struct bnxt_qplib_mrw qplib_fmr; | |
| 106 | +}; | |
| 107 | + | |
| 108 | +struct bnxt_re_mw { | |
| 109 | + struct bnxt_re_dev *rdev; | |
| 110 | + struct ib_mw ib_mw; | |
| 111 | + struct bnxt_qplib_mrw qplib_mw; | |
| 112 | +}; | |
| 113 | + | |
| 114 | +struct bnxt_re_ucontext { | |
| 115 | + struct bnxt_re_dev *rdev; | |
| 116 | + struct ib_ucontext ib_uctx; | |
| 117 | + struct bnxt_qplib_dpi *dpi; | |
| 118 | + void *shpg; | |
| 119 | + spinlock_t sh_lock; /* protect shpg */ | |
| 120 | +}; | |
| 121 | + | |
| 122 | +struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num); | |
| 123 | + | |
| 124 | +int bnxt_re_query_device(struct ib_device *ibdev, | |
| 125 | + struct ib_device_attr *ib_attr, | |
| 126 | + struct ib_udata *udata); | |
| 127 | +int bnxt_re_modify_device(struct ib_device *ibdev, | |
| 128 | + int device_modify_mask, | |
| 129 | + struct ib_device_modify *device_modify); | |
| 130 | +int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, | |
| 131 | + struct ib_port_attr *port_attr); | |
| 132 | +int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, | |
| 133 | + int port_modify_mask, | |
| 134 | + struct ib_port_modify *port_modify); | |
| 135 | +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, | |
| 136 | + struct ib_port_immutable *immutable); | |
| 137 | +int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, | |
| 138 | + u16 index, u16 *pkey); | |
| 139 | +int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, | |
| 140 | + unsigned int index, void **context); | |
| 141 | +int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, | |
| 142 | + unsigned int index, const union ib_gid *gid, | |
| 143 | + const struct ib_gid_attr *attr, void **context); | |
| 144 | +int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, | |
| 145 | + int index, union ib_gid *gid); | |
| 146 | +enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, | |
| 147 | + u8 port_num); | |
| 148 | +struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev, | |
| 149 | + struct ib_ucontext *context, | |
| 150 | + struct ib_udata *udata); | |
| 151 | +int bnxt_re_dealloc_pd(struct ib_pd *pd); | |
| 152 | +struct ib_ah *bnxt_re_create_ah(struct ib_pd *pd, | |
| 153 | + struct ib_ah_attr *ah_attr, | |
| 154 | + struct ib_udata *udata); | |
| 155 | +int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); | |
| 156 | +int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); | |
| 157 | +int bnxt_re_destroy_ah(struct ib_ah *ah); | |
| 158 | +struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd, | |
| 159 | + struct ib_qp_init_attr *qp_init_attr, | |
| 160 | + struct ib_udata *udata); | |
| 161 | +int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, | |
| 162 | + int qp_attr_mask, struct ib_udata *udata); | |
| 163 | +int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, | |
| 164 | + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); | |
| 165 | +int bnxt_re_destroy_qp(struct ib_qp *qp); | |
| 166 | +int bnxt_re_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr, | |
| 167 | + struct ib_send_wr **bad_send_wr); | |
| 168 | +int bnxt_re_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr, | |
| 169 | + struct ib_recv_wr **bad_recv_wr); | |
| 170 | +struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, | |
| 171 | + const struct ib_cq_init_attr *attr, | |
| 172 | + struct ib_ucontext *context, | |
| 173 | + struct ib_udata *udata); | |
| 174 | +int bnxt_re_destroy_cq(struct ib_cq *cq); | |
| 175 | +int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); | |
| 176 | +int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); | |
| 177 | +struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); | |
| 178 | + | |
| 179 | +int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, | |
| 180 | + unsigned int *sg_offset); | |
| 181 | +struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type, | |
| 182 | + u32 max_num_sg); | |
| 183 | +int bnxt_re_dereg_mr(struct ib_mr *mr); | |
| 184 | +struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *pd, int mr_access_flags, | |
| 185 | + struct ib_fmr_attr *fmr_attr); | |
| 186 | +int bnxt_re_map_phys_fmr(struct ib_fmr *fmr, u64 *page_list, int list_len, | |
| 187 | + u64 iova); | |
| 188 | +int bnxt_re_unmap_fmr(struct list_head *fmr_list); | |
| 189 | +int bnxt_re_dealloc_fmr(struct ib_fmr *fmr); | |
| 190 | +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, | |
| 191 | + u64 virt_addr, int mr_access_flags, | |
| 192 | + struct ib_udata *udata); | |
| 193 | +struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev, | |
| 194 | + struct ib_udata *udata); | |
| 195 | +int bnxt_re_dealloc_ucontext(struct ib_ucontext *context); | |
| 196 | +int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); | |
| 197 | +#endif /* __BNXT_RE_IB_VERBS_H__ */ | |
| @@ -0,0 +1,1315 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: Main component of the bnxt_re driver | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#include <linux/module.h> | |
| 40 | +#include <linux/netdevice.h> | |
| 41 | +#include <linux/ethtool.h> | |
| 42 | +#include <linux/mutex.h> | |
| 43 | +#include <linux/list.h> | |
| 44 | +#include <linux/rculist.h> | |
| 45 | +#include <linux/spinlock.h> | |
| 46 | +#include <linux/pci.h> | |
| 47 | +#include <net/dcbnl.h> | |
| 48 | +#include <net/ipv6.h> | |
| 49 | +#include <net/addrconf.h> | |
| 50 | +#include <linux/if_ether.h> | |
| 51 | + | |
| 52 | +#include <rdma/ib_verbs.h> | |
| 53 | +#include <rdma/ib_user_verbs.h> | |
| 54 | +#include <rdma/ib_umem.h> | |
| 55 | +#include <rdma/ib_addr.h> | |
| 56 | + | |
| 57 | +#include "bnxt_ulp.h" | |
| 58 | +#include "roce_hsi.h" | |
| 59 | +#include "qplib_res.h" | |
| 60 | +#include "qplib_sp.h" | |
| 61 | +#include "qplib_fp.h" | |
| 62 | +#include "qplib_rcfw.h" | |
| 63 | +#include "bnxt_re.h" | |
| 64 | +#include "ib_verbs.h" | |
| 65 | +#include <rdma/bnxt_re-abi.h> | |
| 66 | +#include "bnxt.h" | |
| 67 | +static char version[] = | |
| 68 | + BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n"; | |
| 69 | + | |
| 70 | +MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>"); | |
| 71 | +MODULE_DESCRIPTION(BNXT_RE_DESC " Driver"); | |
| 72 | +MODULE_LICENSE("Dual BSD/GPL"); | |
| 73 | +MODULE_VERSION(ROCE_DRV_MODULE_VERSION); | |
| 74 | + | |
| 75 | +/* globals */ | |
| 76 | +static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list); | |
| 77 | +/* Mutex to protect the list of bnxt_re devices added */ | |
| 78 | +static DEFINE_MUTEX(bnxt_re_dev_lock); | |
| 79 | +static struct workqueue_struct *bnxt_re_wq; | |
| 80 | + | |
| 81 | +/* for handling bnxt_en callbacks later */ | |
| 82 | +static void bnxt_re_stop(void *p) | |
| 83 | +{ | |
| 84 | +} | |
| 85 | + | |
| 86 | +static void bnxt_re_start(void *p) | |
| 87 | +{ | |
| 88 | +} | |
| 89 | + | |
| 90 | +static void bnxt_re_sriov_config(void *p, int num_vfs) | |
| 91 | +{ | |
| 92 | +} | |
| 93 | + | |
| 94 | +static struct bnxt_ulp_ops bnxt_re_ulp_ops = { | |
| 95 | + .ulp_async_notifier = NULL, | |
| 96 | + .ulp_stop = bnxt_re_stop, | |
| 97 | + .ulp_start = bnxt_re_start, | |
| 98 | + .ulp_sriov_config = bnxt_re_sriov_config | |
| 99 | +}; | |
| 100 | + | |
| 101 | +/* RoCE -> Net driver */ | |
| 102 | + | |
| 103 | +/* Driver registration routines used to let the networking driver (bnxt_en) | |
| 104 | + * to know that the RoCE driver is now installed | |
| 105 | + */ | |
| 106 | +static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev, bool lock_wait) | |
| 107 | +{ | |
| 108 | + struct bnxt_en_dev *en_dev; | |
| 109 | + int rc; | |
| 110 | + | |
| 111 | + if (!rdev) | |
| 112 | + return -EINVAL; | |
| 113 | + | |
| 114 | + en_dev = rdev->en_dev; | |
| 115 | + /* Acquire rtnl lock if it is not invokded from netdev event */ | |
| 116 | + if (lock_wait) | |
| 117 | + rtnl_lock(); | |
| 118 | + | |
| 119 | + rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, | |
| 120 | + BNXT_ROCE_ULP); | |
| 121 | + if (lock_wait) | |
| 122 | + rtnl_unlock(); | |
| 123 | + return rc; | |
| 124 | +} | |
| 125 | + | |
| 126 | +static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) | |
| 127 | +{ | |
| 128 | + struct bnxt_en_dev *en_dev; | |
| 129 | + int rc = 0; | |
| 130 | + | |
| 131 | + if (!rdev) | |
| 132 | + return -EINVAL; | |
| 133 | + | |
| 134 | + en_dev = rdev->en_dev; | |
| 135 | + | |
| 136 | + rtnl_lock(); | |
| 137 | + rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP, | |
| 138 | + &bnxt_re_ulp_ops, rdev); | |
| 139 | + rtnl_unlock(); | |
| 140 | + return rc; | |
| 141 | +} | |
| 142 | + | |
| 143 | +static int bnxt_re_free_msix(struct bnxt_re_dev *rdev, bool lock_wait) | |
| 144 | +{ | |
| 145 | + struct bnxt_en_dev *en_dev; | |
| 146 | + int rc; | |
| 147 | + | |
| 148 | + if (!rdev) | |
| 149 | + return -EINVAL; | |
| 150 | + | |
| 151 | + en_dev = rdev->en_dev; | |
| 152 | + | |
| 153 | + if (lock_wait) | |
| 154 | + rtnl_lock(); | |
| 155 | + | |
| 156 | + rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); | |
| 157 | + | |
| 158 | + if (lock_wait) | |
| 159 | + rtnl_unlock(); | |
| 160 | + return rc; | |
| 161 | +} | |
| 162 | + | |
| 163 | +static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) | |
| 164 | +{ | |
| 165 | + int rc = 0, num_msix_want = BNXT_RE_MIN_MSIX, num_msix_got; | |
| 166 | + struct bnxt_en_dev *en_dev; | |
| 167 | + | |
| 168 | + if (!rdev) | |
| 169 | + return -EINVAL; | |
| 170 | + | |
| 171 | + en_dev = rdev->en_dev; | |
| 172 | + | |
| 173 | + rtnl_lock(); | |
| 174 | + num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, | |
| 175 | + rdev->msix_entries, | |
| 176 | + num_msix_want); | |
| 177 | + if (num_msix_got < BNXT_RE_MIN_MSIX) { | |
| 178 | + rc = -EINVAL; | |
| 179 | + goto done; | |
| 180 | + } | |
| 181 | + if (num_msix_got != num_msix_want) { | |
| 182 | + dev_warn(rdev_to_dev(rdev), | |
| 183 | + "Requested %d MSI-X vectors, got %d\n", | |
| 184 | + num_msix_want, num_msix_got); | |
| 185 | + } | |
| 186 | + rdev->num_msix = num_msix_got; | |
| 187 | +done: | |
| 188 | + rtnl_unlock(); | |
| 189 | + return rc; | |
| 190 | +} | |
| 191 | + | |
| 192 | +static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr, | |
| 193 | + u16 opcd, u16 crid, u16 trid) | |
| 194 | +{ | |
| 195 | + hdr->req_type = cpu_to_le16(opcd); | |
| 196 | + hdr->cmpl_ring = cpu_to_le16(crid); | |
| 197 | + hdr->target_id = cpu_to_le16(trid); | |
| 198 | +} | |
| 199 | + | |
| 200 | +static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg, | |
| 201 | + int msg_len, void *resp, int resp_max_len, | |
| 202 | + int timeout) | |
| 203 | +{ | |
| 204 | + fw_msg->msg = msg; | |
| 205 | + fw_msg->msg_len = msg_len; | |
| 206 | + fw_msg->resp = resp; | |
| 207 | + fw_msg->resp_max_len = resp_max_len; | |
| 208 | + fw_msg->timeout = timeout; | |
| 209 | +} | |
| 210 | + | |
| 211 | +static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id, | |
| 212 | + bool lock_wait) | |
| 213 | +{ | |
| 214 | + struct bnxt_en_dev *en_dev = rdev->en_dev; | |
| 215 | + struct hwrm_ring_free_input req = {0}; | |
| 216 | + struct hwrm_ring_free_output resp; | |
| 217 | + struct bnxt_fw_msg fw_msg; | |
| 218 | + bool do_unlock = false; | |
| 219 | + int rc = -EINVAL; | |
| 220 | + | |
| 221 | + if (!en_dev) | |
| 222 | + return rc; | |
| 223 | + | |
| 224 | + memset(&fw_msg, 0, sizeof(fw_msg)); | |
| 225 | + if (lock_wait) { | |
| 226 | + rtnl_lock(); | |
| 227 | + do_unlock = true; | |
| 228 | + } | |
| 229 | + | |
| 230 | + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); | |
| 231 | + req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL; | |
| 232 | + req.ring_id = cpu_to_le16(fw_ring_id); | |
| 233 | + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
| 234 | + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
| 235 | + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
| 236 | + if (rc) | |
| 237 | + dev_err(rdev_to_dev(rdev), | |
| 238 | + "Failed to free HW ring:%d :%#x", req.ring_id, rc); | |
| 239 | + if (do_unlock) | |
| 240 | + rtnl_unlock(); | |
| 241 | + return rc; | |
| 242 | +} | |
| 243 | + | |
| 244 | +static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr, | |
| 245 | + int pages, int type, u32 ring_mask, | |
| 246 | + u32 map_index, u16 *fw_ring_id) | |
| 247 | +{ | |
| 248 | + struct bnxt_en_dev *en_dev = rdev->en_dev; | |
| 249 | + struct hwrm_ring_alloc_input req = {0}; | |
| 250 | + struct hwrm_ring_alloc_output resp; | |
| 251 | + struct bnxt_fw_msg fw_msg; | |
| 252 | + int rc = -EINVAL; | |
| 253 | + | |
| 254 | + if (!en_dev) | |
| 255 | + return rc; | |
| 256 | + | |
| 257 | + memset(&fw_msg, 0, sizeof(fw_msg)); | |
| 258 | + rtnl_lock(); | |
| 259 | + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); | |
| 260 | + req.enables = 0; | |
| 261 | + req.page_tbl_addr = cpu_to_le64(dma_arr[0]); | |
| 262 | + if (pages > 1) { | |
| 263 | + /* Page size is in log2 units */ | |
| 264 | + req.page_size = BNXT_PAGE_SHIFT; | |
| 265 | + req.page_tbl_depth = 1; | |
| 266 | + } | |
| 267 | + req.fbo = 0; | |
| 268 | + /* Association of ring index with doorbell index and MSIX number */ | |
| 269 | + req.logical_id = cpu_to_le16(map_index); | |
| 270 | + req.length = cpu_to_le32(ring_mask + 1); | |
| 271 | + req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL; | |
| 272 | + req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; | |
| 273 | + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
| 274 | + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
| 275 | + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
| 276 | + if (!rc) | |
| 277 | + *fw_ring_id = le16_to_cpu(resp.ring_id); | |
| 278 | + | |
| 279 | + rtnl_unlock(); | |
| 280 | + return rc; | |
| 281 | +} | |
| 282 | + | |
| 283 | +static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, | |
| 284 | + u32 fw_stats_ctx_id, bool lock_wait) | |
| 285 | +{ | |
| 286 | + struct bnxt_en_dev *en_dev = rdev->en_dev; | |
| 287 | + struct hwrm_stat_ctx_free_input req = {0}; | |
| 288 | + struct bnxt_fw_msg fw_msg; | |
| 289 | + bool do_unlock = false; | |
| 290 | + int rc = -EINVAL; | |
| 291 | + | |
| 292 | + if (!en_dev) | |
| 293 | + return rc; | |
| 294 | + | |
| 295 | + memset(&fw_msg, 0, sizeof(fw_msg)); | |
| 296 | + if (lock_wait) { | |
| 297 | + rtnl_lock(); | |
| 298 | + do_unlock = true; | |
| 299 | + } | |
| 300 | + | |
| 301 | + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1); | |
| 302 | + req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); | |
| 303 | + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&req, | |
| 304 | + sizeof(req), DFLT_HWRM_CMD_TIMEOUT); | |
| 305 | + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
| 306 | + if (rc) | |
| 307 | + dev_err(rdev_to_dev(rdev), | |
| 308 | + "Failed to free HW stats context %#x", rc); | |
| 309 | + | |
| 310 | + if (do_unlock) | |
| 311 | + rtnl_unlock(); | |
| 312 | + return rc; | |
| 313 | +} | |
| 314 | + | |
| 315 | +static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, | |
| 316 | + dma_addr_t dma_map, | |
| 317 | + u32 *fw_stats_ctx_id) | |
| 318 | +{ | |
| 319 | + struct hwrm_stat_ctx_alloc_output resp = {0}; | |
| 320 | + struct hwrm_stat_ctx_alloc_input req = {0}; | |
| 321 | + struct bnxt_en_dev *en_dev = rdev->en_dev; | |
| 322 | + struct bnxt_fw_msg fw_msg; | |
| 323 | + int rc = -EINVAL; | |
| 324 | + | |
| 325 | + *fw_stats_ctx_id = INVALID_STATS_CTX_ID; | |
| 326 | + | |
| 327 | + if (!en_dev) | |
| 328 | + return rc; | |
| 329 | + | |
| 330 | + memset(&fw_msg, 0, sizeof(fw_msg)); | |
| 331 | + rtnl_lock(); | |
| 332 | + | |
| 333 | + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1); | |
| 334 | + req.update_period_ms = cpu_to_le32(1000); | |
| 335 | + req.stats_dma_addr = cpu_to_le64(dma_map); | |
| 336 | + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
| 337 | + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
| 338 | + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
| 339 | + if (!rc) | |
| 340 | + *fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id); | |
| 341 | + | |
| 342 | + rtnl_unlock(); | |
| 343 | + return rc; | |
| 344 | +} | |
| 345 | + | |
| 346 | +/* Device */ | |
| 347 | + | |
| 348 | +static bool is_bnxt_re_dev(struct net_device *netdev) | |
| 349 | +{ | |
| 350 | + struct ethtool_drvinfo drvinfo; | |
| 351 | + | |
| 352 | + if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) { | |
| 353 | + memset(&drvinfo, 0, sizeof(drvinfo)); | |
| 354 | + netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo); | |
| 355 | + | |
| 356 | + if (strcmp(drvinfo.driver, "bnxt_en")) | |
| 357 | + return false; | |
| 358 | + return true; | |
| 359 | + } | |
| 360 | + return false; | |
| 361 | +} | |
| 362 | + | |
| 363 | +static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) | |
| 364 | +{ | |
| 365 | + struct bnxt_re_dev *rdev; | |
| 366 | + | |
| 367 | + rcu_read_lock(); | |
| 368 | + list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { | |
| 369 | + if (rdev->netdev == netdev) { | |
| 370 | + rcu_read_unlock(); | |
| 371 | + return rdev; | |
| 372 | + } | |
| 373 | + } | |
| 374 | + rcu_read_unlock(); | |
| 375 | + return NULL; | |
| 376 | +} | |
| 377 | + | |
| 378 | +static void bnxt_re_dev_unprobe(struct net_device *netdev, | |
| 379 | + struct bnxt_en_dev *en_dev) | |
| 380 | +{ | |
| 381 | + dev_put(netdev); | |
| 382 | + module_put(en_dev->pdev->driver->driver.owner); | |
| 383 | +} | |
| 384 | + | |
| 385 | +static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev) | |
| 386 | +{ | |
| 387 | + struct bnxt *bp = netdev_priv(netdev); | |
| 388 | + struct bnxt_en_dev *en_dev; | |
| 389 | + struct pci_dev *pdev; | |
| 390 | + | |
| 391 | + /* Call bnxt_en's RoCE probe via indirect API */ | |
| 392 | + if (!bp->ulp_probe) | |
| 393 | + return ERR_PTR(-EINVAL); | |
| 394 | + | |
| 395 | + en_dev = bp->ulp_probe(netdev); | |
| 396 | + if (IS_ERR(en_dev)) | |
| 397 | + return en_dev; | |
| 398 | + | |
| 399 | + pdev = en_dev->pdev; | |
| 400 | + if (!pdev) | |
| 401 | + return ERR_PTR(-EINVAL); | |
| 402 | + | |
| 403 | + if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) { | |
| 404 | + dev_dbg(&pdev->dev, | |
| 405 | + "%s: probe error: RoCE is not supported on this device", | |
| 406 | + ROCE_DRV_MODULE_NAME); | |
| 407 | + return ERR_PTR(-ENODEV); | |
| 408 | + } | |
| 409 | + | |
| 410 | + /* Bump net device reference count */ | |
| 411 | + if (!try_module_get(pdev->driver->driver.owner)) | |
| 412 | + return ERR_PTR(-ENODEV); | |
| 413 | + | |
| 414 | + dev_hold(netdev); | |
| 415 | + | |
| 416 | + return en_dev; | |
| 417 | +} | |
| 418 | + | |
| 419 | +static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) | |
| 420 | +{ | |
| 421 | + ib_unregister_device(&rdev->ibdev); | |
| 422 | +} | |
| 423 | + | |
| 424 | +static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) | |
| 425 | +{ | |
| 426 | + struct ib_device *ibdev = &rdev->ibdev; | |
| 427 | + | |
| 428 | + /* ib device init */ | |
| 429 | + ibdev->owner = THIS_MODULE; | |
| 430 | + ibdev->node_type = RDMA_NODE_IB_CA; | |
| 431 | + strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX); | |
| 432 | + strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", | |
| 433 | + strlen(BNXT_RE_DESC) + 5); | |
| 434 | + ibdev->phys_port_cnt = 1; | |
| 435 | + | |
| 436 | + bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid); | |
| 437 | + | |
| 438 | + ibdev->num_comp_vectors = 1; | |
| 439 | + ibdev->dma_device = &rdev->en_dev->pdev->dev; | |
| 440 | + ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; | |
| 441 | + | |
| 442 | + /* User space */ | |
| 443 | + ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; | |
| 444 | + ibdev->uverbs_cmd_mask = | |
| 445 | + (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | | |
| 446 | + (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | | |
| 447 | + (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | | |
| 448 | + (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | | |
| 449 | + (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | | |
| 450 | + (1ull << IB_USER_VERBS_CMD_REG_MR) | | |
| 451 | + (1ull << IB_USER_VERBS_CMD_REREG_MR) | | |
| 452 | + (1ull << IB_USER_VERBS_CMD_DEREG_MR) | | |
| 453 | + (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | | |
| 454 | + (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | | |
| 455 | + (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | | |
| 456 | + (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | | |
| 457 | + (1ull << IB_USER_VERBS_CMD_CREATE_QP) | | |
| 458 | + (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | | |
| 459 | + (1ull << IB_USER_VERBS_CMD_QUERY_QP) | | |
| 460 | + (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | | |
| 461 | + (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | | |
| 462 | + (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | | |
| 463 | + (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | | |
| 464 | + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | | |
| 465 | + (1ull << IB_USER_VERBS_CMD_CREATE_AH) | | |
| 466 | + (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | | |
| 467 | + (1ull << IB_USER_VERBS_CMD_QUERY_AH) | | |
| 468 | + (1ull << IB_USER_VERBS_CMD_DESTROY_AH); | |
| 469 | + /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */ | |
| 470 | + | |
| 471 | + /* Kernel verbs */ | |
| 472 | + ibdev->query_device = bnxt_re_query_device; | |
| 473 | + ibdev->modify_device = bnxt_re_modify_device; | |
| 474 | + | |
| 475 | + ibdev->query_port = bnxt_re_query_port; | |
| 476 | + ibdev->modify_port = bnxt_re_modify_port; | |
| 477 | + ibdev->get_port_immutable = bnxt_re_get_port_immutable; | |
| 478 | + ibdev->query_pkey = bnxt_re_query_pkey; | |
| 479 | + ibdev->query_gid = bnxt_re_query_gid; | |
| 480 | + ibdev->get_netdev = bnxt_re_get_netdev; | |
| 481 | + ibdev->add_gid = bnxt_re_add_gid; | |
| 482 | + ibdev->del_gid = bnxt_re_del_gid; | |
| 483 | + ibdev->get_link_layer = bnxt_re_get_link_layer; | |
| 484 | + | |
| 485 | + ibdev->alloc_pd = bnxt_re_alloc_pd; | |
| 486 | + ibdev->dealloc_pd = bnxt_re_dealloc_pd; | |
| 487 | + | |
| 488 | + ibdev->create_ah = bnxt_re_create_ah; | |
| 489 | + ibdev->modify_ah = bnxt_re_modify_ah; | |
| 490 | + ibdev->query_ah = bnxt_re_query_ah; | |
| 491 | + ibdev->destroy_ah = bnxt_re_destroy_ah; | |
| 492 | + | |
| 493 | + ibdev->create_qp = bnxt_re_create_qp; | |
| 494 | + ibdev->modify_qp = bnxt_re_modify_qp; | |
| 495 | + ibdev->query_qp = bnxt_re_query_qp; | |
| 496 | + ibdev->destroy_qp = bnxt_re_destroy_qp; | |
| 497 | + | |
| 498 | + ibdev->post_send = bnxt_re_post_send; | |
| 499 | + ibdev->post_recv = bnxt_re_post_recv; | |
| 500 | + | |
| 501 | + ibdev->create_cq = bnxt_re_create_cq; | |
| 502 | + ibdev->destroy_cq = bnxt_re_destroy_cq; | |
| 503 | + ibdev->poll_cq = bnxt_re_poll_cq; | |
| 504 | + ibdev->req_notify_cq = bnxt_re_req_notify_cq; | |
| 505 | + | |
| 506 | + ibdev->get_dma_mr = bnxt_re_get_dma_mr; | |
| 507 | + ibdev->dereg_mr = bnxt_re_dereg_mr; | |
| 508 | + ibdev->alloc_mr = bnxt_re_alloc_mr; | |
| 509 | + ibdev->map_mr_sg = bnxt_re_map_mr_sg; | |
| 510 | + ibdev->alloc_fmr = bnxt_re_alloc_fmr; | |
| 511 | + ibdev->map_phys_fmr = bnxt_re_map_phys_fmr; | |
| 512 | + ibdev->unmap_fmr = bnxt_re_unmap_fmr; | |
| 513 | + ibdev->dealloc_fmr = bnxt_re_dealloc_fmr; | |
| 514 | + | |
| 515 | + ibdev->reg_user_mr = bnxt_re_reg_user_mr; | |
| 516 | + ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; | |
| 517 | + ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; | |
| 518 | + ibdev->mmap = bnxt_re_mmap; | |
| 519 | + | |
| 520 | + return ib_register_device(ibdev, NULL); | |
| 521 | +} | |
| 522 | + | |
| 523 | +static ssize_t show_rev(struct device *device, struct device_attribute *attr, | |
| 524 | + char *buf) | |
| 525 | +{ | |
| 526 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
| 527 | + | |
| 528 | + return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); | |
| 529 | +} | |
| 530 | + | |
| 531 | +static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, | |
| 532 | + char *buf) | |
| 533 | +{ | |
| 534 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
| 535 | + | |
| 536 | + return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->dev_attr.fw_ver); | |
| 537 | +} | |
| 538 | + | |
| 539 | +static ssize_t show_hca(struct device *device, struct device_attribute *attr, | |
| 540 | + char *buf) | |
| 541 | +{ | |
| 542 | + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
| 543 | + | |
| 544 | + return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); | |
| 545 | +} | |
| 546 | + | |
| 547 | +static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); | |
| 548 | +static DEVICE_ATTR(fw_rev, 0444, show_fw_ver, NULL); | |
| 549 | +static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); | |
| 550 | + | |
| 551 | +static struct device_attribute *bnxt_re_attributes[] = { | |
| 552 | + &dev_attr_hw_rev, | |
| 553 | + &dev_attr_fw_rev, | |
| 554 | + &dev_attr_hca_type | |
| 555 | +}; | |
| 556 | + | |
| 557 | +static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) | |
| 558 | +{ | |
| 559 | + dev_put(rdev->netdev); | |
| 560 | + rdev->netdev = NULL; | |
| 561 | + | |
| 562 | + mutex_lock(&bnxt_re_dev_lock); | |
| 563 | + list_del_rcu(&rdev->list); | |
| 564 | + mutex_unlock(&bnxt_re_dev_lock); | |
| 565 | + | |
| 566 | + synchronize_rcu(); | |
| 567 | + flush_workqueue(bnxt_re_wq); | |
| 568 | + | |
| 569 | + ib_dealloc_device(&rdev->ibdev); | |
| 570 | + /* rdev is gone */ | |
| 571 | +} | |
| 572 | + | |
| 573 | +static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev, | |
| 574 | + struct bnxt_en_dev *en_dev) | |
| 575 | +{ | |
| 576 | + struct bnxt_re_dev *rdev; | |
| 577 | + | |
| 578 | + /* Allocate bnxt_re_dev instance here */ | |
| 579 | + rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev)); | |
| 580 | + if (!rdev) { | |
| 581 | + dev_err(NULL, "%s: bnxt_re_dev allocation failure!", | |
| 582 | + ROCE_DRV_MODULE_NAME); | |
| 583 | + return NULL; | |
| 584 | + } | |
| 585 | + /* Default values */ | |
| 586 | + rdev->netdev = netdev; | |
| 587 | + dev_hold(rdev->netdev); | |
| 588 | + rdev->en_dev = en_dev; | |
| 589 | + rdev->id = rdev->en_dev->pdev->devfn; | |
| 590 | + INIT_LIST_HEAD(&rdev->qp_list); | |
| 591 | + mutex_init(&rdev->qp_lock); | |
| 592 | + atomic_set(&rdev->qp_count, 0); | |
| 593 | + atomic_set(&rdev->cq_count, 0); | |
| 594 | + atomic_set(&rdev->srq_count, 0); | |
| 595 | + atomic_set(&rdev->mr_count, 0); | |
| 596 | + atomic_set(&rdev->mw_count, 0); | |
| 597 | + rdev->cosq[0] = 0xFFFF; | |
| 598 | + rdev->cosq[1] = 0xFFFF; | |
| 599 | + | |
| 600 | + mutex_lock(&bnxt_re_dev_lock); | |
| 601 | + list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); | |
| 602 | + mutex_unlock(&bnxt_re_dev_lock); | |
| 603 | + return rdev; | |
| 604 | +} | |
| 605 | + | |
| 606 | +static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, | |
| 607 | + struct creq_func_event *aeqe) | |
| 608 | +{ | |
| 609 | + switch (aeqe->event) { | |
| 610 | + case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: | |
| 611 | + break; | |
| 612 | + case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: | |
| 613 | + break; | |
| 614 | + case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: | |
| 615 | + break; | |
| 616 | + case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: | |
| 617 | + break; | |
| 618 | + case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: | |
| 619 | + break; | |
| 620 | + case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: | |
| 621 | + break; | |
| 622 | + case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: | |
| 623 | + break; | |
| 624 | + case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: | |
| 625 | + break; | |
| 626 | + case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: | |
| 627 | + break; | |
| 628 | + case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: | |
| 629 | + break; | |
| 630 | + case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: | |
| 631 | + break; | |
| 632 | + default: | |
| 633 | + return -EINVAL; | |
| 634 | + } | |
| 635 | + return 0; | |
| 636 | +} | |
| 637 | + | |
| 638 | +static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, | |
| 639 | + struct bnxt_qplib_cq *handle) | |
| 640 | +{ | |
| 641 | + struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq, | |
| 642 | + qplib_cq); | |
| 643 | + | |
| 644 | + if (!cq) { | |
| 645 | + dev_err(NULL, "%s: CQ is NULL, CQN not handled", | |
| 646 | + ROCE_DRV_MODULE_NAME); | |
| 647 | + return -EINVAL; | |
| 648 | + } | |
| 649 | + if (cq->ib_cq.comp_handler) { | |
| 650 | + /* Lock comp_handler? */ | |
| 651 | + (*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context); | |
| 652 | + } | |
| 653 | + | |
| 654 | + return 0; | |
| 655 | +} | |
| 656 | + | |
| 657 | +static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev) | |
| 658 | +{ | |
| 659 | + if (rdev->nq.hwq.max_elements) | |
| 660 | + bnxt_qplib_disable_nq(&rdev->nq); | |
| 661 | + | |
| 662 | + if (rdev->qplib_res.rcfw) | |
| 663 | + bnxt_qplib_cleanup_res(&rdev->qplib_res); | |
| 664 | +} | |
| 665 | + | |
| 666 | +static int bnxt_re_init_res(struct bnxt_re_dev *rdev) | |
| 667 | +{ | |
| 668 | + int rc = 0; | |
| 669 | + | |
| 670 | + bnxt_qplib_init_res(&rdev->qplib_res); | |
| 671 | + | |
| 672 | + if (rdev->msix_entries[BNXT_RE_NQ_IDX].vector <= 0) | |
| 673 | + return -EINVAL; | |
| 674 | + | |
| 675 | + rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq, | |
| 676 | + rdev->msix_entries[BNXT_RE_NQ_IDX].vector, | |
| 677 | + rdev->msix_entries[BNXT_RE_NQ_IDX].db_offset, | |
| 678 | + &bnxt_re_cqn_handler, | |
| 679 | + NULL); | |
| 680 | + | |
| 681 | + if (rc) | |
| 682 | + dev_err(rdev_to_dev(rdev), "Failed to enable NQ: %#x", rc); | |
| 683 | + | |
| 684 | + return rc; | |
| 685 | +} | |
| 686 | + | |
| 687 | +static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait) | |
| 688 | +{ | |
| 689 | + if (rdev->nq.hwq.max_elements) { | |
| 690 | + bnxt_re_net_ring_free(rdev, rdev->nq.ring_id, lock_wait); | |
| 691 | + bnxt_qplib_free_nq(&rdev->nq); | |
| 692 | + } | |
| 693 | + if (rdev->qplib_res.dpi_tbl.max) { | |
| 694 | + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, | |
| 695 | + &rdev->qplib_res.dpi_tbl, | |
| 696 | + &rdev->dpi_privileged); | |
| 697 | + } | |
| 698 | + if (rdev->qplib_res.rcfw) { | |
| 699 | + bnxt_qplib_free_res(&rdev->qplib_res); | |
| 700 | + rdev->qplib_res.rcfw = NULL; | |
| 701 | + } | |
| 702 | +} | |
| 703 | + | |
| 704 | +static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) | |
| 705 | +{ | |
| 706 | + int rc = 0; | |
| 707 | + | |
| 708 | + /* Configure and allocate resources for qplib */ | |
| 709 | + rdev->qplib_res.rcfw = &rdev->rcfw; | |
| 710 | + rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); | |
| 711 | + if (rc) | |
| 712 | + goto fail; | |
| 713 | + | |
| 714 | + rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev, | |
| 715 | + rdev->netdev, &rdev->dev_attr); | |
| 716 | + if (rc) | |
| 717 | + goto fail; | |
| 718 | + | |
| 719 | + rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, | |
| 720 | + &rdev->dpi_privileged, | |
| 721 | + rdev); | |
| 722 | + if (rc) | |
| 723 | + goto fail; | |
| 724 | + | |
| 725 | + rdev->nq.hwq.max_elements = BNXT_RE_MAX_CQ_COUNT + | |
| 726 | + BNXT_RE_MAX_SRQC_COUNT + 2; | |
| 727 | + rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq); | |
| 728 | + if (rc) { | |
| 729 | + dev_err(rdev_to_dev(rdev), | |
| 730 | + "Failed to allocate NQ memory: %#x", rc); | |
| 731 | + goto fail; | |
| 732 | + } | |
| 733 | + rc = bnxt_re_net_ring_alloc | |
| 734 | + (rdev, rdev->nq.hwq.pbl[PBL_LVL_0].pg_map_arr, | |
| 735 | + rdev->nq.hwq.pbl[rdev->nq.hwq.level].pg_count, | |
| 736 | + HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_NQE_MAX_CNT - 1, | |
| 737 | + rdev->msix_entries[BNXT_RE_NQ_IDX].ring_idx, | |
| 738 | + &rdev->nq.ring_id); | |
| 739 | + if (rc) { | |
| 740 | + dev_err(rdev_to_dev(rdev), | |
| 741 | + "Failed to allocate NQ ring: %#x", rc); | |
| 742 | + goto free_nq; | |
| 743 | + } | |
| 744 | + return 0; | |
| 745 | +free_nq: | |
| 746 | + bnxt_qplib_free_nq(&rdev->nq); | |
| 747 | +fail: | |
| 748 | + rdev->qplib_res.rcfw = NULL; | |
| 749 | + return rc; | |
| 750 | +} | |
| 751 | + | |
| 752 | +static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, | |
| 753 | + u8 port_num, enum ib_event_type event) | |
| 754 | +{ | |
| 755 | + struct ib_event ib_event; | |
| 756 | + | |
| 757 | + ib_event.device = ibdev; | |
| 758 | + if (qp) | |
| 759 | + ib_event.element.qp = qp; | |
| 760 | + else | |
| 761 | + ib_event.element.port_num = port_num; | |
| 762 | + ib_event.event = event; | |
| 763 | + ib_dispatch_event(&ib_event); | |
| 764 | +} | |
| 765 | + | |
| 766 | +#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02 | |
| 767 | +static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir, | |
| 768 | + u64 *cid_map) | |
| 769 | +{ | |
| 770 | + struct hwrm_queue_pri2cos_qcfg_input req = {0}; | |
| 771 | + struct bnxt *bp = netdev_priv(rdev->netdev); | |
| 772 | + struct hwrm_queue_pri2cos_qcfg_output resp; | |
| 773 | + struct bnxt_en_dev *en_dev = rdev->en_dev; | |
| 774 | + struct bnxt_fw_msg fw_msg; | |
| 775 | + u32 flags = 0; | |
| 776 | + u8 *qcfgmap, *tmp_map; | |
| 777 | + int rc = 0, i; | |
| 778 | + | |
| 779 | + if (!cid_map) | |
| 780 | + return -EINVAL; | |
| 781 | + | |
| 782 | + memset(&fw_msg, 0, sizeof(fw_msg)); | |
| 783 | + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, | |
| 784 | + HWRM_QUEUE_PRI2COS_QCFG, -1, -1); | |
| 785 | + flags |= (dir & 0x01); | |
| 786 | + flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN; | |
| 787 | + req.flags = cpu_to_le32(flags); | |
| 788 | + req.port_id = bp->pf.port_id; | |
| 789 | + | |
| 790 | + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
| 791 | + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
| 792 | + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
| 793 | + if (rc) | |
| 794 | + return rc; | |
| 795 | + | |
| 796 | + if (resp.queue_cfg_info) { | |
| 797 | + dev_warn(rdev_to_dev(rdev), | |
| 798 | + "Asymmetric cos queue configuration detected"); | |
| 799 | + dev_warn(rdev_to_dev(rdev), | |
| 800 | + " on device, QoS may not be fully functional\n"); | |
| 801 | + } | |
| 802 | + qcfgmap = &resp.pri0_cos_queue_id; | |
| 803 | + tmp_map = (u8 *)cid_map; | |
| 804 | + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) | |
| 805 | + tmp_map[i] = qcfgmap[i]; | |
| 806 | + | |
| 807 | + return rc; | |
| 808 | +} | |
| 809 | + | |
| 810 | +static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, | |
| 811 | + struct bnxt_re_qp *qp) | |
| 812 | +{ | |
| 813 | + return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp); | |
| 814 | +} | |
| 815 | + | |
| 816 | +static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev) | |
| 817 | +{ | |
| 818 | + int mask = IB_QP_STATE; | |
| 819 | + struct ib_qp_attr qp_attr; | |
| 820 | + struct bnxt_re_qp *qp; | |
| 821 | + | |
| 822 | + qp_attr.qp_state = IB_QPS_ERR; | |
| 823 | + mutex_lock(&rdev->qp_lock); | |
| 824 | + list_for_each_entry(qp, &rdev->qp_list, list) { | |
| 825 | + /* Modify the state of all QPs except QP1/Shadow QP */ | |
| 826 | + if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { | |
| 827 | + if (qp->qplib_qp.state != | |
| 828 | + CMDQ_MODIFY_QP_NEW_STATE_RESET && | |
| 829 | + qp->qplib_qp.state != | |
| 830 | + CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
| 831 | + bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, | |
| 832 | + 1, IB_EVENT_QP_FATAL); | |
| 833 | + bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, | |
| 834 | + NULL); | |
| 835 | + } | |
| 836 | + } | |
| 837 | + } | |
| 838 | + mutex_unlock(&rdev->qp_lock); | |
| 839 | +} | |
| 840 | + | |
| 841 | +static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev) | |
| 842 | +{ | |
| 843 | + u32 prio_map = 0, tmp_map = 0; | |
| 844 | + struct net_device *netdev; | |
| 845 | + struct dcb_app app; | |
| 846 | + | |
| 847 | + netdev = rdev->netdev; | |
| 848 | + | |
| 849 | + memset(&app, 0, sizeof(app)); | |
| 850 | + app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE; | |
| 851 | + app.protocol = ETH_P_IBOE; | |
| 852 | + tmp_map = dcb_ieee_getapp_mask(netdev, &app); | |
| 853 | + prio_map = tmp_map; | |
| 854 | + | |
| 855 | + app.selector = IEEE_8021QAZ_APP_SEL_DGRAM; | |
| 856 | + app.protocol = ROCE_V2_UDP_DPORT; | |
| 857 | + tmp_map = dcb_ieee_getapp_mask(netdev, &app); | |
| 858 | + prio_map |= tmp_map; | |
| 859 | + | |
| 860 | + if (!prio_map) | |
| 861 | + prio_map = -EFAULT; | |
| 862 | + return prio_map; | |
| 863 | +} | |
| 864 | + | |
| 865 | +static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq) | |
| 866 | +{ | |
| 867 | + u16 prio; | |
| 868 | + u8 id; | |
| 869 | + | |
| 870 | + for (prio = 0, id = 0; prio < 8; prio++) { | |
| 871 | + if (prio_map & (1 << prio)) { | |
| 872 | + cosq[id] = cid_map[prio]; | |
| 873 | + id++; | |
| 874 | + if (id == 2) /* Max 2 tcs supported */ | |
| 875 | + break; | |
| 876 | + } | |
| 877 | + } | |
| 878 | +} | |
| 879 | + | |
| 880 | +static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev) | |
| 881 | +{ | |
| 882 | + u8 prio_map = 0; | |
| 883 | + u64 cid_map; | |
| 884 | + int rc; | |
| 885 | + | |
| 886 | + /* Get priority for roce */ | |
| 887 | + rc = bnxt_re_get_priority_mask(rdev); | |
| 888 | + if (rc < 0) | |
| 889 | + return rc; | |
| 890 | + prio_map = (u8)rc; | |
| 891 | + | |
| 892 | + if (prio_map == rdev->cur_prio_map) | |
| 893 | + return 0; | |
| 894 | + rdev->cur_prio_map = prio_map; | |
| 895 | + /* Get cosq id for this priority */ | |
| 896 | + rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map); | |
| 897 | + if (rc) { | |
| 898 | + dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map); | |
| 899 | + return rc; | |
| 900 | + } | |
| 901 | + /* Parse CoS IDs for app priority */ | |
| 902 | + bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq); | |
| 903 | + | |
| 904 | + /* Config BONO. */ | |
| 905 | + rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq); | |
| 906 | + if (rc) { | |
| 907 | + dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n", | |
| 908 | + rdev->cosq[0], rdev->cosq[1]); | |
| 909 | + return rc; | |
| 910 | + } | |
| 911 | + | |
| 912 | + return 0; | |
| 913 | +} | |
| 914 | + | |
| 915 | +static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait) | |
| 916 | +{ | |
| 917 | + int i, rc; | |
| 918 | + | |
| 919 | + if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { | |
| 920 | + for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) | |
| 921 | + device_remove_file(&rdev->ibdev.dev, | |
| 922 | + bnxt_re_attributes[i]); | |
| 923 | + /* Cleanup ib dev */ | |
| 924 | + bnxt_re_unregister_ib(rdev); | |
| 925 | + } | |
| 926 | + if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags)) | |
| 927 | + cancel_delayed_work(&rdev->worker); | |
| 928 | + | |
| 929 | + bnxt_re_cleanup_res(rdev); | |
| 930 | + bnxt_re_free_res(rdev, lock_wait); | |
| 931 | + | |
| 932 | + if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) { | |
| 933 | + rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); | |
| 934 | + if (rc) | |
| 935 | + dev_warn(rdev_to_dev(rdev), | |
| 936 | + "Failed to deinitialize RCFW: %#x", rc); | |
| 937 | + bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, | |
| 938 | + lock_wait); | |
| 939 | + bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); | |
| 940 | + bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); | |
| 941 | + bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, lock_wait); | |
| 942 | + bnxt_qplib_free_rcfw_channel(&rdev->rcfw); | |
| 943 | + } | |
| 944 | + if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) { | |
| 945 | + rc = bnxt_re_free_msix(rdev, lock_wait); | |
| 946 | + if (rc) | |
| 947 | + dev_warn(rdev_to_dev(rdev), | |
| 948 | + "Failed to free MSI-X vectors: %#x", rc); | |
| 949 | + } | |
| 950 | + if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { | |
| 951 | + rc = bnxt_re_unregister_netdev(rdev, lock_wait); | |
| 952 | + if (rc) | |
| 953 | + dev_warn(rdev_to_dev(rdev), | |
| 954 | + "Failed to unregister with netdev: %#x", rc); | |
| 955 | + } | |
| 956 | +} | |
| 957 | + | |
| 958 | +static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) | |
| 959 | +{ | |
| 960 | + u32 i; | |
| 961 | + | |
| 962 | + rdev->qplib_ctx.qpc_count = BNXT_RE_MAX_QPC_COUNT; | |
| 963 | + rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT; | |
| 964 | + rdev->qplib_ctx.srqc_count = BNXT_RE_MAX_SRQC_COUNT; | |
| 965 | + rdev->qplib_ctx.cq_count = BNXT_RE_MAX_CQ_COUNT; | |
| 966 | + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) | |
| 967 | + rdev->qplib_ctx.tqm_count[i] = | |
| 968 | + rdev->dev_attr.tqm_alloc_reqs[i]; | |
| 969 | +} | |
| 970 | + | |
| 971 | +/* worker thread for polling periodic events. Now used for QoS programming*/ | |
| 972 | +static void bnxt_re_worker(struct work_struct *work) | |
| 973 | +{ | |
| 974 | + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, | |
| 975 | + worker.work); | |
| 976 | + | |
| 977 | + bnxt_re_setup_qos(rdev); | |
| 978 | + schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); | |
| 979 | +} | |
| 980 | + | |
| 981 | +static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev) | |
| 982 | +{ | |
| 983 | + int i, j, rc; | |
| 984 | + | |
| 985 | + /* Registered a new RoCE device instance to netdev */ | |
| 986 | + rc = bnxt_re_register_netdev(rdev); | |
| 987 | + if (rc) { | |
| 988 | + pr_err("Failed to register with netedev: %#x\n", rc); | |
| 989 | + return -EINVAL; | |
| 990 | + } | |
| 991 | + set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); | |
| 992 | + | |
| 993 | + rc = bnxt_re_request_msix(rdev); | |
| 994 | + if (rc) { | |
| 995 | + pr_err("Failed to get MSI-X vectors: %#x\n", rc); | |
| 996 | + rc = -EINVAL; | |
| 997 | + goto fail; | |
| 998 | + } | |
| 999 | + set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); | |
| 1000 | + | |
| 1001 | + /* Establish RCFW Communication Channel to initialize the context | |
| 1002 | + * memory for the function and all child VFs | |
| 1003 | + */ | |
| 1004 | + rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw); | |
| 1005 | + if (rc) | |
| 1006 | + goto fail; | |
| 1007 | + | |
| 1008 | + rc = bnxt_re_net_ring_alloc | |
| 1009 | + (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr, | |
| 1010 | + rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count, | |
| 1011 | + HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1, | |
| 1012 | + rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx, | |
| 1013 | + &rdev->rcfw.creq_ring_id); | |
| 1014 | + if (rc) { | |
| 1015 | + pr_err("Failed to allocate CREQ: %#x\n", rc); | |
| 1016 | + goto free_rcfw; | |
| 1017 | + } | |
| 1018 | + rc = bnxt_qplib_enable_rcfw_channel | |
| 1019 | + (rdev->en_dev->pdev, &rdev->rcfw, | |
| 1020 | + rdev->msix_entries[BNXT_RE_AEQ_IDX].vector, | |
| 1021 | + rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset, | |
| 1022 | + 0, &bnxt_re_aeq_handler); | |
| 1023 | + if (rc) { | |
| 1024 | + pr_err("Failed to enable RCFW channel: %#x\n", rc); | |
| 1025 | + goto free_ring; | |
| 1026 | + } | |
| 1027 | + | |
| 1028 | + rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); | |
| 1029 | + if (rc) | |
| 1030 | + goto disable_rcfw; | |
| 1031 | + bnxt_re_set_resource_limits(rdev); | |
| 1032 | + | |
| 1033 | + rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0); | |
| 1034 | + if (rc) { | |
| 1035 | + pr_err("Failed to allocate QPLIB context: %#x\n", rc); | |
| 1036 | + goto disable_rcfw; | |
| 1037 | + } | |
| 1038 | + rc = bnxt_re_net_stats_ctx_alloc(rdev, | |
| 1039 | + rdev->qplib_ctx.stats.dma_map, | |
| 1040 | + &rdev->qplib_ctx.stats.fw_id); | |
| 1041 | + if (rc) { | |
| 1042 | + pr_err("Failed to allocate stats context: %#x\n", rc); | |
| 1043 | + goto free_ctx; | |
| 1044 | + } | |
| 1045 | + | |
| 1046 | + rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, 0); | |
| 1047 | + if (rc) { | |
| 1048 | + pr_err("Failed to initialize RCFW: %#x\n", rc); | |
| 1049 | + goto free_sctx; | |
| 1050 | + } | |
| 1051 | + set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); | |
| 1052 | + | |
| 1053 | + /* Resources based on the 'new' device caps */ | |
| 1054 | + rc = bnxt_re_alloc_res(rdev); | |
| 1055 | + if (rc) { | |
| 1056 | + pr_err("Failed to allocate resources: %#x\n", rc); | |
| 1057 | + goto fail; | |
| 1058 | + } | |
| 1059 | + rc = bnxt_re_init_res(rdev); | |
| 1060 | + if (rc) { | |
| 1061 | + pr_err("Failed to initialize resources: %#x\n", rc); | |
| 1062 | + goto fail; | |
| 1063 | + } | |
| 1064 | + | |
| 1065 | + rc = bnxt_re_setup_qos(rdev); | |
| 1066 | + if (rc) | |
| 1067 | + pr_info("RoCE priority not yet configured\n"); | |
| 1068 | + | |
| 1069 | + INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); | |
| 1070 | + set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags); | |
| 1071 | + schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); | |
| 1072 | + | |
| 1073 | + /* Register ib dev */ | |
| 1074 | + rc = bnxt_re_register_ib(rdev); | |
| 1075 | + if (rc) { | |
| 1076 | + pr_err("Failed to register with IB: %#x\n", rc); | |
| 1077 | + goto fail; | |
| 1078 | + } | |
| 1079 | + dev_info(rdev_to_dev(rdev), "Device registered successfully"); | |
| 1080 | + for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { | |
| 1081 | + rc = device_create_file(&rdev->ibdev.dev, | |
| 1082 | + bnxt_re_attributes[i]); | |
| 1083 | + if (rc) { | |
| 1084 | + dev_err(rdev_to_dev(rdev), | |
| 1085 | + "Failed to create IB sysfs: %#x", rc); | |
| 1086 | + /* Must clean up all created device files */ | |
| 1087 | + for (j = 0; j < i; j++) | |
| 1088 | + device_remove_file(&rdev->ibdev.dev, | |
| 1089 | + bnxt_re_attributes[j]); | |
| 1090 | + bnxt_re_unregister_ib(rdev); | |
| 1091 | + goto fail; | |
| 1092 | + } | |
| 1093 | + } | |
| 1094 | + set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); | |
| 1095 | + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); | |
| 1096 | + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); | |
| 1097 | + | |
| 1098 | + return 0; | |
| 1099 | +free_sctx: | |
| 1100 | + bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, true); | |
| 1101 | +free_ctx: | |
| 1102 | + bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); | |
| 1103 | +disable_rcfw: | |
| 1104 | + bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); | |
| 1105 | +free_ring: | |
| 1106 | + bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, true); | |
| 1107 | +free_rcfw: | |
| 1108 | + bnxt_qplib_free_rcfw_channel(&rdev->rcfw); | |
| 1109 | +fail: | |
| 1110 | + bnxt_re_ib_unreg(rdev, true); | |
| 1111 | + return rc; | |
| 1112 | +} | |
| 1113 | + | |
| 1114 | +static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) | |
| 1115 | +{ | |
| 1116 | + struct bnxt_en_dev *en_dev = rdev->en_dev; | |
| 1117 | + struct net_device *netdev = rdev->netdev; | |
| 1118 | + | |
| 1119 | + bnxt_re_dev_remove(rdev); | |
| 1120 | + | |
| 1121 | + if (netdev) | |
| 1122 | + bnxt_re_dev_unprobe(netdev, en_dev); | |
| 1123 | +} | |
| 1124 | + | |
| 1125 | +static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev) | |
| 1126 | +{ | |
| 1127 | + struct bnxt_en_dev *en_dev; | |
| 1128 | + int rc = 0; | |
| 1129 | + | |
| 1130 | + if (!is_bnxt_re_dev(netdev)) | |
| 1131 | + return -ENODEV; | |
| 1132 | + | |
| 1133 | + en_dev = bnxt_re_dev_probe(netdev); | |
| 1134 | + if (IS_ERR(en_dev)) { | |
| 1135 | + if (en_dev != ERR_PTR(-ENODEV)) | |
| 1136 | + pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME); | |
| 1137 | + rc = PTR_ERR(en_dev); | |
| 1138 | + goto exit; | |
| 1139 | + } | |
| 1140 | + *rdev = bnxt_re_dev_add(netdev, en_dev); | |
| 1141 | + if (!*rdev) { | |
| 1142 | + rc = -ENOMEM; | |
| 1143 | + bnxt_re_dev_unprobe(netdev, en_dev); | |
| 1144 | + goto exit; | |
| 1145 | + } | |
| 1146 | +exit: | |
| 1147 | + return rc; | |
| 1148 | +} | |
| 1149 | + | |
| 1150 | +static void bnxt_re_remove_one(struct bnxt_re_dev *rdev) | |
| 1151 | +{ | |
| 1152 | + pci_dev_put(rdev->en_dev->pdev); | |
| 1153 | +} | |
| 1154 | + | |
| 1155 | +/* Handle all deferred netevents tasks */ | |
| 1156 | +static void bnxt_re_task(struct work_struct *work) | |
| 1157 | +{ | |
| 1158 | + struct bnxt_re_work *re_work; | |
| 1159 | + struct bnxt_re_dev *rdev; | |
| 1160 | + int rc = 0; | |
| 1161 | + | |
| 1162 | + re_work = container_of(work, struct bnxt_re_work, work); | |
| 1163 | + rdev = re_work->rdev; | |
| 1164 | + | |
| 1165 | + if (re_work->event != NETDEV_REGISTER && | |
| 1166 | + !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) | |
| 1167 | + return; | |
| 1168 | + | |
| 1169 | + switch (re_work->event) { | |
| 1170 | + case NETDEV_REGISTER: | |
| 1171 | + rc = bnxt_re_ib_reg(rdev); | |
| 1172 | + if (rc) | |
| 1173 | + dev_err(rdev_to_dev(rdev), | |
| 1174 | + "Failed to register with IB: %#x", rc); | |
| 1175 | + break; | |
| 1176 | + case NETDEV_UP: | |
| 1177 | + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, | |
| 1178 | + IB_EVENT_PORT_ACTIVE); | |
| 1179 | + break; | |
| 1180 | + case NETDEV_DOWN: | |
| 1181 | + bnxt_re_dev_stop(rdev); | |
| 1182 | + break; | |
| 1183 | + case NETDEV_CHANGE: | |
| 1184 | + if (!netif_carrier_ok(rdev->netdev)) | |
| 1185 | + bnxt_re_dev_stop(rdev); | |
| 1186 | + else if (netif_carrier_ok(rdev->netdev)) | |
| 1187 | + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, | |
| 1188 | + IB_EVENT_PORT_ACTIVE); | |
| 1189 | + break; | |
| 1190 | + default: | |
| 1191 | + break; | |
| 1192 | + } | |
| 1193 | + kfree(re_work); | |
| 1194 | +} | |
| 1195 | + | |
| 1196 | +static void bnxt_re_init_one(struct bnxt_re_dev *rdev) | |
| 1197 | +{ | |
| 1198 | + pci_dev_get(rdev->en_dev->pdev); | |
| 1199 | +} | |
| 1200 | + | |
| 1201 | +/* | |
| 1202 | + * "Notifier chain callback can be invoked for the same chain from | |
| 1203 | + * different CPUs at the same time". | |
| 1204 | + * | |
| 1205 | + * For cases when the netdev is already present, our call to the | |
| 1206 | + * register_netdevice_notifier() will actually get the rtnl_lock() | |
| 1207 | + * before sending NETDEV_REGISTER and (if up) NETDEV_UP | |
| 1208 | + * events. | |
| 1209 | + * | |
| 1210 | + * But for cases when the netdev is not already present, the notifier | |
| 1211 | + * chain is subjected to be invoked from different CPUs simultaneously. | |
| 1212 | + * | |
| 1213 | + * This is protected by the netdev_mutex. | |
| 1214 | + */ | |
| 1215 | +static int bnxt_re_netdev_event(struct notifier_block *notifier, | |
| 1216 | + unsigned long event, void *ptr) | |
| 1217 | +{ | |
| 1218 | + struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr); | |
| 1219 | + struct bnxt_re_work *re_work; | |
| 1220 | + struct bnxt_re_dev *rdev; | |
| 1221 | + int rc = 0; | |
| 1222 | + bool sch_work = false; | |
| 1223 | + | |
| 1224 | + real_dev = rdma_vlan_dev_real_dev(netdev); | |
| 1225 | + if (!real_dev) | |
| 1226 | + real_dev = netdev; | |
| 1227 | + | |
| 1228 | + rdev = bnxt_re_from_netdev(real_dev); | |
| 1229 | + if (!rdev && event != NETDEV_REGISTER) | |
| 1230 | + goto exit; | |
| 1231 | + if (real_dev != netdev) | |
| 1232 | + goto exit; | |
| 1233 | + | |
| 1234 | + switch (event) { | |
| 1235 | + case NETDEV_REGISTER: | |
| 1236 | + if (rdev) | |
| 1237 | + break; | |
| 1238 | + rc = bnxt_re_dev_reg(&rdev, real_dev); | |
| 1239 | + if (rc == -ENODEV) | |
| 1240 | + break; | |
| 1241 | + if (rc) { | |
| 1242 | + pr_err("Failed to register with the device %s: %#x\n", | |
| 1243 | + real_dev->name, rc); | |
| 1244 | + break; | |
| 1245 | + } | |
| 1246 | + bnxt_re_init_one(rdev); | |
| 1247 | + sch_work = true; | |
| 1248 | + break; | |
| 1249 | + | |
| 1250 | + case NETDEV_UNREGISTER: | |
| 1251 | + bnxt_re_ib_unreg(rdev, false); | |
| 1252 | + bnxt_re_remove_one(rdev); | |
| 1253 | + bnxt_re_dev_unreg(rdev); | |
| 1254 | + break; | |
| 1255 | + | |
| 1256 | + default: | |
| 1257 | + sch_work = true; | |
| 1258 | + break; | |
| 1259 | + } | |
| 1260 | + if (sch_work) { | |
| 1261 | + /* Allocate for the deferred task */ | |
| 1262 | + re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC); | |
| 1263 | + if (re_work) { | |
| 1264 | + re_work->rdev = rdev; | |
| 1265 | + re_work->event = event; | |
| 1266 | + re_work->vlan_dev = (real_dev == netdev ? | |
| 1267 | + NULL : netdev); | |
| 1268 | + INIT_WORK(&re_work->work, bnxt_re_task); | |
| 1269 | + queue_work(bnxt_re_wq, &re_work->work); | |
| 1270 | + } | |
| 1271 | + } | |
| 1272 | + | |
| 1273 | +exit: | |
| 1274 | + return NOTIFY_DONE; | |
| 1275 | +} | |
| 1276 | + | |
| 1277 | +static struct notifier_block bnxt_re_netdev_notifier = { | |
| 1278 | + .notifier_call = bnxt_re_netdev_event | |
| 1279 | +}; | |
| 1280 | + | |
| 1281 | +static int __init bnxt_re_mod_init(void) | |
| 1282 | +{ | |
| 1283 | + int rc = 0; | |
| 1284 | + | |
| 1285 | + pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version); | |
| 1286 | + | |
| 1287 | + bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); | |
| 1288 | + if (!bnxt_re_wq) | |
| 1289 | + return -ENOMEM; | |
| 1290 | + | |
| 1291 | + INIT_LIST_HEAD(&bnxt_re_dev_list); | |
| 1292 | + | |
| 1293 | + rc = register_netdevice_notifier(&bnxt_re_netdev_notifier); | |
| 1294 | + if (rc) { | |
| 1295 | + pr_err("%s: Cannot register to netdevice_notifier", | |
| 1296 | + ROCE_DRV_MODULE_NAME); | |
| 1297 | + goto err_netdev; | |
| 1298 | + } | |
| 1299 | + return 0; | |
| 1300 | + | |
| 1301 | +err_netdev: | |
| 1302 | + destroy_workqueue(bnxt_re_wq); | |
| 1303 | + | |
| 1304 | + return rc; | |
| 1305 | +} | |
| 1306 | + | |
| 1307 | +static void __exit bnxt_re_mod_exit(void) | |
| 1308 | +{ | |
| 1309 | + unregister_netdevice_notifier(&bnxt_re_netdev_notifier); | |
| 1310 | + if (bnxt_re_wq) | |
| 1311 | + destroy_workqueue(bnxt_re_wq); | |
| 1312 | +} | |
| 1313 | + | |
| 1314 | +module_init(bnxt_re_mod_init); | |
| 1315 | +module_exit(bnxt_re_mod_exit); | |
| @@ -0,0 +1,2167 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: Fast Path Operators | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#include <linux/interrupt.h> | |
| 40 | +#include <linux/spinlock.h> | |
| 41 | +#include <linux/sched.h> | |
| 42 | +#include <linux/slab.h> | |
| 43 | +#include <linux/pci.h> | |
| 44 | +#include <linux/prefetch.h> | |
| 45 | + | |
| 46 | +#include "roce_hsi.h" | |
| 47 | + | |
| 48 | +#include "qplib_res.h" | |
| 49 | +#include "qplib_rcfw.h" | |
| 50 | +#include "qplib_sp.h" | |
| 51 | +#include "qplib_fp.h" | |
| 52 | + | |
| 53 | +static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq); | |
| 54 | + | |
| 55 | +static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, | |
| 56 | + struct bnxt_qplib_qp *qp) | |
| 57 | +{ | |
| 58 | + struct bnxt_qplib_q *rq = &qp->rq; | |
| 59 | + struct bnxt_qplib_q *sq = &qp->sq; | |
| 60 | + | |
| 61 | + if (qp->rq_hdr_buf) | |
| 62 | + dma_free_coherent(&res->pdev->dev, | |
| 63 | + rq->hwq.max_elements * qp->rq_hdr_buf_size, | |
| 64 | + qp->rq_hdr_buf, qp->rq_hdr_buf_map); | |
| 65 | + if (qp->sq_hdr_buf) | |
| 66 | + dma_free_coherent(&res->pdev->dev, | |
| 67 | + sq->hwq.max_elements * qp->sq_hdr_buf_size, | |
| 68 | + qp->sq_hdr_buf, qp->sq_hdr_buf_map); | |
| 69 | + qp->rq_hdr_buf = NULL; | |
| 70 | + qp->sq_hdr_buf = NULL; | |
| 71 | + qp->rq_hdr_buf_map = 0; | |
| 72 | + qp->sq_hdr_buf_map = 0; | |
| 73 | + qp->sq_hdr_buf_size = 0; | |
| 74 | + qp->rq_hdr_buf_size = 0; | |
| 75 | +} | |
| 76 | + | |
| 77 | +static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, | |
| 78 | + struct bnxt_qplib_qp *qp) | |
| 79 | +{ | |
| 80 | + struct bnxt_qplib_q *rq = &qp->rq; | |
| 81 | + struct bnxt_qplib_q *sq = &qp->rq; | |
| 82 | + int rc = 0; | |
| 83 | + | |
| 84 | + if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { | |
| 85 | + qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, | |
| 86 | + sq->hwq.max_elements * | |
| 87 | + qp->sq_hdr_buf_size, | |
| 88 | + &qp->sq_hdr_buf_map, GFP_KERNEL); | |
| 89 | + if (!qp->sq_hdr_buf) { | |
| 90 | + rc = -ENOMEM; | |
| 91 | + dev_err(&res->pdev->dev, | |
| 92 | + "QPLIB: Failed to create sq_hdr_buf"); | |
| 93 | + goto fail; | |
| 94 | + } | |
| 95 | + } | |
| 96 | + | |
| 97 | + if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { | |
| 98 | + qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, | |
| 99 | + rq->hwq.max_elements * | |
| 100 | + qp->rq_hdr_buf_size, | |
| 101 | + &qp->rq_hdr_buf_map, | |
| 102 | + GFP_KERNEL); | |
| 103 | + if (!qp->rq_hdr_buf) { | |
| 104 | + rc = -ENOMEM; | |
| 105 | + dev_err(&res->pdev->dev, | |
| 106 | + "QPLIB: Failed to create rq_hdr_buf"); | |
| 107 | + goto fail; | |
| 108 | + } | |
| 109 | + } | |
| 110 | + return 0; | |
| 111 | + | |
| 112 | +fail: | |
| 113 | + bnxt_qplib_free_qp_hdr_buf(res, qp); | |
| 114 | + return rc; | |
| 115 | +} | |
| 116 | + | |
| 117 | +static void bnxt_qplib_service_nq(unsigned long data) | |
| 118 | +{ | |
| 119 | + struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; | |
| 120 | + struct bnxt_qplib_hwq *hwq = &nq->hwq; | |
| 121 | + struct nq_base *nqe, **nq_ptr; | |
| 122 | + int num_cqne_processed = 0; | |
| 123 | + u32 sw_cons, raw_cons; | |
| 124 | + u16 type; | |
| 125 | + int budget = nq->budget; | |
| 126 | + u64 q_handle; | |
| 127 | + | |
| 128 | + /* Service the NQ until empty */ | |
| 129 | + raw_cons = hwq->cons; | |
| 130 | + while (budget--) { | |
| 131 | + sw_cons = HWQ_CMP(raw_cons, hwq); | |
| 132 | + nq_ptr = (struct nq_base **)hwq->pbl_ptr; | |
| 133 | + nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]; | |
| 134 | + if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) | |
| 135 | + break; | |
| 136 | + | |
| 137 | + type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; | |
| 138 | + switch (type) { | |
| 139 | + case NQ_BASE_TYPE_CQ_NOTIFICATION: | |
| 140 | + { | |
| 141 | + struct nq_cn *nqcne = (struct nq_cn *)nqe; | |
| 142 | + | |
| 143 | + q_handle = le32_to_cpu(nqcne->cq_handle_low); | |
| 144 | + q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) | |
| 145 | + << 32; | |
| 146 | + bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *) | |
| 147 | + ((unsigned long)q_handle)); | |
| 148 | + if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *) | |
| 149 | + ((unsigned long)q_handle))) | |
| 150 | + num_cqne_processed++; | |
| 151 | + else | |
| 152 | + dev_warn(&nq->pdev->dev, | |
| 153 | + "QPLIB: cqn - type 0x%x not handled", | |
| 154 | + type); | |
| 155 | + break; | |
| 156 | + } | |
| 157 | + case NQ_BASE_TYPE_DBQ_EVENT: | |
| 158 | + break; | |
| 159 | + default: | |
| 160 | + dev_warn(&nq->pdev->dev, | |
| 161 | + "QPLIB: nqe with type = 0x%x not handled", | |
| 162 | + type); | |
| 163 | + break; | |
| 164 | + } | |
| 165 | + raw_cons++; | |
| 166 | + } | |
| 167 | + if (hwq->cons != raw_cons) { | |
| 168 | + hwq->cons = raw_cons; | |
| 169 | + NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements); | |
| 170 | + } | |
| 171 | +} | |
| 172 | + | |
| 173 | +static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) | |
| 174 | +{ | |
| 175 | + struct bnxt_qplib_nq *nq = dev_instance; | |
| 176 | + struct bnxt_qplib_hwq *hwq = &nq->hwq; | |
| 177 | + struct nq_base **nq_ptr; | |
| 178 | + u32 sw_cons; | |
| 179 | + | |
| 180 | + /* Prefetch the NQ element */ | |
| 181 | + sw_cons = HWQ_CMP(hwq->cons, hwq); | |
| 182 | + nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr; | |
| 183 | + prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]); | |
| 184 | + | |
| 185 | + /* Fan out to CPU affinitized kthreads? */ | |
| 186 | + tasklet_schedule(&nq->worker); | |
| 187 | + | |
| 188 | + return IRQ_HANDLED; | |
| 189 | +} | |
| 190 | + | |
| 191 | +void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) | |
| 192 | +{ | |
| 193 | + /* Make sure the HW is stopped! */ | |
| 194 | + synchronize_irq(nq->vector); | |
| 195 | + tasklet_disable(&nq->worker); | |
| 196 | + tasklet_kill(&nq->worker); | |
| 197 | + | |
| 198 | + if (nq->requested) { | |
| 199 | + free_irq(nq->vector, nq); | |
| 200 | + nq->requested = false; | |
| 201 | + } | |
| 202 | + if (nq->bar_reg_iomem) | |
| 203 | + iounmap(nq->bar_reg_iomem); | |
| 204 | + nq->bar_reg_iomem = NULL; | |
| 205 | + | |
| 206 | + nq->cqn_handler = NULL; | |
| 207 | + nq->srqn_handler = NULL; | |
| 208 | + nq->vector = 0; | |
| 209 | +} | |
| 210 | + | |
| 211 | +int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, | |
| 212 | + int msix_vector, int bar_reg_offset, | |
| 213 | + int (*cqn_handler)(struct bnxt_qplib_nq *nq, | |
| 214 | + struct bnxt_qplib_cq *), | |
| 215 | + int (*srqn_handler)(struct bnxt_qplib_nq *nq, | |
| 216 | + void *, u8 event)) | |
| 217 | +{ | |
| 218 | + resource_size_t nq_base; | |
| 219 | + int rc; | |
| 220 | + | |
| 221 | + nq->pdev = pdev; | |
| 222 | + nq->vector = msix_vector; | |
| 223 | + | |
| 224 | + nq->cqn_handler = cqn_handler; | |
| 225 | + | |
| 226 | + nq->srqn_handler = srqn_handler; | |
| 227 | + | |
| 228 | + tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq); | |
| 229 | + | |
| 230 | + nq->requested = false; | |
| 231 | + rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq); | |
| 232 | + if (rc) { | |
| 233 | + dev_err(&nq->pdev->dev, | |
| 234 | + "Failed to request IRQ for NQ: %#x", rc); | |
| 235 | + bnxt_qplib_disable_nq(nq); | |
| 236 | + goto fail; | |
| 237 | + } | |
| 238 | + nq->requested = true; | |
| 239 | + nq->bar_reg = NQ_CONS_PCI_BAR_REGION; | |
| 240 | + nq->bar_reg_off = bar_reg_offset; | |
| 241 | + nq_base = pci_resource_start(pdev, nq->bar_reg); | |
| 242 | + if (!nq_base) { | |
| 243 | + rc = -ENOMEM; | |
| 244 | + goto fail; | |
| 245 | + } | |
| 246 | + nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4); | |
| 247 | + if (!nq->bar_reg_iomem) { | |
| 248 | + rc = -ENOMEM; | |
| 249 | + goto fail; | |
| 250 | + } | |
| 251 | + NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); | |
| 252 | + | |
| 253 | + return 0; | |
| 254 | +fail: | |
| 255 | + bnxt_qplib_disable_nq(nq); | |
| 256 | + return rc; | |
| 257 | +} | |
| 258 | + | |
| 259 | +void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq) | |
| 260 | +{ | |
| 261 | + if (nq->hwq.max_elements) | |
| 262 | + bnxt_qplib_free_hwq(nq->pdev, &nq->hwq); | |
| 263 | +} | |
| 264 | + | |
| 265 | +int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq) | |
| 266 | +{ | |
| 267 | + nq->pdev = pdev; | |
| 268 | + if (!nq->hwq.max_elements || | |
| 269 | + nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) | |
| 270 | + nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; | |
| 271 | + | |
| 272 | + if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0, | |
| 273 | + &nq->hwq.max_elements, | |
| 274 | + BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0, | |
| 275 | + PAGE_SIZE, HWQ_TYPE_L2_CMPL)) | |
| 276 | + return -ENOMEM; | |
| 277 | + | |
| 278 | + nq->budget = 8; | |
| 279 | + return 0; | |
| 280 | +} | |
| 281 | + | |
| 282 | +/* QP */ | |
| 283 | +int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | |
| 284 | +{ | |
| 285 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 286 | + struct cmdq_create_qp1 req; | |
| 287 | + struct creq_create_qp1_resp *resp; | |
| 288 | + struct bnxt_qplib_pbl *pbl; | |
| 289 | + struct bnxt_qplib_q *sq = &qp->sq; | |
| 290 | + struct bnxt_qplib_q *rq = &qp->rq; | |
| 291 | + int rc; | |
| 292 | + u16 cmd_flags = 0; | |
| 293 | + u32 qp_flags = 0; | |
| 294 | + | |
| 295 | + RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags); | |
| 296 | + | |
| 297 | + /* General */ | |
| 298 | + req.type = qp->type; | |
| 299 | + req.dpi = cpu_to_le32(qp->dpi->dpi); | |
| 300 | + req.qp_handle = cpu_to_le64(qp->qp_handle); | |
| 301 | + | |
| 302 | + /* SQ */ | |
| 303 | + sq->hwq.max_elements = sq->max_wqe; | |
| 304 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0, | |
| 305 | + &sq->hwq.max_elements, | |
| 306 | + BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0, | |
| 307 | + PAGE_SIZE, HWQ_TYPE_QUEUE); | |
| 308 | + if (rc) | |
| 309 | + goto exit; | |
| 310 | + | |
| 311 | + sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); | |
| 312 | + if (!sq->swq) { | |
| 313 | + rc = -ENOMEM; | |
| 314 | + goto fail_sq; | |
| 315 | + } | |
| 316 | + pbl = &sq->hwq.pbl[PBL_LVL_0]; | |
| 317 | + req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
| 318 | + req.sq_pg_size_sq_lvl = | |
| 319 | + ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) | |
| 320 | + << CMDQ_CREATE_QP1_SQ_LVL_SFT) | | |
| 321 | + (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
| 322 | + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K : | |
| 323 | + pbl->pg_size == ROCE_PG_SIZE_8K ? | |
| 324 | + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K : | |
| 325 | + pbl->pg_size == ROCE_PG_SIZE_64K ? | |
| 326 | + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K : | |
| 327 | + pbl->pg_size == ROCE_PG_SIZE_2M ? | |
| 328 | + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M : | |
| 329 | + pbl->pg_size == ROCE_PG_SIZE_8M ? | |
| 330 | + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M : | |
| 331 | + pbl->pg_size == ROCE_PG_SIZE_1G ? | |
| 332 | + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G : | |
| 333 | + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K); | |
| 334 | + | |
| 335 | + if (qp->scq) | |
| 336 | + req.scq_cid = cpu_to_le32(qp->scq->id); | |
| 337 | + | |
| 338 | + qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; | |
| 339 | + | |
| 340 | + /* RQ */ | |
| 341 | + if (rq->max_wqe) { | |
| 342 | + rq->hwq.max_elements = qp->rq.max_wqe; | |
| 343 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0, | |
| 344 | + &rq->hwq.max_elements, | |
| 345 | + BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, | |
| 346 | + PAGE_SIZE, HWQ_TYPE_QUEUE); | |
| 347 | + if (rc) | |
| 348 | + goto fail_sq; | |
| 349 | + | |
| 350 | + rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), | |
| 351 | + GFP_KERNEL); | |
| 352 | + if (!rq->swq) { | |
| 353 | + rc = -ENOMEM; | |
| 354 | + goto fail_rq; | |
| 355 | + } | |
| 356 | + pbl = &rq->hwq.pbl[PBL_LVL_0]; | |
| 357 | + req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
| 358 | + req.rq_pg_size_rq_lvl = | |
| 359 | + ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << | |
| 360 | + CMDQ_CREATE_QP1_RQ_LVL_SFT) | | |
| 361 | + (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
| 362 | + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K : | |
| 363 | + pbl->pg_size == ROCE_PG_SIZE_8K ? | |
| 364 | + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K : | |
| 365 | + pbl->pg_size == ROCE_PG_SIZE_64K ? | |
| 366 | + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K : | |
| 367 | + pbl->pg_size == ROCE_PG_SIZE_2M ? | |
| 368 | + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M : | |
| 369 | + pbl->pg_size == ROCE_PG_SIZE_8M ? | |
| 370 | + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M : | |
| 371 | + pbl->pg_size == ROCE_PG_SIZE_1G ? | |
| 372 | + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G : | |
| 373 | + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K); | |
| 374 | + if (qp->rcq) | |
| 375 | + req.rcq_cid = cpu_to_le32(qp->rcq->id); | |
| 376 | + } | |
| 377 | + | |
| 378 | + /* Header buffer - allow hdr_buf pass in */ | |
| 379 | + rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); | |
| 380 | + if (rc) { | |
| 381 | + rc = -ENOMEM; | |
| 382 | + goto fail; | |
| 383 | + } | |
| 384 | + req.qp_flags = cpu_to_le32(qp_flags); | |
| 385 | + req.sq_size = cpu_to_le32(sq->hwq.max_elements); | |
| 386 | + req.rq_size = cpu_to_le32(rq->hwq.max_elements); | |
| 387 | + | |
| 388 | + req.sq_fwo_sq_sge = | |
| 389 | + cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << | |
| 390 | + CMDQ_CREATE_QP1_SQ_SGE_SFT); | |
| 391 | + req.rq_fwo_rq_sge = | |
| 392 | + cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << | |
| 393 | + CMDQ_CREATE_QP1_RQ_SGE_SFT); | |
| 394 | + | |
| 395 | + req.pd_id = cpu_to_le32(qp->pd->id); | |
| 396 | + | |
| 397 | + resp = (struct creq_create_qp1_resp *) | |
| 398 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 399 | + NULL, 0); | |
| 400 | + if (!resp) { | |
| 401 | + dev_err(&res->pdev->dev, "QPLIB: FP: CREATE_QP1 send failed"); | |
| 402 | + rc = -EINVAL; | |
| 403 | + goto fail; | |
| 404 | + } | |
| 405 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 406 | + /* Cmd timed out */ | |
| 407 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 timed out"); | |
| 408 | + rc = -ETIMEDOUT; | |
| 409 | + goto fail; | |
| 410 | + } | |
| 411 | + if (resp->status || | |
| 412 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 413 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 failed "); | |
| 414 | + dev_err(&rcfw->pdev->dev, | |
| 415 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 416 | + resp->status, le16_to_cpu(req.cookie), | |
| 417 | + le16_to_cpu(resp->cookie)); | |
| 418 | + rc = -EINVAL; | |
| 419 | + goto fail; | |
| 420 | + } | |
| 421 | + qp->id = le32_to_cpu(resp->xid); | |
| 422 | + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; | |
| 423 | + sq->flush_in_progress = false; | |
| 424 | + rq->flush_in_progress = false; | |
| 425 | + | |
| 426 | + return 0; | |
| 427 | + | |
| 428 | +fail: | |
| 429 | + bnxt_qplib_free_qp_hdr_buf(res, qp); | |
| 430 | +fail_rq: | |
| 431 | + bnxt_qplib_free_hwq(res->pdev, &rq->hwq); | |
| 432 | + kfree(rq->swq); | |
| 433 | +fail_sq: | |
| 434 | + bnxt_qplib_free_hwq(res->pdev, &sq->hwq); | |
| 435 | + kfree(sq->swq); | |
| 436 | +exit: | |
| 437 | + return rc; | |
| 438 | +} | |
| 439 | + | |
| 440 | +int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | |
| 441 | +{ | |
| 442 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 443 | + struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; | |
| 444 | + struct cmdq_create_qp req; | |
| 445 | + struct creq_create_qp_resp *resp; | |
| 446 | + struct bnxt_qplib_pbl *pbl; | |
| 447 | + struct sq_psn_search **psn_search_ptr; | |
| 448 | + unsigned long int psn_search, poff = 0; | |
| 449 | + struct bnxt_qplib_q *sq = &qp->sq; | |
| 450 | + struct bnxt_qplib_q *rq = &qp->rq; | |
| 451 | + struct bnxt_qplib_hwq *xrrq; | |
| 452 | + int i, rc, req_size, psn_sz; | |
| 453 | + u16 cmd_flags = 0, max_ssge; | |
| 454 | + u32 sw_prod, qp_flags = 0; | |
| 455 | + | |
| 456 | + RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); | |
| 457 | + | |
| 458 | + /* General */ | |
| 459 | + req.type = qp->type; | |
| 460 | + req.dpi = cpu_to_le32(qp->dpi->dpi); | |
| 461 | + req.qp_handle = cpu_to_le64(qp->qp_handle); | |
| 462 | + | |
| 463 | + /* SQ */ | |
| 464 | + psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ? | |
| 465 | + sizeof(struct sq_psn_search) : 0; | |
| 466 | + sq->hwq.max_elements = sq->max_wqe; | |
| 467 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist, | |
| 468 | + sq->nmap, &sq->hwq.max_elements, | |
| 469 | + BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, | |
| 470 | + psn_sz, | |
| 471 | + PAGE_SIZE, HWQ_TYPE_QUEUE); | |
| 472 | + if (rc) | |
| 473 | + goto exit; | |
| 474 | + | |
| 475 | + sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); | |
| 476 | + if (!sq->swq) { | |
| 477 | + rc = -ENOMEM; | |
| 478 | + goto fail_sq; | |
| 479 | + } | |
| 480 | + hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | |
| 481 | + if (psn_sz) { | |
| 482 | + psn_search_ptr = (struct sq_psn_search **) | |
| 483 | + &hw_sq_send_ptr[get_sqe_pg | |
| 484 | + (sq->hwq.max_elements)]; | |
| 485 | + psn_search = (unsigned long int) | |
| 486 | + &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)] | |
| 487 | + [get_sqe_idx(sq->hwq.max_elements)]; | |
| 488 | + if (psn_search & ~PAGE_MASK) { | |
| 489 | + /* If the psn_search does not start on a page boundary, | |
| 490 | + * then calculate the offset | |
| 491 | + */ | |
| 492 | + poff = (psn_search & ~PAGE_MASK) / | |
| 493 | + BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE; | |
| 494 | + } | |
| 495 | + for (i = 0; i < sq->hwq.max_elements; i++) | |
| 496 | + sq->swq[i].psn_search = | |
| 497 | + &psn_search_ptr[get_psne_pg(i + poff)] | |
| 498 | + [get_psne_idx(i + poff)]; | |
| 499 | + } | |
| 500 | + pbl = &sq->hwq.pbl[PBL_LVL_0]; | |
| 501 | + req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
| 502 | + req.sq_pg_size_sq_lvl = | |
| 503 | + ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) | |
| 504 | + << CMDQ_CREATE_QP_SQ_LVL_SFT) | | |
| 505 | + (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
| 506 | + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K : | |
| 507 | + pbl->pg_size == ROCE_PG_SIZE_8K ? | |
| 508 | + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K : | |
| 509 | + pbl->pg_size == ROCE_PG_SIZE_64K ? | |
| 510 | + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K : | |
| 511 | + pbl->pg_size == ROCE_PG_SIZE_2M ? | |
| 512 | + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M : | |
| 513 | + pbl->pg_size == ROCE_PG_SIZE_8M ? | |
| 514 | + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M : | |
| 515 | + pbl->pg_size == ROCE_PG_SIZE_1G ? | |
| 516 | + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G : | |
| 517 | + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K); | |
| 518 | + | |
| 519 | + /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */ | |
| 520 | + hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | |
| 521 | + for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) { | |
| 522 | + hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] | |
| 523 | + [get_sqe_idx(sw_prod)]; | |
| 524 | + hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID; | |
| 525 | + } | |
| 526 | + | |
| 527 | + if (qp->scq) | |
| 528 | + req.scq_cid = cpu_to_le32(qp->scq->id); | |
| 529 | + | |
| 530 | + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; | |
| 531 | + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; | |
| 532 | + if (qp->sig_type) | |
| 533 | + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; | |
| 534 | + | |
| 535 | + /* RQ */ | |
| 536 | + if (rq->max_wqe) { | |
| 537 | + rq->hwq.max_elements = rq->max_wqe; | |
| 538 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist, | |
| 539 | + rq->nmap, &rq->hwq.max_elements, | |
| 540 | + BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, | |
| 541 | + PAGE_SIZE, HWQ_TYPE_QUEUE); | |
| 542 | + if (rc) | |
| 543 | + goto fail_sq; | |
| 544 | + | |
| 545 | + rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), | |
| 546 | + GFP_KERNEL); | |
| 547 | + if (!rq->swq) { | |
| 548 | + rc = -ENOMEM; | |
| 549 | + goto fail_rq; | |
| 550 | + } | |
| 551 | + pbl = &rq->hwq.pbl[PBL_LVL_0]; | |
| 552 | + req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
| 553 | + req.rq_pg_size_rq_lvl = | |
| 554 | + ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << | |
| 555 | + CMDQ_CREATE_QP_RQ_LVL_SFT) | | |
| 556 | + (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
| 557 | + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K : | |
| 558 | + pbl->pg_size == ROCE_PG_SIZE_8K ? | |
| 559 | + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K : | |
| 560 | + pbl->pg_size == ROCE_PG_SIZE_64K ? | |
| 561 | + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K : | |
| 562 | + pbl->pg_size == ROCE_PG_SIZE_2M ? | |
| 563 | + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M : | |
| 564 | + pbl->pg_size == ROCE_PG_SIZE_8M ? | |
| 565 | + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M : | |
| 566 | + pbl->pg_size == ROCE_PG_SIZE_1G ? | |
| 567 | + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G : | |
| 568 | + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K); | |
| 569 | + } | |
| 570 | + | |
| 571 | + if (qp->rcq) | |
| 572 | + req.rcq_cid = cpu_to_le32(qp->rcq->id); | |
| 573 | + req.qp_flags = cpu_to_le32(qp_flags); | |
| 574 | + req.sq_size = cpu_to_le32(sq->hwq.max_elements); | |
| 575 | + req.rq_size = cpu_to_le32(rq->hwq.max_elements); | |
| 576 | + qp->sq_hdr_buf = NULL; | |
| 577 | + qp->rq_hdr_buf = NULL; | |
| 578 | + | |
| 579 | + rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); | |
| 580 | + if (rc) | |
| 581 | + goto fail_rq; | |
| 582 | + | |
| 583 | + /* CTRL-22434: Irrespective of the requested SGE count on the SQ | |
| 584 | + * always create the QP with max send sges possible if the requested | |
| 585 | + * inline size is greater than 0. | |
| 586 | + */ | |
| 587 | + max_ssge = qp->max_inline_data ? 6 : sq->max_sge; | |
| 588 | + req.sq_fwo_sq_sge = cpu_to_le16( | |
| 589 | + ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) | |
| 590 | + << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); | |
| 591 | + req.rq_fwo_rq_sge = cpu_to_le16( | |
| 592 | + ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK) | |
| 593 | + << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); | |
| 594 | + /* ORRQ and IRRQ */ | |
| 595 | + if (psn_sz) { | |
| 596 | + xrrq = &qp->orrq; | |
| 597 | + xrrq->max_elements = | |
| 598 | + ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); | |
| 599 | + req_size = xrrq->max_elements * | |
| 600 | + BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; | |
| 601 | + req_size &= ~(PAGE_SIZE - 1); | |
| 602 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, | |
| 603 | + &xrrq->max_elements, | |
| 604 | + BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE, | |
| 605 | + 0, req_size, HWQ_TYPE_CTX); | |
| 606 | + if (rc) | |
| 607 | + goto fail_buf_free; | |
| 608 | + pbl = &xrrq->pbl[PBL_LVL_0]; | |
| 609 | + req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); | |
| 610 | + | |
| 611 | + xrrq = &qp->irrq; | |
| 612 | + xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( | |
| 613 | + qp->max_dest_rd_atomic); | |
| 614 | + req_size = xrrq->max_elements * | |
| 615 | + BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; | |
| 616 | + req_size &= ~(PAGE_SIZE - 1); | |
| 617 | + | |
| 618 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, | |
| 619 | + &xrrq->max_elements, | |
| 620 | + BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE, | |
| 621 | + 0, req_size, HWQ_TYPE_CTX); | |
| 622 | + if (rc) | |
| 623 | + goto fail_orrq; | |
| 624 | + | |
| 625 | + pbl = &xrrq->pbl[PBL_LVL_0]; | |
| 626 | + req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); | |
| 627 | + } | |
| 628 | + req.pd_id = cpu_to_le32(qp->pd->id); | |
| 629 | + | |
| 630 | + resp = (struct creq_create_qp_resp *) | |
| 631 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 632 | + NULL, 0); | |
| 633 | + if (!resp) { | |
| 634 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP send failed"); | |
| 635 | + rc = -EINVAL; | |
| 636 | + goto fail; | |
| 637 | + } | |
| 638 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 639 | + /* Cmd timed out */ | |
| 640 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP timed out"); | |
| 641 | + rc = -ETIMEDOUT; | |
| 642 | + goto fail; | |
| 643 | + } | |
| 644 | + if (resp->status || | |
| 645 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 646 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP failed "); | |
| 647 | + dev_err(&rcfw->pdev->dev, | |
| 648 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 649 | + resp->status, le16_to_cpu(req.cookie), | |
| 650 | + le16_to_cpu(resp->cookie)); | |
| 651 | + rc = -EINVAL; | |
| 652 | + goto fail; | |
| 653 | + } | |
| 654 | + qp->id = le32_to_cpu(resp->xid); | |
| 655 | + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; | |
| 656 | + sq->flush_in_progress = false; | |
| 657 | + rq->flush_in_progress = false; | |
| 658 | + | |
| 659 | + return 0; | |
| 660 | + | |
| 661 | +fail: | |
| 662 | + if (qp->irrq.max_elements) | |
| 663 | + bnxt_qplib_free_hwq(res->pdev, &qp->irrq); | |
| 664 | +fail_orrq: | |
| 665 | + if (qp->orrq.max_elements) | |
| 666 | + bnxt_qplib_free_hwq(res->pdev, &qp->orrq); | |
| 667 | +fail_buf_free: | |
| 668 | + bnxt_qplib_free_qp_hdr_buf(res, qp); | |
| 669 | +fail_rq: | |
| 670 | + bnxt_qplib_free_hwq(res->pdev, &rq->hwq); | |
| 671 | + kfree(rq->swq); | |
| 672 | +fail_sq: | |
| 673 | + bnxt_qplib_free_hwq(res->pdev, &sq->hwq); | |
| 674 | + kfree(sq->swq); | |
| 675 | +exit: | |
| 676 | + return rc; | |
| 677 | +} | |
| 678 | + | |
| 679 | +static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp) | |
| 680 | +{ | |
| 681 | + switch (qp->state) { | |
| 682 | + case CMDQ_MODIFY_QP_NEW_STATE_RTR: | |
| 683 | + /* INIT->RTR, configure the path_mtu to the default | |
| 684 | + * 2048 if not being requested | |
| 685 | + */ | |
| 686 | + if (!(qp->modify_flags & | |
| 687 | + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { | |
| 688 | + qp->modify_flags |= | |
| 689 | + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; | |
| 690 | + qp->path_mtu = | |
| 691 | + CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; | |
| 692 | + } | |
| 693 | + qp->modify_flags &= | |
| 694 | + ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; | |
| 695 | + /* Bono FW require the max_dest_rd_atomic to be >= 1 */ | |
| 696 | + if (qp->max_dest_rd_atomic < 1) | |
| 697 | + qp->max_dest_rd_atomic = 1; | |
| 698 | + qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; | |
| 699 | + /* Bono FW 20.6.5 requires SGID_INDEX configuration */ | |
| 700 | + if (!(qp->modify_flags & | |
| 701 | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { | |
| 702 | + qp->modify_flags |= | |
| 703 | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; | |
| 704 | + qp->ah.sgid_index = 0; | |
| 705 | + } | |
| 706 | + break; | |
| 707 | + default: | |
| 708 | + break; | |
| 709 | + } | |
| 710 | +} | |
| 711 | + | |
| 712 | +static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp) | |
| 713 | +{ | |
| 714 | + switch (qp->state) { | |
| 715 | + case CMDQ_MODIFY_QP_NEW_STATE_RTS: | |
| 716 | + /* Bono FW requires the max_rd_atomic to be >= 1 */ | |
| 717 | + if (qp->max_rd_atomic < 1) | |
| 718 | + qp->max_rd_atomic = 1; | |
| 719 | + /* Bono FW does not allow PKEY_INDEX, | |
| 720 | + * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT, | |
| 721 | + * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN, | |
| 722 | + * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID | |
| 723 | + * modification | |
| 724 | + */ | |
| 725 | + qp->modify_flags &= | |
| 726 | + ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | | |
| 727 | + CMDQ_MODIFY_QP_MODIFY_MASK_DGID | | |
| 728 | + CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | | |
| 729 | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | | |
| 730 | + CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | | |
| 731 | + CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | | |
| 732 | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | | |
| 733 | + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | | |
| 734 | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | | |
| 735 | + CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | | |
| 736 | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | | |
| 737 | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); | |
| 738 | + break; | |
| 739 | + default: | |
| 740 | + break; | |
| 741 | + } | |
| 742 | +} | |
| 743 | + | |
| 744 | +static void __filter_modify_flags(struct bnxt_qplib_qp *qp) | |
| 745 | +{ | |
| 746 | + switch (qp->cur_qp_state) { | |
| 747 | + case CMDQ_MODIFY_QP_NEW_STATE_RESET: | |
| 748 | + break; | |
| 749 | + case CMDQ_MODIFY_QP_NEW_STATE_INIT: | |
| 750 | + __modify_flags_from_init_state(qp); | |
| 751 | + break; | |
| 752 | + case CMDQ_MODIFY_QP_NEW_STATE_RTR: | |
| 753 | + __modify_flags_from_rtr_state(qp); | |
| 754 | + break; | |
| 755 | + case CMDQ_MODIFY_QP_NEW_STATE_RTS: | |
| 756 | + break; | |
| 757 | + case CMDQ_MODIFY_QP_NEW_STATE_SQD: | |
| 758 | + break; | |
| 759 | + case CMDQ_MODIFY_QP_NEW_STATE_SQE: | |
| 760 | + break; | |
| 761 | + case CMDQ_MODIFY_QP_NEW_STATE_ERR: | |
| 762 | + break; | |
| 763 | + default: | |
| 764 | + break; | |
| 765 | + } | |
| 766 | +} | |
| 767 | + | |
| 768 | +int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | |
| 769 | +{ | |
| 770 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 771 | + struct cmdq_modify_qp req; | |
| 772 | + struct creq_modify_qp_resp *resp; | |
| 773 | + u16 cmd_flags = 0, pkey; | |
| 774 | + u32 temp32[4]; | |
| 775 | + u32 bmask; | |
| 776 | + | |
| 777 | + RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags); | |
| 778 | + | |
| 779 | + /* Filter out the qp_attr_mask based on the state->new transition */ | |
| 780 | + __filter_modify_flags(qp); | |
| 781 | + bmask = qp->modify_flags; | |
| 782 | + req.modify_mask = cpu_to_le32(qp->modify_flags); | |
| 783 | + req.qp_cid = cpu_to_le32(qp->id); | |
| 784 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { | |
| 785 | + req.network_type_en_sqd_async_notify_new_state = | |
| 786 | + (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | | |
| 787 | + (qp->en_sqd_async_notify ? | |
| 788 | + CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); | |
| 789 | + } | |
| 790 | + req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; | |
| 791 | + | |
| 792 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) | |
| 793 | + req.access = qp->access; | |
| 794 | + | |
| 795 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) { | |
| 796 | + if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl, | |
| 797 | + qp->pkey_index, &pkey)) | |
| 798 | + req.pkey = cpu_to_le16(pkey); | |
| 799 | + } | |
| 800 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) | |
| 801 | + req.qkey = cpu_to_le32(qp->qkey); | |
| 802 | + | |
| 803 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { | |
| 804 | + memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); | |
| 805 | + req.dgid[0] = cpu_to_le32(temp32[0]); | |
| 806 | + req.dgid[1] = cpu_to_le32(temp32[1]); | |
| 807 | + req.dgid[2] = cpu_to_le32(temp32[2]); | |
| 808 | + req.dgid[3] = cpu_to_le32(temp32[3]); | |
| 809 | + } | |
| 810 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) | |
| 811 | + req.flow_label = cpu_to_le32(qp->ah.flow_label); | |
| 812 | + | |
| 813 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) | |
| 814 | + req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id | |
| 815 | + [qp->ah.sgid_index]); | |
| 816 | + | |
| 817 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) | |
| 818 | + req.hop_limit = qp->ah.hop_limit; | |
| 819 | + | |
| 820 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) | |
| 821 | + req.traffic_class = qp->ah.traffic_class; | |
| 822 | + | |
| 823 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) | |
| 824 | + memcpy(req.dest_mac, qp->ah.dmac, 6); | |
| 825 | + | |
| 826 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) | |
| 827 | + req.path_mtu = qp->path_mtu; | |
| 828 | + | |
| 829 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) | |
| 830 | + req.timeout = qp->timeout; | |
| 831 | + | |
| 832 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) | |
| 833 | + req.retry_cnt = qp->retry_cnt; | |
| 834 | + | |
| 835 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) | |
| 836 | + req.rnr_retry = qp->rnr_retry; | |
| 837 | + | |
| 838 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) | |
| 839 | + req.min_rnr_timer = qp->min_rnr_timer; | |
| 840 | + | |
| 841 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) | |
| 842 | + req.rq_psn = cpu_to_le32(qp->rq.psn); | |
| 843 | + | |
| 844 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) | |
| 845 | + req.sq_psn = cpu_to_le32(qp->sq.psn); | |
| 846 | + | |
| 847 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) | |
| 848 | + req.max_rd_atomic = | |
| 849 | + ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); | |
| 850 | + | |
| 851 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) | |
| 852 | + req.max_dest_rd_atomic = | |
| 853 | + IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); | |
| 854 | + | |
| 855 | + req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); | |
| 856 | + req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); | |
| 857 | + req.sq_sge = cpu_to_le16(qp->sq.max_sge); | |
| 858 | + req.rq_sge = cpu_to_le16(qp->rq.max_sge); | |
| 859 | + req.max_inline_data = cpu_to_le32(qp->max_inline_data); | |
| 860 | + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) | |
| 861 | + req.dest_qp_id = cpu_to_le32(qp->dest_qpn); | |
| 862 | + | |
| 863 | + req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); | |
| 864 | + | |
| 865 | + resp = (struct creq_modify_qp_resp *) | |
| 866 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 867 | + NULL, 0); | |
| 868 | + if (!resp) { | |
| 869 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP send failed"); | |
| 870 | + return -EINVAL; | |
| 871 | + } | |
| 872 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 873 | + /* Cmd timed out */ | |
| 874 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP timed out"); | |
| 875 | + return -ETIMEDOUT; | |
| 876 | + } | |
| 877 | + if (resp->status || | |
| 878 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 879 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP failed "); | |
| 880 | + dev_err(&rcfw->pdev->dev, | |
| 881 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 882 | + resp->status, le16_to_cpu(req.cookie), | |
| 883 | + le16_to_cpu(resp->cookie)); | |
| 884 | + return -EINVAL; | |
| 885 | + } | |
| 886 | + qp->cur_qp_state = qp->state; | |
| 887 | + return 0; | |
| 888 | +} | |
| 889 | + | |
| 890 | +int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | |
| 891 | +{ | |
| 892 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 893 | + struct cmdq_query_qp req; | |
| 894 | + struct creq_query_qp_resp *resp; | |
| 895 | + struct creq_query_qp_resp_sb *sb; | |
| 896 | + u16 cmd_flags = 0; | |
| 897 | + u32 temp32[4]; | |
| 898 | + int i; | |
| 899 | + | |
| 900 | + RCFW_CMD_PREP(req, QUERY_QP, cmd_flags); | |
| 901 | + | |
| 902 | + req.qp_cid = cpu_to_le32(qp->id); | |
| 903 | + req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; | |
| 904 | + resp = (struct creq_query_qp_resp *) | |
| 905 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 906 | + (void **)&sb, 0); | |
| 907 | + if (!resp) { | |
| 908 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP send failed"); | |
| 909 | + return -EINVAL; | |
| 910 | + } | |
| 911 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 912 | + /* Cmd timed out */ | |
| 913 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP timed out"); | |
| 914 | + return -ETIMEDOUT; | |
| 915 | + } | |
| 916 | + if (resp->status || | |
| 917 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 918 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP failed "); | |
| 919 | + dev_err(&rcfw->pdev->dev, | |
| 920 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 921 | + resp->status, le16_to_cpu(req.cookie), | |
| 922 | + le16_to_cpu(resp->cookie)); | |
| 923 | + return -EINVAL; | |
| 924 | + } | |
| 925 | + /* Extract the context from the side buffer */ | |
| 926 | + qp->state = sb->en_sqd_async_notify_state & | |
| 927 | + CREQ_QUERY_QP_RESP_SB_STATE_MASK; | |
| 928 | + qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & | |
| 929 | + CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? | |
| 930 | + true : false; | |
| 931 | + qp->access = sb->access; | |
| 932 | + qp->pkey_index = le16_to_cpu(sb->pkey); | |
| 933 | + qp->qkey = le32_to_cpu(sb->qkey); | |
| 934 | + | |
| 935 | + temp32[0] = le32_to_cpu(sb->dgid[0]); | |
| 936 | + temp32[1] = le32_to_cpu(sb->dgid[1]); | |
| 937 | + temp32[2] = le32_to_cpu(sb->dgid[2]); | |
| 938 | + temp32[3] = le32_to_cpu(sb->dgid[3]); | |
| 939 | + memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); | |
| 940 | + | |
| 941 | + qp->ah.flow_label = le32_to_cpu(sb->flow_label); | |
| 942 | + | |
| 943 | + qp->ah.sgid_index = 0; | |
| 944 | + for (i = 0; i < res->sgid_tbl.max; i++) { | |
| 945 | + if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { | |
| 946 | + qp->ah.sgid_index = i; | |
| 947 | + break; | |
| 948 | + } | |
| 949 | + } | |
| 950 | + if (i == res->sgid_tbl.max) | |
| 951 | + dev_warn(&res->pdev->dev, "QPLIB: SGID not found??"); | |
| 952 | + | |
| 953 | + qp->ah.hop_limit = sb->hop_limit; | |
| 954 | + qp->ah.traffic_class = sb->traffic_class; | |
| 955 | + memcpy(qp->ah.dmac, sb->dest_mac, 6); | |
| 956 | + qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & | |
| 957 | + CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >> | |
| 958 | + CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; | |
| 959 | + qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & | |
| 960 | + CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >> | |
| 961 | + CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT; | |
| 962 | + qp->timeout = sb->timeout; | |
| 963 | + qp->retry_cnt = sb->retry_cnt; | |
| 964 | + qp->rnr_retry = sb->rnr_retry; | |
| 965 | + qp->min_rnr_timer = sb->min_rnr_timer; | |
| 966 | + qp->rq.psn = le32_to_cpu(sb->rq_psn); | |
| 967 | + qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); | |
| 968 | + qp->sq.psn = le32_to_cpu(sb->sq_psn); | |
| 969 | + qp->max_dest_rd_atomic = | |
| 970 | + IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); | |
| 971 | + qp->sq.max_wqe = qp->sq.hwq.max_elements; | |
| 972 | + qp->rq.max_wqe = qp->rq.hwq.max_elements; | |
| 973 | + qp->sq.max_sge = le16_to_cpu(sb->sq_sge); | |
| 974 | + qp->rq.max_sge = le16_to_cpu(sb->rq_sge); | |
| 975 | + qp->max_inline_data = le32_to_cpu(sb->max_inline_data); | |
| 976 | + qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); | |
| 977 | + memcpy(qp->smac, sb->src_mac, 6); | |
| 978 | + qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); | |
| 979 | + return 0; | |
| 980 | +} | |
| 981 | + | |
| 982 | +static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) | |
| 983 | +{ | |
| 984 | + struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; | |
| 985 | + struct cq_base *hw_cqe, **hw_cqe_ptr; | |
| 986 | + int i; | |
| 987 | + | |
| 988 | + for (i = 0; i < cq_hwq->max_elements; i++) { | |
| 989 | + hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr; | |
| 990 | + hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)]; | |
| 991 | + if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements)) | |
| 992 | + continue; | |
| 993 | + switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { | |
| 994 | + case CQ_BASE_CQE_TYPE_REQ: | |
| 995 | + case CQ_BASE_CQE_TYPE_TERMINAL: | |
| 996 | + { | |
| 997 | + struct cq_req *cqe = (struct cq_req *)hw_cqe; | |
| 998 | + | |
| 999 | + if (qp == le64_to_cpu(cqe->qp_handle)) | |
| 1000 | + cqe->qp_handle = 0; | |
| 1001 | + break; | |
| 1002 | + } | |
| 1003 | + case CQ_BASE_CQE_TYPE_RES_RC: | |
| 1004 | + case CQ_BASE_CQE_TYPE_RES_UD: | |
| 1005 | + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: | |
| 1006 | + { | |
| 1007 | + struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; | |
| 1008 | + | |
| 1009 | + if (qp == le64_to_cpu(cqe->qp_handle)) | |
| 1010 | + cqe->qp_handle = 0; | |
| 1011 | + break; | |
| 1012 | + } | |
| 1013 | + default: | |
| 1014 | + break; | |
| 1015 | + } | |
| 1016 | + } | |
| 1017 | +} | |
| 1018 | + | |
| 1019 | +int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, | |
| 1020 | + struct bnxt_qplib_qp *qp) | |
| 1021 | +{ | |
| 1022 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 1023 | + struct cmdq_destroy_qp req; | |
| 1024 | + struct creq_destroy_qp_resp *resp; | |
| 1025 | + unsigned long flags; | |
| 1026 | + u16 cmd_flags = 0; | |
| 1027 | + | |
| 1028 | + RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags); | |
| 1029 | + | |
| 1030 | + req.qp_cid = cpu_to_le32(qp->id); | |
| 1031 | + resp = (struct creq_destroy_qp_resp *) | |
| 1032 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 1033 | + NULL, 0); | |
| 1034 | + if (!resp) { | |
| 1035 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP send failed"); | |
| 1036 | + return -EINVAL; | |
| 1037 | + } | |
| 1038 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 1039 | + /* Cmd timed out */ | |
| 1040 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP timed out"); | |
| 1041 | + return -ETIMEDOUT; | |
| 1042 | + } | |
| 1043 | + if (resp->status || | |
| 1044 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 1045 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP failed "); | |
| 1046 | + dev_err(&rcfw->pdev->dev, | |
| 1047 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 1048 | + resp->status, le16_to_cpu(req.cookie), | |
| 1049 | + le16_to_cpu(resp->cookie)); | |
| 1050 | + return -EINVAL; | |
| 1051 | + } | |
| 1052 | + | |
| 1053 | + /* Must walk the associated CQs to nullified the QP ptr */ | |
| 1054 | + spin_lock_irqsave(&qp->scq->hwq.lock, flags); | |
| 1055 | + | |
| 1056 | + __clean_cq(qp->scq, (u64)(unsigned long)qp); | |
| 1057 | + | |
| 1058 | + if (qp->rcq && qp->rcq != qp->scq) { | |
| 1059 | + spin_lock(&qp->rcq->hwq.lock); | |
| 1060 | + __clean_cq(qp->rcq, (u64)(unsigned long)qp); | |
| 1061 | + spin_unlock(&qp->rcq->hwq.lock); | |
| 1062 | + } | |
| 1063 | + | |
| 1064 | + spin_unlock_irqrestore(&qp->scq->hwq.lock, flags); | |
| 1065 | + | |
| 1066 | + bnxt_qplib_free_qp_hdr_buf(res, qp); | |
| 1067 | + bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq); | |
| 1068 | + kfree(qp->sq.swq); | |
| 1069 | + | |
| 1070 | + bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq); | |
| 1071 | + kfree(qp->rq.swq); | |
| 1072 | + | |
| 1073 | + if (qp->irrq.max_elements) | |
| 1074 | + bnxt_qplib_free_hwq(res->pdev, &qp->irrq); | |
| 1075 | + if (qp->orrq.max_elements) | |
| 1076 | + bnxt_qplib_free_hwq(res->pdev, &qp->orrq); | |
| 1077 | + | |
| 1078 | + return 0; | |
| 1079 | +} | |
| 1080 | + | |
| 1081 | +void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, | |
| 1082 | + struct bnxt_qplib_sge *sge) | |
| 1083 | +{ | |
| 1084 | + struct bnxt_qplib_q *sq = &qp->sq; | |
| 1085 | + u32 sw_prod; | |
| 1086 | + | |
| 1087 | + memset(sge, 0, sizeof(*sge)); | |
| 1088 | + | |
| 1089 | + if (qp->sq_hdr_buf) { | |
| 1090 | + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
| 1091 | + sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + | |
| 1092 | + sw_prod * qp->sq_hdr_buf_size); | |
| 1093 | + sge->lkey = 0xFFFFFFFF; | |
| 1094 | + sge->size = qp->sq_hdr_buf_size; | |
| 1095 | + return qp->sq_hdr_buf + sw_prod * sge->size; | |
| 1096 | + } | |
| 1097 | + return NULL; | |
| 1098 | +} | |
| 1099 | + | |
| 1100 | +u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) | |
| 1101 | +{ | |
| 1102 | + struct bnxt_qplib_q *rq = &qp->rq; | |
| 1103 | + | |
| 1104 | + return HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1105 | +} | |
| 1106 | + | |
| 1107 | +dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) | |
| 1108 | +{ | |
| 1109 | + return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); | |
| 1110 | +} | |
| 1111 | + | |
| 1112 | +void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, | |
| 1113 | + struct bnxt_qplib_sge *sge) | |
| 1114 | +{ | |
| 1115 | + struct bnxt_qplib_q *rq = &qp->rq; | |
| 1116 | + u32 sw_prod; | |
| 1117 | + | |
| 1118 | + memset(sge, 0, sizeof(*sge)); | |
| 1119 | + | |
| 1120 | + if (qp->rq_hdr_buf) { | |
| 1121 | + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1122 | + sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + | |
| 1123 | + sw_prod * qp->rq_hdr_buf_size); | |
| 1124 | + sge->lkey = 0xFFFFFFFF; | |
| 1125 | + sge->size = qp->rq_hdr_buf_size; | |
| 1126 | + return qp->rq_hdr_buf + sw_prod * sge->size; | |
| 1127 | + } | |
| 1128 | + return NULL; | |
| 1129 | +} | |
| 1130 | + | |
| 1131 | +void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) | |
| 1132 | +{ | |
| 1133 | + struct bnxt_qplib_q *sq = &qp->sq; | |
| 1134 | + struct dbr_dbr db_msg = { 0 }; | |
| 1135 | + u32 sw_prod; | |
| 1136 | + | |
| 1137 | + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
| 1138 | + | |
| 1139 | + db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & | |
| 1140 | + DBR_DBR_INDEX_MASK); | |
| 1141 | + db_msg.type_xid = | |
| 1142 | + cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | |
| 1143 | + DBR_DBR_TYPE_SQ); | |
| 1144 | + /* Flush all the WQE writes to HW */ | |
| 1145 | + wmb(); | |
| 1146 | + __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | |
| 1147 | +} | |
| 1148 | + | |
| 1149 | +int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, | |
| 1150 | + struct bnxt_qplib_swqe *wqe) | |
| 1151 | +{ | |
| 1152 | + struct bnxt_qplib_q *sq = &qp->sq; | |
| 1153 | + struct bnxt_qplib_swq *swq; | |
| 1154 | + struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; | |
| 1155 | + struct sq_sge *hw_sge; | |
| 1156 | + u32 sw_prod; | |
| 1157 | + u8 wqe_size16; | |
| 1158 | + int i, rc = 0, data_len = 0, pkt_num = 0; | |
| 1159 | + __le32 temp32; | |
| 1160 | + | |
| 1161 | + if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { | |
| 1162 | + rc = -EINVAL; | |
| 1163 | + goto done; | |
| 1164 | + } | |
| 1165 | + if (HWQ_CMP((sq->hwq.prod + 1), &sq->hwq) == | |
| 1166 | + HWQ_CMP(sq->hwq.cons, &sq->hwq)) { | |
| 1167 | + rc = -ENOMEM; | |
| 1168 | + goto done; | |
| 1169 | + } | |
| 1170 | + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
| 1171 | + swq = &sq->swq[sw_prod]; | |
| 1172 | + swq->wr_id = wqe->wr_id; | |
| 1173 | + swq->type = wqe->type; | |
| 1174 | + swq->flags = wqe->flags; | |
| 1175 | + if (qp->sig_type) | |
| 1176 | + swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; | |
| 1177 | + swq->start_psn = sq->psn & BTH_PSN_MASK; | |
| 1178 | + | |
| 1179 | + hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | |
| 1180 | + hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] | |
| 1181 | + [get_sqe_idx(sw_prod)]; | |
| 1182 | + | |
| 1183 | + memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); | |
| 1184 | + | |
| 1185 | + if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { | |
| 1186 | + /* Copy the inline data */ | |
| 1187 | + if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { | |
| 1188 | + dev_warn(&sq->hwq.pdev->dev, | |
| 1189 | + "QPLIB: Inline data length > 96 detected"); | |
| 1190 | + data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH; | |
| 1191 | + } else { | |
| 1192 | + data_len = wqe->inline_len; | |
| 1193 | + } | |
| 1194 | + memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); | |
| 1195 | + wqe_size16 = (data_len + 15) >> 4; | |
| 1196 | + } else { | |
| 1197 | + for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data; | |
| 1198 | + i < wqe->num_sge; i++, hw_sge++) { | |
| 1199 | + hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); | |
| 1200 | + hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); | |
| 1201 | + hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); | |
| 1202 | + data_len += wqe->sg_list[i].size; | |
| 1203 | + } | |
| 1204 | + /* Each SGE entry = 1 WQE size16 */ | |
| 1205 | + wqe_size16 = wqe->num_sge; | |
| 1206 | + } | |
| 1207 | + | |
| 1208 | + /* Specifics */ | |
| 1209 | + switch (wqe->type) { | |
| 1210 | + case BNXT_QPLIB_SWQE_TYPE_SEND: | |
| 1211 | + if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { | |
| 1212 | + /* Assemble info for Raw Ethertype QPs */ | |
| 1213 | + struct sq_send_raweth_qp1 *sqe = | |
| 1214 | + (struct sq_send_raweth_qp1 *)hw_sq_send_hdr; | |
| 1215 | + | |
| 1216 | + sqe->wqe_type = wqe->type; | |
| 1217 | + sqe->flags = wqe->flags; | |
| 1218 | + sqe->wqe_size = wqe_size16 + | |
| 1219 | + ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
| 1220 | + sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); | |
| 1221 | + sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); | |
| 1222 | + sqe->length = cpu_to_le32(data_len); | |
| 1223 | + sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & | |
| 1224 | + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << | |
| 1225 | + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); | |
| 1226 | + | |
| 1227 | + break; | |
| 1228 | + } | |
| 1229 | + /* else, just fall thru */ | |
| 1230 | + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: | |
| 1231 | + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: | |
| 1232 | + { | |
| 1233 | + struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr; | |
| 1234 | + | |
| 1235 | + sqe->wqe_type = wqe->type; | |
| 1236 | + sqe->flags = wqe->flags; | |
| 1237 | + sqe->wqe_size = wqe_size16 + | |
| 1238 | + ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
| 1239 | + sqe->inv_key_or_imm_data = cpu_to_le32( | |
| 1240 | + wqe->send.inv_key); | |
| 1241 | + if (qp->type == CMDQ_CREATE_QP_TYPE_UD) { | |
| 1242 | + sqe->q_key = cpu_to_le32(wqe->send.q_key); | |
| 1243 | + sqe->dst_qp = cpu_to_le32( | |
| 1244 | + wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); | |
| 1245 | + sqe->length = cpu_to_le32(data_len); | |
| 1246 | + sqe->avid = cpu_to_le32(wqe->send.avid & | |
| 1247 | + SQ_SEND_AVID_MASK); | |
| 1248 | + sq->psn = (sq->psn + 1) & BTH_PSN_MASK; | |
| 1249 | + } else { | |
| 1250 | + sqe->length = cpu_to_le32(data_len); | |
| 1251 | + sqe->dst_qp = 0; | |
| 1252 | + sqe->avid = 0; | |
| 1253 | + if (qp->mtu) | |
| 1254 | + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | |
| 1255 | + if (!pkt_num) | |
| 1256 | + pkt_num = 1; | |
| 1257 | + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | |
| 1258 | + } | |
| 1259 | + break; | |
| 1260 | + } | |
| 1261 | + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: | |
| 1262 | + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: | |
| 1263 | + case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: | |
| 1264 | + { | |
| 1265 | + struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr; | |
| 1266 | + | |
| 1267 | + sqe->wqe_type = wqe->type; | |
| 1268 | + sqe->flags = wqe->flags; | |
| 1269 | + sqe->wqe_size = wqe_size16 + | |
| 1270 | + ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
| 1271 | + sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); | |
| 1272 | + sqe->length = cpu_to_le32((u32)data_len); | |
| 1273 | + sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); | |
| 1274 | + sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); | |
| 1275 | + if (qp->mtu) | |
| 1276 | + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | |
| 1277 | + if (!pkt_num) | |
| 1278 | + pkt_num = 1; | |
| 1279 | + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | |
| 1280 | + break; | |
| 1281 | + } | |
| 1282 | + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: | |
| 1283 | + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: | |
| 1284 | + { | |
| 1285 | + struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr; | |
| 1286 | + | |
| 1287 | + sqe->wqe_type = wqe->type; | |
| 1288 | + sqe->flags = wqe->flags; | |
| 1289 | + sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); | |
| 1290 | + sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); | |
| 1291 | + sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); | |
| 1292 | + sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); | |
| 1293 | + if (qp->mtu) | |
| 1294 | + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | |
| 1295 | + if (!pkt_num) | |
| 1296 | + pkt_num = 1; | |
| 1297 | + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | |
| 1298 | + break; | |
| 1299 | + } | |
| 1300 | + case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: | |
| 1301 | + { | |
| 1302 | + struct sq_localinvalidate *sqe = | |
| 1303 | + (struct sq_localinvalidate *)hw_sq_send_hdr; | |
| 1304 | + | |
| 1305 | + sqe->wqe_type = wqe->type; | |
| 1306 | + sqe->flags = wqe->flags; | |
| 1307 | + sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); | |
| 1308 | + | |
| 1309 | + break; | |
| 1310 | + } | |
| 1311 | + case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: | |
| 1312 | + { | |
| 1313 | + struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr; | |
| 1314 | + | |
| 1315 | + sqe->wqe_type = wqe->type; | |
| 1316 | + sqe->flags = wqe->flags; | |
| 1317 | + sqe->access_cntl = wqe->frmr.access_cntl | | |
| 1318 | + SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; | |
| 1319 | + sqe->zero_based_page_size_log = | |
| 1320 | + (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << | |
| 1321 | + SQ_FR_PMR_PAGE_SIZE_LOG_SFT | | |
| 1322 | + (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); | |
| 1323 | + sqe->l_key = cpu_to_le32(wqe->frmr.l_key); | |
| 1324 | + temp32 = cpu_to_le32(wqe->frmr.length); | |
| 1325 | + memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); | |
| 1326 | + sqe->numlevels_pbl_page_size_log = | |
| 1327 | + ((wqe->frmr.pbl_pg_sz_log << | |
| 1328 | + SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & | |
| 1329 | + SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | | |
| 1330 | + ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & | |
| 1331 | + SQ_FR_PMR_NUMLEVELS_MASK); | |
| 1332 | + | |
| 1333 | + for (i = 0; i < wqe->frmr.page_list_len; i++) | |
| 1334 | + wqe->frmr.pbl_ptr[i] = cpu_to_le64( | |
| 1335 | + wqe->frmr.page_list[i] | | |
| 1336 | + PTU_PTE_VALID); | |
| 1337 | + sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); | |
| 1338 | + sqe->va = cpu_to_le64(wqe->frmr.va); | |
| 1339 | + | |
| 1340 | + break; | |
| 1341 | + } | |
| 1342 | + case BNXT_QPLIB_SWQE_TYPE_BIND_MW: | |
| 1343 | + { | |
| 1344 | + struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr; | |
| 1345 | + | |
| 1346 | + sqe->wqe_type = wqe->type; | |
| 1347 | + sqe->flags = wqe->flags; | |
| 1348 | + sqe->access_cntl = wqe->bind.access_cntl; | |
| 1349 | + sqe->mw_type_zero_based = wqe->bind.mw_type | | |
| 1350 | + (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); | |
| 1351 | + sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); | |
| 1352 | + sqe->l_key = cpu_to_le32(wqe->bind.r_key); | |
| 1353 | + sqe->va = cpu_to_le64(wqe->bind.va); | |
| 1354 | + temp32 = cpu_to_le32(wqe->bind.length); | |
| 1355 | + memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); | |
| 1356 | + break; | |
| 1357 | + } | |
| 1358 | + default: | |
| 1359 | + /* Bad wqe, return error */ | |
| 1360 | + rc = -EINVAL; | |
| 1361 | + goto done; | |
| 1362 | + } | |
| 1363 | + swq->next_psn = sq->psn & BTH_PSN_MASK; | |
| 1364 | + if (swq->psn_search) { | |
| 1365 | + swq->psn_search->opcode_start_psn = cpu_to_le32( | |
| 1366 | + ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & | |
| 1367 | + SQ_PSN_SEARCH_START_PSN_MASK) | | |
| 1368 | + ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & | |
| 1369 | + SQ_PSN_SEARCH_OPCODE_MASK)); | |
| 1370 | + swq->psn_search->flags_next_psn = cpu_to_le32( | |
| 1371 | + ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & | |
| 1372 | + SQ_PSN_SEARCH_NEXT_PSN_MASK)); | |
| 1373 | + } | |
| 1374 | + | |
| 1375 | + sq->hwq.prod++; | |
| 1376 | +done: | |
| 1377 | + return rc; | |
| 1378 | +} | |
| 1379 | + | |
| 1380 | +void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) | |
| 1381 | +{ | |
| 1382 | + struct bnxt_qplib_q *rq = &qp->rq; | |
| 1383 | + struct dbr_dbr db_msg = { 0 }; | |
| 1384 | + u32 sw_prod; | |
| 1385 | + | |
| 1386 | + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1387 | + db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & | |
| 1388 | + DBR_DBR_INDEX_MASK); | |
| 1389 | + db_msg.type_xid = | |
| 1390 | + cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | |
| 1391 | + DBR_DBR_TYPE_RQ); | |
| 1392 | + | |
| 1393 | + /* Flush the writes to HW Rx WQE before the ringing Rx DB */ | |
| 1394 | + wmb(); | |
| 1395 | + __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | |
| 1396 | +} | |
| 1397 | + | |
| 1398 | +int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, | |
| 1399 | + struct bnxt_qplib_swqe *wqe) | |
| 1400 | +{ | |
| 1401 | + struct bnxt_qplib_q *rq = &qp->rq; | |
| 1402 | + struct rq_wqe *rqe, **rqe_ptr; | |
| 1403 | + struct sq_sge *hw_sge; | |
| 1404 | + u32 sw_prod; | |
| 1405 | + int i, rc = 0; | |
| 1406 | + | |
| 1407 | + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
| 1408 | + dev_err(&rq->hwq.pdev->dev, | |
| 1409 | + "QPLIB: FP: QP (0x%x) is in the 0x%x state", | |
| 1410 | + qp->id, qp->state); | |
| 1411 | + rc = -EINVAL; | |
| 1412 | + goto done; | |
| 1413 | + } | |
| 1414 | + if (HWQ_CMP((rq->hwq.prod + 1), &rq->hwq) == | |
| 1415 | + HWQ_CMP(rq->hwq.cons, &rq->hwq)) { | |
| 1416 | + dev_err(&rq->hwq.pdev->dev, | |
| 1417 | + "QPLIB: FP: QP (0x%x) RQ is full!", qp->id); | |
| 1418 | + rc = -EINVAL; | |
| 1419 | + goto done; | |
| 1420 | + } | |
| 1421 | + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1422 | + rq->swq[sw_prod].wr_id = wqe->wr_id; | |
| 1423 | + | |
| 1424 | + rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr; | |
| 1425 | + rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)]; | |
| 1426 | + | |
| 1427 | + memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); | |
| 1428 | + | |
| 1429 | + /* Calculate wqe_size16 and data_len */ | |
| 1430 | + for (i = 0, hw_sge = (struct sq_sge *)rqe->data; | |
| 1431 | + i < wqe->num_sge; i++, hw_sge++) { | |
| 1432 | + hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); | |
| 1433 | + hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); | |
| 1434 | + hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); | |
| 1435 | + } | |
| 1436 | + rqe->wqe_type = wqe->type; | |
| 1437 | + rqe->flags = wqe->flags; | |
| 1438 | + rqe->wqe_size = wqe->num_sge + | |
| 1439 | + ((offsetof(typeof(*rqe), data) + 15) >> 4); | |
| 1440 | + | |
| 1441 | + /* Supply the rqe->wr_id index to the wr_id_tbl for now */ | |
| 1442 | + rqe->wr_id[0] = cpu_to_le32(sw_prod); | |
| 1443 | + | |
| 1444 | + rq->hwq.prod++; | |
| 1445 | +done: | |
| 1446 | + return rc; | |
| 1447 | +} | |
| 1448 | + | |
| 1449 | +/* CQ */ | |
| 1450 | + | |
| 1451 | +/* Spinlock must be held */ | |
| 1452 | +static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq) | |
| 1453 | +{ | |
| 1454 | + struct dbr_dbr db_msg = { 0 }; | |
| 1455 | + | |
| 1456 | + db_msg.type_xid = | |
| 1457 | + cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | |
| 1458 | + DBR_DBR_TYPE_CQ_ARMENA); | |
| 1459 | + /* Flush memory writes before enabling the CQ */ | |
| 1460 | + wmb(); | |
| 1461 | + __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64)); | |
| 1462 | +} | |
| 1463 | + | |
| 1464 | +static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type) | |
| 1465 | +{ | |
| 1466 | + struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; | |
| 1467 | + struct dbr_dbr db_msg = { 0 }; | |
| 1468 | + u32 sw_cons; | |
| 1469 | + | |
| 1470 | + /* Ring DB */ | |
| 1471 | + sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq); | |
| 1472 | + db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) & | |
| 1473 | + DBR_DBR_INDEX_MASK); | |
| 1474 | + db_msg.type_xid = | |
| 1475 | + cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | |
| 1476 | + arm_type); | |
| 1477 | + /* flush memory writes before arming the CQ */ | |
| 1478 | + wmb(); | |
| 1479 | + __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | |
| 1480 | +} | |
| 1481 | + | |
| 1482 | +int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) | |
| 1483 | +{ | |
| 1484 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 1485 | + struct cmdq_create_cq req; | |
| 1486 | + struct creq_create_cq_resp *resp; | |
| 1487 | + struct bnxt_qplib_pbl *pbl; | |
| 1488 | + u16 cmd_flags = 0; | |
| 1489 | + int rc; | |
| 1490 | + | |
| 1491 | + cq->hwq.max_elements = cq->max_wqe; | |
| 1492 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead, | |
| 1493 | + cq->nmap, &cq->hwq.max_elements, | |
| 1494 | + BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0, | |
| 1495 | + PAGE_SIZE, HWQ_TYPE_QUEUE); | |
| 1496 | + if (rc) | |
| 1497 | + goto exit; | |
| 1498 | + | |
| 1499 | + RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags); | |
| 1500 | + | |
| 1501 | + if (!cq->dpi) { | |
| 1502 | + dev_err(&rcfw->pdev->dev, | |
| 1503 | + "QPLIB: FP: CREATE_CQ failed due to NULL DPI"); | |
| 1504 | + return -EINVAL; | |
| 1505 | + } | |
| 1506 | + req.dpi = cpu_to_le32(cq->dpi->dpi); | |
| 1507 | + req.cq_handle = cpu_to_le64(cq->cq_handle); | |
| 1508 | + | |
| 1509 | + req.cq_size = cpu_to_le32(cq->hwq.max_elements); | |
| 1510 | + pbl = &cq->hwq.pbl[PBL_LVL_0]; | |
| 1511 | + req.pg_size_lvl = cpu_to_le32( | |
| 1512 | + ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << | |
| 1513 | + CMDQ_CREATE_CQ_LVL_SFT) | | |
| 1514 | + (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K : | |
| 1515 | + pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K : | |
| 1516 | + pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K : | |
| 1517 | + pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M : | |
| 1518 | + pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M : | |
| 1519 | + pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G : | |
| 1520 | + CMDQ_CREATE_CQ_PG_SIZE_PG_4K)); | |
| 1521 | + | |
| 1522 | + req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); | |
| 1523 | + | |
| 1524 | + req.cq_fco_cnq_id = cpu_to_le32( | |
| 1525 | + (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << | |
| 1526 | + CMDQ_CREATE_CQ_CNQ_ID_SFT); | |
| 1527 | + | |
| 1528 | + resp = (struct creq_create_cq_resp *) | |
| 1529 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 1530 | + NULL, 0); | |
| 1531 | + if (!resp) { | |
| 1532 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ send failed"); | |
| 1533 | + return -EINVAL; | |
| 1534 | + } | |
| 1535 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 1536 | + /* Cmd timed out */ | |
| 1537 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ timed out"); | |
| 1538 | + rc = -ETIMEDOUT; | |
| 1539 | + goto fail; | |
| 1540 | + } | |
| 1541 | + if (resp->status || | |
| 1542 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 1543 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ failed "); | |
| 1544 | + dev_err(&rcfw->pdev->dev, | |
| 1545 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 1546 | + resp->status, le16_to_cpu(req.cookie), | |
| 1547 | + le16_to_cpu(resp->cookie)); | |
| 1548 | + rc = -EINVAL; | |
| 1549 | + goto fail; | |
| 1550 | + } | |
| 1551 | + cq->id = le32_to_cpu(resp->xid); | |
| 1552 | + cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem; | |
| 1553 | + cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; | |
| 1554 | + init_waitqueue_head(&cq->waitq); | |
| 1555 | + | |
| 1556 | + bnxt_qplib_arm_cq_enable(cq); | |
| 1557 | + return 0; | |
| 1558 | + | |
| 1559 | +fail: | |
| 1560 | + bnxt_qplib_free_hwq(res->pdev, &cq->hwq); | |
| 1561 | +exit: | |
| 1562 | + return rc; | |
| 1563 | +} | |
| 1564 | + | |
| 1565 | +int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) | |
| 1566 | +{ | |
| 1567 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 1568 | + struct cmdq_destroy_cq req; | |
| 1569 | + struct creq_destroy_cq_resp *resp; | |
| 1570 | + u16 cmd_flags = 0; | |
| 1571 | + | |
| 1572 | + RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags); | |
| 1573 | + | |
| 1574 | + req.cq_cid = cpu_to_le32(cq->id); | |
| 1575 | + resp = (struct creq_destroy_cq_resp *) | |
| 1576 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 1577 | + NULL, 0); | |
| 1578 | + if (!resp) { | |
| 1579 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ send failed"); | |
| 1580 | + return -EINVAL; | |
| 1581 | + } | |
| 1582 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 1583 | + /* Cmd timed out */ | |
| 1584 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ timed out"); | |
| 1585 | + return -ETIMEDOUT; | |
| 1586 | + } | |
| 1587 | + if (resp->status || | |
| 1588 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 1589 | + dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ failed "); | |
| 1590 | + dev_err(&rcfw->pdev->dev, | |
| 1591 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 1592 | + resp->status, le16_to_cpu(req.cookie), | |
| 1593 | + le16_to_cpu(resp->cookie)); | |
| 1594 | + return -EINVAL; | |
| 1595 | + } | |
| 1596 | + bnxt_qplib_free_hwq(res->pdev, &cq->hwq); | |
| 1597 | + return 0; | |
| 1598 | +} | |
| 1599 | + | |
| 1600 | +static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, | |
| 1601 | + struct bnxt_qplib_cqe **pcqe, int *budget) | |
| 1602 | +{ | |
| 1603 | + u32 sw_prod, sw_cons; | |
| 1604 | + struct bnxt_qplib_cqe *cqe; | |
| 1605 | + int rc = 0; | |
| 1606 | + | |
| 1607 | + /* Now complete all outstanding SQEs with FLUSHED_ERR */ | |
| 1608 | + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
| 1609 | + cqe = *pcqe; | |
| 1610 | + while (*budget) { | |
| 1611 | + sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | |
| 1612 | + if (sw_cons == sw_prod) { | |
| 1613 | + sq->flush_in_progress = false; | |
| 1614 | + break; | |
| 1615 | + } | |
| 1616 | + memset(cqe, 0, sizeof(*cqe)); | |
| 1617 | + cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; | |
| 1618 | + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | |
| 1619 | + cqe->qp_handle = (u64)(unsigned long)qp; | |
| 1620 | + cqe->wr_id = sq->swq[sw_cons].wr_id; | |
| 1621 | + cqe->src_qp = qp->id; | |
| 1622 | + cqe->type = sq->swq[sw_cons].type; | |
| 1623 | + cqe++; | |
| 1624 | + (*budget)--; | |
| 1625 | + sq->hwq.cons++; | |
| 1626 | + } | |
| 1627 | + *pcqe = cqe; | |
| 1628 | + if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) | |
| 1629 | + /* Out of budget */ | |
| 1630 | + rc = -EAGAIN; | |
| 1631 | + | |
| 1632 | + return rc; | |
| 1633 | +} | |
| 1634 | + | |
| 1635 | +static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, | |
| 1636 | + int opcode, struct bnxt_qplib_cqe **pcqe, int *budget) | |
| 1637 | +{ | |
| 1638 | + struct bnxt_qplib_cqe *cqe; | |
| 1639 | + u32 sw_prod, sw_cons; | |
| 1640 | + int rc = 0; | |
| 1641 | + | |
| 1642 | + /* Flush the rest of the RQ */ | |
| 1643 | + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1644 | + cqe = *pcqe; | |
| 1645 | + while (*budget) { | |
| 1646 | + sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); | |
| 1647 | + if (sw_cons == sw_prod) | |
| 1648 | + break; | |
| 1649 | + memset(cqe, 0, sizeof(*cqe)); | |
| 1650 | + cqe->status = | |
| 1651 | + CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; | |
| 1652 | + cqe->opcode = opcode; | |
| 1653 | + cqe->qp_handle = (unsigned long)qp; | |
| 1654 | + cqe->wr_id = rq->swq[sw_cons].wr_id; | |
| 1655 | + cqe++; | |
| 1656 | + (*budget)--; | |
| 1657 | + rq->hwq.cons++; | |
| 1658 | + } | |
| 1659 | + *pcqe = cqe; | |
| 1660 | + if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) | |
| 1661 | + /* Out of budget */ | |
| 1662 | + rc = -EAGAIN; | |
| 1663 | + | |
| 1664 | + return rc; | |
| 1665 | +} | |
| 1666 | + | |
| 1667 | +static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, | |
| 1668 | + struct cq_req *hwcqe, | |
| 1669 | + struct bnxt_qplib_cqe **pcqe, int *budget) | |
| 1670 | +{ | |
| 1671 | + struct bnxt_qplib_qp *qp; | |
| 1672 | + struct bnxt_qplib_q *sq; | |
| 1673 | + struct bnxt_qplib_cqe *cqe; | |
| 1674 | + u32 sw_cons, cqe_cons; | |
| 1675 | + int rc = 0; | |
| 1676 | + | |
| 1677 | + qp = (struct bnxt_qplib_qp *)((unsigned long) | |
| 1678 | + le64_to_cpu(hwcqe->qp_handle)); | |
| 1679 | + if (!qp) { | |
| 1680 | + dev_err(&cq->hwq.pdev->dev, | |
| 1681 | + "QPLIB: FP: Process Req qp is NULL"); | |
| 1682 | + return -EINVAL; | |
| 1683 | + } | |
| 1684 | + sq = &qp->sq; | |
| 1685 | + | |
| 1686 | + cqe_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); | |
| 1687 | + if (cqe_cons > sq->hwq.max_elements) { | |
| 1688 | + dev_err(&cq->hwq.pdev->dev, | |
| 1689 | + "QPLIB: FP: CQ Process req reported "); | |
| 1690 | + dev_err(&cq->hwq.pdev->dev, | |
| 1691 | + "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", | |
| 1692 | + cqe_cons, sq->hwq.max_elements); | |
| 1693 | + return -EINVAL; | |
| 1694 | + } | |
| 1695 | + /* If we were in the middle of flushing the SQ, continue */ | |
| 1696 | + if (sq->flush_in_progress) | |
| 1697 | + goto flush; | |
| 1698 | + | |
| 1699 | + /* Require to walk the sq's swq to fabricate CQEs for all previously | |
| 1700 | + * signaled SWQEs due to CQE aggregation from the current sq cons | |
| 1701 | + * to the cqe_cons | |
| 1702 | + */ | |
| 1703 | + cqe = *pcqe; | |
| 1704 | + while (*budget) { | |
| 1705 | + sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | |
| 1706 | + if (sw_cons == cqe_cons) | |
| 1707 | + break; | |
| 1708 | + memset(cqe, 0, sizeof(*cqe)); | |
| 1709 | + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | |
| 1710 | + cqe->qp_handle = (u64)(unsigned long)qp; | |
| 1711 | + cqe->src_qp = qp->id; | |
| 1712 | + cqe->wr_id = sq->swq[sw_cons].wr_id; | |
| 1713 | + cqe->type = sq->swq[sw_cons].type; | |
| 1714 | + | |
| 1715 | + /* For the last CQE, check for status. For errors, regardless | |
| 1716 | + * of the request being signaled or not, it must complete with | |
| 1717 | + * the hwcqe error status | |
| 1718 | + */ | |
| 1719 | + if (HWQ_CMP((sw_cons + 1), &sq->hwq) == cqe_cons && | |
| 1720 | + hwcqe->status != CQ_REQ_STATUS_OK) { | |
| 1721 | + cqe->status = hwcqe->status; | |
| 1722 | + dev_err(&cq->hwq.pdev->dev, | |
| 1723 | + "QPLIB: FP: CQ Processed Req "); | |
| 1724 | + dev_err(&cq->hwq.pdev->dev, | |
| 1725 | + "QPLIB: wr_id[%d] = 0x%llx with status 0x%x", | |
| 1726 | + sw_cons, cqe->wr_id, cqe->status); | |
| 1727 | + cqe++; | |
| 1728 | + (*budget)--; | |
| 1729 | + sq->flush_in_progress = true; | |
| 1730 | + /* Must block new posting of SQ and RQ */ | |
| 1731 | + qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; | |
| 1732 | + } else { | |
| 1733 | + if (sq->swq[sw_cons].flags & | |
| 1734 | + SQ_SEND_FLAGS_SIGNAL_COMP) { | |
| 1735 | + cqe->status = CQ_REQ_STATUS_OK; | |
| 1736 | + cqe++; | |
| 1737 | + (*budget)--; | |
| 1738 | + } | |
| 1739 | + } | |
| 1740 | + sq->hwq.cons++; | |
| 1741 | + } | |
| 1742 | + *pcqe = cqe; | |
| 1743 | + if (!*budget && HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_cons) { | |
| 1744 | + /* Out of budget */ | |
| 1745 | + rc = -EAGAIN; | |
| 1746 | + goto done; | |
| 1747 | + } | |
| 1748 | + if (!sq->flush_in_progress) | |
| 1749 | + goto done; | |
| 1750 | +flush: | |
| 1751 | + /* Require to walk the sq's swq to fabricate CQEs for all | |
| 1752 | + * previously posted SWQEs due to the error CQE received | |
| 1753 | + */ | |
| 1754 | + rc = __flush_sq(sq, qp, pcqe, budget); | |
| 1755 | + if (!rc) | |
| 1756 | + sq->flush_in_progress = false; | |
| 1757 | +done: | |
| 1758 | + return rc; | |
| 1759 | +} | |
| 1760 | + | |
| 1761 | +static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, | |
| 1762 | + struct cq_res_rc *hwcqe, | |
| 1763 | + struct bnxt_qplib_cqe **pcqe, | |
| 1764 | + int *budget) | |
| 1765 | +{ | |
| 1766 | + struct bnxt_qplib_qp *qp; | |
| 1767 | + struct bnxt_qplib_q *rq; | |
| 1768 | + struct bnxt_qplib_cqe *cqe; | |
| 1769 | + u32 wr_id_idx; | |
| 1770 | + int rc = 0; | |
| 1771 | + | |
| 1772 | + qp = (struct bnxt_qplib_qp *)((unsigned long) | |
| 1773 | + le64_to_cpu(hwcqe->qp_handle)); | |
| 1774 | + if (!qp) { | |
| 1775 | + dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL"); | |
| 1776 | + return -EINVAL; | |
| 1777 | + } | |
| 1778 | + cqe = *pcqe; | |
| 1779 | + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | |
| 1780 | + cqe->length = le32_to_cpu(hwcqe->length); | |
| 1781 | + cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); | |
| 1782 | + cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); | |
| 1783 | + cqe->flags = le16_to_cpu(hwcqe->flags); | |
| 1784 | + cqe->status = hwcqe->status; | |
| 1785 | + cqe->qp_handle = (u64)(unsigned long)qp; | |
| 1786 | + | |
| 1787 | + wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & | |
| 1788 | + CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; | |
| 1789 | + rq = &qp->rq; | |
| 1790 | + if (wr_id_idx > rq->hwq.max_elements) { | |
| 1791 | + dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC "); | |
| 1792 | + dev_err(&cq->hwq.pdev->dev, | |
| 1793 | + "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x", | |
| 1794 | + wr_id_idx, rq->hwq.max_elements); | |
| 1795 | + return -EINVAL; | |
| 1796 | + } | |
| 1797 | + if (rq->flush_in_progress) | |
| 1798 | + goto flush_rq; | |
| 1799 | + | |
| 1800 | + cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
| 1801 | + cqe++; | |
| 1802 | + (*budget)--; | |
| 1803 | + rq->hwq.cons++; | |
| 1804 | + *pcqe = cqe; | |
| 1805 | + | |
| 1806 | + if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | |
| 1807 | + rq->flush_in_progress = true; | |
| 1808 | +flush_rq: | |
| 1809 | + rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget); | |
| 1810 | + if (!rc) | |
| 1811 | + rq->flush_in_progress = false; | |
| 1812 | + } | |
| 1813 | + return rc; | |
| 1814 | +} | |
| 1815 | + | |
| 1816 | +static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, | |
| 1817 | + struct cq_res_ud *hwcqe, | |
| 1818 | + struct bnxt_qplib_cqe **pcqe, | |
| 1819 | + int *budget) | |
| 1820 | +{ | |
| 1821 | + struct bnxt_qplib_qp *qp; | |
| 1822 | + struct bnxt_qplib_q *rq; | |
| 1823 | + struct bnxt_qplib_cqe *cqe; | |
| 1824 | + u32 wr_id_idx; | |
| 1825 | + int rc = 0; | |
| 1826 | + | |
| 1827 | + qp = (struct bnxt_qplib_qp *)((unsigned long) | |
| 1828 | + le64_to_cpu(hwcqe->qp_handle)); | |
| 1829 | + if (!qp) { | |
| 1830 | + dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL"); | |
| 1831 | + return -EINVAL; | |
| 1832 | + } | |
| 1833 | + cqe = *pcqe; | |
| 1834 | + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | |
| 1835 | + cqe->length = le32_to_cpu(hwcqe->length); | |
| 1836 | + cqe->invrkey = le32_to_cpu(hwcqe->imm_data); | |
| 1837 | + cqe->flags = le16_to_cpu(hwcqe->flags); | |
| 1838 | + cqe->status = hwcqe->status; | |
| 1839 | + cqe->qp_handle = (u64)(unsigned long)qp; | |
| 1840 | + memcpy(cqe->smac, hwcqe->src_mac, 6); | |
| 1841 | + wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) | |
| 1842 | + & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK; | |
| 1843 | + cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | | |
| 1844 | + ((le32_to_cpu( | |
| 1845 | + hwcqe->src_qp_high_srq_or_rq_wr_id) & | |
| 1846 | + CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8); | |
| 1847 | + | |
| 1848 | + rq = &qp->rq; | |
| 1849 | + if (wr_id_idx > rq->hwq.max_elements) { | |
| 1850 | + dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD "); | |
| 1851 | + dev_err(&cq->hwq.pdev->dev, | |
| 1852 | + "QPLIB: wr_id idx %#x exceeded RQ max %#x", | |
| 1853 | + wr_id_idx, rq->hwq.max_elements); | |
| 1854 | + return -EINVAL; | |
| 1855 | + } | |
| 1856 | + if (rq->flush_in_progress) | |
| 1857 | + goto flush_rq; | |
| 1858 | + | |
| 1859 | + cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
| 1860 | + cqe++; | |
| 1861 | + (*budget)--; | |
| 1862 | + rq->hwq.cons++; | |
| 1863 | + *pcqe = cqe; | |
| 1864 | + | |
| 1865 | + if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | |
| 1866 | + rq->flush_in_progress = true; | |
| 1867 | +flush_rq: | |
| 1868 | + rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget); | |
| 1869 | + if (!rc) | |
| 1870 | + rq->flush_in_progress = false; | |
| 1871 | + } | |
| 1872 | + return rc; | |
| 1873 | +} | |
| 1874 | + | |
| 1875 | +static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, | |
| 1876 | + struct cq_res_raweth_qp1 *hwcqe, | |
| 1877 | + struct bnxt_qplib_cqe **pcqe, | |
| 1878 | + int *budget) | |
| 1879 | +{ | |
| 1880 | + struct bnxt_qplib_qp *qp; | |
| 1881 | + struct bnxt_qplib_q *rq; | |
| 1882 | + struct bnxt_qplib_cqe *cqe; | |
| 1883 | + u32 wr_id_idx; | |
| 1884 | + int rc = 0; | |
| 1885 | + | |
| 1886 | + qp = (struct bnxt_qplib_qp *)((unsigned long) | |
| 1887 | + le64_to_cpu(hwcqe->qp_handle)); | |
| 1888 | + if (!qp) { | |
| 1889 | + dev_err(&cq->hwq.pdev->dev, | |
| 1890 | + "QPLIB: process_cq Raw/QP1 qp is NULL"); | |
| 1891 | + return -EINVAL; | |
| 1892 | + } | |
| 1893 | + cqe = *pcqe; | |
| 1894 | + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | |
| 1895 | + cqe->flags = le16_to_cpu(hwcqe->flags); | |
| 1896 | + cqe->qp_handle = (u64)(unsigned long)qp; | |
| 1897 | + | |
| 1898 | + wr_id_idx = | |
| 1899 | + le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) | |
| 1900 | + & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; | |
| 1901 | + cqe->src_qp = qp->id; | |
| 1902 | + if (qp->id == 1 && !cqe->length) { | |
| 1903 | + /* Add workaround for the length misdetection */ | |
| 1904 | + cqe->length = 296; | |
| 1905 | + } else { | |
| 1906 | + cqe->length = le16_to_cpu(hwcqe->length); | |
| 1907 | + } | |
| 1908 | + cqe->pkey_index = qp->pkey_index; | |
| 1909 | + memcpy(cqe->smac, qp->smac, 6); | |
| 1910 | + | |
| 1911 | + cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); | |
| 1912 | + cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); | |
| 1913 | + | |
| 1914 | + rq = &qp->rq; | |
| 1915 | + if (wr_id_idx > rq->hwq.max_elements) { | |
| 1916 | + dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id "); | |
| 1917 | + dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x", | |
| 1918 | + wr_id_idx, rq->hwq.max_elements); | |
| 1919 | + return -EINVAL; | |
| 1920 | + } | |
| 1921 | + if (rq->flush_in_progress) | |
| 1922 | + goto flush_rq; | |
| 1923 | + | |
| 1924 | + cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
| 1925 | + cqe++; | |
| 1926 | + (*budget)--; | |
| 1927 | + rq->hwq.cons++; | |
| 1928 | + *pcqe = cqe; | |
| 1929 | + | |
| 1930 | + if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | |
| 1931 | + rq->flush_in_progress = true; | |
| 1932 | +flush_rq: | |
| 1933 | + rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe, | |
| 1934 | + budget); | |
| 1935 | + if (!rc) | |
| 1936 | + rq->flush_in_progress = false; | |
| 1937 | + } | |
| 1938 | + return rc; | |
| 1939 | +} | |
| 1940 | + | |
| 1941 | +static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, | |
| 1942 | + struct cq_terminal *hwcqe, | |
| 1943 | + struct bnxt_qplib_cqe **pcqe, | |
| 1944 | + int *budget) | |
| 1945 | +{ | |
| 1946 | + struct bnxt_qplib_qp *qp; | |
| 1947 | + struct bnxt_qplib_q *sq, *rq; | |
| 1948 | + struct bnxt_qplib_cqe *cqe; | |
| 1949 | + u32 sw_cons = 0, cqe_cons; | |
| 1950 | + int rc = 0; | |
| 1951 | + u8 opcode = 0; | |
| 1952 | + | |
| 1953 | + /* Check the Status */ | |
| 1954 | + if (hwcqe->status != CQ_TERMINAL_STATUS_OK) | |
| 1955 | + dev_warn(&cq->hwq.pdev->dev, | |
| 1956 | + "QPLIB: FP: CQ Process Terminal Error status = 0x%x", | |
| 1957 | + hwcqe->status); | |
| 1958 | + | |
| 1959 | + qp = (struct bnxt_qplib_qp *)((unsigned long) | |
| 1960 | + le64_to_cpu(hwcqe->qp_handle)); | |
| 1961 | + if (!qp) { | |
| 1962 | + dev_err(&cq->hwq.pdev->dev, | |
| 1963 | + "QPLIB: FP: CQ Process terminal qp is NULL"); | |
| 1964 | + return -EINVAL; | |
| 1965 | + } | |
| 1966 | + /* Must block new posting of SQ and RQ */ | |
| 1967 | + qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; | |
| 1968 | + | |
| 1969 | + sq = &qp->sq; | |
| 1970 | + rq = &qp->rq; | |
| 1971 | + | |
| 1972 | + cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); | |
| 1973 | + if (cqe_cons == 0xFFFF) | |
| 1974 | + goto do_rq; | |
| 1975 | + | |
| 1976 | + if (cqe_cons > sq->hwq.max_elements) { | |
| 1977 | + dev_err(&cq->hwq.pdev->dev, | |
| 1978 | + "QPLIB: FP: CQ Process terminal reported "); | |
| 1979 | + dev_err(&cq->hwq.pdev->dev, | |
| 1980 | + "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", | |
| 1981 | + cqe_cons, sq->hwq.max_elements); | |
| 1982 | + goto do_rq; | |
| 1983 | + } | |
| 1984 | + /* If we were in the middle of flushing, continue */ | |
| 1985 | + if (sq->flush_in_progress) | |
| 1986 | + goto flush_sq; | |
| 1987 | + | |
| 1988 | + /* Terminal CQE can also include aggregated successful CQEs prior. | |
| 1989 | + * So we must complete all CQEs from the current sq's cons to the | |
| 1990 | + * cq_cons with status OK | |
| 1991 | + */ | |
| 1992 | + cqe = *pcqe; | |
| 1993 | + while (*budget) { | |
| 1994 | + sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | |
| 1995 | + if (sw_cons == cqe_cons) | |
| 1996 | + break; | |
| 1997 | + if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { | |
| 1998 | + memset(cqe, 0, sizeof(*cqe)); | |
| 1999 | + cqe->status = CQ_REQ_STATUS_OK; | |
| 2000 | + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | |
| 2001 | + cqe->qp_handle = (u64)(unsigned long)qp; | |
| 2002 | + cqe->src_qp = qp->id; | |
| 2003 | + cqe->wr_id = sq->swq[sw_cons].wr_id; | |
| 2004 | + cqe->type = sq->swq[sw_cons].type; | |
| 2005 | + cqe++; | |
| 2006 | + (*budget)--; | |
| 2007 | + } | |
| 2008 | + sq->hwq.cons++; | |
| 2009 | + } | |
| 2010 | + *pcqe = cqe; | |
| 2011 | + if (!(*budget) && sw_cons != cqe_cons) { | |
| 2012 | + /* Out of budget */ | |
| 2013 | + rc = -EAGAIN; | |
| 2014 | + goto sq_done; | |
| 2015 | + } | |
| 2016 | + sq->flush_in_progress = true; | |
| 2017 | +flush_sq: | |
| 2018 | + rc = __flush_sq(sq, qp, pcqe, budget); | |
| 2019 | + if (!rc) | |
| 2020 | + sq->flush_in_progress = false; | |
| 2021 | +sq_done: | |
| 2022 | + if (rc) | |
| 2023 | + return rc; | |
| 2024 | +do_rq: | |
| 2025 | + cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); | |
| 2026 | + if (cqe_cons == 0xFFFF) { | |
| 2027 | + goto done; | |
| 2028 | + } else if (cqe_cons > rq->hwq.max_elements) { | |
| 2029 | + dev_err(&cq->hwq.pdev->dev, | |
| 2030 | + "QPLIB: FP: CQ Processed terminal "); | |
| 2031 | + dev_err(&cq->hwq.pdev->dev, | |
| 2032 | + "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x", | |
| 2033 | + cqe_cons, rq->hwq.max_elements); | |
| 2034 | + goto done; | |
| 2035 | + } | |
| 2036 | + /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR | |
| 2037 | + * from the current rq->cons to the rq->prod regardless what the | |
| 2038 | + * rq->cons the terminal CQE indicates | |
| 2039 | + */ | |
| 2040 | + rq->flush_in_progress = true; | |
| 2041 | + switch (qp->type) { | |
| 2042 | + case CMDQ_CREATE_QP1_TYPE_GSI: | |
| 2043 | + opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; | |
| 2044 | + break; | |
| 2045 | + case CMDQ_CREATE_QP_TYPE_RC: | |
| 2046 | + opcode = CQ_BASE_CQE_TYPE_RES_RC; | |
| 2047 | + break; | |
| 2048 | + case CMDQ_CREATE_QP_TYPE_UD: | |
| 2049 | + opcode = CQ_BASE_CQE_TYPE_RES_UD; | |
| 2050 | + break; | |
| 2051 | + } | |
| 2052 | + | |
| 2053 | + rc = __flush_rq(rq, qp, opcode, pcqe, budget); | |
| 2054 | + if (!rc) | |
| 2055 | + rq->flush_in_progress = false; | |
| 2056 | +done: | |
| 2057 | + return rc; | |
| 2058 | +} | |
| 2059 | + | |
| 2060 | +static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, | |
| 2061 | + struct cq_cutoff *hwcqe) | |
| 2062 | +{ | |
| 2063 | + /* Check the Status */ | |
| 2064 | + if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { | |
| 2065 | + dev_err(&cq->hwq.pdev->dev, | |
| 2066 | + "QPLIB: FP: CQ Process Cutoff Error status = 0x%x", | |
| 2067 | + hwcqe->status); | |
| 2068 | + return -EINVAL; | |
| 2069 | + } | |
| 2070 | + clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); | |
| 2071 | + wake_up_interruptible(&cq->waitq); | |
| 2072 | + | |
| 2073 | + return 0; | |
| 2074 | +} | |
| 2075 | + | |
| 2076 | +int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, | |
| 2077 | + int num_cqes) | |
| 2078 | +{ | |
| 2079 | + struct cq_base *hw_cqe, **hw_cqe_ptr; | |
| 2080 | + unsigned long flags; | |
| 2081 | + u32 sw_cons, raw_cons; | |
| 2082 | + int budget, rc = 0; | |
| 2083 | + | |
| 2084 | + spin_lock_irqsave(&cq->hwq.lock, flags); | |
| 2085 | + raw_cons = cq->hwq.cons; | |
| 2086 | + budget = num_cqes; | |
| 2087 | + | |
| 2088 | + while (budget) { | |
| 2089 | + sw_cons = HWQ_CMP(raw_cons, &cq->hwq); | |
| 2090 | + hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; | |
| 2091 | + hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)]; | |
| 2092 | + | |
| 2093 | + /* Check for Valid bit */ | |
| 2094 | + if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) | |
| 2095 | + break; | |
| 2096 | + | |
| 2097 | + /* From the device's respective CQE format to qplib_wc*/ | |
| 2098 | + switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { | |
| 2099 | + case CQ_BASE_CQE_TYPE_REQ: | |
| 2100 | + rc = bnxt_qplib_cq_process_req(cq, | |
| 2101 | + (struct cq_req *)hw_cqe, | |
| 2102 | + &cqe, &budget); | |
| 2103 | + break; | |
| 2104 | + case CQ_BASE_CQE_TYPE_RES_RC: | |
| 2105 | + rc = bnxt_qplib_cq_process_res_rc(cq, | |
| 2106 | + (struct cq_res_rc *) | |
| 2107 | + hw_cqe, &cqe, | |
| 2108 | + &budget); | |
| 2109 | + break; | |
| 2110 | + case CQ_BASE_CQE_TYPE_RES_UD: | |
| 2111 | + rc = bnxt_qplib_cq_process_res_ud | |
| 2112 | + (cq, (struct cq_res_ud *)hw_cqe, &cqe, | |
| 2113 | + &budget); | |
| 2114 | + break; | |
| 2115 | + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: | |
| 2116 | + rc = bnxt_qplib_cq_process_res_raweth_qp1 | |
| 2117 | + (cq, (struct cq_res_raweth_qp1 *) | |
| 2118 | + hw_cqe, &cqe, &budget); | |
| 2119 | + break; | |
| 2120 | + case CQ_BASE_CQE_TYPE_TERMINAL: | |
| 2121 | + rc = bnxt_qplib_cq_process_terminal | |
| 2122 | + (cq, (struct cq_terminal *)hw_cqe, | |
| 2123 | + &cqe, &budget); | |
| 2124 | + break; | |
| 2125 | + case CQ_BASE_CQE_TYPE_CUT_OFF: | |
| 2126 | + bnxt_qplib_cq_process_cutoff | |
| 2127 | + (cq, (struct cq_cutoff *)hw_cqe); | |
| 2128 | + /* Done processing this CQ */ | |
| 2129 | + goto exit; | |
| 2130 | + default: | |
| 2131 | + dev_err(&cq->hwq.pdev->dev, | |
| 2132 | + "QPLIB: process_cq unknown type 0x%lx", | |
| 2133 | + hw_cqe->cqe_type_toggle & | |
| 2134 | + CQ_BASE_CQE_TYPE_MASK); | |
| 2135 | + rc = -EINVAL; | |
| 2136 | + break; | |
| 2137 | + } | |
| 2138 | + if (rc < 0) { | |
| 2139 | + if (rc == -EAGAIN) | |
| 2140 | + break; | |
| 2141 | + /* Error while processing the CQE, just skip to the | |
| 2142 | + * next one | |
| 2143 | + */ | |
| 2144 | + dev_err(&cq->hwq.pdev->dev, | |
| 2145 | + "QPLIB: process_cqe error rc = 0x%x", rc); | |
| 2146 | + } | |
| 2147 | + raw_cons++; | |
| 2148 | + } | |
| 2149 | + if (cq->hwq.cons != raw_cons) { | |
| 2150 | + cq->hwq.cons = raw_cons; | |
| 2151 | + bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ); | |
| 2152 | + } | |
| 2153 | +exit: | |
| 2154 | + spin_unlock_irqrestore(&cq->hwq.lock, flags); | |
| 2155 | + return num_cqes - budget; | |
| 2156 | +} | |
| 2157 | + | |
| 2158 | +void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) | |
| 2159 | +{ | |
| 2160 | + unsigned long flags; | |
| 2161 | + | |
| 2162 | + spin_lock_irqsave(&cq->hwq.lock, flags); | |
| 2163 | + if (arm_type) | |
| 2164 | + bnxt_qplib_arm_cq(cq, arm_type); | |
| 2165 | + | |
| 2166 | + spin_unlock_irqrestore(&cq->hwq.lock, flags); | |
| 2167 | +} | |
| @@ -0,0 +1,439 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: Fast Path Operators (header) | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#ifndef __BNXT_QPLIB_FP_H__ | |
| 40 | +#define __BNXT_QPLIB_FP_H__ | |
| 41 | + | |
| 42 | +struct bnxt_qplib_sge { | |
| 43 | + u64 addr; | |
| 44 | + u32 lkey; | |
| 45 | + u32 size; | |
| 46 | +}; | |
| 47 | + | |
| 48 | +#define BNXT_QPLIB_MAX_SQE_ENTRY_SIZE sizeof(struct sq_send) | |
| 49 | + | |
| 50 | +#define SQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_SQE_ENTRY_SIZE) | |
| 51 | +#define SQE_MAX_IDX_PER_PG (SQE_CNT_PER_PG - 1) | |
| 52 | + | |
| 53 | +static inline u32 get_sqe_pg(u32 val) | |
| 54 | +{ | |
| 55 | + return ((val & ~SQE_MAX_IDX_PER_PG) / SQE_CNT_PER_PG); | |
| 56 | +} | |
| 57 | + | |
| 58 | +static inline u32 get_sqe_idx(u32 val) | |
| 59 | +{ | |
| 60 | + return (val & SQE_MAX_IDX_PER_PG); | |
| 61 | +} | |
| 62 | + | |
| 63 | +#define BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE sizeof(struct sq_psn_search) | |
| 64 | + | |
| 65 | +#define PSNE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE) | |
| 66 | +#define PSNE_MAX_IDX_PER_PG (PSNE_CNT_PER_PG - 1) | |
| 67 | + | |
| 68 | +static inline u32 get_psne_pg(u32 val) | |
| 69 | +{ | |
| 70 | + return ((val & ~PSNE_MAX_IDX_PER_PG) / PSNE_CNT_PER_PG); | |
| 71 | +} | |
| 72 | + | |
| 73 | +static inline u32 get_psne_idx(u32 val) | |
| 74 | +{ | |
| 75 | + return (val & PSNE_MAX_IDX_PER_PG); | |
| 76 | +} | |
| 77 | + | |
| 78 | +#define BNXT_QPLIB_QP_MAX_SGL 6 | |
| 79 | + | |
| 80 | +struct bnxt_qplib_swq { | |
| 81 | + u64 wr_id; | |
| 82 | + u8 type; | |
| 83 | + u8 flags; | |
| 84 | + u32 start_psn; | |
| 85 | + u32 next_psn; | |
| 86 | + struct sq_psn_search *psn_search; | |
| 87 | +}; | |
| 88 | + | |
| 89 | +struct bnxt_qplib_swqe { | |
| 90 | + /* General */ | |
| 91 | + u64 wr_id; | |
| 92 | + u8 reqs_type; | |
| 93 | + u8 type; | |
| 94 | +#define BNXT_QPLIB_SWQE_TYPE_SEND 0 | |
| 95 | +#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 | |
| 96 | +#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 | |
| 97 | +#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 | |
| 98 | +#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 | |
| 99 | +#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 | |
| 100 | +#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 | |
| 101 | +#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 | |
| 102 | +#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 | |
| 103 | +#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 | |
| 104 | +#define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 | |
| 105 | +#define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 | |
| 106 | +#define BNXT_QPLIB_SWQE_TYPE_RECV 128 | |
| 107 | +#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 | |
| 108 | + u8 flags; | |
| 109 | +#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0) | |
| 110 | +#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1) | |
| 111 | +#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2) | |
| 112 | +#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3) | |
| 113 | +#define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4) | |
| 114 | + struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL]; | |
| 115 | + int num_sge; | |
| 116 | + /* Max inline data is 96 bytes */ | |
| 117 | + u32 inline_len; | |
| 118 | +#define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96 | |
| 119 | + u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH]; | |
| 120 | + | |
| 121 | + union { | |
| 122 | + /* Send, with imm, inval key */ | |
| 123 | + struct { | |
| 124 | + union { | |
| 125 | + __be32 imm_data; | |
| 126 | + u32 inv_key; | |
| 127 | + }; | |
| 128 | + u32 q_key; | |
| 129 | + u32 dst_qp; | |
| 130 | + u16 avid; | |
| 131 | + } send; | |
| 132 | + | |
| 133 | + /* Send Raw Ethernet and QP1 */ | |
| 134 | + struct { | |
| 135 | + u16 lflags; | |
| 136 | + u16 cfa_action; | |
| 137 | + u32 cfa_meta; | |
| 138 | + } rawqp1; | |
| 139 | + | |
| 140 | + /* RDMA write, with imm, read */ | |
| 141 | + struct { | |
| 142 | + union { | |
| 143 | + __be32 imm_data; | |
| 144 | + u32 inv_key; | |
| 145 | + }; | |
| 146 | + u64 remote_va; | |
| 147 | + u32 r_key; | |
| 148 | + } rdma; | |
| 149 | + | |
| 150 | + /* Atomic cmp/swap, fetch/add */ | |
| 151 | + struct { | |
| 152 | + u64 remote_va; | |
| 153 | + u32 r_key; | |
| 154 | + u64 swap_data; | |
| 155 | + u64 cmp_data; | |
| 156 | + } atomic; | |
| 157 | + | |
| 158 | + /* Local Invalidate */ | |
| 159 | + struct { | |
| 160 | + u32 inv_l_key; | |
| 161 | + } local_inv; | |
| 162 | + | |
| 163 | + /* FR-PMR */ | |
| 164 | + struct { | |
| 165 | + u8 access_cntl; | |
| 166 | + u8 pg_sz_log; | |
| 167 | + bool zero_based; | |
| 168 | + u32 l_key; | |
| 169 | + u32 length; | |
| 170 | + u8 pbl_pg_sz_log; | |
| 171 | +#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 | |
| 172 | +#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 | |
| 173 | +#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 | |
| 174 | +#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 | |
| 175 | +#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 | |
| 176 | +#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 | |
| 177 | +#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 | |
| 178 | +#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 | |
| 179 | + u8 levels; | |
| 180 | +#define PAGE_SHIFT_4K 12 | |
| 181 | + __le64 *pbl_ptr; | |
| 182 | + dma_addr_t pbl_dma_ptr; | |
| 183 | + u64 *page_list; | |
| 184 | + u16 page_list_len; | |
| 185 | + u64 va; | |
| 186 | + } frmr; | |
| 187 | + | |
| 188 | + /* Bind */ | |
| 189 | + struct { | |
| 190 | + u8 access_cntl; | |
| 191 | +#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0) | |
| 192 | +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1) | |
| 193 | +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2) | |
| 194 | +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3) | |
| 195 | +#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4) | |
| 196 | + bool zero_based; | |
| 197 | + u8 mw_type; | |
| 198 | + u32 parent_l_key; | |
| 199 | + u32 r_key; | |
| 200 | + u64 va; | |
| 201 | + u32 length; | |
| 202 | + } bind; | |
| 203 | + }; | |
| 204 | +}; | |
| 205 | + | |
| 206 | +#define BNXT_QPLIB_MAX_RQE_ENTRY_SIZE sizeof(struct rq_wqe) | |
| 207 | + | |
| 208 | +#define RQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_RQE_ENTRY_SIZE) | |
| 209 | +#define RQE_MAX_IDX_PER_PG (RQE_CNT_PER_PG - 1) | |
| 210 | +#define RQE_PG(x) (((x) & ~RQE_MAX_IDX_PER_PG) / RQE_CNT_PER_PG) | |
| 211 | +#define RQE_IDX(x) ((x) & RQE_MAX_IDX_PER_PG) | |
| 212 | + | |
| 213 | +struct bnxt_qplib_q { | |
| 214 | + struct bnxt_qplib_hwq hwq; | |
| 215 | + struct bnxt_qplib_swq *swq; | |
| 216 | + struct scatterlist *sglist; | |
| 217 | + u32 nmap; | |
| 218 | + u32 max_wqe; | |
| 219 | + u16 max_sge; | |
| 220 | + u32 psn; | |
| 221 | + bool flush_in_progress; | |
| 222 | +}; | |
| 223 | + | |
| 224 | +struct bnxt_qplib_qp { | |
| 225 | + struct bnxt_qplib_pd *pd; | |
| 226 | + struct bnxt_qplib_dpi *dpi; | |
| 227 | + u64 qp_handle; | |
| 228 | + u32 id; | |
| 229 | + u8 type; | |
| 230 | + u8 sig_type; | |
| 231 | + u32 modify_flags; | |
| 232 | + u8 state; | |
| 233 | + u8 cur_qp_state; | |
| 234 | + u32 max_inline_data; | |
| 235 | + u32 mtu; | |
| 236 | + u8 path_mtu; | |
| 237 | + bool en_sqd_async_notify; | |
| 238 | + u16 pkey_index; | |
| 239 | + u32 qkey; | |
| 240 | + u32 dest_qp_id; | |
| 241 | + u8 access; | |
| 242 | + u8 timeout; | |
| 243 | + u8 retry_cnt; | |
| 244 | + u8 rnr_retry; | |
| 245 | + u32 min_rnr_timer; | |
| 246 | + u32 max_rd_atomic; | |
| 247 | + u32 max_dest_rd_atomic; | |
| 248 | + u32 dest_qpn; | |
| 249 | + u8 smac[6]; | |
| 250 | + u16 vlan_id; | |
| 251 | + u8 nw_type; | |
| 252 | + struct bnxt_qplib_ah ah; | |
| 253 | + | |
| 254 | +#define BTH_PSN_MASK ((1 << 24) - 1) | |
| 255 | + /* SQ */ | |
| 256 | + struct bnxt_qplib_q sq; | |
| 257 | + /* RQ */ | |
| 258 | + struct bnxt_qplib_q rq; | |
| 259 | + /* SRQ */ | |
| 260 | + struct bnxt_qplib_srq *srq; | |
| 261 | + /* CQ */ | |
| 262 | + struct bnxt_qplib_cq *scq; | |
| 263 | + struct bnxt_qplib_cq *rcq; | |
| 264 | + /* IRRQ and ORRQ */ | |
| 265 | + struct bnxt_qplib_hwq irrq; | |
| 266 | + struct bnxt_qplib_hwq orrq; | |
| 267 | + /* Header buffer for QP1 */ | |
| 268 | + int sq_hdr_buf_size; | |
| 269 | + int rq_hdr_buf_size; | |
| 270 | +/* | |
| 271 | + * Buffer space for ETH(14), IP or GRH(40), UDP header(8) | |
| 272 | + * and ib_bth + ib_deth (20). | |
| 273 | + * Max required is 82 when RoCE V2 is enabled | |
| 274 | + */ | |
| 275 | +#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 | |
| 276 | + /* Ethernet header = 14 */ | |
| 277 | + /* ib_grh = 40 (provided by MAD) */ | |
| 278 | + /* ib_bth + ib_deth = 20 */ | |
| 279 | + /* MAD = 256 (provided by MAD) */ | |
| 280 | + /* iCRC = 4 */ | |
| 281 | +#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 | |
| 282 | +#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 | |
| 283 | +#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 | |
| 284 | +#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 | |
| 285 | +#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 | |
| 286 | + void *sq_hdr_buf; | |
| 287 | + dma_addr_t sq_hdr_buf_map; | |
| 288 | + void *rq_hdr_buf; | |
| 289 | + dma_addr_t rq_hdr_buf_map; | |
| 290 | +}; | |
| 291 | + | |
| 292 | +#define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) | |
| 293 | + | |
| 294 | +#define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE) | |
| 295 | +#define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1) | |
| 296 | +#define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG) | |
| 297 | +#define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG) | |
| 298 | + | |
| 299 | +#define ROCE_CQE_CMP_V 0 | |
| 300 | +#define CQE_CMP_VALID(hdr, raw_cons, cp_bit) \ | |
| 301 | + (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ | |
| 302 | + !((raw_cons) & (cp_bit))) | |
| 303 | + | |
| 304 | +struct bnxt_qplib_cqe { | |
| 305 | + u8 status; | |
| 306 | + u8 type; | |
| 307 | + u8 opcode; | |
| 308 | + u32 length; | |
| 309 | + u64 wr_id; | |
| 310 | + union { | |
| 311 | + __be32 immdata; | |
| 312 | + u32 invrkey; | |
| 313 | + }; | |
| 314 | + u64 qp_handle; | |
| 315 | + u64 mr_handle; | |
| 316 | + u16 flags; | |
| 317 | + u8 smac[6]; | |
| 318 | + u32 src_qp; | |
| 319 | + u16 raweth_qp1_flags; | |
| 320 | + u16 raweth_qp1_errors; | |
| 321 | + u16 raweth_qp1_cfa_code; | |
| 322 | + u32 raweth_qp1_flags2; | |
| 323 | + u32 raweth_qp1_metadata; | |
| 324 | + u8 raweth_qp1_payload_offset; | |
| 325 | + u16 pkey_index; | |
| 326 | +}; | |
| 327 | + | |
| 328 | +#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 | |
| 329 | +struct bnxt_qplib_cq { | |
| 330 | + struct bnxt_qplib_dpi *dpi; | |
| 331 | + void __iomem *dbr_base; | |
| 332 | + u32 max_wqe; | |
| 333 | + u32 id; | |
| 334 | + u16 count; | |
| 335 | + u16 period; | |
| 336 | + struct bnxt_qplib_hwq hwq; | |
| 337 | + u32 cnq_hw_ring_id; | |
| 338 | + bool resize_in_progress; | |
| 339 | + struct scatterlist *sghead; | |
| 340 | + u32 nmap; | |
| 341 | + u64 cq_handle; | |
| 342 | + | |
| 343 | +#define CQ_RESIZE_WAIT_TIME_MS 500 | |
| 344 | + unsigned long flags; | |
| 345 | +#define CQ_FLAGS_RESIZE_IN_PROG 1 | |
| 346 | + wait_queue_head_t waitq; | |
| 347 | +}; | |
| 348 | + | |
| 349 | +#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) | |
| 350 | +#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) | |
| 351 | +#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2) | |
| 352 | +#define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1) | |
| 353 | +#define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1) | |
| 354 | +#define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1) | |
| 355 | + | |
| 356 | +#define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base) | |
| 357 | + | |
| 358 | +#define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE) | |
| 359 | +#define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1) | |
| 360 | +#define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG) | |
| 361 | +#define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG) | |
| 362 | + | |
| 363 | +#define NQE_CMP_VALID(hdr, raw_cons, cp_bit) \ | |
| 364 | + (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \ | |
| 365 | + !((raw_cons) & (cp_bit))) | |
| 366 | + | |
| 367 | +#define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) | |
| 368 | + | |
| 369 | +#define NQ_CONS_PCI_BAR_REGION 2 | |
| 370 | +#define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) | |
| 371 | +#define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID | |
| 372 | +#define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK | |
| 373 | +#define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \ | |
| 374 | + NQ_DB_IDX_VALID) | |
| 375 | +#define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \ | |
| 376 | + NQ_DB_IDX_VALID | \ | |
| 377 | + NQ_DB_IRQ_DIS) | |
| 378 | +#define NQ_DB_REARM(db, raw_cons, cp_bit) \ | |
| 379 | + writel(NQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db) | |
| 380 | +#define NQ_DB(db, raw_cons, cp_bit) \ | |
| 381 | + writel(NQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db) | |
| 382 | + | |
| 383 | +struct bnxt_qplib_nq { | |
| 384 | + struct pci_dev *pdev; | |
| 385 | + | |
| 386 | + int vector; | |
| 387 | + int budget; | |
| 388 | + bool requested; | |
| 389 | + struct tasklet_struct worker; | |
| 390 | + struct bnxt_qplib_hwq hwq; | |
| 391 | + | |
| 392 | + u16 bar_reg; | |
| 393 | + u16 bar_reg_off; | |
| 394 | + u16 ring_id; | |
| 395 | + void __iomem *bar_reg_iomem; | |
| 396 | + | |
| 397 | + int (*cqn_handler) | |
| 398 | + (struct bnxt_qplib_nq *nq, | |
| 399 | + struct bnxt_qplib_cq *cq); | |
| 400 | + int (*srqn_handler) | |
| 401 | + (struct bnxt_qplib_nq *nq, | |
| 402 | + void *srq, | |
| 403 | + u8 event); | |
| 404 | +}; | |
| 405 | + | |
| 406 | +void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); | |
| 407 | +int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, | |
| 408 | + int msix_vector, int bar_reg_offset, | |
| 409 | + int (*cqn_handler)(struct bnxt_qplib_nq *nq, | |
| 410 | + struct bnxt_qplib_cq *cq), | |
| 411 | + int (*srqn_handler)(struct bnxt_qplib_nq *nq, | |
| 412 | + void *srq, | |
| 413 | + u8 event)); | |
| 414 | +int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
| 415 | +int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
| 416 | +int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
| 417 | +int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
| 418 | +int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
| 419 | +void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, | |
| 420 | + struct bnxt_qplib_sge *sge); | |
| 421 | +void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, | |
| 422 | + struct bnxt_qplib_sge *sge); | |
| 423 | +u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); | |
| 424 | +dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, | |
| 425 | + u32 index); | |
| 426 | +void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); | |
| 427 | +int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, | |
| 428 | + struct bnxt_qplib_swqe *wqe); | |
| 429 | +void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); | |
| 430 | +int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, | |
| 431 | + struct bnxt_qplib_swqe *wqe); | |
| 432 | +int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); | |
| 433 | +int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); | |
| 434 | +int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, | |
| 435 | + int num); | |
| 436 | +void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); | |
| 437 | +void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq); | |
| 438 | +int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq); | |
| 439 | +#endif /* __BNXT_QPLIB_FP_H__ */ | |
| @@ -0,0 +1,694 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: RDMA Controller HW interface | |
| 37 | + */ | |
| 38 | +#include <linux/interrupt.h> | |
| 39 | +#include <linux/spinlock.h> | |
| 40 | +#include <linux/pci.h> | |
| 41 | +#include <linux/prefetch.h> | |
| 42 | +#include "roce_hsi.h" | |
| 43 | +#include "qplib_res.h" | |
| 44 | +#include "qplib_rcfw.h" | |
| 45 | +static void bnxt_qplib_service_creq(unsigned long data); | |
| 46 | + | |
| 47 | +/* Hardware communication channel */ | |
| 48 | +int bnxt_qplib_rcfw_wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) | |
| 49 | +{ | |
| 50 | + u16 cbit; | |
| 51 | + int rc; | |
| 52 | + | |
| 53 | + cookie &= RCFW_MAX_COOKIE_VALUE; | |
| 54 | + cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; | |
| 55 | + if (!test_bit(cbit, rcfw->cmdq_bitmap)) | |
| 56 | + dev_warn(&rcfw->pdev->dev, | |
| 57 | + "QPLIB: CMD bit %d for cookie 0x%x is not set?", | |
| 58 | + cbit, cookie); | |
| 59 | + | |
| 60 | + rc = wait_event_timeout(rcfw->waitq, | |
| 61 | + !test_bit(cbit, rcfw->cmdq_bitmap), | |
| 62 | + msecs_to_jiffies(RCFW_CMD_WAIT_TIME_MS)); | |
| 63 | + if (!rc) { | |
| 64 | + dev_warn(&rcfw->pdev->dev, | |
| 65 | + "QPLIB: Bono Error: timeout %d msec, msg {0x%x}\n", | |
| 66 | + RCFW_CMD_WAIT_TIME_MS, cookie); | |
| 67 | + } | |
| 68 | + | |
| 69 | + return rc; | |
| 70 | +}; | |
| 71 | + | |
| 72 | +int bnxt_qplib_rcfw_block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) | |
| 73 | +{ | |
| 74 | + u32 count = -1; | |
| 75 | + u16 cbit; | |
| 76 | + | |
| 77 | + cookie &= RCFW_MAX_COOKIE_VALUE; | |
| 78 | + cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; | |
| 79 | + if (!test_bit(cbit, rcfw->cmdq_bitmap)) | |
| 80 | + goto done; | |
| 81 | + do { | |
| 82 | + bnxt_qplib_service_creq((unsigned long)rcfw); | |
| 83 | + } while (test_bit(cbit, rcfw->cmdq_bitmap) && --count); | |
| 84 | +done: | |
| 85 | + return count; | |
| 86 | +}; | |
| 87 | + | |
| 88 | +void *bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, | |
| 89 | + struct cmdq_base *req, void **crsbe, | |
| 90 | + u8 is_block) | |
| 91 | +{ | |
| 92 | + struct bnxt_qplib_crsq *crsq = &rcfw->crsq; | |
| 93 | + struct bnxt_qplib_cmdqe *cmdqe, **cmdq_ptr; | |
| 94 | + struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq; | |
| 95 | + struct bnxt_qplib_hwq *crsb = &rcfw->crsb; | |
| 96 | + struct bnxt_qplib_crsqe *crsqe = NULL; | |
| 97 | + struct bnxt_qplib_crsbe **crsb_ptr; | |
| 98 | + u32 sw_prod, cmdq_prod; | |
| 99 | + u8 retry_cnt = 0xFF; | |
| 100 | + dma_addr_t dma_addr; | |
| 101 | + unsigned long flags; | |
| 102 | + u32 size, opcode; | |
| 103 | + u16 cookie, cbit; | |
| 104 | + int pg, idx; | |
| 105 | + u8 *preq; | |
| 106 | + | |
| 107 | +retry: | |
| 108 | + opcode = req->opcode; | |
| 109 | + if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) && | |
| 110 | + (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC && | |
| 111 | + opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW)) { | |
| 112 | + dev_err(&rcfw->pdev->dev, | |
| 113 | + "QPLIB: RCFW not initialized, reject opcode 0x%x", | |
| 114 | + opcode); | |
| 115 | + return NULL; | |
| 116 | + } | |
| 117 | + | |
| 118 | + if (test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) && | |
| 119 | + opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) { | |
| 120 | + dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!"); | |
| 121 | + return NULL; | |
| 122 | + } | |
| 123 | + | |
| 124 | + /* Cmdq are in 16-byte units, each request can consume 1 or more | |
| 125 | + * cmdqe | |
| 126 | + */ | |
| 127 | + spin_lock_irqsave(&cmdq->lock, flags); | |
| 128 | + if (req->cmd_size > cmdq->max_elements - | |
| 129 | + ((HWQ_CMP(cmdq->prod, cmdq) - HWQ_CMP(cmdq->cons, cmdq)) & | |
| 130 | + (cmdq->max_elements - 1))) { | |
| 131 | + dev_err(&rcfw->pdev->dev, "QPLIB: RCFW: CMDQ is full!"); | |
| 132 | + spin_unlock_irqrestore(&cmdq->lock, flags); | |
| 133 | + | |
| 134 | + if (!retry_cnt--) | |
| 135 | + return NULL; | |
| 136 | + goto retry; | |
| 137 | + } | |
| 138 | + | |
| 139 | + retry_cnt = 0xFF; | |
| 140 | + | |
| 141 | + cookie = atomic_inc_return(&rcfw->seq_num) & RCFW_MAX_COOKIE_VALUE; | |
| 142 | + cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; | |
| 143 | + if (is_block) | |
| 144 | + cookie |= RCFW_CMD_IS_BLOCKING; | |
| 145 | + req->cookie = cpu_to_le16(cookie); | |
| 146 | + if (test_and_set_bit(cbit, rcfw->cmdq_bitmap)) { | |
| 147 | + dev_err(&rcfw->pdev->dev, | |
| 148 | + "QPLIB: RCFW MAX outstanding cmd reached!"); | |
| 149 | + atomic_dec(&rcfw->seq_num); | |
| 150 | + spin_unlock_irqrestore(&cmdq->lock, flags); | |
| 151 | + | |
| 152 | + if (!retry_cnt--) | |
| 153 | + return NULL; | |
| 154 | + goto retry; | |
| 155 | + } | |
| 156 | + /* Reserve a resp buffer slot if requested */ | |
| 157 | + if (req->resp_size && crsbe) { | |
| 158 | + spin_lock(&crsb->lock); | |
| 159 | + sw_prod = HWQ_CMP(crsb->prod, crsb); | |
| 160 | + crsb_ptr = (struct bnxt_qplib_crsbe **)crsb->pbl_ptr; | |
| 161 | + *crsbe = (void *)&crsb_ptr[get_crsb_pg(sw_prod)] | |
| 162 | + [get_crsb_idx(sw_prod)]; | |
| 163 | + bnxt_qplib_crsb_dma_next(crsb->pbl_dma_ptr, sw_prod, &dma_addr); | |
| 164 | + req->resp_addr = cpu_to_le64(dma_addr); | |
| 165 | + crsb->prod++; | |
| 166 | + spin_unlock(&crsb->lock); | |
| 167 | + | |
| 168 | + req->resp_size = (sizeof(struct bnxt_qplib_crsbe) + | |
| 169 | + BNXT_QPLIB_CMDQE_UNITS - 1) / | |
| 170 | + BNXT_QPLIB_CMDQE_UNITS; | |
| 171 | + } | |
| 172 | + cmdq_ptr = (struct bnxt_qplib_cmdqe **)cmdq->pbl_ptr; | |
| 173 | + preq = (u8 *)req; | |
| 174 | + size = req->cmd_size * BNXT_QPLIB_CMDQE_UNITS; | |
| 175 | + do { | |
| 176 | + pg = 0; | |
| 177 | + idx = 0; | |
| 178 | + | |
| 179 | + /* Locate the next cmdq slot */ | |
| 180 | + sw_prod = HWQ_CMP(cmdq->prod, cmdq); | |
| 181 | + cmdqe = &cmdq_ptr[get_cmdq_pg(sw_prod)][get_cmdq_idx(sw_prod)]; | |
| 182 | + if (!cmdqe) { | |
| 183 | + dev_err(&rcfw->pdev->dev, | |
| 184 | + "QPLIB: RCFW request failed with no cmdqe!"); | |
| 185 | + goto done; | |
| 186 | + } | |
| 187 | + /* Copy a segment of the req cmd to the cmdq */ | |
| 188 | + memset(cmdqe, 0, sizeof(*cmdqe)); | |
| 189 | + memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe))); | |
| 190 | + preq += min_t(u32, size, sizeof(*cmdqe)); | |
| 191 | + size -= min_t(u32, size, sizeof(*cmdqe)); | |
| 192 | + cmdq->prod++; | |
| 193 | + } while (size > 0); | |
| 194 | + | |
| 195 | + cmdq_prod = cmdq->prod; | |
| 196 | + if (rcfw->flags & FIRMWARE_FIRST_FLAG) { | |
| 197 | + /* The very first doorbell write is required to set this flag | |
| 198 | + * which prompts the FW to reset its internal pointers | |
| 199 | + */ | |
| 200 | + cmdq_prod |= FIRMWARE_FIRST_FLAG; | |
| 201 | + rcfw->flags &= ~FIRMWARE_FIRST_FLAG; | |
| 202 | + } | |
| 203 | + sw_prod = HWQ_CMP(crsq->prod, crsq); | |
| 204 | + crsqe = &crsq->crsq[sw_prod]; | |
| 205 | + memset(crsqe, 0, sizeof(*crsqe)); | |
| 206 | + crsq->prod++; | |
| 207 | + crsqe->req_size = req->cmd_size; | |
| 208 | + | |
| 209 | + /* ring CMDQ DB */ | |
| 210 | + writel(cmdq_prod, rcfw->cmdq_bar_reg_iomem + | |
| 211 | + rcfw->cmdq_bar_reg_prod_off); | |
| 212 | + writel(RCFW_CMDQ_TRIG_VAL, rcfw->cmdq_bar_reg_iomem + | |
| 213 | + rcfw->cmdq_bar_reg_trig_off); | |
| 214 | +done: | |
| 215 | + spin_unlock_irqrestore(&cmdq->lock, flags); | |
| 216 | + /* Return the CREQ response pointer */ | |
| 217 | + return crsqe ? &crsqe->qp_event : NULL; | |
| 218 | +} | |
| 219 | + | |
| 220 | +/* Completions */ | |
| 221 | +static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw, | |
| 222 | + struct creq_func_event *func_event) | |
| 223 | +{ | |
| 224 | + switch (func_event->event) { | |
| 225 | + case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: | |
| 226 | + break; | |
| 227 | + case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: | |
| 228 | + break; | |
| 229 | + case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: | |
| 230 | + break; | |
| 231 | + case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: | |
| 232 | + break; | |
| 233 | + case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: | |
| 234 | + break; | |
| 235 | + case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: | |
| 236 | + break; | |
| 237 | + case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: | |
| 238 | + break; | |
| 239 | + case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: | |
| 240 | + /* SRQ ctx error, call srq_handler?? | |
| 241 | + * But there's no SRQ handle! | |
| 242 | + */ | |
| 243 | + break; | |
| 244 | + case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: | |
| 245 | + break; | |
| 246 | + case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: | |
| 247 | + break; | |
| 248 | + case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: | |
| 249 | + break; | |
| 250 | + case CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST: | |
| 251 | + break; | |
| 252 | + case CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED: | |
| 253 | + break; | |
| 254 | + default: | |
| 255 | + return -EINVAL; | |
| 256 | + } | |
| 257 | + return 0; | |
| 258 | +} | |
| 259 | + | |
| 260 | +static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, | |
| 261 | + struct creq_qp_event *qp_event) | |
| 262 | +{ | |
| 263 | + struct bnxt_qplib_crsq *crsq = &rcfw->crsq; | |
| 264 | + struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq; | |
| 265 | + struct bnxt_qplib_crsqe *crsqe; | |
| 266 | + u16 cbit, cookie, blocked = 0; | |
| 267 | + unsigned long flags; | |
| 268 | + u32 sw_cons; | |
| 269 | + | |
| 270 | + switch (qp_event->event) { | |
| 271 | + case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: | |
| 272 | + dev_dbg(&rcfw->pdev->dev, | |
| 273 | + "QPLIB: Received QP error notification"); | |
| 274 | + break; | |
| 275 | + default: | |
| 276 | + /* Command Response */ | |
| 277 | + spin_lock_irqsave(&cmdq->lock, flags); | |
| 278 | + sw_cons = HWQ_CMP(crsq->cons, crsq); | |
| 279 | + crsqe = &crsq->crsq[sw_cons]; | |
| 280 | + crsq->cons++; | |
| 281 | + memcpy(&crsqe->qp_event, qp_event, sizeof(crsqe->qp_event)); | |
| 282 | + | |
| 283 | + cookie = le16_to_cpu(crsqe->qp_event.cookie); | |
| 284 | + blocked = cookie & RCFW_CMD_IS_BLOCKING; | |
| 285 | + cookie &= RCFW_MAX_COOKIE_VALUE; | |
| 286 | + cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; | |
| 287 | + if (!test_and_clear_bit(cbit, rcfw->cmdq_bitmap)) | |
| 288 | + dev_warn(&rcfw->pdev->dev, | |
| 289 | + "QPLIB: CMD bit %d was not requested", cbit); | |
| 290 | + | |
| 291 | + cmdq->cons += crsqe->req_size; | |
| 292 | + spin_unlock_irqrestore(&cmdq->lock, flags); | |
| 293 | + if (!blocked) | |
| 294 | + wake_up(&rcfw->waitq); | |
| 295 | + break; | |
| 296 | + } | |
| 297 | + return 0; | |
| 298 | +} | |
| 299 | + | |
| 300 | +/* SP - CREQ Completion handlers */ | |
| 301 | +static void bnxt_qplib_service_creq(unsigned long data) | |
| 302 | +{ | |
| 303 | + struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data; | |
| 304 | + struct bnxt_qplib_hwq *creq = &rcfw->creq; | |
| 305 | + struct creq_base *creqe, **creq_ptr; | |
| 306 | + u32 sw_cons, raw_cons; | |
| 307 | + unsigned long flags; | |
| 308 | + u32 type; | |
| 309 | + | |
| 310 | + /* Service the CREQ until empty */ | |
| 311 | + spin_lock_irqsave(&creq->lock, flags); | |
| 312 | + raw_cons = creq->cons; | |
| 313 | + while (1) { | |
| 314 | + sw_cons = HWQ_CMP(raw_cons, creq); | |
| 315 | + creq_ptr = (struct creq_base **)creq->pbl_ptr; | |
| 316 | + creqe = &creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]; | |
| 317 | + if (!CREQ_CMP_VALID(creqe, raw_cons, creq->max_elements)) | |
| 318 | + break; | |
| 319 | + | |
| 320 | + type = creqe->type & CREQ_BASE_TYPE_MASK; | |
| 321 | + switch (type) { | |
| 322 | + case CREQ_BASE_TYPE_QP_EVENT: | |
| 323 | + if (!bnxt_qplib_process_qp_event | |
| 324 | + (rcfw, (struct creq_qp_event *)creqe)) | |
| 325 | + rcfw->creq_qp_event_processed++; | |
| 326 | + else { | |
| 327 | + dev_warn(&rcfw->pdev->dev, "QPLIB: crsqe with"); | |
| 328 | + dev_warn(&rcfw->pdev->dev, | |
| 329 | + "QPLIB: type = 0x%x not handled", | |
| 330 | + type); | |
| 331 | + } | |
| 332 | + break; | |
| 333 | + case CREQ_BASE_TYPE_FUNC_EVENT: | |
| 334 | + if (!bnxt_qplib_process_func_event | |
| 335 | + (rcfw, (struct creq_func_event *)creqe)) | |
| 336 | + rcfw->creq_func_event_processed++; | |
| 337 | + else | |
| 338 | + dev_warn | |
| 339 | + (&rcfw->pdev->dev, "QPLIB:aeqe:%#x Not handled", | |
| 340 | + type); | |
| 341 | + break; | |
| 342 | + default: | |
| 343 | + dev_warn(&rcfw->pdev->dev, "QPLIB: creqe with "); | |
| 344 | + dev_warn(&rcfw->pdev->dev, | |
| 345 | + "QPLIB: op_event = 0x%x not handled", type); | |
| 346 | + break; | |
| 347 | + } | |
| 348 | + raw_cons++; | |
| 349 | + } | |
| 350 | + if (creq->cons != raw_cons) { | |
| 351 | + creq->cons = raw_cons; | |
| 352 | + CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, raw_cons, | |
| 353 | + creq->max_elements); | |
| 354 | + } | |
| 355 | + spin_unlock_irqrestore(&creq->lock, flags); | |
| 356 | +} | |
| 357 | + | |
| 358 | +static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance) | |
| 359 | +{ | |
| 360 | + struct bnxt_qplib_rcfw *rcfw = dev_instance; | |
| 361 | + struct bnxt_qplib_hwq *creq = &rcfw->creq; | |
| 362 | + struct creq_base **creq_ptr; | |
| 363 | + u32 sw_cons; | |
| 364 | + | |
| 365 | + /* Prefetch the CREQ element */ | |
| 366 | + sw_cons = HWQ_CMP(creq->cons, creq); | |
| 367 | + creq_ptr = (struct creq_base **)rcfw->creq.pbl_ptr; | |
| 368 | + prefetch(&creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]); | |
| 369 | + | |
| 370 | + tasklet_schedule(&rcfw->worker); | |
| 371 | + | |
| 372 | + return IRQ_HANDLED; | |
| 373 | +} | |
| 374 | + | |
| 375 | +/* RCFW */ | |
| 376 | +int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw) | |
| 377 | +{ | |
| 378 | + struct creq_deinitialize_fw_resp *resp; | |
| 379 | + struct cmdq_deinitialize_fw req; | |
| 380 | + u16 cmd_flags = 0; | |
| 381 | + | |
| 382 | + RCFW_CMD_PREP(req, DEINITIALIZE_FW, cmd_flags); | |
| 383 | + resp = (struct creq_deinitialize_fw_resp *) | |
| 384 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 385 | + NULL, 0); | |
| 386 | + if (!resp) | |
| 387 | + return -EINVAL; | |
| 388 | + | |
| 389 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) | |
| 390 | + return -ETIMEDOUT; | |
| 391 | + | |
| 392 | + if (resp->status || | |
| 393 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) | |
| 394 | + return -EFAULT; | |
| 395 | + | |
| 396 | + clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags); | |
| 397 | + return 0; | |
| 398 | +} | |
| 399 | + | |
| 400 | +static int __get_pbl_pg_idx(struct bnxt_qplib_pbl *pbl) | |
| 401 | +{ | |
| 402 | + return (pbl->pg_size == ROCE_PG_SIZE_4K ? | |
| 403 | + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K : | |
| 404 | + pbl->pg_size == ROCE_PG_SIZE_8K ? | |
| 405 | + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K : | |
| 406 | + pbl->pg_size == ROCE_PG_SIZE_64K ? | |
| 407 | + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K : | |
| 408 | + pbl->pg_size == ROCE_PG_SIZE_2M ? | |
| 409 | + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M : | |
| 410 | + pbl->pg_size == ROCE_PG_SIZE_8M ? | |
| 411 | + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M : | |
| 412 | + pbl->pg_size == ROCE_PG_SIZE_1G ? | |
| 413 | + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G : | |
| 414 | + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K); | |
| 415 | +} | |
| 416 | + | |
| 417 | +int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, | |
| 418 | + struct bnxt_qplib_ctx *ctx, int is_virtfn) | |
| 419 | +{ | |
| 420 | + struct creq_initialize_fw_resp *resp; | |
| 421 | + struct cmdq_initialize_fw req; | |
| 422 | + u16 cmd_flags = 0, level; | |
| 423 | + | |
| 424 | + RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags); | |
| 425 | + | |
| 426 | + /* | |
| 427 | + * VFs need not setup the HW context area, PF | |
| 428 | + * shall setup this area for VF. Skipping the | |
| 429 | + * HW programming | |
| 430 | + */ | |
| 431 | + if (is_virtfn) | |
| 432 | + goto skip_ctx_setup; | |
| 433 | + | |
| 434 | + level = ctx->qpc_tbl.level; | |
| 435 | + req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) | | |
| 436 | + __get_pbl_pg_idx(&ctx->qpc_tbl.pbl[level]); | |
| 437 | + level = ctx->mrw_tbl.level; | |
| 438 | + req.mrw_pg_size_mrw_lvl = (level << CMDQ_INITIALIZE_FW_MRW_LVL_SFT) | | |
| 439 | + __get_pbl_pg_idx(&ctx->mrw_tbl.pbl[level]); | |
| 440 | + level = ctx->srqc_tbl.level; | |
| 441 | + req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) | | |
| 442 | + __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]); | |
| 443 | + level = ctx->cq_tbl.level; | |
| 444 | + req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) | | |
| 445 | + __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]); | |
| 446 | + level = ctx->srqc_tbl.level; | |
| 447 | + req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) | | |
| 448 | + __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]); | |
| 449 | + level = ctx->cq_tbl.level; | |
| 450 | + req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) | | |
| 451 | + __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]); | |
| 452 | + level = ctx->tim_tbl.level; | |
| 453 | + req.tim_pg_size_tim_lvl = (level << CMDQ_INITIALIZE_FW_TIM_LVL_SFT) | | |
| 454 | + __get_pbl_pg_idx(&ctx->tim_tbl.pbl[level]); | |
| 455 | + level = ctx->tqm_pde_level; | |
| 456 | + req.tqm_pg_size_tqm_lvl = (level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) | | |
| 457 | + __get_pbl_pg_idx(&ctx->tqm_pde.pbl[level]); | |
| 458 | + | |
| 459 | + req.qpc_page_dir = | |
| 460 | + cpu_to_le64(ctx->qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | |
| 461 | + req.mrw_page_dir = | |
| 462 | + cpu_to_le64(ctx->mrw_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | |
| 463 | + req.srq_page_dir = | |
| 464 | + cpu_to_le64(ctx->srqc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | |
| 465 | + req.cq_page_dir = | |
| 466 | + cpu_to_le64(ctx->cq_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | |
| 467 | + req.tim_page_dir = | |
| 468 | + cpu_to_le64(ctx->tim_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | |
| 469 | + req.tqm_page_dir = | |
| 470 | + cpu_to_le64(ctx->tqm_pde.pbl[PBL_LVL_0].pg_map_arr[0]); | |
| 471 | + | |
| 472 | + req.number_of_qp = cpu_to_le32(ctx->qpc_tbl.max_elements); | |
| 473 | + req.number_of_mrw = cpu_to_le32(ctx->mrw_tbl.max_elements); | |
| 474 | + req.number_of_srq = cpu_to_le32(ctx->srqc_tbl.max_elements); | |
| 475 | + req.number_of_cq = cpu_to_le32(ctx->cq_tbl.max_elements); | |
| 476 | + | |
| 477 | + req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf); | |
| 478 | + req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf); | |
| 479 | + req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf); | |
| 480 | + req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf); | |
| 481 | + req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf); | |
| 482 | + | |
| 483 | +skip_ctx_setup: | |
| 484 | + req.stat_ctx_id = cpu_to_le32(ctx->stats.fw_id); | |
| 485 | + resp = (struct creq_initialize_fw_resp *) | |
| 486 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 487 | + NULL, 0); | |
| 488 | + if (!resp) { | |
| 489 | + dev_err(&rcfw->pdev->dev, | |
| 490 | + "QPLIB: RCFW: INITIALIZE_FW send failed"); | |
| 491 | + return -EINVAL; | |
| 492 | + } | |
| 493 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 494 | + /* Cmd timed out */ | |
| 495 | + dev_err(&rcfw->pdev->dev, | |
| 496 | + "QPLIB: RCFW: INITIALIZE_FW timed out"); | |
| 497 | + return -ETIMEDOUT; | |
| 498 | + } | |
| 499 | + if (resp->status || | |
| 500 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 501 | + dev_err(&rcfw->pdev->dev, | |
| 502 | + "QPLIB: RCFW: INITIALIZE_FW failed"); | |
| 503 | + return -EINVAL; | |
| 504 | + } | |
| 505 | + set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags); | |
| 506 | + return 0; | |
| 507 | +} | |
| 508 | + | |
| 509 | +void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) | |
| 510 | +{ | |
| 511 | + bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->crsb); | |
| 512 | + kfree(rcfw->crsq.crsq); | |
| 513 | + bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->cmdq); | |
| 514 | + bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->creq); | |
| 515 | + | |
| 516 | + rcfw->pdev = NULL; | |
| 517 | +} | |
| 518 | + | |
| 519 | +int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev, | |
| 520 | + struct bnxt_qplib_rcfw *rcfw) | |
| 521 | +{ | |
| 522 | + rcfw->pdev = pdev; | |
| 523 | + rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT; | |
| 524 | + if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL, 0, | |
| 525 | + &rcfw->creq.max_elements, | |
| 526 | + BNXT_QPLIB_CREQE_UNITS, 0, PAGE_SIZE, | |
| 527 | + HWQ_TYPE_L2_CMPL)) { | |
| 528 | + dev_err(&rcfw->pdev->dev, | |
| 529 | + "QPLIB: HW channel CREQ allocation failed"); | |
| 530 | + goto fail; | |
| 531 | + } | |
| 532 | + rcfw->cmdq.max_elements = BNXT_QPLIB_CMDQE_MAX_CNT; | |
| 533 | + if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->cmdq, NULL, 0, | |
| 534 | + &rcfw->cmdq.max_elements, | |
| 535 | + BNXT_QPLIB_CMDQE_UNITS, 0, PAGE_SIZE, | |
| 536 | + HWQ_TYPE_CTX)) { | |
| 537 | + dev_err(&rcfw->pdev->dev, | |
| 538 | + "QPLIB: HW channel CMDQ allocation failed"); | |
| 539 | + goto fail; | |
| 540 | + } | |
| 541 | + | |
| 542 | + rcfw->crsq.max_elements = rcfw->cmdq.max_elements; | |
| 543 | + rcfw->crsq.crsq = kcalloc(rcfw->crsq.max_elements, | |
| 544 | + sizeof(*rcfw->crsq.crsq), GFP_KERNEL); | |
| 545 | + if (!rcfw->crsq.crsq) | |
| 546 | + goto fail; | |
| 547 | + | |
| 548 | + rcfw->crsb.max_elements = BNXT_QPLIB_CRSBE_MAX_CNT; | |
| 549 | + if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->crsb, NULL, 0, | |
| 550 | + &rcfw->crsb.max_elements, | |
| 551 | + BNXT_QPLIB_CRSBE_UNITS, 0, PAGE_SIZE, | |
| 552 | + HWQ_TYPE_CTX)) { | |
| 553 | + dev_err(&rcfw->pdev->dev, | |
| 554 | + "QPLIB: HW channel CRSB allocation failed"); | |
| 555 | + goto fail; | |
| 556 | + } | |
| 557 | + return 0; | |
| 558 | + | |
| 559 | +fail: | |
| 560 | + bnxt_qplib_free_rcfw_channel(rcfw); | |
| 561 | + return -ENOMEM; | |
| 562 | +} | |
| 563 | + | |
| 564 | +void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) | |
| 565 | +{ | |
| 566 | + unsigned long indx; | |
| 567 | + | |
| 568 | + /* Make sure the HW channel is stopped! */ | |
| 569 | + synchronize_irq(rcfw->vector); | |
| 570 | + tasklet_disable(&rcfw->worker); | |
| 571 | + tasklet_kill(&rcfw->worker); | |
| 572 | + | |
| 573 | + if (rcfw->requested) { | |
| 574 | + free_irq(rcfw->vector, rcfw); | |
| 575 | + rcfw->requested = false; | |
| 576 | + } | |
| 577 | + if (rcfw->cmdq_bar_reg_iomem) | |
| 578 | + iounmap(rcfw->cmdq_bar_reg_iomem); | |
| 579 | + rcfw->cmdq_bar_reg_iomem = NULL; | |
| 580 | + | |
| 581 | + if (rcfw->creq_bar_reg_iomem) | |
| 582 | + iounmap(rcfw->creq_bar_reg_iomem); | |
| 583 | + rcfw->creq_bar_reg_iomem = NULL; | |
| 584 | + | |
| 585 | + indx = find_first_bit(rcfw->cmdq_bitmap, rcfw->bmap_size); | |
| 586 | + if (indx != rcfw->bmap_size) | |
| 587 | + dev_err(&rcfw->pdev->dev, | |
| 588 | + "QPLIB: disabling RCFW with pending cmd-bit %lx", indx); | |
| 589 | + kfree(rcfw->cmdq_bitmap); | |
| 590 | + rcfw->bmap_size = 0; | |
| 591 | + | |
| 592 | + rcfw->aeq_handler = NULL; | |
| 593 | + rcfw->vector = 0; | |
| 594 | +} | |
| 595 | + | |
| 596 | +int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, | |
| 597 | + struct bnxt_qplib_rcfw *rcfw, | |
| 598 | + int msix_vector, | |
| 599 | + int cp_bar_reg_off, int virt_fn, | |
| 600 | + int (*aeq_handler)(struct bnxt_qplib_rcfw *, | |
| 601 | + struct creq_func_event *)) | |
| 602 | +{ | |
| 603 | + resource_size_t res_base; | |
| 604 | + struct cmdq_init init; | |
| 605 | + u16 bmap_size; | |
| 606 | + int rc; | |
| 607 | + | |
| 608 | + /* General */ | |
| 609 | + atomic_set(&rcfw->seq_num, 0); | |
| 610 | + rcfw->flags = FIRMWARE_FIRST_FLAG; | |
| 611 | + bmap_size = BITS_TO_LONGS(RCFW_MAX_OUTSTANDING_CMD * | |
| 612 | + sizeof(unsigned long)); | |
| 613 | + rcfw->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL); | |
| 614 | + if (!rcfw->cmdq_bitmap) | |
| 615 | + return -ENOMEM; | |
| 616 | + rcfw->bmap_size = bmap_size; | |
| 617 | + | |
| 618 | + /* CMDQ */ | |
| 619 | + rcfw->cmdq_bar_reg = RCFW_COMM_PCI_BAR_REGION; | |
| 620 | + res_base = pci_resource_start(pdev, rcfw->cmdq_bar_reg); | |
| 621 | + if (!res_base) | |
| 622 | + return -ENOMEM; | |
| 623 | + | |
| 624 | + rcfw->cmdq_bar_reg_iomem = ioremap_nocache(res_base + | |
| 625 | + RCFW_COMM_BASE_OFFSET, | |
| 626 | + RCFW_COMM_SIZE); | |
| 627 | + if (!rcfw->cmdq_bar_reg_iomem) { | |
| 628 | + dev_err(&rcfw->pdev->dev, | |
| 629 | + "QPLIB: CMDQ BAR region %d mapping failed", | |
| 630 | + rcfw->cmdq_bar_reg); | |
| 631 | + return -ENOMEM; | |
| 632 | + } | |
| 633 | + | |
| 634 | + rcfw->cmdq_bar_reg_prod_off = virt_fn ? RCFW_VF_COMM_PROD_OFFSET : | |
| 635 | + RCFW_PF_COMM_PROD_OFFSET; | |
| 636 | + | |
| 637 | + rcfw->cmdq_bar_reg_trig_off = RCFW_COMM_TRIG_OFFSET; | |
| 638 | + | |
| 639 | + /* CRSQ */ | |
| 640 | + rcfw->crsq.prod = 0; | |
| 641 | + rcfw->crsq.cons = 0; | |
| 642 | + | |
| 643 | + /* CREQ */ | |
| 644 | + rcfw->creq_bar_reg = RCFW_COMM_CONS_PCI_BAR_REGION; | |
| 645 | + res_base = pci_resource_start(pdev, rcfw->creq_bar_reg); | |
| 646 | + if (!res_base) | |
| 647 | + dev_err(&rcfw->pdev->dev, | |
| 648 | + "QPLIB: CREQ BAR region %d resc start is 0!", | |
| 649 | + rcfw->creq_bar_reg); | |
| 650 | + rcfw->creq_bar_reg_iomem = ioremap_nocache(res_base + cp_bar_reg_off, | |
| 651 | + 4); | |
| 652 | + if (!rcfw->creq_bar_reg_iomem) { | |
| 653 | + dev_err(&rcfw->pdev->dev, | |
| 654 | + "QPLIB: CREQ BAR region %d mapping failed", | |
| 655 | + rcfw->creq_bar_reg); | |
| 656 | + return -ENOMEM; | |
| 657 | + } | |
| 658 | + rcfw->creq_qp_event_processed = 0; | |
| 659 | + rcfw->creq_func_event_processed = 0; | |
| 660 | + | |
| 661 | + rcfw->vector = msix_vector; | |
| 662 | + if (aeq_handler) | |
| 663 | + rcfw->aeq_handler = aeq_handler; | |
| 664 | + | |
| 665 | + tasklet_init(&rcfw->worker, bnxt_qplib_service_creq, | |
| 666 | + (unsigned long)rcfw); | |
| 667 | + | |
| 668 | + rcfw->requested = false; | |
| 669 | + rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0, | |
| 670 | + "bnxt_qplib_creq", rcfw); | |
| 671 | + if (rc) { | |
| 672 | + dev_err(&rcfw->pdev->dev, | |
| 673 | + "QPLIB: Failed to request IRQ for CREQ rc = 0x%x", rc); | |
| 674 | + bnxt_qplib_disable_rcfw_channel(rcfw); | |
| 675 | + return rc; | |
| 676 | + } | |
| 677 | + rcfw->requested = true; | |
| 678 | + | |
| 679 | + init_waitqueue_head(&rcfw->waitq); | |
| 680 | + | |
| 681 | + CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, 0, rcfw->creq.max_elements); | |
| 682 | + | |
| 683 | + init.cmdq_pbl = cpu_to_le64(rcfw->cmdq.pbl[PBL_LVL_0].pg_map_arr[0]); | |
| 684 | + init.cmdq_size_cmdq_lvl = cpu_to_le16( | |
| 685 | + ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) & | |
| 686 | + CMDQ_INIT_CMDQ_SIZE_MASK) | | |
| 687 | + ((rcfw->cmdq.level << CMDQ_INIT_CMDQ_LVL_SFT) & | |
| 688 | + CMDQ_INIT_CMDQ_LVL_MASK)); | |
| 689 | + init.creq_ring_id = cpu_to_le16(rcfw->creq_ring_id); | |
| 690 | + | |
| 691 | + /* Write to the Bono mailbox register */ | |
| 692 | + __iowrite32_copy(rcfw->cmdq_bar_reg_iomem, &init, sizeof(init) / 4); | |
| 693 | + return 0; | |
| 694 | +} | |
| @@ -0,0 +1,231 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: RDMA Controller HW interface (header) | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#ifndef __BNXT_QPLIB_RCFW_H__ | |
| 40 | +#define __BNXT_QPLIB_RCFW_H__ | |
| 41 | + | |
| 42 | +#define RCFW_CMDQ_TRIG_VAL 1 | |
| 43 | +#define RCFW_COMM_PCI_BAR_REGION 0 | |
| 44 | +#define RCFW_COMM_CONS_PCI_BAR_REGION 2 | |
| 45 | +#define RCFW_COMM_BASE_OFFSET 0x600 | |
| 46 | +#define RCFW_PF_COMM_PROD_OFFSET 0xc | |
| 47 | +#define RCFW_VF_COMM_PROD_OFFSET 0xc | |
| 48 | +#define RCFW_COMM_TRIG_OFFSET 0x100 | |
| 49 | +#define RCFW_COMM_SIZE 0x104 | |
| 50 | + | |
| 51 | +#define RCFW_DBR_PCI_BAR_REGION 2 | |
| 52 | + | |
| 53 | +#define RCFW_CMD_PREP(req, CMD, cmd_flags) \ | |
| 54 | + do { \ | |
| 55 | + memset(&(req), 0, sizeof((req))); \ | |
| 56 | + (req).opcode = CMDQ_BASE_OPCODE_##CMD; \ | |
| 57 | + (req).cmd_size = (sizeof((req)) + \ | |
| 58 | + BNXT_QPLIB_CMDQE_UNITS - 1) / \ | |
| 59 | + BNXT_QPLIB_CMDQE_UNITS; \ | |
| 60 | + (req).flags = cpu_to_le16(cmd_flags); \ | |
| 61 | + } while (0) | |
| 62 | + | |
| 63 | +#define RCFW_CMD_WAIT_TIME_MS 20000 /* 20 Seconds timeout */ | |
| 64 | + | |
| 65 | +/* CMDQ elements */ | |
| 66 | +#define BNXT_QPLIB_CMDQE_MAX_CNT 256 | |
| 67 | +#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe) | |
| 68 | +#define BNXT_QPLIB_CMDQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CMDQE_UNITS) | |
| 69 | + | |
| 70 | +#define MAX_CMDQ_IDX (BNXT_QPLIB_CMDQE_MAX_CNT - 1) | |
| 71 | +#define MAX_CMDQ_IDX_PER_PG (BNXT_QPLIB_CMDQE_CNT_PER_PG - 1) | |
| 72 | + | |
| 73 | +#define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT | |
| 74 | +#define RCFW_MAX_COOKIE_VALUE 0x7FFF | |
| 75 | +#define RCFW_CMD_IS_BLOCKING 0x8000 | |
| 76 | + | |
| 77 | +/* Cmdq contains a fix number of a 16-Byte slots */ | |
| 78 | +struct bnxt_qplib_cmdqe { | |
| 79 | + u8 data[16]; | |
| 80 | +}; | |
| 81 | + | |
| 82 | +static inline u32 get_cmdq_pg(u32 val) | |
| 83 | +{ | |
| 84 | + return (val & ~MAX_CMDQ_IDX_PER_PG) / BNXT_QPLIB_CMDQE_CNT_PER_PG; | |
| 85 | +} | |
| 86 | + | |
| 87 | +static inline u32 get_cmdq_idx(u32 val) | |
| 88 | +{ | |
| 89 | + return val & MAX_CMDQ_IDX_PER_PG; | |
| 90 | +} | |
| 91 | + | |
| 92 | +/* Crsq buf is 1024-Byte */ | |
| 93 | +struct bnxt_qplib_crsbe { | |
| 94 | + u8 data[1024]; | |
| 95 | +}; | |
| 96 | + | |
| 97 | +/* CRSQ SB */ | |
| 98 | +#define BNXT_QPLIB_CRSBE_MAX_CNT 4 | |
| 99 | +#define BNXT_QPLIB_CRSBE_UNITS sizeof(struct bnxt_qplib_crsbe) | |
| 100 | +#define BNXT_QPLIB_CRSBE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CRSBE_UNITS) | |
| 101 | + | |
| 102 | +#define MAX_CRSB_IDX (BNXT_QPLIB_CRSBE_MAX_CNT - 1) | |
| 103 | +#define MAX_CRSB_IDX_PER_PG (BNXT_QPLIB_CRSBE_CNT_PER_PG - 1) | |
| 104 | + | |
| 105 | +static inline u32 get_crsb_pg(u32 val) | |
| 106 | +{ | |
| 107 | + return (val & ~MAX_CRSB_IDX_PER_PG) / BNXT_QPLIB_CRSBE_CNT_PER_PG; | |
| 108 | +} | |
| 109 | + | |
| 110 | +static inline u32 get_crsb_idx(u32 val) | |
| 111 | +{ | |
| 112 | + return val & MAX_CRSB_IDX_PER_PG; | |
| 113 | +} | |
| 114 | + | |
| 115 | +static inline void bnxt_qplib_crsb_dma_next(dma_addr_t *pg_map_arr, | |
| 116 | + u32 prod, dma_addr_t *dma_addr) | |
| 117 | +{ | |
| 118 | + *dma_addr = pg_map_arr[(prod) / BNXT_QPLIB_CRSBE_CNT_PER_PG]; | |
| 119 | + *dma_addr += ((prod) % BNXT_QPLIB_CRSBE_CNT_PER_PG) * | |
| 120 | + BNXT_QPLIB_CRSBE_UNITS; | |
| 121 | +} | |
| 122 | + | |
| 123 | +/* CREQ */ | |
| 124 | +/* Allocate 1 per QP for async error notification for now */ | |
| 125 | +#define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024) | |
| 126 | +#define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */ | |
| 127 | +#define BNXT_QPLIB_CREQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CREQE_UNITS) | |
| 128 | + | |
| 129 | +#define MAX_CREQ_IDX (BNXT_QPLIB_CREQE_MAX_CNT - 1) | |
| 130 | +#define MAX_CREQ_IDX_PER_PG (BNXT_QPLIB_CREQE_CNT_PER_PG - 1) | |
| 131 | + | |
| 132 | +static inline u32 get_creq_pg(u32 val) | |
| 133 | +{ | |
| 134 | + return (val & ~MAX_CREQ_IDX_PER_PG) / BNXT_QPLIB_CREQE_CNT_PER_PG; | |
| 135 | +} | |
| 136 | + | |
| 137 | +static inline u32 get_creq_idx(u32 val) | |
| 138 | +{ | |
| 139 | + return val & MAX_CREQ_IDX_PER_PG; | |
| 140 | +} | |
| 141 | + | |
| 142 | +#define BNXT_QPLIB_CREQE_PER_PG (PAGE_SIZE / sizeof(struct creq_base)) | |
| 143 | + | |
| 144 | +#define CREQ_CMP_VALID(hdr, raw_cons, cp_bit) \ | |
| 145 | + (!!((hdr)->v & CREQ_BASE_V) == \ | |
| 146 | + !((raw_cons) & (cp_bit))) | |
| 147 | + | |
| 148 | +#define CREQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) | |
| 149 | +#define CREQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID | |
| 150 | +#define CREQ_DB_IRQ_DIS CMPL_DOORBELL_MASK | |
| 151 | +#define CREQ_DB_CP_FLAGS_REARM (CREQ_DB_KEY_CP | \ | |
| 152 | + CREQ_DB_IDX_VALID) | |
| 153 | +#define CREQ_DB_CP_FLAGS (CREQ_DB_KEY_CP | \ | |
| 154 | + CREQ_DB_IDX_VALID | \ | |
| 155 | + CREQ_DB_IRQ_DIS) | |
| 156 | +#define CREQ_DB_REARM(db, raw_cons, cp_bit) \ | |
| 157 | + writel(CREQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db) | |
| 158 | +#define CREQ_DB(db, raw_cons, cp_bit) \ | |
| 159 | + writel(CREQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db) | |
| 160 | + | |
| 161 | +/* HWQ */ | |
| 162 | +struct bnxt_qplib_crsqe { | |
| 163 | + struct creq_qp_event qp_event; | |
| 164 | + u32 req_size; | |
| 165 | +}; | |
| 166 | + | |
| 167 | +struct bnxt_qplib_crsq { | |
| 168 | + struct bnxt_qplib_crsqe *crsq; | |
| 169 | + u32 prod; | |
| 170 | + u32 cons; | |
| 171 | + u32 max_elements; | |
| 172 | +}; | |
| 173 | + | |
| 174 | +/* RCFW Communication Channels */ | |
| 175 | +struct bnxt_qplib_rcfw { | |
| 176 | + struct pci_dev *pdev; | |
| 177 | + int vector; | |
| 178 | + struct tasklet_struct worker; | |
| 179 | + bool requested; | |
| 180 | + unsigned long *cmdq_bitmap; | |
| 181 | + u32 bmap_size; | |
| 182 | + unsigned long flags; | |
| 183 | +#define FIRMWARE_INITIALIZED_FLAG 1 | |
| 184 | +#define FIRMWARE_FIRST_FLAG BIT(31) | |
| 185 | + wait_queue_head_t waitq; | |
| 186 | + int (*aeq_handler)(struct bnxt_qplib_rcfw *, | |
| 187 | + struct creq_func_event *); | |
| 188 | + atomic_t seq_num; | |
| 189 | + | |
| 190 | + /* Bar region info */ | |
| 191 | + void __iomem *cmdq_bar_reg_iomem; | |
| 192 | + u16 cmdq_bar_reg; | |
| 193 | + u16 cmdq_bar_reg_prod_off; | |
| 194 | + u16 cmdq_bar_reg_trig_off; | |
| 195 | + u16 creq_ring_id; | |
| 196 | + u16 creq_bar_reg; | |
| 197 | + void __iomem *creq_bar_reg_iomem; | |
| 198 | + | |
| 199 | + /* Cmd-Resp and Async Event notification queue */ | |
| 200 | + struct bnxt_qplib_hwq creq; | |
| 201 | + u64 creq_qp_event_processed; | |
| 202 | + u64 creq_func_event_processed; | |
| 203 | + | |
| 204 | + /* Actual Cmd and Resp Queues */ | |
| 205 | + struct bnxt_qplib_hwq cmdq; | |
| 206 | + struct bnxt_qplib_crsq crsq; | |
| 207 | + struct bnxt_qplib_hwq crsb; | |
| 208 | +}; | |
| 209 | + | |
| 210 | +void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); | |
| 211 | +int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev, | |
| 212 | + struct bnxt_qplib_rcfw *rcfw); | |
| 213 | +void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); | |
| 214 | +int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, | |
| 215 | + struct bnxt_qplib_rcfw *rcfw, | |
| 216 | + int msix_vector, | |
| 217 | + int cp_bar_reg_off, int virt_fn, | |
| 218 | + int (*aeq_handler) | |
| 219 | + (struct bnxt_qplib_rcfw *, | |
| 220 | + struct creq_func_event *)); | |
| 221 | + | |
| 222 | +int bnxt_qplib_rcfw_block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie); | |
| 223 | +int bnxt_qplib_rcfw_wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie); | |
| 224 | +void *bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, | |
| 225 | + struct cmdq_base *req, void **crsbe, | |
| 226 | + u8 is_block); | |
| 227 | + | |
| 228 | +int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw); | |
| 229 | +int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, | |
| 230 | + struct bnxt_qplib_ctx *ctx, int is_virtfn); | |
| 231 | +#endif /* __BNXT_QPLIB_RCFW_H__ */ | |
| @@ -0,0 +1,825 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: QPLib resource manager | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#include <linux/spinlock.h> | |
| 40 | +#include <linux/pci.h> | |
| 41 | +#include <linux/interrupt.h> | |
| 42 | +#include <linux/inetdevice.h> | |
| 43 | +#include <linux/dma-mapping.h> | |
| 44 | +#include <linux/if_vlan.h> | |
| 45 | +#include "roce_hsi.h" | |
| 46 | +#include "qplib_res.h" | |
| 47 | +#include "qplib_sp.h" | |
| 48 | +#include "qplib_rcfw.h" | |
| 49 | + | |
| 50 | +static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev, | |
| 51 | + struct bnxt_qplib_stats *stats); | |
| 52 | +static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev, | |
| 53 | + struct bnxt_qplib_stats *stats); | |
| 54 | + | |
| 55 | +/* PBL */ | |
| 56 | +static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl, | |
| 57 | + bool is_umem) | |
| 58 | +{ | |
| 59 | + int i; | |
| 60 | + | |
| 61 | + if (!is_umem) { | |
| 62 | + for (i = 0; i < pbl->pg_count; i++) { | |
| 63 | + if (pbl->pg_arr[i]) | |
| 64 | + dma_free_coherent(&pdev->dev, pbl->pg_size, | |
| 65 | + (void *)((unsigned long) | |
| 66 | + pbl->pg_arr[i] & | |
| 67 | + PAGE_MASK), | |
| 68 | + pbl->pg_map_arr[i]); | |
| 69 | + else | |
| 70 | + dev_warn(&pdev->dev, | |
| 71 | + "QPLIB: PBL free pg_arr[%d] empty?!", | |
| 72 | + i); | |
| 73 | + pbl->pg_arr[i] = NULL; | |
| 74 | + } | |
| 75 | + } | |
| 76 | + kfree(pbl->pg_arr); | |
| 77 | + pbl->pg_arr = NULL; | |
| 78 | + kfree(pbl->pg_map_arr); | |
| 79 | + pbl->pg_map_arr = NULL; | |
| 80 | + pbl->pg_count = 0; | |
| 81 | + pbl->pg_size = 0; | |
| 82 | +} | |
| 83 | + | |
| 84 | +static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl, | |
| 85 | + struct scatterlist *sghead, u32 pages, u32 pg_size) | |
| 86 | +{ | |
| 87 | + struct scatterlist *sg; | |
| 88 | + bool is_umem = false; | |
| 89 | + int i; | |
| 90 | + | |
| 91 | + /* page ptr arrays */ | |
| 92 | + pbl->pg_arr = kcalloc(pages, sizeof(void *), GFP_KERNEL); | |
| 93 | + if (!pbl->pg_arr) | |
| 94 | + return -ENOMEM; | |
| 95 | + | |
| 96 | + pbl->pg_map_arr = kcalloc(pages, sizeof(dma_addr_t), GFP_KERNEL); | |
| 97 | + if (!pbl->pg_map_arr) { | |
| 98 | + kfree(pbl->pg_arr); | |
| 99 | + pbl->pg_arr = NULL; | |
| 100 | + return -ENOMEM; | |
| 101 | + } | |
| 102 | + pbl->pg_count = 0; | |
| 103 | + pbl->pg_size = pg_size; | |
| 104 | + | |
| 105 | + if (!sghead) { | |
| 106 | + for (i = 0; i < pages; i++) { | |
| 107 | + pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev, | |
| 108 | + pbl->pg_size, | |
| 109 | + &pbl->pg_map_arr[i], | |
| 110 | + GFP_KERNEL); | |
| 111 | + if (!pbl->pg_arr[i]) | |
| 112 | + goto fail; | |
| 113 | + memset(pbl->pg_arr[i], 0, pbl->pg_size); | |
| 114 | + pbl->pg_count++; | |
| 115 | + } | |
| 116 | + } else { | |
| 117 | + i = 0; | |
| 118 | + is_umem = true; | |
| 119 | + for_each_sg(sghead, sg, pages, i) { | |
| 120 | + pbl->pg_map_arr[i] = sg_dma_address(sg); | |
| 121 | + pbl->pg_arr[i] = sg_virt(sg); | |
| 122 | + if (!pbl->pg_arr[i]) | |
| 123 | + goto fail; | |
| 124 | + | |
| 125 | + pbl->pg_count++; | |
| 126 | + } | |
| 127 | + } | |
| 128 | + | |
| 129 | + return 0; | |
| 130 | + | |
| 131 | +fail: | |
| 132 | + __free_pbl(pdev, pbl, is_umem); | |
| 133 | + return -ENOMEM; | |
| 134 | +} | |
| 135 | + | |
| 136 | +/* HWQ */ | |
| 137 | +void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq) | |
| 138 | +{ | |
| 139 | + int i; | |
| 140 | + | |
| 141 | + if (!hwq->max_elements) | |
| 142 | + return; | |
| 143 | + if (hwq->level >= PBL_LVL_MAX) | |
| 144 | + return; | |
| 145 | + | |
| 146 | + for (i = 0; i < hwq->level + 1; i++) { | |
| 147 | + if (i == hwq->level) | |
| 148 | + __free_pbl(pdev, &hwq->pbl[i], hwq->is_user); | |
| 149 | + else | |
| 150 | + __free_pbl(pdev, &hwq->pbl[i], false); | |
| 151 | + } | |
| 152 | + | |
| 153 | + hwq->level = PBL_LVL_MAX; | |
| 154 | + hwq->max_elements = 0; | |
| 155 | + hwq->element_size = 0; | |
| 156 | + hwq->prod = 0; | |
| 157 | + hwq->cons = 0; | |
| 158 | + hwq->cp_bit = 0; | |
| 159 | +} | |
| 160 | + | |
| 161 | +/* All HWQs are power of 2 in size */ | |
| 162 | +int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq, | |
| 163 | + struct scatterlist *sghead, int nmap, | |
| 164 | + u32 *elements, u32 element_size, u32 aux, | |
| 165 | + u32 pg_size, enum bnxt_qplib_hwq_type hwq_type) | |
| 166 | +{ | |
| 167 | + u32 pages, slots, size, aux_pages = 0, aux_size = 0; | |
| 168 | + dma_addr_t *src_phys_ptr, **dst_virt_ptr; | |
| 169 | + int i, rc; | |
| 170 | + | |
| 171 | + hwq->level = PBL_LVL_MAX; | |
| 172 | + | |
| 173 | + slots = roundup_pow_of_two(*elements); | |
| 174 | + if (aux) { | |
| 175 | + aux_size = roundup_pow_of_two(aux); | |
| 176 | + aux_pages = (slots * aux_size) / pg_size; | |
| 177 | + if ((slots * aux_size) % pg_size) | |
| 178 | + aux_pages++; | |
| 179 | + } | |
| 180 | + size = roundup_pow_of_two(element_size); | |
| 181 | + | |
| 182 | + if (!sghead) { | |
| 183 | + hwq->is_user = false; | |
| 184 | + pages = (slots * size) / pg_size + aux_pages; | |
| 185 | + if ((slots * size) % pg_size) | |
| 186 | + pages++; | |
| 187 | + if (!pages) | |
| 188 | + return -EINVAL; | |
| 189 | + } else { | |
| 190 | + hwq->is_user = true; | |
| 191 | + pages = nmap; | |
| 192 | + } | |
| 193 | + | |
| 194 | + /* Alloc the 1st memory block; can be a PDL/PTL/PBL */ | |
| 195 | + if (sghead && (pages == MAX_PBL_LVL_0_PGS)) | |
| 196 | + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], sghead, | |
| 197 | + pages, pg_size); | |
| 198 | + else | |
| 199 | + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], NULL, 1, pg_size); | |
| 200 | + if (rc) | |
| 201 | + goto fail; | |
| 202 | + | |
| 203 | + hwq->level = PBL_LVL_0; | |
| 204 | + | |
| 205 | + if (pages > MAX_PBL_LVL_0_PGS) { | |
| 206 | + if (pages > MAX_PBL_LVL_1_PGS) { | |
| 207 | + /* 2 levels of indirection */ | |
| 208 | + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], NULL, | |
| 209 | + MAX_PBL_LVL_1_PGS_FOR_LVL_2, pg_size); | |
| 210 | + if (rc) | |
| 211 | + goto fail; | |
| 212 | + /* Fill in lvl0 PBL */ | |
| 213 | + dst_virt_ptr = | |
| 214 | + (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; | |
| 215 | + src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; | |
| 216 | + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) | |
| 217 | + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = | |
| 218 | + src_phys_ptr[i] | PTU_PDE_VALID; | |
| 219 | + hwq->level = PBL_LVL_1; | |
| 220 | + | |
| 221 | + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_2], sghead, | |
| 222 | + pages, pg_size); | |
| 223 | + if (rc) | |
| 224 | + goto fail; | |
| 225 | + | |
| 226 | + /* Fill in lvl1 PBL */ | |
| 227 | + dst_virt_ptr = | |
| 228 | + (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr; | |
| 229 | + src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr; | |
| 230 | + for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) { | |
| 231 | + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = | |
| 232 | + src_phys_ptr[i] | PTU_PTE_VALID; | |
| 233 | + } | |
| 234 | + if (hwq_type == HWQ_TYPE_QUEUE) { | |
| 235 | + /* Find the last pg of the size */ | |
| 236 | + i = hwq->pbl[PBL_LVL_2].pg_count; | |
| 237 | + dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= | |
| 238 | + PTU_PTE_LAST; | |
| 239 | + if (i > 1) | |
| 240 | + dst_virt_ptr[PTR_PG(i - 2)] | |
| 241 | + [PTR_IDX(i - 2)] |= | |
| 242 | + PTU_PTE_NEXT_TO_LAST; | |
| 243 | + } | |
| 244 | + hwq->level = PBL_LVL_2; | |
| 245 | + } else { | |
| 246 | + u32 flag = hwq_type == HWQ_TYPE_L2_CMPL ? 0 : | |
| 247 | + PTU_PTE_VALID; | |
| 248 | + | |
| 249 | + /* 1 level of indirection */ | |
| 250 | + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], sghead, | |
| 251 | + pages, pg_size); | |
| 252 | + if (rc) | |
| 253 | + goto fail; | |
| 254 | + /* Fill in lvl0 PBL */ | |
| 255 | + dst_virt_ptr = | |
| 256 | + (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; | |
| 257 | + src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; | |
| 258 | + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) { | |
| 259 | + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = | |
| 260 | + src_phys_ptr[i] | flag; | |
| 261 | + } | |
| 262 | + if (hwq_type == HWQ_TYPE_QUEUE) { | |
| 263 | + /* Find the last pg of the size */ | |
| 264 | + i = hwq->pbl[PBL_LVL_1].pg_count; | |
| 265 | + dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= | |
| 266 | + PTU_PTE_LAST; | |
| 267 | + if (i > 1) | |
| 268 | + dst_virt_ptr[PTR_PG(i - 2)] | |
| 269 | + [PTR_IDX(i - 2)] |= | |
| 270 | + PTU_PTE_NEXT_TO_LAST; | |
| 271 | + } | |
| 272 | + hwq->level = PBL_LVL_1; | |
| 273 | + } | |
| 274 | + } | |
| 275 | + hwq->pdev = pdev; | |
| 276 | + spin_lock_init(&hwq->lock); | |
| 277 | + hwq->prod = 0; | |
| 278 | + hwq->cons = 0; | |
| 279 | + *elements = hwq->max_elements = slots; | |
| 280 | + hwq->element_size = size; | |
| 281 | + | |
| 282 | + /* For direct access to the elements */ | |
| 283 | + hwq->pbl_ptr = hwq->pbl[hwq->level].pg_arr; | |
| 284 | + hwq->pbl_dma_ptr = hwq->pbl[hwq->level].pg_map_arr; | |
| 285 | + | |
| 286 | + return 0; | |
| 287 | + | |
| 288 | +fail: | |
| 289 | + bnxt_qplib_free_hwq(pdev, hwq); | |
| 290 | + return -ENOMEM; | |
| 291 | +} | |
| 292 | + | |
| 293 | +/* Context Tables */ | |
| 294 | +void bnxt_qplib_free_ctx(struct pci_dev *pdev, | |
| 295 | + struct bnxt_qplib_ctx *ctx) | |
| 296 | +{ | |
| 297 | + int i; | |
| 298 | + | |
| 299 | + bnxt_qplib_free_hwq(pdev, &ctx->qpc_tbl); | |
| 300 | + bnxt_qplib_free_hwq(pdev, &ctx->mrw_tbl); | |
| 301 | + bnxt_qplib_free_hwq(pdev, &ctx->srqc_tbl); | |
| 302 | + bnxt_qplib_free_hwq(pdev, &ctx->cq_tbl); | |
| 303 | + bnxt_qplib_free_hwq(pdev, &ctx->tim_tbl); | |
| 304 | + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) | |
| 305 | + bnxt_qplib_free_hwq(pdev, &ctx->tqm_tbl[i]); | |
| 306 | + bnxt_qplib_free_hwq(pdev, &ctx->tqm_pde); | |
| 307 | + bnxt_qplib_free_stats_ctx(pdev, &ctx->stats); | |
| 308 | +} | |
| 309 | + | |
| 310 | +/* | |
| 311 | + * Routine: bnxt_qplib_alloc_ctx | |
| 312 | + * Description: | |
| 313 | + * Context tables are memories which are used by the chip fw. | |
| 314 | + * The 6 tables defined are: | |
| 315 | + * QPC ctx - holds QP states | |
| 316 | + * MRW ctx - holds memory region and window | |
| 317 | + * SRQ ctx - holds shared RQ states | |
| 318 | + * CQ ctx - holds completion queue states | |
| 319 | + * TQM ctx - holds Tx Queue Manager context | |
| 320 | + * TIM ctx - holds timer context | |
| 321 | + * Depending on the size of the tbl requested, either a 1 Page Buffer List | |
| 322 | + * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used | |
| 323 | + * instead. | |
| 324 | + * Table might be employed as follows: | |
| 325 | + * For 0 < ctx size <= 1 PAGE, 0 level of ind is used | |
| 326 | + * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used | |
| 327 | + * For 512 < ctx size <= MAX, 2 levels of ind is used | |
| 328 | + * Returns: | |
| 329 | + * 0 if success, else -ERRORS | |
| 330 | + */ | |
| 331 | +int bnxt_qplib_alloc_ctx(struct pci_dev *pdev, | |
| 332 | + struct bnxt_qplib_ctx *ctx, | |
| 333 | + bool virt_fn) | |
| 334 | +{ | |
| 335 | + int i, j, k, rc = 0; | |
| 336 | + int fnz_idx = -1; | |
| 337 | + __le64 **pbl_ptr; | |
| 338 | + | |
| 339 | + if (virt_fn) | |
| 340 | + goto stats_alloc; | |
| 341 | + | |
| 342 | + /* QPC Tables */ | |
| 343 | + ctx->qpc_tbl.max_elements = ctx->qpc_count; | |
| 344 | + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->qpc_tbl, NULL, 0, | |
| 345 | + &ctx->qpc_tbl.max_elements, | |
| 346 | + BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE, 0, | |
| 347 | + PAGE_SIZE, HWQ_TYPE_CTX); | |
| 348 | + if (rc) | |
| 349 | + goto fail; | |
| 350 | + | |
| 351 | + /* MRW Tables */ | |
| 352 | + ctx->mrw_tbl.max_elements = ctx->mrw_count; | |
| 353 | + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->mrw_tbl, NULL, 0, | |
| 354 | + &ctx->mrw_tbl.max_elements, | |
| 355 | + BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE, 0, | |
| 356 | + PAGE_SIZE, HWQ_TYPE_CTX); | |
| 357 | + if (rc) | |
| 358 | + goto fail; | |
| 359 | + | |
| 360 | + /* SRQ Tables */ | |
| 361 | + ctx->srqc_tbl.max_elements = ctx->srqc_count; | |
| 362 | + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->srqc_tbl, NULL, 0, | |
| 363 | + &ctx->srqc_tbl.max_elements, | |
| 364 | + BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE, 0, | |
| 365 | + PAGE_SIZE, HWQ_TYPE_CTX); | |
| 366 | + if (rc) | |
| 367 | + goto fail; | |
| 368 | + | |
| 369 | + /* CQ Tables */ | |
| 370 | + ctx->cq_tbl.max_elements = ctx->cq_count; | |
| 371 | + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->cq_tbl, NULL, 0, | |
| 372 | + &ctx->cq_tbl.max_elements, | |
| 373 | + BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE, 0, | |
| 374 | + PAGE_SIZE, HWQ_TYPE_CTX); | |
| 375 | + if (rc) | |
| 376 | + goto fail; | |
| 377 | + | |
| 378 | + /* TQM Buffer */ | |
| 379 | + ctx->tqm_pde.max_elements = 512; | |
| 380 | + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_pde, NULL, 0, | |
| 381 | + &ctx->tqm_pde.max_elements, sizeof(u64), | |
| 382 | + 0, PAGE_SIZE, HWQ_TYPE_CTX); | |
| 383 | + if (rc) | |
| 384 | + goto fail; | |
| 385 | + | |
| 386 | + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) { | |
| 387 | + if (!ctx->tqm_count[i]) | |
| 388 | + continue; | |
| 389 | + ctx->tqm_tbl[i].max_elements = ctx->qpc_count * | |
| 390 | + ctx->tqm_count[i]; | |
| 391 | + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_tbl[i], NULL, 0, | |
| 392 | + &ctx->tqm_tbl[i].max_elements, 1, | |
| 393 | + 0, PAGE_SIZE, HWQ_TYPE_CTX); | |
| 394 | + if (rc) | |
| 395 | + goto fail; | |
| 396 | + } | |
| 397 | + pbl_ptr = (__le64 **)ctx->tqm_pde.pbl_ptr; | |
| 398 | + for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ; | |
| 399 | + i++, j += MAX_TQM_ALLOC_BLK_SIZE) { | |
| 400 | + if (!ctx->tqm_tbl[i].max_elements) | |
| 401 | + continue; | |
| 402 | + if (fnz_idx == -1) | |
| 403 | + fnz_idx = i; | |
| 404 | + switch (ctx->tqm_tbl[i].level) { | |
| 405 | + case PBL_LVL_2: | |
| 406 | + for (k = 0; k < ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_count; | |
| 407 | + k++) | |
| 408 | + pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)] = | |
| 409 | + cpu_to_le64( | |
| 410 | + ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_map_arr[k] | |
| 411 | + | PTU_PTE_VALID); | |
| 412 | + break; | |
| 413 | + case PBL_LVL_1: | |
| 414 | + case PBL_LVL_0: | |
| 415 | + default: | |
| 416 | + pbl_ptr[PTR_PG(j)][PTR_IDX(j)] = cpu_to_le64( | |
| 417 | + ctx->tqm_tbl[i].pbl[PBL_LVL_0].pg_map_arr[0] | | |
| 418 | + PTU_PTE_VALID); | |
| 419 | + break; | |
| 420 | + } | |
| 421 | + } | |
| 422 | + if (fnz_idx == -1) | |
| 423 | + fnz_idx = 0; | |
| 424 | + ctx->tqm_pde_level = ctx->tqm_tbl[fnz_idx].level == PBL_LVL_2 ? | |
| 425 | + PBL_LVL_2 : ctx->tqm_tbl[fnz_idx].level + 1; | |
| 426 | + | |
| 427 | + /* TIM Buffer */ | |
| 428 | + ctx->tim_tbl.max_elements = ctx->qpc_count * 16; | |
| 429 | + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tim_tbl, NULL, 0, | |
| 430 | + &ctx->tim_tbl.max_elements, 1, | |
| 431 | + 0, PAGE_SIZE, HWQ_TYPE_CTX); | |
| 432 | + if (rc) | |
| 433 | + goto fail; | |
| 434 | + | |
| 435 | +stats_alloc: | |
| 436 | + /* Stats */ | |
| 437 | + rc = bnxt_qplib_alloc_stats_ctx(pdev, &ctx->stats); | |
| 438 | + if (rc) | |
| 439 | + goto fail; | |
| 440 | + | |
| 441 | + return 0; | |
| 442 | + | |
| 443 | +fail: | |
| 444 | + bnxt_qplib_free_ctx(pdev, ctx); | |
| 445 | + return rc; | |
| 446 | +} | |
| 447 | + | |
| 448 | +/* GUID */ | |
| 449 | +void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid) | |
| 450 | +{ | |
| 451 | + u8 mac[ETH_ALEN]; | |
| 452 | + | |
| 453 | + /* MAC-48 to EUI-64 mapping */ | |
| 454 | + memcpy(mac, dev_addr, ETH_ALEN); | |
| 455 | + guid[0] = mac[0] ^ 2; | |
| 456 | + guid[1] = mac[1]; | |
| 457 | + guid[2] = mac[2]; | |
| 458 | + guid[3] = 0xff; | |
| 459 | + guid[4] = 0xfe; | |
| 460 | + guid[5] = mac[3]; | |
| 461 | + guid[6] = mac[4]; | |
| 462 | + guid[7] = mac[5]; | |
| 463 | +} | |
| 464 | + | |
| 465 | +static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res, | |
| 466 | + struct bnxt_qplib_sgid_tbl *sgid_tbl) | |
| 467 | +{ | |
| 468 | + kfree(sgid_tbl->tbl); | |
| 469 | + kfree(sgid_tbl->hw_id); | |
| 470 | + kfree(sgid_tbl->ctx); | |
| 471 | + sgid_tbl->tbl = NULL; | |
| 472 | + sgid_tbl->hw_id = NULL; | |
| 473 | + sgid_tbl->ctx = NULL; | |
| 474 | + sgid_tbl->max = 0; | |
| 475 | + sgid_tbl->active = 0; | |
| 476 | +} | |
| 477 | + | |
| 478 | +static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res, | |
| 479 | + struct bnxt_qplib_sgid_tbl *sgid_tbl, | |
| 480 | + u16 max) | |
| 481 | +{ | |
| 482 | + sgid_tbl->tbl = kcalloc(max, sizeof(struct bnxt_qplib_gid), GFP_KERNEL); | |
| 483 | + if (!sgid_tbl->tbl) | |
| 484 | + return -ENOMEM; | |
| 485 | + | |
| 486 | + sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL); | |
| 487 | + if (!sgid_tbl->hw_id) | |
| 488 | + goto out_free1; | |
| 489 | + | |
| 490 | + sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL); | |
| 491 | + if (!sgid_tbl->ctx) | |
| 492 | + goto out_free2; | |
| 493 | + | |
| 494 | + sgid_tbl->max = max; | |
| 495 | + return 0; | |
| 496 | +out_free2: | |
| 497 | + kfree(sgid_tbl->hw_id); | |
| 498 | + sgid_tbl->hw_id = NULL; | |
| 499 | +out_free1: | |
| 500 | + kfree(sgid_tbl->tbl); | |
| 501 | + sgid_tbl->tbl = NULL; | |
| 502 | + return -ENOMEM; | |
| 503 | +}; | |
| 504 | + | |
| 505 | +static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res, | |
| 506 | + struct bnxt_qplib_sgid_tbl *sgid_tbl) | |
| 507 | +{ | |
| 508 | + int i; | |
| 509 | + | |
| 510 | + for (i = 0; i < sgid_tbl->max; i++) { | |
| 511 | + if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, | |
| 512 | + sizeof(bnxt_qplib_gid_zero))) | |
| 513 | + bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i], true); | |
| 514 | + } | |
| 515 | + memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max); | |
| 516 | + memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); | |
| 517 | + sgid_tbl->active = 0; | |
| 518 | +} | |
| 519 | + | |
| 520 | +static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, | |
| 521 | + struct net_device *netdev) | |
| 522 | +{ | |
| 523 | + memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max); | |
| 524 | + memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); | |
| 525 | +} | |
| 526 | + | |
| 527 | +static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res, | |
| 528 | + struct bnxt_qplib_pkey_tbl *pkey_tbl) | |
| 529 | +{ | |
| 530 | + if (!pkey_tbl->tbl) | |
| 531 | + dev_dbg(&res->pdev->dev, "QPLIB: PKEY tbl not present"); | |
| 532 | + else | |
| 533 | + kfree(pkey_tbl->tbl); | |
| 534 | + | |
| 535 | + pkey_tbl->tbl = NULL; | |
| 536 | + pkey_tbl->max = 0; | |
| 537 | + pkey_tbl->active = 0; | |
| 538 | +} | |
| 539 | + | |
| 540 | +static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res, | |
| 541 | + struct bnxt_qplib_pkey_tbl *pkey_tbl, | |
| 542 | + u16 max) | |
| 543 | +{ | |
| 544 | + pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL); | |
| 545 | + if (!pkey_tbl->tbl) | |
| 546 | + return -ENOMEM; | |
| 547 | + | |
| 548 | + pkey_tbl->max = max; | |
| 549 | + return 0; | |
| 550 | +}; | |
| 551 | + | |
| 552 | +/* PDs */ | |
| 553 | +int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd) | |
| 554 | +{ | |
| 555 | + u32 bit_num; | |
| 556 | + | |
| 557 | + bit_num = find_first_bit(pdt->tbl, pdt->max); | |
| 558 | + if (bit_num == pdt->max) | |
| 559 | + return -ENOMEM; | |
| 560 | + | |
| 561 | + /* Found unused PD */ | |
| 562 | + clear_bit(bit_num, pdt->tbl); | |
| 563 | + pd->id = bit_num; | |
| 564 | + return 0; | |
| 565 | +} | |
| 566 | + | |
| 567 | +int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, | |
| 568 | + struct bnxt_qplib_pd_tbl *pdt, | |
| 569 | + struct bnxt_qplib_pd *pd) | |
| 570 | +{ | |
| 571 | + if (test_and_set_bit(pd->id, pdt->tbl)) { | |
| 572 | + dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d", | |
| 573 | + pd->id); | |
| 574 | + return -EINVAL; | |
| 575 | + } | |
| 576 | + pd->id = 0; | |
| 577 | + return 0; | |
| 578 | +} | |
| 579 | + | |
| 580 | +static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt) | |
| 581 | +{ | |
| 582 | + kfree(pdt->tbl); | |
| 583 | + pdt->tbl = NULL; | |
| 584 | + pdt->max = 0; | |
| 585 | +} | |
| 586 | + | |
| 587 | +static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, | |
| 588 | + struct bnxt_qplib_pd_tbl *pdt, | |
| 589 | + u32 max) | |
| 590 | +{ | |
| 591 | + u32 bytes; | |
| 592 | + | |
| 593 | + bytes = max >> 3; | |
| 594 | + if (!bytes) | |
| 595 | + bytes = 1; | |
| 596 | + pdt->tbl = kmalloc(bytes, GFP_KERNEL); | |
| 597 | + if (!pdt->tbl) | |
| 598 | + return -ENOMEM; | |
| 599 | + | |
| 600 | + pdt->max = max; | |
| 601 | + memset((u8 *)pdt->tbl, 0xFF, bytes); | |
| 602 | + | |
| 603 | + return 0; | |
| 604 | +} | |
| 605 | + | |
| 606 | +/* DPIs */ | |
| 607 | +int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit, | |
| 608 | + struct bnxt_qplib_dpi *dpi, | |
| 609 | + void *app) | |
| 610 | +{ | |
| 611 | + u32 bit_num; | |
| 612 | + | |
| 613 | + bit_num = find_first_bit(dpit->tbl, dpit->max); | |
| 614 | + if (bit_num == dpit->max) | |
| 615 | + return -ENOMEM; | |
| 616 | + | |
| 617 | + /* Found unused DPI */ | |
| 618 | + clear_bit(bit_num, dpit->tbl); | |
| 619 | + dpit->app_tbl[bit_num] = app; | |
| 620 | + | |
| 621 | + dpi->dpi = bit_num; | |
| 622 | + dpi->dbr = dpit->dbr_bar_reg_iomem + (bit_num * PAGE_SIZE); | |
| 623 | + dpi->umdbr = dpit->unmapped_dbr + (bit_num * PAGE_SIZE); | |
| 624 | + | |
| 625 | + return 0; | |
| 626 | +} | |
| 627 | + | |
| 628 | +int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, | |
| 629 | + struct bnxt_qplib_dpi_tbl *dpit, | |
| 630 | + struct bnxt_qplib_dpi *dpi) | |
| 631 | +{ | |
| 632 | + if (dpi->dpi >= dpit->max) { | |
| 633 | + dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d", dpi->dpi); | |
| 634 | + return -EINVAL; | |
| 635 | + } | |
| 636 | + if (test_and_set_bit(dpi->dpi, dpit->tbl)) { | |
| 637 | + dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d", | |
| 638 | + dpi->dpi); | |
| 639 | + return -EINVAL; | |
| 640 | + } | |
| 641 | + if (dpit->app_tbl) | |
| 642 | + dpit->app_tbl[dpi->dpi] = NULL; | |
| 643 | + memset(dpi, 0, sizeof(*dpi)); | |
| 644 | + | |
| 645 | + return 0; | |
| 646 | +} | |
| 647 | + | |
| 648 | +static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res *res, | |
| 649 | + struct bnxt_qplib_dpi_tbl *dpit) | |
| 650 | +{ | |
| 651 | + kfree(dpit->tbl); | |
| 652 | + kfree(dpit->app_tbl); | |
| 653 | + if (dpit->dbr_bar_reg_iomem) | |
| 654 | + pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); | |
| 655 | + memset(dpit, 0, sizeof(*dpit)); | |
| 656 | +} | |
| 657 | + | |
| 658 | +static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, | |
| 659 | + struct bnxt_qplib_dpi_tbl *dpit, | |
| 660 | + u32 dbr_offset) | |
| 661 | +{ | |
| 662 | + u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION; | |
| 663 | + resource_size_t bar_reg_base; | |
| 664 | + u32 dbr_len, bytes; | |
| 665 | + | |
| 666 | + if (dpit->dbr_bar_reg_iomem) { | |
| 667 | + dev_err(&res->pdev->dev, | |
| 668 | + "QPLIB: DBR BAR region %d already mapped", dbr_bar_reg); | |
| 669 | + return -EALREADY; | |
| 670 | + } | |
| 671 | + | |
| 672 | + bar_reg_base = pci_resource_start(res->pdev, dbr_bar_reg); | |
| 673 | + if (!bar_reg_base) { | |
| 674 | + dev_err(&res->pdev->dev, | |
| 675 | + "QPLIB: BAR region %d resc start failed", dbr_bar_reg); | |
| 676 | + return -ENOMEM; | |
| 677 | + } | |
| 678 | + | |
| 679 | + dbr_len = pci_resource_len(res->pdev, dbr_bar_reg) - dbr_offset; | |
| 680 | + if (!dbr_len || ((dbr_len & (PAGE_SIZE - 1)) != 0)) { | |
| 681 | + dev_err(&res->pdev->dev, "QPLIB: Invalid DBR length %d", | |
| 682 | + dbr_len); | |
| 683 | + return -ENOMEM; | |
| 684 | + } | |
| 685 | + | |
| 686 | + dpit->dbr_bar_reg_iomem = ioremap_nocache(bar_reg_base + dbr_offset, | |
| 687 | + dbr_len); | |
| 688 | + if (!dpit->dbr_bar_reg_iomem) { | |
| 689 | + dev_err(&res->pdev->dev, | |
| 690 | + "QPLIB: FP: DBR BAR region %d mapping failed", | |
| 691 | + dbr_bar_reg); | |
| 692 | + return -ENOMEM; | |
| 693 | + } | |
| 694 | + | |
| 695 | + dpit->unmapped_dbr = bar_reg_base + dbr_offset; | |
| 696 | + dpit->max = dbr_len / PAGE_SIZE; | |
| 697 | + | |
| 698 | + dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL); | |
| 699 | + if (!dpit->app_tbl) { | |
| 700 | + pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); | |
| 701 | + dev_err(&res->pdev->dev, | |
| 702 | + "QPLIB: DPI app tbl allocation failed"); | |
| 703 | + return -ENOMEM; | |
| 704 | + } | |
| 705 | + | |
| 706 | + bytes = dpit->max >> 3; | |
| 707 | + if (!bytes) | |
| 708 | + bytes = 1; | |
| 709 | + | |
| 710 | + dpit->tbl = kmalloc(bytes, GFP_KERNEL); | |
| 711 | + if (!dpit->tbl) { | |
| 712 | + pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); | |
| 713 | + kfree(dpit->app_tbl); | |
| 714 | + dpit->app_tbl = NULL; | |
| 715 | + dev_err(&res->pdev->dev, | |
| 716 | + "QPLIB: DPI tbl allocation failed for size = %d", | |
| 717 | + bytes); | |
| 718 | + return -ENOMEM; | |
| 719 | + } | |
| 720 | + | |
| 721 | + memset((u8 *)dpit->tbl, 0xFF, bytes); | |
| 722 | + | |
| 723 | + return 0; | |
| 724 | +} | |
| 725 | + | |
| 726 | +/* PKEYs */ | |
| 727 | +static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl) | |
| 728 | +{ | |
| 729 | + memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max); | |
| 730 | + pkey_tbl->active = 0; | |
| 731 | +} | |
| 732 | + | |
| 733 | +static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res, | |
| 734 | + struct bnxt_qplib_pkey_tbl *pkey_tbl) | |
| 735 | +{ | |
| 736 | + u16 pkey = 0xFFFF; | |
| 737 | + | |
| 738 | + memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max); | |
| 739 | + | |
| 740 | + /* pkey default = 0xFFFF */ | |
| 741 | + bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false); | |
| 742 | +} | |
| 743 | + | |
| 744 | +/* Stats */ | |
| 745 | +static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev, | |
| 746 | + struct bnxt_qplib_stats *stats) | |
| 747 | +{ | |
| 748 | + if (stats->dma) { | |
| 749 | + dma_free_coherent(&pdev->dev, stats->size, | |
| 750 | + stats->dma, stats->dma_map); | |
| 751 | + } | |
| 752 | + memset(stats, 0, sizeof(*stats)); | |
| 753 | + stats->fw_id = -1; | |
| 754 | +} | |
| 755 | + | |
| 756 | +static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev, | |
| 757 | + struct bnxt_qplib_stats *stats) | |
| 758 | +{ | |
| 759 | + memset(stats, 0, sizeof(*stats)); | |
| 760 | + stats->fw_id = -1; | |
| 761 | + stats->size = sizeof(struct ctx_hw_stats); | |
| 762 | + stats->dma = dma_alloc_coherent(&pdev->dev, stats->size, | |
| 763 | + &stats->dma_map, GFP_KERNEL); | |
| 764 | + if (!stats->dma) { | |
| 765 | + dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed"); | |
| 766 | + return -ENOMEM; | |
| 767 | + } | |
| 768 | + return 0; | |
| 769 | +} | |
| 770 | + | |
| 771 | +void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res) | |
| 772 | +{ | |
| 773 | + bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl); | |
| 774 | + bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl); | |
| 775 | +} | |
| 776 | + | |
| 777 | +int bnxt_qplib_init_res(struct bnxt_qplib_res *res) | |
| 778 | +{ | |
| 779 | + bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev); | |
| 780 | + bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl); | |
| 781 | + | |
| 782 | + return 0; | |
| 783 | +} | |
| 784 | + | |
| 785 | +void bnxt_qplib_free_res(struct bnxt_qplib_res *res) | |
| 786 | +{ | |
| 787 | + bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl); | |
| 788 | + bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl); | |
| 789 | + bnxt_qplib_free_pd_tbl(&res->pd_tbl); | |
| 790 | + bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl); | |
| 791 | + | |
| 792 | + res->netdev = NULL; | |
| 793 | + res->pdev = NULL; | |
| 794 | +} | |
| 795 | + | |
| 796 | +int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, | |
| 797 | + struct net_device *netdev, | |
| 798 | + struct bnxt_qplib_dev_attr *dev_attr) | |
| 799 | +{ | |
| 800 | + int rc = 0; | |
| 801 | + | |
| 802 | + res->pdev = pdev; | |
| 803 | + res->netdev = netdev; | |
| 804 | + | |
| 805 | + rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid); | |
| 806 | + if (rc) | |
| 807 | + goto fail; | |
| 808 | + | |
| 809 | + rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey); | |
| 810 | + if (rc) | |
| 811 | + goto fail; | |
| 812 | + | |
| 813 | + rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd); | |
| 814 | + if (rc) | |
| 815 | + goto fail; | |
| 816 | + | |
| 817 | + rc = bnxt_qplib_alloc_dpi_tbl(res, &res->dpi_tbl, dev_attr->l2_db_size); | |
| 818 | + if (rc) | |
| 819 | + goto fail; | |
| 820 | + | |
| 821 | + return 0; | |
| 822 | +fail: | |
| 823 | + bnxt_qplib_free_res(res); | |
| 824 | + return rc; | |
| 825 | +} | |
| @@ -0,0 +1,223 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: QPLib resource manager (header) | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#ifndef __BNXT_QPLIB_RES_H__ | |
| 40 | +#define __BNXT_QPLIB_RES_H__ | |
| 41 | + | |
| 42 | +extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; | |
| 43 | + | |
| 44 | +#define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *)) | |
| 45 | +#define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1) | |
| 46 | +#define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG) | |
| 47 | +#define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG) | |
| 48 | + | |
| 49 | +#define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1)) | |
| 50 | + | |
| 51 | +enum bnxt_qplib_hwq_type { | |
| 52 | + HWQ_TYPE_CTX, | |
| 53 | + HWQ_TYPE_QUEUE, | |
| 54 | + HWQ_TYPE_L2_CMPL | |
| 55 | +}; | |
| 56 | + | |
| 57 | +#define MAX_PBL_LVL_0_PGS 1 | |
| 58 | +#define MAX_PBL_LVL_1_PGS 512 | |
| 59 | +#define MAX_PBL_LVL_1_PGS_SHIFT 9 | |
| 60 | +#define MAX_PBL_LVL_1_PGS_FOR_LVL_2 256 | |
| 61 | +#define MAX_PBL_LVL_2_PGS (256 * 512) | |
| 62 | + | |
| 63 | +enum bnxt_qplib_pbl_lvl { | |
| 64 | + PBL_LVL_0, | |
| 65 | + PBL_LVL_1, | |
| 66 | + PBL_LVL_2, | |
| 67 | + PBL_LVL_MAX | |
| 68 | +}; | |
| 69 | + | |
| 70 | +#define ROCE_PG_SIZE_4K (4 * 1024) | |
| 71 | +#define ROCE_PG_SIZE_8K (8 * 1024) | |
| 72 | +#define ROCE_PG_SIZE_64K (64 * 1024) | |
| 73 | +#define ROCE_PG_SIZE_2M (2 * 1024 * 1024) | |
| 74 | +#define ROCE_PG_SIZE_8M (8 * 1024 * 1024) | |
| 75 | +#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024) | |
| 76 | + | |
| 77 | +struct bnxt_qplib_pbl { | |
| 78 | + u32 pg_count; | |
| 79 | + u32 pg_size; | |
| 80 | + void **pg_arr; | |
| 81 | + dma_addr_t *pg_map_arr; | |
| 82 | +}; | |
| 83 | + | |
| 84 | +struct bnxt_qplib_hwq { | |
| 85 | + struct pci_dev *pdev; | |
| 86 | + /* lock to protect qplib_hwq */ | |
| 87 | + spinlock_t lock; | |
| 88 | + struct bnxt_qplib_pbl pbl[PBL_LVL_MAX]; | |
| 89 | + enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */ | |
| 90 | + /* ptr for easy access to the PBL entries */ | |
| 91 | + void **pbl_ptr; | |
| 92 | + /* ptr for easy access to the dma_addr */ | |
| 93 | + dma_addr_t *pbl_dma_ptr; | |
| 94 | + u32 max_elements; | |
| 95 | + u16 element_size; /* Size of each entry */ | |
| 96 | + | |
| 97 | + u32 prod; /* raw */ | |
| 98 | + u32 cons; /* raw */ | |
| 99 | + u8 cp_bit; | |
| 100 | + u8 is_user; | |
| 101 | +}; | |
| 102 | + | |
| 103 | +/* Tables */ | |
| 104 | +struct bnxt_qplib_pd_tbl { | |
| 105 | + unsigned long *tbl; | |
| 106 | + u32 max; | |
| 107 | +}; | |
| 108 | + | |
| 109 | +struct bnxt_qplib_sgid_tbl { | |
| 110 | + struct bnxt_qplib_gid *tbl; | |
| 111 | + u16 *hw_id; | |
| 112 | + u16 max; | |
| 113 | + u16 active; | |
| 114 | + void *ctx; | |
| 115 | +}; | |
| 116 | + | |
| 117 | +struct bnxt_qplib_pkey_tbl { | |
| 118 | + u16 *tbl; | |
| 119 | + u16 max; | |
| 120 | + u16 active; | |
| 121 | +}; | |
| 122 | + | |
| 123 | +struct bnxt_qplib_dpi { | |
| 124 | + u32 dpi; | |
| 125 | + void __iomem *dbr; | |
| 126 | + u64 umdbr; | |
| 127 | +}; | |
| 128 | + | |
| 129 | +struct bnxt_qplib_dpi_tbl { | |
| 130 | + void **app_tbl; | |
| 131 | + unsigned long *tbl; | |
| 132 | + u16 max; | |
| 133 | + void __iomem *dbr_bar_reg_iomem; | |
| 134 | + u64 unmapped_dbr; | |
| 135 | +}; | |
| 136 | + | |
| 137 | +struct bnxt_qplib_stats { | |
| 138 | + dma_addr_t dma_map; | |
| 139 | + void *dma; | |
| 140 | + u32 size; | |
| 141 | + u32 fw_id; | |
| 142 | +}; | |
| 143 | + | |
| 144 | +struct bnxt_qplib_vf_res { | |
| 145 | + u32 max_qp_per_vf; | |
| 146 | + u32 max_mrw_per_vf; | |
| 147 | + u32 max_srq_per_vf; | |
| 148 | + u32 max_cq_per_vf; | |
| 149 | + u32 max_gid_per_vf; | |
| 150 | +}; | |
| 151 | + | |
| 152 | +#define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448 | |
| 153 | +#define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64 | |
| 154 | +#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64 | |
| 155 | +#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128 | |
| 156 | + | |
| 157 | +struct bnxt_qplib_ctx { | |
| 158 | + u32 qpc_count; | |
| 159 | + struct bnxt_qplib_hwq qpc_tbl; | |
| 160 | + u32 mrw_count; | |
| 161 | + struct bnxt_qplib_hwq mrw_tbl; | |
| 162 | + u32 srqc_count; | |
| 163 | + struct bnxt_qplib_hwq srqc_tbl; | |
| 164 | + u32 cq_count; | |
| 165 | + struct bnxt_qplib_hwq cq_tbl; | |
| 166 | + struct bnxt_qplib_hwq tim_tbl; | |
| 167 | +#define MAX_TQM_ALLOC_REQ 32 | |
| 168 | +#define MAX_TQM_ALLOC_BLK_SIZE 8 | |
| 169 | + u8 tqm_count[MAX_TQM_ALLOC_REQ]; | |
| 170 | + struct bnxt_qplib_hwq tqm_pde; | |
| 171 | + u32 tqm_pde_level; | |
| 172 | + struct bnxt_qplib_hwq tqm_tbl[MAX_TQM_ALLOC_REQ]; | |
| 173 | + struct bnxt_qplib_stats stats; | |
| 174 | + struct bnxt_qplib_vf_res vf_res; | |
| 175 | +}; | |
| 176 | + | |
| 177 | +struct bnxt_qplib_res { | |
| 178 | + struct pci_dev *pdev; | |
| 179 | + struct net_device *netdev; | |
| 180 | + | |
| 181 | + struct bnxt_qplib_rcfw *rcfw; | |
| 182 | + | |
| 183 | + struct bnxt_qplib_pd_tbl pd_tbl; | |
| 184 | + struct bnxt_qplib_sgid_tbl sgid_tbl; | |
| 185 | + struct bnxt_qplib_pkey_tbl pkey_tbl; | |
| 186 | + struct bnxt_qplib_dpi_tbl dpi_tbl; | |
| 187 | +}; | |
| 188 | + | |
| 189 | +#define to_bnxt_qplib(ptr, type, member) \ | |
| 190 | + container_of(ptr, type, member) | |
| 191 | + | |
| 192 | +struct bnxt_qplib_pd; | |
| 193 | +struct bnxt_qplib_dev_attr; | |
| 194 | + | |
| 195 | +void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq); | |
| 196 | +int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq, | |
| 197 | + struct scatterlist *sl, int nmap, u32 *elements, | |
| 198 | + u32 elements_per_page, u32 aux, u32 pg_size, | |
| 199 | + enum bnxt_qplib_hwq_type hwq_type); | |
| 200 | +void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid); | |
| 201 | +int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pd_tbl, | |
| 202 | + struct bnxt_qplib_pd *pd); | |
| 203 | +int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, | |
| 204 | + struct bnxt_qplib_pd_tbl *pd_tbl, | |
| 205 | + struct bnxt_qplib_pd *pd); | |
| 206 | +int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit, | |
| 207 | + struct bnxt_qplib_dpi *dpi, | |
| 208 | + void *app); | |
| 209 | +int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, | |
| 210 | + struct bnxt_qplib_dpi_tbl *dpi_tbl, | |
| 211 | + struct bnxt_qplib_dpi *dpi); | |
| 212 | +void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res); | |
| 213 | +int bnxt_qplib_init_res(struct bnxt_qplib_res *res); | |
| 214 | +void bnxt_qplib_free_res(struct bnxt_qplib_res *res); | |
| 215 | +int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, | |
| 216 | + struct net_device *netdev, | |
| 217 | + struct bnxt_qplib_dev_attr *dev_attr); | |
| 218 | +void bnxt_qplib_free_ctx(struct pci_dev *pdev, | |
| 219 | + struct bnxt_qplib_ctx *ctx); | |
| 220 | +int bnxt_qplib_alloc_ctx(struct pci_dev *pdev, | |
| 221 | + struct bnxt_qplib_ctx *ctx, | |
| 222 | + bool virt_fn); | |
| 223 | +#endif /* __BNXT_QPLIB_RES_H__ */ | |
| @@ -0,0 +1,838 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: Slow Path Operators | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#include <linux/interrupt.h> | |
| 40 | +#include <linux/spinlock.h> | |
| 41 | +#include <linux/sched.h> | |
| 42 | +#include <linux/pci.h> | |
| 43 | + | |
| 44 | +#include "roce_hsi.h" | |
| 45 | + | |
| 46 | +#include "qplib_res.h" | |
| 47 | +#include "qplib_rcfw.h" | |
| 48 | +#include "qplib_sp.h" | |
| 49 | + | |
| 50 | +const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0, | |
| 51 | + 0, 0, 0, 0, 0, 0, 0, 0 } }; | |
| 52 | + | |
| 53 | +/* Device */ | |
| 54 | +int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, | |
| 55 | + struct bnxt_qplib_dev_attr *attr) | |
| 56 | +{ | |
| 57 | + struct cmdq_query_func req; | |
| 58 | + struct creq_query_func_resp *resp; | |
| 59 | + struct creq_query_func_resp_sb *sb; | |
| 60 | + u16 cmd_flags = 0; | |
| 61 | + u32 temp; | |
| 62 | + u8 *tqm_alloc; | |
| 63 | + int i; | |
| 64 | + | |
| 65 | + RCFW_CMD_PREP(req, QUERY_FUNC, cmd_flags); | |
| 66 | + | |
| 67 | + req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; | |
| 68 | + resp = (struct creq_query_func_resp *) | |
| 69 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void **)&sb, | |
| 70 | + 0); | |
| 71 | + if (!resp) { | |
| 72 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC send failed"); | |
| 73 | + return -EINVAL; | |
| 74 | + } | |
| 75 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 76 | + /* Cmd timed out */ | |
| 77 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC timed out"); | |
| 78 | + return -ETIMEDOUT; | |
| 79 | + } | |
| 80 | + if (resp->status || | |
| 81 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 82 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC failed "); | |
| 83 | + dev_err(&rcfw->pdev->dev, | |
| 84 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 85 | + resp->status, le16_to_cpu(req.cookie), | |
| 86 | + le16_to_cpu(resp->cookie)); | |
| 87 | + return -EINVAL; | |
| 88 | + } | |
| 89 | + /* Extract the context from the side buffer */ | |
| 90 | + attr->max_qp = le32_to_cpu(sb->max_qp); | |
| 91 | + attr->max_qp_rd_atom = | |
| 92 | + sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? | |
| 93 | + BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom; | |
| 94 | + attr->max_qp_init_rd_atom = | |
| 95 | + sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? | |
| 96 | + BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom; | |
| 97 | + attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr); | |
| 98 | + attr->max_qp_sges = sb->max_sge; | |
| 99 | + attr->max_cq = le32_to_cpu(sb->max_cq); | |
| 100 | + attr->max_cq_wqes = le32_to_cpu(sb->max_cqe); | |
| 101 | + attr->max_cq_sges = attr->max_qp_sges; | |
| 102 | + attr->max_mr = le32_to_cpu(sb->max_mr); | |
| 103 | + attr->max_mw = le32_to_cpu(sb->max_mw); | |
| 104 | + | |
| 105 | + attr->max_mr_size = le64_to_cpu(sb->max_mr_size); | |
| 106 | + attr->max_pd = 64 * 1024; | |
| 107 | + attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp); | |
| 108 | + attr->max_ah = le32_to_cpu(sb->max_ah); | |
| 109 | + | |
| 110 | + attr->max_fmr = le32_to_cpu(sb->max_fmr); | |
| 111 | + attr->max_map_per_fmr = sb->max_map_per_fmr; | |
| 112 | + | |
| 113 | + attr->max_srq = le16_to_cpu(sb->max_srq); | |
| 114 | + attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; | |
| 115 | + attr->max_srq_sges = sb->max_srq_sge; | |
| 116 | + /* Bono only reports 1 PKEY for now, but it can support > 1 */ | |
| 117 | + attr->max_pkey = le32_to_cpu(sb->max_pkeys); | |
| 118 | + | |
| 119 | + attr->max_inline_data = le32_to_cpu(sb->max_inline_data); | |
| 120 | + attr->l2_db_size = (sb->l2_db_space_size + 1) * PAGE_SIZE; | |
| 121 | + attr->max_sgid = le32_to_cpu(sb->max_gid); | |
| 122 | + | |
| 123 | + strlcpy(attr->fw_ver, "20.6.28.0", sizeof(attr->fw_ver)); | |
| 124 | + | |
| 125 | + for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) { | |
| 126 | + temp = le32_to_cpu(sb->tqm_alloc_reqs[i]); | |
| 127 | + tqm_alloc = (u8 *)&temp; | |
| 128 | + attr->tqm_alloc_reqs[i * 4] = *tqm_alloc; | |
| 129 | + attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc); | |
| 130 | + attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc); | |
| 131 | + attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc); | |
| 132 | + } | |
| 133 | + return 0; | |
| 134 | +} | |
| 135 | + | |
| 136 | +/* SGID */ | |
| 137 | +int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, | |
| 138 | + struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, | |
| 139 | + struct bnxt_qplib_gid *gid) | |
| 140 | +{ | |
| 141 | + if (index > sgid_tbl->max) { | |
| 142 | + dev_err(&res->pdev->dev, | |
| 143 | + "QPLIB: Index %d exceeded SGID table max (%d)", | |
| 144 | + index, sgid_tbl->max); | |
| 145 | + return -EINVAL; | |
| 146 | + } | |
| 147 | + memcpy(gid, &sgid_tbl->tbl[index], sizeof(*gid)); | |
| 148 | + return 0; | |
| 149 | +} | |
| 150 | + | |
| 151 | +int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, | |
| 152 | + struct bnxt_qplib_gid *gid, bool update) | |
| 153 | +{ | |
| 154 | + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, | |
| 155 | + struct bnxt_qplib_res, | |
| 156 | + sgid_tbl); | |
| 157 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 158 | + int index; | |
| 159 | + | |
| 160 | + if (!sgid_tbl) { | |
| 161 | + dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated"); | |
| 162 | + return -EINVAL; | |
| 163 | + } | |
| 164 | + /* Do we need a sgid_lock here? */ | |
| 165 | + if (!sgid_tbl->active) { | |
| 166 | + dev_err(&res->pdev->dev, | |
| 167 | + "QPLIB: SGID table has no active entries"); | |
| 168 | + return -ENOMEM; | |
| 169 | + } | |
| 170 | + for (index = 0; index < sgid_tbl->max; index++) { | |
| 171 | + if (!memcmp(&sgid_tbl->tbl[index], gid, sizeof(*gid))) | |
| 172 | + break; | |
| 173 | + } | |
| 174 | + if (index == sgid_tbl->max) { | |
| 175 | + dev_warn(&res->pdev->dev, "GID not found in the SGID table"); | |
| 176 | + return 0; | |
| 177 | + } | |
| 178 | + /* Remove GID from the SGID table */ | |
| 179 | + if (update) { | |
| 180 | + struct cmdq_delete_gid req; | |
| 181 | + struct creq_delete_gid_resp *resp; | |
| 182 | + u16 cmd_flags = 0; | |
| 183 | + | |
| 184 | + RCFW_CMD_PREP(req, DELETE_GID, cmd_flags); | |
| 185 | + if (sgid_tbl->hw_id[index] == 0xFFFF) { | |
| 186 | + dev_err(&res->pdev->dev, | |
| 187 | + "QPLIB: GID entry contains an invalid HW id"); | |
| 188 | + return -EINVAL; | |
| 189 | + } | |
| 190 | + req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]); | |
| 191 | + resp = (struct creq_delete_gid_resp *) | |
| 192 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, NULL, | |
| 193 | + 0); | |
| 194 | + if (!resp) { | |
| 195 | + dev_err(&res->pdev->dev, | |
| 196 | + "QPLIB: SP: DELETE_GID send failed"); | |
| 197 | + return -EINVAL; | |
| 198 | + } | |
| 199 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, | |
| 200 | + le16_to_cpu(req.cookie))) { | |
| 201 | + /* Cmd timed out */ | |
| 202 | + dev_err(&res->pdev->dev, | |
| 203 | + "QPLIB: SP: DELETE_GID timed out"); | |
| 204 | + return -ETIMEDOUT; | |
| 205 | + } | |
| 206 | + if (resp->status || | |
| 207 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 208 | + dev_err(&res->pdev->dev, | |
| 209 | + "QPLIB: SP: DELETE_GID failed "); | |
| 210 | + dev_err(&res->pdev->dev, | |
| 211 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 212 | + resp->status, le16_to_cpu(req.cookie), | |
| 213 | + le16_to_cpu(resp->cookie)); | |
| 214 | + return -EINVAL; | |
| 215 | + } | |
| 216 | + } | |
| 217 | + memcpy(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, | |
| 218 | + sizeof(bnxt_qplib_gid_zero)); | |
| 219 | + sgid_tbl->active--; | |
| 220 | + dev_dbg(&res->pdev->dev, | |
| 221 | + "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x", | |
| 222 | + index, sgid_tbl->hw_id[index], sgid_tbl->active); | |
| 223 | + sgid_tbl->hw_id[index] = (u16)-1; | |
| 224 | + | |
| 225 | + /* unlock */ | |
| 226 | + return 0; | |
| 227 | +} | |
| 228 | + | |
| 229 | +int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, | |
| 230 | + struct bnxt_qplib_gid *gid, u8 *smac, u16 vlan_id, | |
| 231 | + bool update, u32 *index) | |
| 232 | +{ | |
| 233 | + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, | |
| 234 | + struct bnxt_qplib_res, | |
| 235 | + sgid_tbl); | |
| 236 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 237 | + int i, free_idx, rc = 0; | |
| 238 | + | |
| 239 | + if (!sgid_tbl) { | |
| 240 | + dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated"); | |
| 241 | + return -EINVAL; | |
| 242 | + } | |
| 243 | + /* Do we need a sgid_lock here? */ | |
| 244 | + if (sgid_tbl->active == sgid_tbl->max) { | |
| 245 | + dev_err(&res->pdev->dev, "QPLIB: SGID table is full"); | |
| 246 | + return -ENOMEM; | |
| 247 | + } | |
| 248 | + free_idx = sgid_tbl->max; | |
| 249 | + for (i = 0; i < sgid_tbl->max; i++) { | |
| 250 | + if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid))) { | |
| 251 | + dev_dbg(&res->pdev->dev, | |
| 252 | + "QPLIB: SGID entry already exist in entry %d!", | |
| 253 | + i); | |
| 254 | + *index = i; | |
| 255 | + return -EALREADY; | |
| 256 | + } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, | |
| 257 | + sizeof(bnxt_qplib_gid_zero)) && | |
| 258 | + free_idx == sgid_tbl->max) { | |
| 259 | + free_idx = i; | |
| 260 | + } | |
| 261 | + } | |
| 262 | + if (free_idx == sgid_tbl->max) { | |
| 263 | + dev_err(&res->pdev->dev, | |
| 264 | + "QPLIB: SGID table is FULL but count is not MAX??"); | |
| 265 | + return -ENOMEM; | |
| 266 | + } | |
| 267 | + if (update) { | |
| 268 | + struct cmdq_add_gid req; | |
| 269 | + struct creq_add_gid_resp *resp; | |
| 270 | + u16 cmd_flags = 0; | |
| 271 | + u32 temp32[4]; | |
| 272 | + u16 temp16[3]; | |
| 273 | + | |
| 274 | + RCFW_CMD_PREP(req, ADD_GID, cmd_flags); | |
| 275 | + | |
| 276 | + memcpy(temp32, gid->data, sizeof(struct bnxt_qplib_gid)); | |
| 277 | + req.gid[0] = cpu_to_be32(temp32[3]); | |
| 278 | + req.gid[1] = cpu_to_be32(temp32[2]); | |
| 279 | + req.gid[2] = cpu_to_be32(temp32[1]); | |
| 280 | + req.gid[3] = cpu_to_be32(temp32[0]); | |
| 281 | + if (vlan_id != 0xFFFF) | |
| 282 | + req.vlan = cpu_to_le16((vlan_id & | |
| 283 | + CMDQ_ADD_GID_VLAN_VLAN_ID_MASK) | | |
| 284 | + CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | | |
| 285 | + CMDQ_ADD_GID_VLAN_VLAN_EN); | |
| 286 | + | |
| 287 | + /* MAC in network format */ | |
| 288 | + memcpy(temp16, smac, 6); | |
| 289 | + req.src_mac[0] = cpu_to_be16(temp16[0]); | |
| 290 | + req.src_mac[1] = cpu_to_be16(temp16[1]); | |
| 291 | + req.src_mac[2] = cpu_to_be16(temp16[2]); | |
| 292 | + | |
| 293 | + resp = (struct creq_add_gid_resp *) | |
| 294 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 295 | + NULL, 0); | |
| 296 | + if (!resp) { | |
| 297 | + dev_err(&res->pdev->dev, | |
| 298 | + "QPLIB: SP: ADD_GID send failed"); | |
| 299 | + return -EINVAL; | |
| 300 | + } | |
| 301 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, | |
| 302 | + le16_to_cpu(req.cookie))) { | |
| 303 | + /* Cmd timed out */ | |
| 304 | + dev_err(&res->pdev->dev, | |
| 305 | + "QPIB: SP: ADD_GID timed out"); | |
| 306 | + return -ETIMEDOUT; | |
| 307 | + } | |
| 308 | + if (resp->status || | |
| 309 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 310 | + dev_err(&res->pdev->dev, "QPLIB: SP: ADD_GID failed "); | |
| 311 | + dev_err(&res->pdev->dev, | |
| 312 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 313 | + resp->status, le16_to_cpu(req.cookie), | |
| 314 | + le16_to_cpu(resp->cookie)); | |
| 315 | + return -EINVAL; | |
| 316 | + } | |
| 317 | + sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp->xid); | |
| 318 | + } | |
| 319 | + /* Add GID to the sgid_tbl */ | |
| 320 | + memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid)); | |
| 321 | + sgid_tbl->active++; | |
| 322 | + dev_dbg(&res->pdev->dev, | |
| 323 | + "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x", | |
| 324 | + free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active); | |
| 325 | + | |
| 326 | + *index = free_idx; | |
| 327 | + /* unlock */ | |
| 328 | + return rc; | |
| 329 | +} | |
| 330 | + | |
| 331 | +/* pkeys */ | |
| 332 | +int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res, | |
| 333 | + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index, | |
| 334 | + u16 *pkey) | |
| 335 | +{ | |
| 336 | + if (index == 0xFFFF) { | |
| 337 | + *pkey = 0xFFFF; | |
| 338 | + return 0; | |
| 339 | + } | |
| 340 | + if (index > pkey_tbl->max) { | |
| 341 | + dev_err(&res->pdev->dev, | |
| 342 | + "QPLIB: Index %d exceeded PKEY table max (%d)", | |
| 343 | + index, pkey_tbl->max); | |
| 344 | + return -EINVAL; | |
| 345 | + } | |
| 346 | + memcpy(pkey, &pkey_tbl->tbl[index], sizeof(*pkey)); | |
| 347 | + return 0; | |
| 348 | +} | |
| 349 | + | |
| 350 | +int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res, | |
| 351 | + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, | |
| 352 | + bool update) | |
| 353 | +{ | |
| 354 | + int i, rc = 0; | |
| 355 | + | |
| 356 | + if (!pkey_tbl) { | |
| 357 | + dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated"); | |
| 358 | + return -EINVAL; | |
| 359 | + } | |
| 360 | + | |
| 361 | + /* Do we need a pkey_lock here? */ | |
| 362 | + if (!pkey_tbl->active) { | |
| 363 | + dev_err(&res->pdev->dev, | |
| 364 | + "QPLIB: PKEY table has no active entries"); | |
| 365 | + return -ENOMEM; | |
| 366 | + } | |
| 367 | + for (i = 0; i < pkey_tbl->max; i++) { | |
| 368 | + if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey))) | |
| 369 | + break; | |
| 370 | + } | |
| 371 | + if (i == pkey_tbl->max) { | |
| 372 | + dev_err(&res->pdev->dev, | |
| 373 | + "QPLIB: PKEY 0x%04x not found in the pkey table", | |
| 374 | + *pkey); | |
| 375 | + return -ENOMEM; | |
| 376 | + } | |
| 377 | + memset(&pkey_tbl->tbl[i], 0, sizeof(*pkey)); | |
| 378 | + pkey_tbl->active--; | |
| 379 | + | |
| 380 | + /* unlock */ | |
| 381 | + return rc; | |
| 382 | +} | |
| 383 | + | |
| 384 | +int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res, | |
| 385 | + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, | |
| 386 | + bool update) | |
| 387 | +{ | |
| 388 | + int i, free_idx, rc = 0; | |
| 389 | + | |
| 390 | + if (!pkey_tbl) { | |
| 391 | + dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated"); | |
| 392 | + return -EINVAL; | |
| 393 | + } | |
| 394 | + | |
| 395 | + /* Do we need a pkey_lock here? */ | |
| 396 | + if (pkey_tbl->active == pkey_tbl->max) { | |
| 397 | + dev_err(&res->pdev->dev, "QPLIB: PKEY table is full"); | |
| 398 | + return -ENOMEM; | |
| 399 | + } | |
| 400 | + free_idx = pkey_tbl->max; | |
| 401 | + for (i = 0; i < pkey_tbl->max; i++) { | |
| 402 | + if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey))) | |
| 403 | + return -EALREADY; | |
| 404 | + else if (!pkey_tbl->tbl[i] && free_idx == pkey_tbl->max) | |
| 405 | + free_idx = i; | |
| 406 | + } | |
| 407 | + if (free_idx == pkey_tbl->max) { | |
| 408 | + dev_err(&res->pdev->dev, | |
| 409 | + "QPLIB: PKEY table is FULL but count is not MAX??"); | |
| 410 | + return -ENOMEM; | |
| 411 | + } | |
| 412 | + /* Add PKEY to the pkey_tbl */ | |
| 413 | + memcpy(&pkey_tbl->tbl[free_idx], pkey, sizeof(*pkey)); | |
| 414 | + pkey_tbl->active++; | |
| 415 | + | |
| 416 | + /* unlock */ | |
| 417 | + return rc; | |
| 418 | +} | |
| 419 | + | |
| 420 | +/* AH */ | |
| 421 | +int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah) | |
| 422 | +{ | |
| 423 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 424 | + struct cmdq_create_ah req; | |
| 425 | + struct creq_create_ah_resp *resp; | |
| 426 | + u16 cmd_flags = 0; | |
| 427 | + u32 temp32[4]; | |
| 428 | + u16 temp16[3]; | |
| 429 | + | |
| 430 | + RCFW_CMD_PREP(req, CREATE_AH, cmd_flags); | |
| 431 | + | |
| 432 | + memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid)); | |
| 433 | + req.dgid[0] = cpu_to_le32(temp32[0]); | |
| 434 | + req.dgid[1] = cpu_to_le32(temp32[1]); | |
| 435 | + req.dgid[2] = cpu_to_le32(temp32[2]); | |
| 436 | + req.dgid[3] = cpu_to_le32(temp32[3]); | |
| 437 | + | |
| 438 | + req.type = ah->nw_type; | |
| 439 | + req.hop_limit = ah->hop_limit; | |
| 440 | + req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]); | |
| 441 | + req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label & | |
| 442 | + CMDQ_CREATE_AH_FLOW_LABEL_MASK) | | |
| 443 | + CMDQ_CREATE_AH_DEST_VLAN_ID_MASK); | |
| 444 | + req.pd_id = cpu_to_le32(ah->pd->id); | |
| 445 | + req.traffic_class = ah->traffic_class; | |
| 446 | + | |
| 447 | + /* MAC in network format */ | |
| 448 | + memcpy(temp16, ah->dmac, 6); | |
| 449 | + req.dest_mac[0] = cpu_to_le16(temp16[0]); | |
| 450 | + req.dest_mac[1] = cpu_to_le16(temp16[1]); | |
| 451 | + req.dest_mac[2] = cpu_to_le16(temp16[2]); | |
| 452 | + | |
| 453 | + resp = (struct creq_create_ah_resp *) | |
| 454 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 455 | + NULL, 1); | |
| 456 | + if (!resp) { | |
| 457 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH send failed"); | |
| 458 | + return -EINVAL; | |
| 459 | + } | |
| 460 | + if (!bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 461 | + /* Cmd timed out */ | |
| 462 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH timed out"); | |
| 463 | + return -ETIMEDOUT; | |
| 464 | + } | |
| 465 | + if (resp->status || | |
| 466 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 467 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH failed "); | |
| 468 | + dev_err(&rcfw->pdev->dev, | |
| 469 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 470 | + resp->status, le16_to_cpu(req.cookie), | |
| 471 | + le16_to_cpu(resp->cookie)); | |
| 472 | + return -EINVAL; | |
| 473 | + } | |
| 474 | + ah->id = le32_to_cpu(resp->xid); | |
| 475 | + return 0; | |
| 476 | +} | |
| 477 | + | |
| 478 | +int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah) | |
| 479 | +{ | |
| 480 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 481 | + struct cmdq_destroy_ah req; | |
| 482 | + struct creq_destroy_ah_resp *resp; | |
| 483 | + u16 cmd_flags = 0; | |
| 484 | + | |
| 485 | + /* Clean up the AH table in the device */ | |
| 486 | + RCFW_CMD_PREP(req, DESTROY_AH, cmd_flags); | |
| 487 | + | |
| 488 | + req.ah_cid = cpu_to_le32(ah->id); | |
| 489 | + | |
| 490 | + resp = (struct creq_destroy_ah_resp *) | |
| 491 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 492 | + NULL, 1); | |
| 493 | + if (!resp) { | |
| 494 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH send failed"); | |
| 495 | + return -EINVAL; | |
| 496 | + } | |
| 497 | + if (!bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 498 | + /* Cmd timed out */ | |
| 499 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH timed out"); | |
| 500 | + return -ETIMEDOUT; | |
| 501 | + } | |
| 502 | + if (resp->status || | |
| 503 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 504 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH failed "); | |
| 505 | + dev_err(&rcfw->pdev->dev, | |
| 506 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 507 | + resp->status, le16_to_cpu(req.cookie), | |
| 508 | + le16_to_cpu(resp->cookie)); | |
| 509 | + return -EINVAL; | |
| 510 | + } | |
| 511 | + return 0; | |
| 512 | +} | |
| 513 | + | |
| 514 | +/* MRW */ | |
| 515 | +int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) | |
| 516 | +{ | |
| 517 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 518 | + struct cmdq_deallocate_key req; | |
| 519 | + struct creq_deallocate_key_resp *resp; | |
| 520 | + u16 cmd_flags = 0; | |
| 521 | + | |
| 522 | + if (mrw->lkey == 0xFFFFFFFF) { | |
| 523 | + dev_info(&res->pdev->dev, | |
| 524 | + "QPLIB: SP: Free a reserved lkey MRW"); | |
| 525 | + return 0; | |
| 526 | + } | |
| 527 | + | |
| 528 | + RCFW_CMD_PREP(req, DEALLOCATE_KEY, cmd_flags); | |
| 529 | + | |
| 530 | + req.mrw_flags = mrw->type; | |
| 531 | + | |
| 532 | + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || | |
| 533 | + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || | |
| 534 | + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) | |
| 535 | + req.key = cpu_to_le32(mrw->rkey); | |
| 536 | + else | |
| 537 | + req.key = cpu_to_le32(mrw->lkey); | |
| 538 | + | |
| 539 | + resp = (struct creq_deallocate_key_resp *) | |
| 540 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 541 | + NULL, 0); | |
| 542 | + if (!resp) { | |
| 543 | + dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR send failed"); | |
| 544 | + return -EINVAL; | |
| 545 | + } | |
| 546 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 547 | + /* Cmd timed out */ | |
| 548 | + dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR timed out"); | |
| 549 | + return -ETIMEDOUT; | |
| 550 | + } | |
| 551 | + if (resp->status || | |
| 552 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 553 | + dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR failed "); | |
| 554 | + dev_err(&res->pdev->dev, | |
| 555 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 556 | + resp->status, le16_to_cpu(req.cookie), | |
| 557 | + le16_to_cpu(resp->cookie)); | |
| 558 | + return -EINVAL; | |
| 559 | + } | |
| 560 | + /* Free the qplib's MRW memory */ | |
| 561 | + if (mrw->hwq.max_elements) | |
| 562 | + bnxt_qplib_free_hwq(res->pdev, &mrw->hwq); | |
| 563 | + | |
| 564 | + return 0; | |
| 565 | +} | |
| 566 | + | |
| 567 | +int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) | |
| 568 | +{ | |
| 569 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 570 | + struct cmdq_allocate_mrw req; | |
| 571 | + struct creq_allocate_mrw_resp *resp; | |
| 572 | + u16 cmd_flags = 0; | |
| 573 | + unsigned long tmp; | |
| 574 | + | |
| 575 | + RCFW_CMD_PREP(req, ALLOCATE_MRW, cmd_flags); | |
| 576 | + | |
| 577 | + req.pd_id = cpu_to_le32(mrw->pd->id); | |
| 578 | + req.mrw_flags = mrw->type; | |
| 579 | + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR && | |
| 580 | + mrw->flags & BNXT_QPLIB_FR_PMR) || | |
| 581 | + mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A || | |
| 582 | + mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B) | |
| 583 | + req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY; | |
| 584 | + tmp = (unsigned long)mrw; | |
| 585 | + req.mrw_handle = cpu_to_le64(tmp); | |
| 586 | + | |
| 587 | + resp = (struct creq_allocate_mrw_resp *) | |
| 588 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 589 | + NULL, 0); | |
| 590 | + if (!resp) { | |
| 591 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW send failed"); | |
| 592 | + return -EINVAL; | |
| 593 | + } | |
| 594 | + if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | |
| 595 | + /* Cmd timed out */ | |
| 596 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW timed out"); | |
| 597 | + return -ETIMEDOUT; | |
| 598 | + } | |
| 599 | + if (resp->status || | |
| 600 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 601 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW failed "); | |
| 602 | + dev_err(&rcfw->pdev->dev, | |
| 603 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 604 | + resp->status, le16_to_cpu(req.cookie), | |
| 605 | + le16_to_cpu(resp->cookie)); | |
| 606 | + return -EINVAL; | |
| 607 | + } | |
| 608 | + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || | |
| 609 | + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || | |
| 610 | + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) | |
| 611 | + mrw->rkey = le32_to_cpu(resp->xid); | |
| 612 | + else | |
| 613 | + mrw->lkey = le32_to_cpu(resp->xid); | |
| 614 | + return 0; | |
| 615 | +} | |
| 616 | + | |
| 617 | +int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, | |
| 618 | + bool block) | |
| 619 | +{ | |
| 620 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 621 | + struct cmdq_deregister_mr req; | |
| 622 | + struct creq_deregister_mr_resp *resp; | |
| 623 | + u16 cmd_flags = 0; | |
| 624 | + int rc; | |
| 625 | + | |
| 626 | + RCFW_CMD_PREP(req, DEREGISTER_MR, cmd_flags); | |
| 627 | + | |
| 628 | + req.lkey = cpu_to_le32(mrw->lkey); | |
| 629 | + resp = (struct creq_deregister_mr_resp *) | |
| 630 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 631 | + NULL, block); | |
| 632 | + if (!resp) { | |
| 633 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: DEREG_MR send failed"); | |
| 634 | + return -EINVAL; | |
| 635 | + } | |
| 636 | + if (block) | |
| 637 | + rc = bnxt_qplib_rcfw_block_for_resp(rcfw, | |
| 638 | + le16_to_cpu(req.cookie)); | |
| 639 | + else | |
| 640 | + rc = bnxt_qplib_rcfw_wait_for_resp(rcfw, | |
| 641 | + le16_to_cpu(req.cookie)); | |
| 642 | + if (!rc) { | |
| 643 | + /* Cmd timed out */ | |
| 644 | + dev_err(&res->pdev->dev, "QPLIB: SP: DEREG_MR timed out"); | |
| 645 | + return -ETIMEDOUT; | |
| 646 | + } | |
| 647 | + if (resp->status || | |
| 648 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 649 | + dev_err(&rcfw->pdev->dev, "QPLIB: SP: DEREG_MR failed "); | |
| 650 | + dev_err(&rcfw->pdev->dev, | |
| 651 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 652 | + resp->status, le16_to_cpu(req.cookie), | |
| 653 | + le16_to_cpu(resp->cookie)); | |
| 654 | + return -EINVAL; | |
| 655 | + } | |
| 656 | + | |
| 657 | + /* Free the qplib's MR memory */ | |
| 658 | + if (mrw->hwq.max_elements) { | |
| 659 | + mrw->va = 0; | |
| 660 | + mrw->total_size = 0; | |
| 661 | + bnxt_qplib_free_hwq(res->pdev, &mrw->hwq); | |
| 662 | + } | |
| 663 | + | |
| 664 | + return 0; | |
| 665 | +} | |
| 666 | + | |
| 667 | +int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, | |
| 668 | + u64 *pbl_tbl, int num_pbls, bool block) | |
| 669 | +{ | |
| 670 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 671 | + struct cmdq_register_mr req; | |
| 672 | + struct creq_register_mr_resp *resp; | |
| 673 | + u16 cmd_flags = 0, level; | |
| 674 | + int pg_ptrs, pages, i, rc; | |
| 675 | + dma_addr_t **pbl_ptr; | |
| 676 | + u32 pg_size; | |
| 677 | + | |
| 678 | + if (num_pbls) { | |
| 679 | + pg_ptrs = roundup_pow_of_two(num_pbls); | |
| 680 | + pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT; | |
| 681 | + if (!pages) | |
| 682 | + pages++; | |
| 683 | + | |
| 684 | + if (pages > MAX_PBL_LVL_1_PGS) { | |
| 685 | + dev_err(&res->pdev->dev, "QPLIB: SP: Reg MR pages "); | |
| 686 | + dev_err(&res->pdev->dev, | |
| 687 | + "requested (0x%x) exceeded max (0x%x)", | |
| 688 | + pages, MAX_PBL_LVL_1_PGS); | |
| 689 | + return -ENOMEM; | |
| 690 | + } | |
| 691 | + /* Free the hwq if it already exist, must be a rereg */ | |
| 692 | + if (mr->hwq.max_elements) | |
| 693 | + bnxt_qplib_free_hwq(res->pdev, &mr->hwq); | |
| 694 | + | |
| 695 | + mr->hwq.max_elements = pages; | |
| 696 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL, 0, | |
| 697 | + &mr->hwq.max_elements, | |
| 698 | + PAGE_SIZE, 0, PAGE_SIZE, | |
| 699 | + HWQ_TYPE_CTX); | |
| 700 | + if (rc) { | |
| 701 | + dev_err(&res->pdev->dev, | |
| 702 | + "SP: Reg MR memory allocation failed"); | |
| 703 | + return -ENOMEM; | |
| 704 | + } | |
| 705 | + /* Write to the hwq */ | |
| 706 | + pbl_ptr = (dma_addr_t **)mr->hwq.pbl_ptr; | |
| 707 | + for (i = 0; i < num_pbls; i++) | |
| 708 | + pbl_ptr[PTR_PG(i)][PTR_IDX(i)] = | |
| 709 | + (pbl_tbl[i] & PAGE_MASK) | PTU_PTE_VALID; | |
| 710 | + } | |
| 711 | + | |
| 712 | + RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags); | |
| 713 | + | |
| 714 | + /* Configure the request */ | |
| 715 | + if (mr->hwq.level == PBL_LVL_MAX) { | |
| 716 | + level = 0; | |
| 717 | + req.pbl = 0; | |
| 718 | + pg_size = PAGE_SIZE; | |
| 719 | + } else { | |
| 720 | + level = mr->hwq.level + 1; | |
| 721 | + req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); | |
| 722 | + pg_size = mr->hwq.pbl[PBL_LVL_0].pg_size; | |
| 723 | + } | |
| 724 | + req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) | | |
| 725 | + ((ilog2(pg_size) << | |
| 726 | + CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) & | |
| 727 | + CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK); | |
| 728 | + req.access = (mr->flags & 0xFFFF); | |
| 729 | + req.va = cpu_to_le64(mr->va); | |
| 730 | + req.key = cpu_to_le32(mr->lkey); | |
| 731 | + req.mr_size = cpu_to_le64(mr->total_size); | |
| 732 | + | |
| 733 | + resp = (struct creq_register_mr_resp *) | |
| 734 | + bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | |
| 735 | + NULL, block); | |
| 736 | + if (!resp) { | |
| 737 | + dev_err(&res->pdev->dev, "SP: REG_MR send failed"); | |
| 738 | + rc = -EINVAL; | |
| 739 | + goto fail; | |
| 740 | + } | |
| 741 | + if (block) | |
| 742 | + rc = bnxt_qplib_rcfw_block_for_resp(rcfw, | |
| 743 | + le16_to_cpu(req.cookie)); | |
| 744 | + else | |
| 745 | + rc = bnxt_qplib_rcfw_wait_for_resp(rcfw, | |
| 746 | + le16_to_cpu(req.cookie)); | |
| 747 | + if (!rc) { | |
| 748 | + /* Cmd timed out */ | |
| 749 | + dev_err(&res->pdev->dev, "SP: REG_MR timed out"); | |
| 750 | + rc = -ETIMEDOUT; | |
| 751 | + goto fail; | |
| 752 | + } | |
| 753 | + if (resp->status || | |
| 754 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 755 | + dev_err(&res->pdev->dev, "QPLIB: SP: REG_MR failed "); | |
| 756 | + dev_err(&res->pdev->dev, | |
| 757 | + "QPLIB: SP: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 758 | + resp->status, le16_to_cpu(req.cookie), | |
| 759 | + le16_to_cpu(resp->cookie)); | |
| 760 | + rc = -EINVAL; | |
| 761 | + goto fail; | |
| 762 | + } | |
| 763 | + return 0; | |
| 764 | + | |
| 765 | +fail: | |
| 766 | + if (mr->hwq.max_elements) | |
| 767 | + bnxt_qplib_free_hwq(res->pdev, &mr->hwq); | |
| 768 | + return rc; | |
| 769 | +} | |
| 770 | + | |
| 771 | +int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, | |
| 772 | + struct bnxt_qplib_frpl *frpl, | |
| 773 | + int max_pg_ptrs) | |
| 774 | +{ | |
| 775 | + int pg_ptrs, pages, rc; | |
| 776 | + | |
| 777 | + /* Re-calculate the max to fit the HWQ allocation model */ | |
| 778 | + pg_ptrs = roundup_pow_of_two(max_pg_ptrs); | |
| 779 | + pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT; | |
| 780 | + if (!pages) | |
| 781 | + pages++; | |
| 782 | + | |
| 783 | + if (pages > MAX_PBL_LVL_1_PGS) | |
| 784 | + return -ENOMEM; | |
| 785 | + | |
| 786 | + frpl->hwq.max_elements = pages; | |
| 787 | + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &frpl->hwq, NULL, 0, | |
| 788 | + &frpl->hwq.max_elements, PAGE_SIZE, 0, | |
| 789 | + PAGE_SIZE, HWQ_TYPE_CTX); | |
| 790 | + if (!rc) | |
| 791 | + frpl->max_pg_ptrs = pg_ptrs; | |
| 792 | + | |
| 793 | + return rc; | |
| 794 | +} | |
| 795 | + | |
| 796 | +int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, | |
| 797 | + struct bnxt_qplib_frpl *frpl) | |
| 798 | +{ | |
| 799 | + bnxt_qplib_free_hwq(res->pdev, &frpl->hwq); | |
| 800 | + return 0; | |
| 801 | +} | |
| 802 | + | |
| 803 | +int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids) | |
| 804 | +{ | |
| 805 | + struct bnxt_qplib_rcfw *rcfw = res->rcfw; | |
| 806 | + struct cmdq_map_tc_to_cos req; | |
| 807 | + struct creq_map_tc_to_cos_resp *resp; | |
| 808 | + u16 cmd_flags = 0; | |
| 809 | + int tleft; | |
| 810 | + | |
| 811 | + RCFW_CMD_PREP(req, MAP_TC_TO_COS, cmd_flags); | |
| 812 | + req.cos0 = cpu_to_le16(cids[0]); | |
| 813 | + req.cos1 = cpu_to_le16(cids[1]); | |
| 814 | + | |
| 815 | + resp = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, NULL, 0); | |
| 816 | + if (!resp) { | |
| 817 | + dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS send failed"); | |
| 818 | + return -EINVAL; | |
| 819 | + } | |
| 820 | + | |
| 821 | + tleft = bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie)); | |
| 822 | + if (!tleft) { | |
| 823 | + dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS timed out"); | |
| 824 | + return -ETIMEDOUT; | |
| 825 | + } | |
| 826 | + | |
| 827 | + if (resp->status || | |
| 828 | + le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | |
| 829 | + dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS failed "); | |
| 830 | + dev_err(&res->pdev->dev, | |
| 831 | + "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | |
| 832 | + resp->status, le16_to_cpu(req.cookie), | |
| 833 | + le16_to_cpu(resp->cookie)); | |
| 834 | + return -EINVAL; | |
| 835 | + } | |
| 836 | + | |
| 837 | + return 0; | |
| 838 | +} | |
| @@ -0,0 +1,160 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: Slow Path Operators (header) | |
| 37 | + * | |
| 38 | + */ | |
| 39 | + | |
| 40 | +#ifndef __BNXT_QPLIB_SP_H__ | |
| 41 | +#define __BNXT_QPLIB_SP_H__ | |
| 42 | + | |
| 43 | +struct bnxt_qplib_dev_attr { | |
| 44 | + char fw_ver[32]; | |
| 45 | + u16 max_sgid; | |
| 46 | + u16 max_mrw; | |
| 47 | + u32 max_qp; | |
| 48 | +#define BNXT_QPLIB_MAX_OUT_RD_ATOM 126 | |
| 49 | + u32 max_qp_rd_atom; | |
| 50 | + u32 max_qp_init_rd_atom; | |
| 51 | + u32 max_qp_wqes; | |
| 52 | + u32 max_qp_sges; | |
| 53 | + u32 max_cq; | |
| 54 | + u32 max_cq_wqes; | |
| 55 | + u32 max_cq_sges; | |
| 56 | + u32 max_mr; | |
| 57 | + u64 max_mr_size; | |
| 58 | + u32 max_pd; | |
| 59 | + u32 max_mw; | |
| 60 | + u32 max_raw_ethy_qp; | |
| 61 | + u32 max_ah; | |
| 62 | + u32 max_fmr; | |
| 63 | + u32 max_map_per_fmr; | |
| 64 | + u32 max_srq; | |
| 65 | + u32 max_srq_wqes; | |
| 66 | + u32 max_srq_sges; | |
| 67 | + u32 max_pkey; | |
| 68 | + u32 max_inline_data; | |
| 69 | + u32 l2_db_size; | |
| 70 | + u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; | |
| 71 | +}; | |
| 72 | + | |
| 73 | +struct bnxt_qplib_pd { | |
| 74 | + u32 id; | |
| 75 | +}; | |
| 76 | + | |
| 77 | +struct bnxt_qplib_gid { | |
| 78 | + u8 data[16]; | |
| 79 | +}; | |
| 80 | + | |
| 81 | +struct bnxt_qplib_ah { | |
| 82 | + struct bnxt_qplib_gid dgid; | |
| 83 | + struct bnxt_qplib_pd *pd; | |
| 84 | + u32 id; | |
| 85 | + u8 sgid_index; | |
| 86 | + /* For Query AH if the hw table and SW table are differnt */ | |
| 87 | + u8 host_sgid_index; | |
| 88 | + u8 traffic_class; | |
| 89 | + u32 flow_label; | |
| 90 | + u8 hop_limit; | |
| 91 | + u8 sl; | |
| 92 | + u8 dmac[6]; | |
| 93 | + u16 vlan_id; | |
| 94 | + u8 nw_type; | |
| 95 | +}; | |
| 96 | + | |
| 97 | +struct bnxt_qplib_mrw { | |
| 98 | + struct bnxt_qplib_pd *pd; | |
| 99 | + int type; | |
| 100 | + u32 flags; | |
| 101 | +#define BNXT_QPLIB_FR_PMR 0x80000000 | |
| 102 | + u32 lkey; | |
| 103 | + u32 rkey; | |
| 104 | +#define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF | |
| 105 | + u64 va; | |
| 106 | + u64 total_size; | |
| 107 | + u32 npages; | |
| 108 | + u64 mr_handle; | |
| 109 | + struct bnxt_qplib_hwq hwq; | |
| 110 | +}; | |
| 111 | + | |
| 112 | +struct bnxt_qplib_frpl { | |
| 113 | + int max_pg_ptrs; | |
| 114 | + struct bnxt_qplib_hwq hwq; | |
| 115 | +}; | |
| 116 | + | |
| 117 | +#define BNXT_QPLIB_ACCESS_LOCAL_WRITE BIT(0) | |
| 118 | +#define BNXT_QPLIB_ACCESS_REMOTE_READ BIT(1) | |
| 119 | +#define BNXT_QPLIB_ACCESS_REMOTE_WRITE BIT(2) | |
| 120 | +#define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC BIT(3) | |
| 121 | +#define BNXT_QPLIB_ACCESS_MW_BIND BIT(4) | |
| 122 | +#define BNXT_QPLIB_ACCESS_ZERO_BASED BIT(5) | |
| 123 | +#define BNXT_QPLIB_ACCESS_ON_DEMAND BIT(6) | |
| 124 | + | |
| 125 | +int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, | |
| 126 | + struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, | |
| 127 | + struct bnxt_qplib_gid *gid); | |
| 128 | +int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, | |
| 129 | + struct bnxt_qplib_gid *gid, bool update); | |
| 130 | +int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, | |
| 131 | + struct bnxt_qplib_gid *gid, u8 *mac, u16 vlan_id, | |
| 132 | + bool update, u32 *index); | |
| 133 | +int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res, | |
| 134 | + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index, | |
| 135 | + u16 *pkey); | |
| 136 | +int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res, | |
| 137 | + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, | |
| 138 | + bool update); | |
| 139 | +int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res, | |
| 140 | + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, | |
| 141 | + bool update); | |
| 142 | +int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, | |
| 143 | + struct bnxt_qplib_dev_attr *attr); | |
| 144 | +int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah); | |
| 145 | +int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah); | |
| 146 | +int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, | |
| 147 | + struct bnxt_qplib_mrw *mrw); | |
| 148 | +int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, | |
| 149 | + bool block); | |
| 150 | +int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, | |
| 151 | + u64 *pbl_tbl, int num_pbls, bool block); | |
| 152 | +int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr); | |
| 153 | +int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res, | |
| 154 | + struct bnxt_qplib_mrw *mr, int max); | |
| 155 | +int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, | |
| 156 | + struct bnxt_qplib_frpl *frpl, int max); | |
| 157 | +int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, | |
| 158 | + struct bnxt_qplib_frpl *frpl); | |
| 159 | +int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids); | |
| 160 | +#endif /* __BNXT_QPLIB_SP_H__*/ | |
| @@ -0,0 +1,2821 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: RoCE HSI File - Autogenerated | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#ifndef __BNXT_RE_HSI_H__ | |
| 40 | +#define __BNXT_RE_HSI_H__ | |
| 41 | + | |
| 42 | +/* include bnxt_hsi.h from bnxt_en driver */ | |
| 43 | +#include "bnxt_hsi.h" | |
| 44 | + | |
| 45 | +/* CMP Door Bell Format (4 bytes) */ | |
| 46 | +struct cmpl_doorbell { | |
| 47 | + __le32 key_mask_valid_idx; | |
| 48 | + #define CMPL_DOORBELL_IDX_MASK 0xffffffUL | |
| 49 | + #define CMPL_DOORBELL_IDX_SFT 0 | |
| 50 | + #define CMPL_DOORBELL_RESERVED_MASK 0x3000000UL | |
| 51 | + #define CMPL_DOORBELL_RESERVED_SFT 24 | |
| 52 | + #define CMPL_DOORBELL_IDX_VALID 0x4000000UL | |
| 53 | + #define CMPL_DOORBELL_MASK 0x8000000UL | |
| 54 | + #define CMPL_DOORBELL_KEY_MASK 0xf0000000UL | |
| 55 | + #define CMPL_DOORBELL_KEY_SFT 28 | |
| 56 | + #define CMPL_DOORBELL_KEY_CMPL (0x2UL << 28) | |
| 57 | +}; | |
| 58 | + | |
| 59 | +/* Status Door Bell Format (4 bytes) */ | |
| 60 | +struct status_doorbell { | |
| 61 | + __le32 key_idx; | |
| 62 | + #define STATUS_DOORBELL_IDX_MASK 0xffffffUL | |
| 63 | + #define STATUS_DOORBELL_IDX_SFT 0 | |
| 64 | + #define STATUS_DOORBELL_RESERVED_MASK 0xf000000UL | |
| 65 | + #define STATUS_DOORBELL_RESERVED_SFT 24 | |
| 66 | + #define STATUS_DOORBELL_KEY_MASK 0xf0000000UL | |
| 67 | + #define STATUS_DOORBELL_KEY_SFT 28 | |
| 68 | + #define STATUS_DOORBELL_KEY_STAT (0x3UL << 28) | |
| 69 | +}; | |
| 70 | + | |
| 71 | +/* RoCE Host Structures */ | |
| 72 | + | |
| 73 | +/* Doorbell Structures */ | |
| 74 | +/* 64b Doorbell Format (8 bytes) */ | |
| 75 | +struct dbr_dbr { | |
| 76 | + __le32 index; | |
| 77 | + #define DBR_DBR_INDEX_MASK 0xfffffUL | |
| 78 | + #define DBR_DBR_INDEX_SFT 0 | |
| 79 | + #define DBR_DBR_RESERVED12_MASK 0xfff00000UL | |
| 80 | + #define DBR_DBR_RESERVED12_SFT 20 | |
| 81 | + __le32 type_xid; | |
| 82 | + #define DBR_DBR_XID_MASK 0xfffffUL | |
| 83 | + #define DBR_DBR_XID_SFT 0 | |
| 84 | + #define DBR_DBR_RESERVED8_MASK 0xff00000UL | |
| 85 | + #define DBR_DBR_RESERVED8_SFT 20 | |
| 86 | + #define DBR_DBR_TYPE_MASK 0xf0000000UL | |
| 87 | + #define DBR_DBR_TYPE_SFT 28 | |
| 88 | + #define DBR_DBR_TYPE_SQ (0x0UL << 28) | |
| 89 | + #define DBR_DBR_TYPE_RQ (0x1UL << 28) | |
| 90 | + #define DBR_DBR_TYPE_SRQ (0x2UL << 28) | |
| 91 | + #define DBR_DBR_TYPE_SRQ_ARM (0x3UL << 28) | |
| 92 | + #define DBR_DBR_TYPE_CQ (0x4UL << 28) | |
| 93 | + #define DBR_DBR_TYPE_CQ_ARMSE (0x5UL << 28) | |
| 94 | + #define DBR_DBR_TYPE_CQ_ARMALL (0x6UL << 28) | |
| 95 | + #define DBR_DBR_TYPE_CQ_ARMENA (0x7UL << 28) | |
| 96 | + #define DBR_DBR_TYPE_SRQ_ARMENA (0x8UL << 28) | |
| 97 | + #define DBR_DBR_TYPE_CQ_CUTOFF_ACK (0x9UL << 28) | |
| 98 | + #define DBR_DBR_TYPE_NULL (0xfUL << 28) | |
| 99 | +}; | |
| 100 | + | |
| 101 | +/* 32b Doorbell Format (4 bytes) */ | |
| 102 | +struct dbr_dbr32 { | |
| 103 | + __le32 type_abs_incr_xid; | |
| 104 | + #define DBR_DBR32_XID_MASK 0xfffffUL | |
| 105 | + #define DBR_DBR32_XID_SFT 0 | |
| 106 | + #define DBR_DBR32_RESERVED4_MASK 0xf00000UL | |
| 107 | + #define DBR_DBR32_RESERVED4_SFT 20 | |
| 108 | + #define DBR_DBR32_INCR_MASK 0xf000000UL | |
| 109 | + #define DBR_DBR32_INCR_SFT 24 | |
| 110 | + #define DBR_DBR32_ABS 0x10000000UL | |
| 111 | + #define DBR_DBR32_TYPE_MASK 0xe0000000UL | |
| 112 | + #define DBR_DBR32_TYPE_SFT 29 | |
| 113 | + #define DBR_DBR32_TYPE_SQ (0x0UL << 29) | |
| 114 | +}; | |
| 115 | + | |
| 116 | +/* SQ WQE Structures */ | |
| 117 | +/* Base SQ WQE (8 bytes) */ | |
| 118 | +struct sq_base { | |
| 119 | + u8 wqe_type; | |
| 120 | + #define SQ_BASE_WQE_TYPE_SEND 0x0UL | |
| 121 | + #define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD 0x1UL | |
| 122 | + #define SQ_BASE_WQE_TYPE_SEND_W_INVALID 0x2UL | |
| 123 | + #define SQ_BASE_WQE_TYPE_WRITE_WQE 0x4UL | |
| 124 | + #define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD 0x5UL | |
| 125 | + #define SQ_BASE_WQE_TYPE_READ_WQE 0x6UL | |
| 126 | + #define SQ_BASE_WQE_TYPE_ATOMIC_CS 0x8UL | |
| 127 | + #define SQ_BASE_WQE_TYPE_ATOMIC_FA 0xbUL | |
| 128 | + #define SQ_BASE_WQE_TYPE_LOCAL_INVALID 0xcUL | |
| 129 | + #define SQ_BASE_WQE_TYPE_FR_PMR 0xdUL | |
| 130 | + #define SQ_BASE_WQE_TYPE_BIND 0xeUL | |
| 131 | + u8 unused_0[7]; | |
| 132 | +}; | |
| 133 | + | |
| 134 | +/* WQE SGE (16 bytes) */ | |
| 135 | +struct sq_sge { | |
| 136 | + __le64 va_or_pa; | |
| 137 | + __le32 l_key; | |
| 138 | + __le32 size; | |
| 139 | +}; | |
| 140 | + | |
| 141 | +/* PSN Search Structure (8 bytes) */ | |
| 142 | +struct sq_psn_search { | |
| 143 | + __le32 opcode_start_psn; | |
| 144 | + #define SQ_PSN_SEARCH_START_PSN_MASK 0xffffffUL | |
| 145 | + #define SQ_PSN_SEARCH_START_PSN_SFT 0 | |
| 146 | + #define SQ_PSN_SEARCH_OPCODE_MASK 0xff000000UL | |
| 147 | + #define SQ_PSN_SEARCH_OPCODE_SFT 24 | |
| 148 | + __le32 flags_next_psn; | |
| 149 | + #define SQ_PSN_SEARCH_NEXT_PSN_MASK 0xffffffUL | |
| 150 | + #define SQ_PSN_SEARCH_NEXT_PSN_SFT 0 | |
| 151 | + #define SQ_PSN_SEARCH_FLAGS_MASK 0xff000000UL | |
| 152 | + #define SQ_PSN_SEARCH_FLAGS_SFT 24 | |
| 153 | +}; | |
| 154 | + | |
| 155 | +/* Send SQ WQE (40 bytes) */ | |
| 156 | +struct sq_send { | |
| 157 | + u8 wqe_type; | |
| 158 | + #define SQ_SEND_WQE_TYPE_SEND 0x0UL | |
| 159 | + #define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD 0x1UL | |
| 160 | + #define SQ_SEND_WQE_TYPE_SEND_W_INVALID 0x2UL | |
| 161 | + u8 flags; | |
| 162 | + #define SQ_SEND_FLAGS_SIGNAL_COMP 0x1UL | |
| 163 | + #define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | |
| 164 | + #define SQ_SEND_FLAGS_UC_FENCE 0x4UL | |
| 165 | + #define SQ_SEND_FLAGS_SE 0x8UL | |
| 166 | + #define SQ_SEND_FLAGS_INLINE 0x10UL | |
| 167 | + u8 wqe_size; | |
| 168 | + u8 reserved8_1; | |
| 169 | + __le32 inv_key_or_imm_data; | |
| 170 | + __le32 length; | |
| 171 | + __le32 q_key; | |
| 172 | + __le32 dst_qp; | |
| 173 | + #define SQ_SEND_DST_QP_MASK 0xffffffUL | |
| 174 | + #define SQ_SEND_DST_QP_SFT 0 | |
| 175 | + #define SQ_SEND_RESERVED8_2_MASK 0xff000000UL | |
| 176 | + #define SQ_SEND_RESERVED8_2_SFT 24 | |
| 177 | + __le32 avid; | |
| 178 | + #define SQ_SEND_AVID_MASK 0xfffffUL | |
| 179 | + #define SQ_SEND_AVID_SFT 0 | |
| 180 | + #define SQ_SEND_RESERVED_AVID_MASK 0xfff00000UL | |
| 181 | + #define SQ_SEND_RESERVED_AVID_SFT 20 | |
| 182 | + __le64 reserved64; | |
| 183 | + __le32 data[24]; | |
| 184 | +}; | |
| 185 | + | |
| 186 | +/* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */ | |
| 187 | +struct sq_send_raweth_qp1 { | |
| 188 | + u8 wqe_type; | |
| 189 | + #define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND 0x0UL | |
| 190 | + u8 flags; | |
| 191 | + #define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP 0x1UL | |
| 192 | + #define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | |
| 193 | + #define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE 0x4UL | |
| 194 | + #define SQ_SEND_RAWETH_QP1_FLAGS_SE 0x8UL | |
| 195 | + #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE 0x10UL | |
| 196 | + u8 wqe_size; | |
| 197 | + u8 reserved8; | |
| 198 | + __le16 lflags; | |
| 199 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM 0x1UL | |
| 200 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM 0x2UL | |
| 201 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC 0x4UL | |
| 202 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP 0x8UL | |
| 203 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM 0x10UL | |
| 204 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_1 0x20UL | |
| 205 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_2 0x40UL | |
| 206 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_3 0x80UL | |
| 207 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC 0x100UL | |
| 208 | + #define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC 0x200UL | |
| 209 | + __le16 cfa_action; | |
| 210 | + __le32 length; | |
| 211 | + __le32 reserved32_1; | |
| 212 | + __le32 cfa_meta; | |
| 213 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK 0xfffUL | |
| 214 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT 0 | |
| 215 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE 0x1000UL | |
| 216 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK 0xe000UL | |
| 217 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT 13 | |
| 218 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK 0x70000UL | |
| 219 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT 16 | |
| 220 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 (0x0UL << 16) | |
| 221 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 (0x1UL << 16) | |
| 222 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 (0x2UL << 16) | |
| 223 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 (0x3UL << 16) | |
| 224 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 (0x4UL << 16) | |
| 225 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG (0x5UL << 16) | |
| 226 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST \ | |
| 227 | + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG | |
| 228 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK 0xff80000UL | |
| 229 | + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19 | |
| 230 | + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK 0xf0000000UL | |
| 231 | + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT 28 | |
| 232 | + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE (0x0UL << 28) | |
| 233 | + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG (0x1UL << 28) | |
| 234 | + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST \ | |
| 235 | + SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG | |
| 236 | + __le32 reserved32_2; | |
| 237 | + __le64 reserved64; | |
| 238 | + __le32 data[24]; | |
| 239 | +}; | |
| 240 | + | |
| 241 | +/* RDMA SQ WQE (40 bytes) */ | |
| 242 | +struct sq_rdma { | |
| 243 | + u8 wqe_type; | |
| 244 | + #define SQ_RDMA_WQE_TYPE_WRITE_WQE 0x4UL | |
| 245 | + #define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD 0x5UL | |
| 246 | + #define SQ_RDMA_WQE_TYPE_READ_WQE 0x6UL | |
| 247 | + u8 flags; | |
| 248 | + #define SQ_RDMA_FLAGS_SIGNAL_COMP 0x1UL | |
| 249 | + #define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | |
| 250 | + #define SQ_RDMA_FLAGS_UC_FENCE 0x4UL | |
| 251 | + #define SQ_RDMA_FLAGS_SE 0x8UL | |
| 252 | + #define SQ_RDMA_FLAGS_INLINE 0x10UL | |
| 253 | + u8 wqe_size; | |
| 254 | + u8 reserved8; | |
| 255 | + __le32 imm_data; | |
| 256 | + __le32 length; | |
| 257 | + __le32 reserved32_1; | |
| 258 | + __le64 remote_va; | |
| 259 | + __le32 remote_key; | |
| 260 | + __le32 reserved32_2; | |
| 261 | + __le32 data[24]; | |
| 262 | +}; | |
| 263 | + | |
| 264 | +/* Atomic SQ WQE (40 bytes) */ | |
| 265 | +struct sq_atomic { | |
| 266 | + u8 wqe_type; | |
| 267 | + #define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS 0x8UL | |
| 268 | + #define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA 0xbUL | |
| 269 | + u8 flags; | |
| 270 | + #define SQ_ATOMIC_FLAGS_SIGNAL_COMP 0x1UL | |
| 271 | + #define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | |
| 272 | + #define SQ_ATOMIC_FLAGS_UC_FENCE 0x4UL | |
| 273 | + #define SQ_ATOMIC_FLAGS_SE 0x8UL | |
| 274 | + #define SQ_ATOMIC_FLAGS_INLINE 0x10UL | |
| 275 | + __le16 reserved16; | |
| 276 | + __le32 remote_key; | |
| 277 | + __le64 remote_va; | |
| 278 | + __le64 swap_data; | |
| 279 | + __le64 cmp_data; | |
| 280 | + __le32 data[24]; | |
| 281 | +}; | |
| 282 | + | |
| 283 | +/* Local Invalidate SQ WQE (40 bytes) */ | |
| 284 | +struct sq_localinvalidate { | |
| 285 | + u8 wqe_type; | |
| 286 | + #define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID 0xcUL | |
| 287 | + u8 flags; | |
| 288 | + #define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP 0x1UL | |
| 289 | + #define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | |
| 290 | + #define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE 0x4UL | |
| 291 | + #define SQ_LOCALINVALIDATE_FLAGS_SE 0x8UL | |
| 292 | + #define SQ_LOCALINVALIDATE_FLAGS_INLINE 0x10UL | |
| 293 | + __le16 reserved16; | |
| 294 | + __le32 inv_l_key; | |
| 295 | + __le64 reserved64; | |
| 296 | + __le32 reserved128[4]; | |
| 297 | + __le32 data[24]; | |
| 298 | +}; | |
| 299 | + | |
| 300 | +/* FR-PMR SQ WQE (40 bytes) */ | |
| 301 | +struct sq_fr_pmr { | |
| 302 | + u8 wqe_type; | |
| 303 | + #define SQ_FR_PMR_WQE_TYPE_FR_PMR 0xdUL | |
| 304 | + u8 flags; | |
| 305 | + #define SQ_FR_PMR_FLAGS_SIGNAL_COMP 0x1UL | |
| 306 | + #define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | |
| 307 | + #define SQ_FR_PMR_FLAGS_UC_FENCE 0x4UL | |
| 308 | + #define SQ_FR_PMR_FLAGS_SE 0x8UL | |
| 309 | + #define SQ_FR_PMR_FLAGS_INLINE 0x10UL | |
| 310 | + u8 access_cntl; | |
| 311 | + #define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE 0x1UL | |
| 312 | + #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ 0x2UL | |
| 313 | + #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE 0x4UL | |
| 314 | + #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL | |
| 315 | + #define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND 0x10UL | |
| 316 | + u8 zero_based_page_size_log; | |
| 317 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_MASK 0x1fUL | |
| 318 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_SFT 0 | |
| 319 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K 0x0UL | |
| 320 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K 0x1UL | |
| 321 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K 0x4UL | |
| 322 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K 0x6UL | |
| 323 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M 0x8UL | |
| 324 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M 0x9UL | |
| 325 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M 0xaUL | |
| 326 | + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G 0x12UL | |
| 327 | + #define SQ_FR_PMR_ZERO_BASED 0x20UL | |
| 328 | + #define SQ_FR_PMR_RESERVED2_MASK 0xc0UL | |
| 329 | + #define SQ_FR_PMR_RESERVED2_SFT 6 | |
| 330 | + __le32 l_key; | |
| 331 | + u8 length[5]; | |
| 332 | + u8 reserved8_1; | |
| 333 | + u8 reserved8_2; | |
| 334 | + u8 numlevels_pbl_page_size_log; | |
| 335 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK 0x1fUL | |
| 336 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT 0 | |
| 337 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K 0x0UL | |
| 338 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K 0x1UL | |
| 339 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K 0x4UL | |
| 340 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K 0x6UL | |
| 341 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M 0x8UL | |
| 342 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M 0x9UL | |
| 343 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M 0xaUL | |
| 344 | + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G 0x12UL | |
| 345 | + #define SQ_FR_PMR_RESERVED1 0x20UL | |
| 346 | + #define SQ_FR_PMR_NUMLEVELS_MASK 0xc0UL | |
| 347 | + #define SQ_FR_PMR_NUMLEVELS_SFT 6 | |
| 348 | + #define SQ_FR_PMR_NUMLEVELS_PHYSICAL (0x0UL << 6) | |
| 349 | + #define SQ_FR_PMR_NUMLEVELS_LAYER1 (0x1UL << 6) | |
| 350 | + #define SQ_FR_PMR_NUMLEVELS_LAYER2 (0x2UL << 6) | |
| 351 | + __le64 pblptr; | |
| 352 | + __le64 va; | |
| 353 | + __le32 data[24]; | |
| 354 | +}; | |
| 355 | + | |
| 356 | +/* Bind SQ WQE (40 bytes) */ | |
| 357 | +struct sq_bind { | |
| 358 | + u8 wqe_type; | |
| 359 | + #define SQ_BIND_WQE_TYPE_BIND 0xeUL | |
| 360 | + u8 flags; | |
| 361 | + #define SQ_BIND_FLAGS_SIGNAL_COMP 0x1UL | |
| 362 | + #define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | |
| 363 | + #define SQ_BIND_FLAGS_UC_FENCE 0x4UL | |
| 364 | + #define SQ_BIND_FLAGS_SE 0x8UL | |
| 365 | + #define SQ_BIND_FLAGS_INLINE 0x10UL | |
| 366 | + u8 access_cntl; | |
| 367 | + #define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE 0x1UL | |
| 368 | + #define SQ_BIND_ACCESS_CNTL_REMOTE_READ 0x2UL | |
| 369 | + #define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE 0x4UL | |
| 370 | + #define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL | |
| 371 | + #define SQ_BIND_ACCESS_CNTL_WINDOW_BIND 0x10UL | |
| 372 | + u8 reserved8_1; | |
| 373 | + u8 mw_type_zero_based; | |
| 374 | + #define SQ_BIND_ZERO_BASED 0x1UL | |
| 375 | + #define SQ_BIND_MW_TYPE 0x2UL | |
| 376 | + #define SQ_BIND_MW_TYPE_TYPE1 (0x0UL << 1) | |
| 377 | + #define SQ_BIND_MW_TYPE_TYPE2 (0x1UL << 1) | |
| 378 | + #define SQ_BIND_RESERVED6_MASK 0xfcUL | |
| 379 | + #define SQ_BIND_RESERVED6_SFT 2 | |
| 380 | + u8 reserved8_2; | |
| 381 | + __le16 reserved16; | |
| 382 | + __le32 parent_l_key; | |
| 383 | + __le32 l_key; | |
| 384 | + __le64 va; | |
| 385 | + u8 length[5]; | |
| 386 | + u8 data_reserved24[99]; | |
| 387 | + #define SQ_BIND_RESERVED24_MASK 0xffffff00UL | |
| 388 | + #define SQ_BIND_RESERVED24_SFT 8 | |
| 389 | + #define SQ_BIND_DATA_MASK 0xffffffffUL | |
| 390 | + #define SQ_BIND_DATA_SFT 0 | |
| 391 | +}; | |
| 392 | + | |
| 393 | +/* RQ/SRQ WQE Structures */ | |
| 394 | +/* RQ/SRQ WQE (40 bytes) */ | |
| 395 | +struct rq_wqe { | |
| 396 | + u8 wqe_type; | |
| 397 | + #define RQ_WQE_WQE_TYPE_RCV 0x80UL | |
| 398 | + u8 flags; | |
| 399 | + u8 wqe_size; | |
| 400 | + u8 reserved8; | |
| 401 | + __le32 reserved32; | |
| 402 | + __le32 wr_id[2]; | |
| 403 | + #define RQ_WQE_WR_ID_MASK 0xfffffUL | |
| 404 | + #define RQ_WQE_WR_ID_SFT 0 | |
| 405 | + #define RQ_WQE_RESERVED44_MASK 0xfff00000UL | |
| 406 | + #define RQ_WQE_RESERVED44_SFT 20 | |
| 407 | + __le32 reserved128[4]; | |
| 408 | + __le32 data[24]; | |
| 409 | +}; | |
| 410 | + | |
| 411 | +/* CQ CQE Structures */ | |
| 412 | +/* Base CQE (32 bytes) */ | |
| 413 | +struct cq_base { | |
| 414 | + __le64 reserved64_1; | |
| 415 | + __le64 reserved64_2; | |
| 416 | + __le64 reserved64_3; | |
| 417 | + u8 cqe_type_toggle; | |
| 418 | + #define CQ_BASE_TOGGLE 0x1UL | |
| 419 | + #define CQ_BASE_CQE_TYPE_MASK 0x1eUL | |
| 420 | + #define CQ_BASE_CQE_TYPE_SFT 1 | |
| 421 | + #define CQ_BASE_CQE_TYPE_REQ (0x0UL << 1) | |
| 422 | + #define CQ_BASE_CQE_TYPE_RES_RC (0x1UL << 1) | |
| 423 | + #define CQ_BASE_CQE_TYPE_RES_UD (0x2UL << 1) | |
| 424 | + #define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1) | |
| 425 | + #define CQ_BASE_CQE_TYPE_TERMINAL (0xeUL << 1) | |
| 426 | + #define CQ_BASE_CQE_TYPE_CUT_OFF (0xfUL << 1) | |
| 427 | + #define CQ_BASE_RESERVED3_MASK 0xe0UL | |
| 428 | + #define CQ_BASE_RESERVED3_SFT 5 | |
| 429 | + u8 status; | |
| 430 | + __le16 reserved16; | |
| 431 | + __le32 reserved32; | |
| 432 | +}; | |
| 433 | + | |
| 434 | +/* Requester CQ CQE (32 bytes) */ | |
| 435 | +struct cq_req { | |
| 436 | + __le64 qp_handle; | |
| 437 | + __le16 sq_cons_idx; | |
| 438 | + __le16 reserved16_1; | |
| 439 | + __le32 reserved32_2; | |
| 440 | + __le64 reserved64; | |
| 441 | + u8 cqe_type_toggle; | |
| 442 | + #define CQ_REQ_TOGGLE 0x1UL | |
| 443 | + #define CQ_REQ_CQE_TYPE_MASK 0x1eUL | |
| 444 | + #define CQ_REQ_CQE_TYPE_SFT 1 | |
| 445 | + #define CQ_REQ_CQE_TYPE_REQ (0x0UL << 1) | |
| 446 | + #define CQ_REQ_RESERVED3_MASK 0xe0UL | |
| 447 | + #define CQ_REQ_RESERVED3_SFT 5 | |
| 448 | + u8 status; | |
| 449 | + #define CQ_REQ_STATUS_OK 0x0UL | |
| 450 | + #define CQ_REQ_STATUS_BAD_RESPONSE_ERR 0x1UL | |
| 451 | + #define CQ_REQ_STATUS_LOCAL_LENGTH_ERR 0x2UL | |
| 452 | + #define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR 0x3UL | |
| 453 | + #define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR 0x4UL | |
| 454 | + #define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL | |
| 455 | + #define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL | |
| 456 | + #define CQ_REQ_STATUS_REMOTE_ACCESS_ERR 0x7UL | |
| 457 | + #define CQ_REQ_STATUS_REMOTE_OPERATION_ERR 0x8UL | |
| 458 | + #define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR 0x9UL | |
| 459 | + #define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR 0xaUL | |
| 460 | + #define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR 0xbUL | |
| 461 | + __le16 reserved16_2; | |
| 462 | + __le32 reserved32_1; | |
| 463 | +}; | |
| 464 | + | |
| 465 | +/* Responder RC CQE (32 bytes) */ | |
| 466 | +struct cq_res_rc { | |
| 467 | + __le32 length; | |
| 468 | + __le32 imm_data_or_inv_r_key; | |
| 469 | + __le64 qp_handle; | |
| 470 | + __le64 mr_handle; | |
| 471 | + u8 cqe_type_toggle; | |
| 472 | + #define CQ_RES_RC_TOGGLE 0x1UL | |
| 473 | + #define CQ_RES_RC_CQE_TYPE_MASK 0x1eUL | |
| 474 | + #define CQ_RES_RC_CQE_TYPE_SFT 1 | |
| 475 | + #define CQ_RES_RC_CQE_TYPE_RES_RC (0x1UL << 1) | |
| 476 | + #define CQ_RES_RC_RESERVED3_MASK 0xe0UL | |
| 477 | + #define CQ_RES_RC_RESERVED3_SFT 5 | |
| 478 | + u8 status; | |
| 479 | + #define CQ_RES_RC_STATUS_OK 0x0UL | |
| 480 | + #define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR 0x1UL | |
| 481 | + #define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR 0x2UL | |
| 482 | + #define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR 0x3UL | |
| 483 | + #define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL | |
| 484 | + #define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL | |
| 485 | + #define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL | |
| 486 | + #define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL | |
| 487 | + #define CQ_RES_RC_STATUS_HW_FLUSH_ERR 0x8UL | |
| 488 | + __le16 flags; | |
| 489 | + #define CQ_RES_RC_FLAGS_SRQ 0x1UL | |
| 490 | + #define CQ_RES_RC_FLAGS_SRQ_RQ (0x0UL << 0) | |
| 491 | + #define CQ_RES_RC_FLAGS_SRQ_SRQ (0x1UL << 0) | |
| 492 | + #define CQ_RES_RC_FLAGS_SRQ_LAST CQ_RES_RC_FLAGS_SRQ_SRQ | |
| 493 | + #define CQ_RES_RC_FLAGS_IMM 0x2UL | |
| 494 | + #define CQ_RES_RC_FLAGS_INV 0x4UL | |
| 495 | + #define CQ_RES_RC_FLAGS_RDMA 0x8UL | |
| 496 | + #define CQ_RES_RC_FLAGS_RDMA_SEND (0x0UL << 3) | |
| 497 | + #define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE (0x1UL << 3) | |
| 498 | + #define CQ_RES_RC_FLAGS_RDMA_LAST CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE | |
| 499 | + __le32 srq_or_rq_wr_id; | |
| 500 | + #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL | |
| 501 | + #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0 | |
| 502 | + #define CQ_RES_RC_RESERVED12_MASK 0xfff00000UL | |
| 503 | + #define CQ_RES_RC_RESERVED12_SFT 20 | |
| 504 | +}; | |
| 505 | + | |
| 506 | +/* Responder UD CQE (32 bytes) */ | |
| 507 | +struct cq_res_ud { | |
| 508 | + __le32 length; | |
| 509 | + #define CQ_RES_UD_LENGTH_MASK 0x3fffUL | |
| 510 | + #define CQ_RES_UD_LENGTH_SFT 0 | |
| 511 | + #define CQ_RES_UD_RESERVED18_MASK 0xffffc000UL | |
| 512 | + #define CQ_RES_UD_RESERVED18_SFT 14 | |
| 513 | + __le32 imm_data; | |
| 514 | + __le64 qp_handle; | |
| 515 | + __le16 src_mac[3]; | |
| 516 | + __le16 src_qp_low; | |
| 517 | + u8 cqe_type_toggle; | |
| 518 | + #define CQ_RES_UD_TOGGLE 0x1UL | |
| 519 | + #define CQ_RES_UD_CQE_TYPE_MASK 0x1eUL | |
| 520 | + #define CQ_RES_UD_CQE_TYPE_SFT 1 | |
| 521 | + #define CQ_RES_UD_CQE_TYPE_RES_UD (0x2UL << 1) | |
| 522 | + #define CQ_RES_UD_RESERVED3_MASK 0xe0UL | |
| 523 | + #define CQ_RES_UD_RESERVED3_SFT 5 | |
| 524 | + u8 status; | |
| 525 | + #define CQ_RES_UD_STATUS_OK 0x0UL | |
| 526 | + #define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR 0x1UL | |
| 527 | + #define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL | |
| 528 | + #define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR 0x3UL | |
| 529 | + #define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL | |
| 530 | + #define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL | |
| 531 | + #define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL | |
| 532 | + #define CQ_RES_UD_STATUS_HW_FLUSH_ERR 0x8UL | |
| 533 | + __le16 flags; | |
| 534 | + #define CQ_RES_UD_FLAGS_SRQ 0x1UL | |
| 535 | + #define CQ_RES_UD_FLAGS_SRQ_RQ (0x0UL << 0) | |
| 536 | + #define CQ_RES_UD_FLAGS_SRQ_SRQ (0x1UL << 0) | |
| 537 | + #define CQ_RES_UD_FLAGS_SRQ_LAST CQ_RES_UD_FLAGS_SRQ_SRQ | |
| 538 | + #define CQ_RES_UD_FLAGS_IMM 0x2UL | |
| 539 | + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK 0xcUL | |
| 540 | + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT 2 | |
| 541 | + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 (0x0UL << 2) | |
| 542 | + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 2) | |
| 543 | + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 2) | |
| 544 | + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST \ | |
| 545 | + CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 | |
| 546 | + __le32 src_qp_high_srq_or_rq_wr_id; | |
| 547 | + #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL | |
| 548 | + #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0 | |
| 549 | + #define CQ_RES_UD_RESERVED4_MASK 0xf00000UL | |
| 550 | + #define CQ_RES_UD_RESERVED4_SFT 20 | |
| 551 | + #define CQ_RES_UD_SRC_QP_HIGH_MASK 0xff000000UL | |
| 552 | + #define CQ_RES_UD_SRC_QP_HIGH_SFT 24 | |
| 553 | +}; | |
| 554 | + | |
| 555 | +/* Responder RawEth and QP1 CQE (32 bytes) */ | |
| 556 | +struct cq_res_raweth_qp1 { | |
| 557 | + __le16 length; | |
| 558 | + #define CQ_RES_RAWETH_QP1_LENGTH_MASK 0x3fffUL | |
| 559 | + #define CQ_RES_RAWETH_QP1_LENGTH_SFT 0 | |
| 560 | + #define CQ_RES_RAWETH_QP1_RESERVED2_MASK 0xc000UL | |
| 561 | + #define CQ_RES_RAWETH_QP1_RESERVED2_SFT 14 | |
| 562 | + __le16 raweth_qp1_flags; | |
| 563 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR 0x1UL | |
| 564 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_MASK 0x3eUL | |
| 565 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_SFT 1 | |
| 566 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK 0x3c0UL | |
| 567 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT 6 | |
| 568 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL << 6) | |
| 569 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP (0x1UL << 6) | |
| 570 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP (0x2UL << 6) | |
| 571 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP (0x3UL << 6) | |
| 572 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE (0x4UL << 6) | |
| 573 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE (0x5UL << 6) | |
| 574 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP (0x7UL << 6) | |
| 575 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP \ | |
| 576 | + (0x8UL << 6) | |
| 577 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP \ | |
| 578 | + (0x9UL << 6) | |
| 579 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST \ | |
| 580 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP | |
| 581 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK 0x3ffUL | |
| 582 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT 0 | |
| 583 | + #define CQ_RES_RAWETH_QP1_RESERVED6_MASK 0xfc00UL | |
| 584 | + #define CQ_RES_RAWETH_QP1_RESERVED6_SFT 10 | |
| 585 | + __le16 raweth_qp1_errors; | |
| 586 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_MASK 0xfUL | |
| 587 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_SFT 0 | |
| 588 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR 0x10UL | |
| 589 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR 0x20UL | |
| 590 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL | |
| 591 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL | |
| 592 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR 0x100UL | |
| 593 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL | |
| 594 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9 | |
| 595 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR \ | |
| 596 | + (0x0UL << 9) | |
| 597 | + #define \ | |
| 598 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \ | |
| 599 | + (0x1UL << 9) | |
| 600 | + #define \ | |
| 601 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \ | |
| 602 | + (0x2UL << 9) | |
| 603 | + #define \ | |
| 604 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \ | |
| 605 | + (0x3UL << 9) | |
| 606 | + #define \ | |
| 607 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \ | |
| 608 | + (0x4UL << 9) | |
| 609 | + #define \ | |
| 610 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \ | |
| 611 | + (0x5UL << 9) | |
| 612 | + #define \ | |
| 613 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \ | |
| 614 | + (0x6UL << 9) | |
| 615 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \ | |
| 616 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL | |
| 617 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL | |
| 618 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12 | |
| 619 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR \ | |
| 620 | + (0x0UL << 12) | |
| 621 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION \ | |
| 622 | + (0x1UL << 12) | |
| 623 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \ | |
| 624 | + (0x2UL << 12) | |
| 625 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL \ | |
| 626 | + (0x3UL << 12) | |
| 627 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \ | |
| 628 | + (0x4UL << 12) | |
| 629 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \ | |
| 630 | + (0x5UL << 12) | |
| 631 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \ | |
| 632 | + (0x6UL << 12) | |
| 633 | + #define \ | |
| 634 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL\ | |
| 635 | + (0x7UL << 12) | |
| 636 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \ | |
| 637 | + (0x8UL << 12) | |
| 638 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \ | |
| 639 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN | |
| 640 | + __le16 raweth_qp1_cfa_code; | |
| 641 | + __le64 qp_handle; | |
| 642 | + __le32 raweth_qp1_flags2; | |
| 643 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC 0x1UL | |
| 644 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC 0x2UL | |
| 645 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC 0x4UL | |
| 646 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC 0x8UL | |
| 647 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL | |
| 648 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4 | |
| 649 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE \ | |
| 650 | + (0x0UL << 4) | |
| 651 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN \ | |
| 652 | + (0x1UL << 4) | |
| 653 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST\ | |
| 654 | + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN | |
| 655 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE 0x100UL | |
| 656 | + __le32 raweth_qp1_metadata; | |
| 657 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK 0xfffUL | |
| 658 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT 0 | |
| 659 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE 0x1000UL | |
| 660 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK 0xe000UL | |
| 661 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT 13 | |
| 662 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK 0xffff0000UL | |
| 663 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT 16 | |
| 664 | + u8 cqe_type_toggle; | |
| 665 | + #define CQ_RES_RAWETH_QP1_TOGGLE 0x1UL | |
| 666 | + #define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK 0x1eUL | |
| 667 | + #define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT 1 | |
| 668 | + #define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1) | |
| 669 | + #define CQ_RES_RAWETH_QP1_RESERVED3_MASK 0xe0UL | |
| 670 | + #define CQ_RES_RAWETH_QP1_RESERVED3_SFT 5 | |
| 671 | + u8 status; | |
| 672 | + #define CQ_RES_RAWETH_QP1_STATUS_OK 0x0UL | |
| 673 | + #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR 0x1UL | |
| 674 | + #define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL | |
| 675 | + #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR 0x3UL | |
| 676 | + #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL | |
| 677 | + #define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL | |
| 678 | + #define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL | |
| 679 | + #define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR 0x8UL | |
| 680 | + __le16 flags; | |
| 681 | + #define CQ_RES_RAWETH_QP1_FLAGS_SRQ 0x1UL | |
| 682 | + #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ 0x0UL | |
| 683 | + #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ 0x1UL | |
| 684 | + #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST \ | |
| 685 | + CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ | |
| 686 | + __le32 raweth_qp1_payload_offset_srq_or_rq_wr_id; | |
| 687 | + #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL | |
| 688 | + #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT 0 | |
| 689 | + #define CQ_RES_RAWETH_QP1_RESERVED4_MASK 0xf00000UL | |
| 690 | + #define CQ_RES_RAWETH_QP1_RESERVED4_SFT 20 | |
| 691 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL | |
| 692 | + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24 | |
| 693 | +}; | |
| 694 | + | |
| 695 | +/* Terminal CQE (32 bytes) */ | |
| 696 | +struct cq_terminal { | |
| 697 | + __le64 qp_handle; | |
| 698 | + __le16 sq_cons_idx; | |
| 699 | + __le16 rq_cons_idx; | |
| 700 | + __le32 reserved32_1; | |
| 701 | + __le64 reserved64_3; | |
| 702 | + u8 cqe_type_toggle; | |
| 703 | + #define CQ_TERMINAL_TOGGLE 0x1UL | |
| 704 | + #define CQ_TERMINAL_CQE_TYPE_MASK 0x1eUL | |
| 705 | + #define CQ_TERMINAL_CQE_TYPE_SFT 1 | |
| 706 | + #define CQ_TERMINAL_CQE_TYPE_TERMINAL (0xeUL << 1) | |
| 707 | + #define CQ_TERMINAL_RESERVED3_MASK 0xe0UL | |
| 708 | + #define CQ_TERMINAL_RESERVED3_SFT 5 | |
| 709 | + u8 status; | |
| 710 | + #define CQ_TERMINAL_STATUS_OK 0x0UL | |
| 711 | + __le16 reserved16; | |
| 712 | + __le32 reserved32_2; | |
| 713 | +}; | |
| 714 | + | |
| 715 | +/* Cutoff CQE (32 bytes) */ | |
| 716 | +struct cq_cutoff { | |
| 717 | + __le64 reserved64_1; | |
| 718 | + __le64 reserved64_2; | |
| 719 | + __le64 reserved64_3; | |
| 720 | + u8 cqe_type_toggle; | |
| 721 | + #define CQ_CUTOFF_TOGGLE 0x1UL | |
| 722 | + #define CQ_CUTOFF_CQE_TYPE_MASK 0x1eUL | |
| 723 | + #define CQ_CUTOFF_CQE_TYPE_SFT 1 | |
| 724 | + #define CQ_CUTOFF_CQE_TYPE_CUT_OFF (0xfUL << 1) | |
| 725 | + #define CQ_CUTOFF_RESERVED3_MASK 0xe0UL | |
| 726 | + #define CQ_CUTOFF_RESERVED3_SFT 5 | |
| 727 | + u8 status; | |
| 728 | + #define CQ_CUTOFF_STATUS_OK 0x0UL | |
| 729 | + __le16 reserved16; | |
| 730 | + __le32 reserved32; | |
| 731 | +}; | |
| 732 | + | |
| 733 | +/* Notification Queue (NQ) Structures */ | |
| 734 | +/* Base NQ Record (16 bytes) */ | |
| 735 | +struct nq_base { | |
| 736 | + __le16 info10_type; | |
| 737 | + #define NQ_BASE_TYPE_MASK 0x3fUL | |
| 738 | + #define NQ_BASE_TYPE_SFT 0 | |
| 739 | + #define NQ_BASE_TYPE_CQ_NOTIFICATION 0x30UL | |
| 740 | + #define NQ_BASE_TYPE_SRQ_EVENT 0x32UL | |
| 741 | + #define NQ_BASE_TYPE_DBQ_EVENT 0x34UL | |
| 742 | + #define NQ_BASE_TYPE_QP_EVENT 0x38UL | |
| 743 | + #define NQ_BASE_TYPE_FUNC_EVENT 0x3aUL | |
| 744 | + #define NQ_BASE_INFO10_MASK 0xffc0UL | |
| 745 | + #define NQ_BASE_INFO10_SFT 6 | |
| 746 | + __le16 info16; | |
| 747 | + __le32 info32; | |
| 748 | + __le32 info63_v[2]; | |
| 749 | + #define NQ_BASE_V 0x1UL | |
| 750 | + #define NQ_BASE_INFO63_MASK 0xfffffffeUL | |
| 751 | + #define NQ_BASE_INFO63_SFT 1 | |
| 752 | +}; | |
| 753 | + | |
| 754 | +/* Completion Queue Notification (16 bytes) */ | |
| 755 | +struct nq_cn { | |
| 756 | + __le16 type; | |
| 757 | + #define NQ_CN_TYPE_MASK 0x3fUL | |
| 758 | + #define NQ_CN_TYPE_SFT 0 | |
| 759 | + #define NQ_CN_TYPE_CQ_NOTIFICATION 0x30UL | |
| 760 | + #define NQ_CN_RESERVED9_MASK 0xffc0UL | |
| 761 | + #define NQ_CN_RESERVED9_SFT 6 | |
| 762 | + __le16 reserved16; | |
| 763 | + __le32 cq_handle_low; | |
| 764 | + __le32 v; | |
| 765 | + #define NQ_CN_V 0x1UL | |
| 766 | + #define NQ_CN_RESERVED31_MASK 0xfffffffeUL | |
| 767 | + #define NQ_CN_RESERVED31_SFT 1 | |
| 768 | + __le32 cq_handle_high; | |
| 769 | +}; | |
| 770 | + | |
| 771 | +/* SRQ Event Notification (16 bytes) */ | |
| 772 | +struct nq_srq_event { | |
| 773 | + u8 type; | |
| 774 | + #define NQ_SRQ_EVENT_TYPE_MASK 0x3fUL | |
| 775 | + #define NQ_SRQ_EVENT_TYPE_SFT 0 | |
| 776 | + #define NQ_SRQ_EVENT_TYPE_SRQ_EVENT 0x32UL | |
| 777 | + #define NQ_SRQ_EVENT_RESERVED1_MASK 0xc0UL | |
| 778 | + #define NQ_SRQ_EVENT_RESERVED1_SFT 6 | |
| 779 | + u8 event; | |
| 780 | + #define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT 0x1UL | |
| 781 | + __le16 reserved16; | |
| 782 | + __le32 srq_handle_low; | |
| 783 | + __le32 v; | |
| 784 | + #define NQ_SRQ_EVENT_V 0x1UL | |
| 785 | + #define NQ_SRQ_EVENT_RESERVED31_MASK 0xfffffffeUL | |
| 786 | + #define NQ_SRQ_EVENT_RESERVED31_SFT 1 | |
| 787 | + __le32 srq_handle_high; | |
| 788 | +}; | |
| 789 | + | |
| 790 | +/* DBQ Async Event Notification (16 bytes) */ | |
| 791 | +struct nq_dbq_event { | |
| 792 | + u8 type; | |
| 793 | + #define NQ_DBQ_EVENT_TYPE_MASK 0x3fUL | |
| 794 | + #define NQ_DBQ_EVENT_TYPE_SFT 0 | |
| 795 | + #define NQ_DBQ_EVENT_TYPE_DBQ_EVENT 0x34UL | |
| 796 | + #define NQ_DBQ_EVENT_RESERVED1_MASK 0xc0UL | |
| 797 | + #define NQ_DBQ_EVENT_RESERVED1_SFT 6 | |
| 798 | + u8 event; | |
| 799 | + #define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT 0x1UL | |
| 800 | + __le16 db_pfid; | |
| 801 | + #define NQ_DBQ_EVENT_DB_PFID_MASK 0xfUL | |
| 802 | + #define NQ_DBQ_EVENT_DB_PFID_SFT 0 | |
| 803 | + #define NQ_DBQ_EVENT_RESERVED12_MASK 0xfff0UL | |
| 804 | + #define NQ_DBQ_EVENT_RESERVED12_SFT 4 | |
| 805 | + __le32 db_dpi; | |
| 806 | + #define NQ_DBQ_EVENT_DB_DPI_MASK 0xfffffUL | |
| 807 | + #define NQ_DBQ_EVENT_DB_DPI_SFT 0 | |
| 808 | + #define NQ_DBQ_EVENT_RESERVED12_2_MASK 0xfff00000UL | |
| 809 | + #define NQ_DBQ_EVENT_RESERVED12_2_SFT 20 | |
| 810 | + __le32 v; | |
| 811 | + #define NQ_DBQ_EVENT_V 0x1UL | |
| 812 | + #define NQ_DBQ_EVENT_RESERVED32_MASK 0xfffffffeUL | |
| 813 | + #define NQ_DBQ_EVENT_RESERVED32_SFT 1 | |
| 814 | + __le32 db_type_db_xid; | |
| 815 | + #define NQ_DBQ_EVENT_DB_XID_MASK 0xfffffUL | |
| 816 | + #define NQ_DBQ_EVENT_DB_XID_SFT 0 | |
| 817 | + #define NQ_DBQ_EVENT_RESERVED8_MASK 0xff00000UL | |
| 818 | + #define NQ_DBQ_EVENT_RESERVED8_SFT 20 | |
| 819 | + #define NQ_DBQ_EVENT_DB_TYPE_MASK 0xf0000000UL | |
| 820 | + #define NQ_DBQ_EVENT_DB_TYPE_SFT 28 | |
| 821 | +}; | |
| 822 | + | |
| 823 | +/* Read Request/Response Queue Structures */ | |
| 824 | +/* Input Read Request Queue (IRRQ) Message (32 bytes) */ | |
| 825 | +struct xrrq_irrq { | |
| 826 | + __le16 credits_type; | |
| 827 | + #define XRRQ_IRRQ_TYPE 0x1UL | |
| 828 | + #define XRRQ_IRRQ_TYPE_READ_REQ 0x0UL | |
| 829 | + #define XRRQ_IRRQ_TYPE_ATOMIC_REQ 0x1UL | |
| 830 | + #define XRRQ_IRRQ_RESERVED10_MASK 0x7feUL | |
| 831 | + #define XRRQ_IRRQ_RESERVED10_SFT 1 | |
| 832 | + #define XRRQ_IRRQ_CREDITS_MASK 0xf800UL | |
| 833 | + #define XRRQ_IRRQ_CREDITS_SFT 11 | |
| 834 | + __le16 reserved16; | |
| 835 | + __le32 reserved32; | |
| 836 | + __le32 psn; | |
| 837 | + #define XRRQ_IRRQ_PSN_MASK 0xffffffUL | |
| 838 | + #define XRRQ_IRRQ_PSN_SFT 0 | |
| 839 | + #define XRRQ_IRRQ_RESERVED8_1_MASK 0xff000000UL | |
| 840 | + #define XRRQ_IRRQ_RESERVED8_1_SFT 24 | |
| 841 | + __le32 msn; | |
| 842 | + #define XRRQ_IRRQ_MSN_MASK 0xffffffUL | |
| 843 | + #define XRRQ_IRRQ_MSN_SFT 0 | |
| 844 | + #define XRRQ_IRRQ_RESERVED8_2_MASK 0xff000000UL | |
| 845 | + #define XRRQ_IRRQ_RESERVED8_2_SFT 24 | |
| 846 | + __le64 va_or_atomic_result; | |
| 847 | + __le32 rdma_r_key; | |
| 848 | + __le32 length; | |
| 849 | +}; | |
| 850 | + | |
| 851 | +/* Output Read Request Queue (ORRQ) Message (32 bytes) */ | |
| 852 | +struct xrrq_orrq { | |
| 853 | + __le16 num_sges_type; | |
| 854 | + #define XRRQ_ORRQ_TYPE 0x1UL | |
| 855 | + #define XRRQ_ORRQ_TYPE_READ_REQ 0x0UL | |
| 856 | + #define XRRQ_ORRQ_TYPE_ATOMIC_REQ 0x1UL | |
| 857 | + #define XRRQ_ORRQ_RESERVED10_MASK 0x7feUL | |
| 858 | + #define XRRQ_ORRQ_RESERVED10_SFT 1 | |
| 859 | + #define XRRQ_ORRQ_NUM_SGES_MASK 0xf800UL | |
| 860 | + #define XRRQ_ORRQ_NUM_SGES_SFT 11 | |
| 861 | + __le16 reserved16; | |
| 862 | + __le32 length; | |
| 863 | + __le32 psn; | |
| 864 | + #define XRRQ_ORRQ_PSN_MASK 0xffffffUL | |
| 865 | + #define XRRQ_ORRQ_PSN_SFT 0 | |
| 866 | + #define XRRQ_ORRQ_RESERVED8_1_MASK 0xff000000UL | |
| 867 | + #define XRRQ_ORRQ_RESERVED8_1_SFT 24 | |
| 868 | + __le32 end_psn; | |
| 869 | + #define XRRQ_ORRQ_END_PSN_MASK 0xffffffUL | |
| 870 | + #define XRRQ_ORRQ_END_PSN_SFT 0 | |
| 871 | + #define XRRQ_ORRQ_RESERVED8_2_MASK 0xff000000UL | |
| 872 | + #define XRRQ_ORRQ_RESERVED8_2_SFT 24 | |
| 873 | + __le64 first_sge_phy_or_sing_sge_va; | |
| 874 | + __le32 single_sge_l_key; | |
| 875 | + __le32 single_sge_size; | |
| 876 | +}; | |
| 877 | + | |
| 878 | +/* Page Buffer List Memory Structures (PBL) */ | |
| 879 | +/* Page Table Entry (PTE) (8 bytes) */ | |
| 880 | +struct ptu_pte { | |
| 881 | + __le32 page_next_to_last_last_valid[2]; | |
| 882 | + #define PTU_PTE_VALID 0x1UL | |
| 883 | + #define PTU_PTE_LAST 0x2UL | |
| 884 | + #define PTU_PTE_NEXT_TO_LAST 0x4UL | |
| 885 | + #define PTU_PTE_PAGE_MASK 0xfffff000UL | |
| 886 | + #define PTU_PTE_PAGE_SFT 12 | |
| 887 | +}; | |
| 888 | + | |
| 889 | +/* Page Directory Entry (PDE) (8 bytes) */ | |
| 890 | +struct ptu_pde { | |
| 891 | + __le32 page_valid[2]; | |
| 892 | + #define PTU_PDE_VALID 0x1UL | |
| 893 | + #define PTU_PDE_PAGE_MASK 0xfffff000UL | |
| 894 | + #define PTU_PDE_PAGE_SFT 12 | |
| 895 | +}; | |
| 896 | + | |
| 897 | +/* RoCE Fastpath Host Structures */ | |
| 898 | +/* Command Queue (CMDQ) Interface */ | |
| 899 | +/* Init CMDQ (16 bytes) */ | |
| 900 | +struct cmdq_init { | |
| 901 | + __le64 cmdq_pbl; | |
| 902 | + __le16 cmdq_size_cmdq_lvl; | |
| 903 | + #define CMDQ_INIT_CMDQ_LVL_MASK 0x3UL | |
| 904 | + #define CMDQ_INIT_CMDQ_LVL_SFT 0 | |
| 905 | + #define CMDQ_INIT_CMDQ_SIZE_MASK 0xfffcUL | |
| 906 | + #define CMDQ_INIT_CMDQ_SIZE_SFT 2 | |
| 907 | + __le16 creq_ring_id; | |
| 908 | + __le32 prod_idx; | |
| 909 | +}; | |
| 910 | + | |
| 911 | +/* Update CMDQ producer index (16 bytes) */ | |
| 912 | +struct cmdq_update { | |
| 913 | + __le64 reserved64; | |
| 914 | + __le32 reserved32; | |
| 915 | + __le32 prod_idx; | |
| 916 | +}; | |
| 917 | + | |
| 918 | +/* CMDQ common header structure (16 bytes) */ | |
| 919 | +struct cmdq_base { | |
| 920 | + u8 opcode; | |
| 921 | + #define CMDQ_BASE_OPCODE_CREATE_QP 0x1UL | |
| 922 | + #define CMDQ_BASE_OPCODE_DESTROY_QP 0x2UL | |
| 923 | + #define CMDQ_BASE_OPCODE_MODIFY_QP 0x3UL | |
| 924 | + #define CMDQ_BASE_OPCODE_QUERY_QP 0x4UL | |
| 925 | + #define CMDQ_BASE_OPCODE_CREATE_SRQ 0x5UL | |
| 926 | + #define CMDQ_BASE_OPCODE_DESTROY_SRQ 0x6UL | |
| 927 | + #define CMDQ_BASE_OPCODE_QUERY_SRQ 0x8UL | |
| 928 | + #define CMDQ_BASE_OPCODE_CREATE_CQ 0x9UL | |
| 929 | + #define CMDQ_BASE_OPCODE_DESTROY_CQ 0xaUL | |
| 930 | + #define CMDQ_BASE_OPCODE_RESIZE_CQ 0xcUL | |
| 931 | + #define CMDQ_BASE_OPCODE_ALLOCATE_MRW 0xdUL | |
| 932 | + #define CMDQ_BASE_OPCODE_DEALLOCATE_KEY 0xeUL | |
| 933 | + #define CMDQ_BASE_OPCODE_REGISTER_MR 0xfUL | |
| 934 | + #define CMDQ_BASE_OPCODE_DEREGISTER_MR 0x10UL | |
| 935 | + #define CMDQ_BASE_OPCODE_ADD_GID 0x11UL | |
| 936 | + #define CMDQ_BASE_OPCODE_DELETE_GID 0x12UL | |
| 937 | + #define CMDQ_BASE_OPCODE_MODIFY_GID 0x17UL | |
| 938 | + #define CMDQ_BASE_OPCODE_QUERY_GID 0x18UL | |
| 939 | + #define CMDQ_BASE_OPCODE_CREATE_QP1 0x13UL | |
| 940 | + #define CMDQ_BASE_OPCODE_DESTROY_QP1 0x14UL | |
| 941 | + #define CMDQ_BASE_OPCODE_CREATE_AH 0x15UL | |
| 942 | + #define CMDQ_BASE_OPCODE_DESTROY_AH 0x16UL | |
| 943 | + #define CMDQ_BASE_OPCODE_INITIALIZE_FW 0x80UL | |
| 944 | + #define CMDQ_BASE_OPCODE_DEINITIALIZE_FW 0x81UL | |
| 945 | + #define CMDQ_BASE_OPCODE_STOP_FUNC 0x82UL | |
| 946 | + #define CMDQ_BASE_OPCODE_QUERY_FUNC 0x83UL | |
| 947 | + #define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES 0x84UL | |
| 948 | + #define CMDQ_BASE_OPCODE_READ_CONTEXT 0x85UL | |
| 949 | + #define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST 0x86UL | |
| 950 | + #define CMDQ_BASE_OPCODE_READ_VF_MEMORY 0x87UL | |
| 951 | + #define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST 0x88UL | |
| 952 | + #define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY 0x89UL | |
| 953 | + #define CMDQ_BASE_OPCODE_MAP_TC_TO_COS 0x8aUL | |
| 954 | + #define CMDQ_BASE_OPCODE_QUERY_VERSION 0x8bUL | |
| 955 | + #define CMDQ_BASE_OPCODE_MODIFY_CC 0x8cUL | |
| 956 | + #define CMDQ_BASE_OPCODE_QUERY_CC 0x8dUL | |
| 957 | + u8 cmd_size; | |
| 958 | + __le16 flags; | |
| 959 | + __le16 cookie; | |
| 960 | + u8 resp_size; | |
| 961 | + u8 reserved8; | |
| 962 | + __le64 resp_addr; | |
| 963 | +}; | |
| 964 | + | |
| 965 | +/* Create QP command (96 bytes) */ | |
| 966 | +struct cmdq_create_qp { | |
| 967 | + u8 opcode; | |
| 968 | + #define CMDQ_CREATE_QP_OPCODE_CREATE_QP 0x1UL | |
| 969 | + u8 cmd_size; | |
| 970 | + __le16 flags; | |
| 971 | + __le16 cookie; | |
| 972 | + u8 resp_size; | |
| 973 | + u8 reserved8; | |
| 974 | + __le64 resp_addr; | |
| 975 | + __le64 qp_handle; | |
| 976 | + __le32 qp_flags; | |
| 977 | + #define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED 0x1UL | |
| 978 | + #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL | |
| 979 | + #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL | |
| 980 | + #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL | |
| 981 | + u8 type; | |
| 982 | + #define CMDQ_CREATE_QP_TYPE_RC 0x2UL | |
| 983 | + #define CMDQ_CREATE_QP_TYPE_UD 0x4UL | |
| 984 | + #define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE 0x6UL | |
| 985 | + u8 sq_pg_size_sq_lvl; | |
| 986 | + #define CMDQ_CREATE_QP_SQ_LVL_MASK 0xfUL | |
| 987 | + #define CMDQ_CREATE_QP_SQ_LVL_SFT 0 | |
| 988 | + #define CMDQ_CREATE_QP_SQ_LVL_LVL_0 0x0UL | |
| 989 | + #define CMDQ_CREATE_QP_SQ_LVL_LVL_1 0x1UL | |
| 990 | + #define CMDQ_CREATE_QP_SQ_LVL_LVL_2 0x2UL | |
| 991 | + #define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK 0xf0UL | |
| 992 | + #define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT 4 | |
| 993 | + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K (0x0UL << 4) | |
| 994 | + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K (0x1UL << 4) | |
| 995 | + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K (0x2UL << 4) | |
| 996 | + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M (0x3UL << 4) | |
| 997 | + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M (0x4UL << 4) | |
| 998 | + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G (0x5UL << 4) | |
| 999 | + u8 rq_pg_size_rq_lvl; | |
| 1000 | + #define CMDQ_CREATE_QP_RQ_LVL_MASK 0xfUL | |
| 1001 | + #define CMDQ_CREATE_QP_RQ_LVL_SFT 0 | |
| 1002 | + #define CMDQ_CREATE_QP_RQ_LVL_LVL_0 0x0UL | |
| 1003 | + #define CMDQ_CREATE_QP_RQ_LVL_LVL_1 0x1UL | |
| 1004 | + #define CMDQ_CREATE_QP_RQ_LVL_LVL_2 0x2UL | |
| 1005 | + #define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK 0xf0UL | |
| 1006 | + #define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT 4 | |
| 1007 | + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1008 | + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1009 | + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1010 | + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1011 | + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1012 | + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1013 | + u8 unused_0; | |
| 1014 | + __le32 dpi; | |
| 1015 | + __le32 sq_size; | |
| 1016 | + __le32 rq_size; | |
| 1017 | + __le16 sq_fwo_sq_sge; | |
| 1018 | + #define CMDQ_CREATE_QP_SQ_SGE_MASK 0xfUL | |
| 1019 | + #define CMDQ_CREATE_QP_SQ_SGE_SFT 0 | |
| 1020 | + #define CMDQ_CREATE_QP_SQ_FWO_MASK 0xfff0UL | |
| 1021 | + #define CMDQ_CREATE_QP_SQ_FWO_SFT 4 | |
| 1022 | + __le16 rq_fwo_rq_sge; | |
| 1023 | + #define CMDQ_CREATE_QP_RQ_SGE_MASK 0xfUL | |
| 1024 | + #define CMDQ_CREATE_QP_RQ_SGE_SFT 0 | |
| 1025 | + #define CMDQ_CREATE_QP_RQ_FWO_MASK 0xfff0UL | |
| 1026 | + #define CMDQ_CREATE_QP_RQ_FWO_SFT 4 | |
| 1027 | + __le32 scq_cid; | |
| 1028 | + __le32 rcq_cid; | |
| 1029 | + __le32 srq_cid; | |
| 1030 | + __le32 pd_id; | |
| 1031 | + __le64 sq_pbl; | |
| 1032 | + __le64 rq_pbl; | |
| 1033 | + __le64 irrq_addr; | |
| 1034 | + __le64 orrq_addr; | |
| 1035 | +}; | |
| 1036 | + | |
| 1037 | +/* Destroy QP command (24 bytes) */ | |
| 1038 | +struct cmdq_destroy_qp { | |
| 1039 | + u8 opcode; | |
| 1040 | + #define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP 0x2UL | |
| 1041 | + u8 cmd_size; | |
| 1042 | + __le16 flags; | |
| 1043 | + __le16 cookie; | |
| 1044 | + u8 resp_size; | |
| 1045 | + u8 reserved8; | |
| 1046 | + __le64 resp_addr; | |
| 1047 | + __le32 qp_cid; | |
| 1048 | + __le32 unused_0; | |
| 1049 | +}; | |
| 1050 | + | |
| 1051 | +/* Modify QP command (112 bytes) */ | |
| 1052 | +struct cmdq_modify_qp { | |
| 1053 | + u8 opcode; | |
| 1054 | + #define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP 0x3UL | |
| 1055 | + u8 cmd_size; | |
| 1056 | + __le16 flags; | |
| 1057 | + __le16 cookie; | |
| 1058 | + u8 resp_size; | |
| 1059 | + u8 reserved8; | |
| 1060 | + __le64 resp_addr; | |
| 1061 | + __le32 modify_mask; | |
| 1062 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_STATE 0x1UL | |
| 1063 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY 0x2UL | |
| 1064 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS 0x4UL | |
| 1065 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY 0x8UL | |
| 1066 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY 0x10UL | |
| 1067 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_DGID 0x20UL | |
| 1068 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL 0x40UL | |
| 1069 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX 0x80UL | |
| 1070 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT 0x100UL | |
| 1071 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS 0x200UL | |
| 1072 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC 0x400UL | |
| 1073 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU 0x1000UL | |
| 1074 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT 0x2000UL | |
| 1075 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT 0x4000UL | |
| 1076 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY 0x8000UL | |
| 1077 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN 0x10000UL | |
| 1078 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC 0x20000UL | |
| 1079 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER 0x40000UL | |
| 1080 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN 0x80000UL | |
| 1081 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC 0x100000UL | |
| 1082 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE 0x200000UL | |
| 1083 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE 0x400000UL | |
| 1084 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE 0x800000UL | |
| 1085 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE 0x1000000UL | |
| 1086 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA 0x2000000UL | |
| 1087 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID 0x4000000UL | |
| 1088 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC 0x8000000UL | |
| 1089 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID 0x10000000UL | |
| 1090 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC 0x20000000UL | |
| 1091 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN 0x40000000UL | |
| 1092 | + #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP 0x80000000UL | |
| 1093 | + __le32 qp_cid; | |
| 1094 | + u8 network_type_en_sqd_async_notify_new_state; | |
| 1095 | + #define CMDQ_MODIFY_QP_NEW_STATE_MASK 0xfUL | |
| 1096 | + #define CMDQ_MODIFY_QP_NEW_STATE_SFT 0 | |
| 1097 | + #define CMDQ_MODIFY_QP_NEW_STATE_RESET 0x0UL | |
| 1098 | + #define CMDQ_MODIFY_QP_NEW_STATE_INIT 0x1UL | |
| 1099 | + #define CMDQ_MODIFY_QP_NEW_STATE_RTR 0x2UL | |
| 1100 | + #define CMDQ_MODIFY_QP_NEW_STATE_RTS 0x3UL | |
| 1101 | + #define CMDQ_MODIFY_QP_NEW_STATE_SQD 0x4UL | |
| 1102 | + #define CMDQ_MODIFY_QP_NEW_STATE_SQE 0x5UL | |
| 1103 | + #define CMDQ_MODIFY_QP_NEW_STATE_ERR 0x6UL | |
| 1104 | + #define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY 0x10UL | |
| 1105 | + #define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK 0xc0UL | |
| 1106 | + #define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT 6 | |
| 1107 | + #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 (0x0UL << 6) | |
| 1108 | + #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 (0x2UL << 6) | |
| 1109 | + #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 (0x3UL << 6) | |
| 1110 | + u8 access; | |
| 1111 | + #define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE 0x1UL | |
| 1112 | + #define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE 0x2UL | |
| 1113 | + #define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ 0x4UL | |
| 1114 | + #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC 0x8UL | |
| 1115 | + __le16 pkey; | |
| 1116 | + __le32 qkey; | |
| 1117 | + __le32 dgid[4]; | |
| 1118 | + __le32 flow_label; | |
| 1119 | + __le16 sgid_index; | |
| 1120 | + u8 hop_limit; | |
| 1121 | + u8 traffic_class; | |
| 1122 | + __le16 dest_mac[3]; | |
| 1123 | + u8 tos_dscp_tos_ecn; | |
| 1124 | + #define CMDQ_MODIFY_QP_TOS_ECN_MASK 0x3UL | |
| 1125 | + #define CMDQ_MODIFY_QP_TOS_ECN_SFT 0 | |
| 1126 | + #define CMDQ_MODIFY_QP_TOS_DSCP_MASK 0xfcUL | |
| 1127 | + #define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2 | |
| 1128 | + u8 path_mtu; | |
| 1129 | + #define CMDQ_MODIFY_QP_PATH_MTU_MASK 0xf0UL | |
| 1130 | + #define CMDQ_MODIFY_QP_PATH_MTU_SFT 4 | |
| 1131 | + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_256 (0x0UL << 4) | |
| 1132 | + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_512 (0x1UL << 4) | |
| 1133 | + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 (0x2UL << 4) | |
| 1134 | + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 (0x3UL << 4) | |
| 1135 | + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 (0x4UL << 4) | |
| 1136 | + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 (0x5UL << 4) | |
| 1137 | + u8 timeout; | |
| 1138 | + u8 retry_cnt; | |
| 1139 | + u8 rnr_retry; | |
| 1140 | + u8 min_rnr_timer; | |
| 1141 | + __le32 rq_psn; | |
| 1142 | + __le32 sq_psn; | |
| 1143 | + u8 max_rd_atomic; | |
| 1144 | + u8 max_dest_rd_atomic; | |
| 1145 | + __le16 enable_cc; | |
| 1146 | + #define CMDQ_MODIFY_QP_ENABLE_CC 0x1UL | |
| 1147 | + __le32 sq_size; | |
| 1148 | + __le32 rq_size; | |
| 1149 | + __le16 sq_sge; | |
| 1150 | + __le16 rq_sge; | |
| 1151 | + __le32 max_inline_data; | |
| 1152 | + __le32 dest_qp_id; | |
| 1153 | + __le32 unused_3; | |
| 1154 | + __le16 src_mac[3]; | |
| 1155 | + __le16 vlan_pcp_vlan_dei_vlan_id; | |
| 1156 | + #define CMDQ_MODIFY_QP_VLAN_ID_MASK 0xfffUL | |
| 1157 | + #define CMDQ_MODIFY_QP_VLAN_ID_SFT 0 | |
| 1158 | + #define CMDQ_MODIFY_QP_VLAN_DEI 0x1000UL | |
| 1159 | + #define CMDQ_MODIFY_QP_VLAN_PCP_MASK 0xe000UL | |
| 1160 | + #define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13 | |
| 1161 | +}; | |
| 1162 | + | |
| 1163 | +/* Query QP command (24 bytes) */ | |
| 1164 | +struct cmdq_query_qp { | |
| 1165 | + u8 opcode; | |
| 1166 | + #define CMDQ_QUERY_QP_OPCODE_QUERY_QP 0x4UL | |
| 1167 | + u8 cmd_size; | |
| 1168 | + __le16 flags; | |
| 1169 | + __le16 cookie; | |
| 1170 | + u8 resp_size; | |
| 1171 | + u8 reserved8; | |
| 1172 | + __le64 resp_addr; | |
| 1173 | + __le32 qp_cid; | |
| 1174 | + __le32 unused_0; | |
| 1175 | +}; | |
| 1176 | + | |
| 1177 | +/* Create SRQ command (48 bytes) */ | |
| 1178 | +struct cmdq_create_srq { | |
| 1179 | + u8 opcode; | |
| 1180 | + #define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ 0x5UL | |
| 1181 | + u8 cmd_size; | |
| 1182 | + __le16 flags; | |
| 1183 | + __le16 cookie; | |
| 1184 | + u8 resp_size; | |
| 1185 | + u8 reserved8; | |
| 1186 | + __le64 resp_addr; | |
| 1187 | + __le64 srq_handle; | |
| 1188 | + __le16 pg_size_lvl; | |
| 1189 | + #define CMDQ_CREATE_SRQ_LVL_MASK 0x3UL | |
| 1190 | + #define CMDQ_CREATE_SRQ_LVL_SFT 0 | |
| 1191 | + #define CMDQ_CREATE_SRQ_LVL_LVL_0 0x0UL | |
| 1192 | + #define CMDQ_CREATE_SRQ_LVL_LVL_1 0x1UL | |
| 1193 | + #define CMDQ_CREATE_SRQ_LVL_LVL_2 0x2UL | |
| 1194 | + #define CMDQ_CREATE_SRQ_PG_SIZE_MASK 0x1cUL | |
| 1195 | + #define CMDQ_CREATE_SRQ_PG_SIZE_SFT 2 | |
| 1196 | + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K (0x0UL << 2) | |
| 1197 | + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K (0x1UL << 2) | |
| 1198 | + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K (0x2UL << 2) | |
| 1199 | + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M (0x3UL << 2) | |
| 1200 | + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M (0x4UL << 2) | |
| 1201 | + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G (0x5UL << 2) | |
| 1202 | + __le16 eventq_id; | |
| 1203 | + #define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK 0xfffUL | |
| 1204 | + #define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0 | |
| 1205 | + __le16 srq_size; | |
| 1206 | + __le16 srq_fwo; | |
| 1207 | + __le32 dpi; | |
| 1208 | + __le32 pd_id; | |
| 1209 | + __le64 pbl; | |
| 1210 | +}; | |
| 1211 | + | |
| 1212 | +/* Destroy SRQ command (24 bytes) */ | |
| 1213 | +struct cmdq_destroy_srq { | |
| 1214 | + u8 opcode; | |
| 1215 | + #define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ 0x6UL | |
| 1216 | + u8 cmd_size; | |
| 1217 | + __le16 flags; | |
| 1218 | + __le16 cookie; | |
| 1219 | + u8 resp_size; | |
| 1220 | + u8 reserved8; | |
| 1221 | + __le64 resp_addr; | |
| 1222 | + __le32 srq_cid; | |
| 1223 | + __le32 unused_0; | |
| 1224 | +}; | |
| 1225 | + | |
| 1226 | +/* Query SRQ command (24 bytes) */ | |
| 1227 | +struct cmdq_query_srq { | |
| 1228 | + u8 opcode; | |
| 1229 | + #define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ 0x8UL | |
| 1230 | + u8 cmd_size; | |
| 1231 | + __le16 flags; | |
| 1232 | + __le16 cookie; | |
| 1233 | + u8 resp_size; | |
| 1234 | + u8 reserved8; | |
| 1235 | + __le64 resp_addr; | |
| 1236 | + __le32 srq_cid; | |
| 1237 | + __le32 unused_0; | |
| 1238 | +}; | |
| 1239 | + | |
| 1240 | +/* Create CQ command (48 bytes) */ | |
| 1241 | +struct cmdq_create_cq { | |
| 1242 | + u8 opcode; | |
| 1243 | + #define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ 0x9UL | |
| 1244 | + u8 cmd_size; | |
| 1245 | + __le16 flags; | |
| 1246 | + __le16 cookie; | |
| 1247 | + u8 resp_size; | |
| 1248 | + u8 reserved8; | |
| 1249 | + __le64 resp_addr; | |
| 1250 | + __le64 cq_handle; | |
| 1251 | + __le32 pg_size_lvl; | |
| 1252 | + #define CMDQ_CREATE_CQ_LVL_MASK 0x3UL | |
| 1253 | + #define CMDQ_CREATE_CQ_LVL_SFT 0 | |
| 1254 | + #define CMDQ_CREATE_CQ_LVL_LVL_0 0x0UL | |
| 1255 | + #define CMDQ_CREATE_CQ_LVL_LVL_1 0x1UL | |
| 1256 | + #define CMDQ_CREATE_CQ_LVL_LVL_2 0x2UL | |
| 1257 | + #define CMDQ_CREATE_CQ_PG_SIZE_MASK 0x1cUL | |
| 1258 | + #define CMDQ_CREATE_CQ_PG_SIZE_SFT 2 | |
| 1259 | + #define CMDQ_CREATE_CQ_PG_SIZE_PG_4K (0x0UL << 2) | |
| 1260 | + #define CMDQ_CREATE_CQ_PG_SIZE_PG_8K (0x1UL << 2) | |
| 1261 | + #define CMDQ_CREATE_CQ_PG_SIZE_PG_64K (0x2UL << 2) | |
| 1262 | + #define CMDQ_CREATE_CQ_PG_SIZE_PG_2M (0x3UL << 2) | |
| 1263 | + #define CMDQ_CREATE_CQ_PG_SIZE_PG_8M (0x4UL << 2) | |
| 1264 | + #define CMDQ_CREATE_CQ_PG_SIZE_PG_1G (0x5UL << 2) | |
| 1265 | + __le32 cq_fco_cnq_id; | |
| 1266 | + #define CMDQ_CREATE_CQ_CNQ_ID_MASK 0xfffUL | |
| 1267 | + #define CMDQ_CREATE_CQ_CNQ_ID_SFT 0 | |
| 1268 | + #define CMDQ_CREATE_CQ_CQ_FCO_MASK 0xfffff000UL | |
| 1269 | + #define CMDQ_CREATE_CQ_CQ_FCO_SFT 12 | |
| 1270 | + __le32 dpi; | |
| 1271 | + __le32 cq_size; | |
| 1272 | + __le64 pbl; | |
| 1273 | +}; | |
| 1274 | + | |
| 1275 | +/* Destroy CQ command (24 bytes) */ | |
| 1276 | +struct cmdq_destroy_cq { | |
| 1277 | + u8 opcode; | |
| 1278 | + #define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ 0xaUL | |
| 1279 | + u8 cmd_size; | |
| 1280 | + __le16 flags; | |
| 1281 | + __le16 cookie; | |
| 1282 | + u8 resp_size; | |
| 1283 | + u8 reserved8; | |
| 1284 | + __le64 resp_addr; | |
| 1285 | + __le32 cq_cid; | |
| 1286 | + __le32 unused_0; | |
| 1287 | +}; | |
| 1288 | + | |
| 1289 | +/* Resize CQ command (40 bytes) */ | |
| 1290 | +struct cmdq_resize_cq { | |
| 1291 | + u8 opcode; | |
| 1292 | + #define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ 0xcUL | |
| 1293 | + u8 cmd_size; | |
| 1294 | + __le16 flags; | |
| 1295 | + __le16 cookie; | |
| 1296 | + u8 resp_size; | |
| 1297 | + u8 reserved8; | |
| 1298 | + __le64 resp_addr; | |
| 1299 | + __le32 cq_cid; | |
| 1300 | + __le32 new_cq_size_pg_size_lvl; | |
| 1301 | + #define CMDQ_RESIZE_CQ_LVL_MASK 0x3UL | |
| 1302 | + #define CMDQ_RESIZE_CQ_LVL_SFT 0 | |
| 1303 | + #define CMDQ_RESIZE_CQ_LVL_LVL_0 0x0UL | |
| 1304 | + #define CMDQ_RESIZE_CQ_LVL_LVL_1 0x1UL | |
| 1305 | + #define CMDQ_RESIZE_CQ_LVL_LVL_2 0x2UL | |
| 1306 | + #define CMDQ_RESIZE_CQ_PG_SIZE_MASK 0x1cUL | |
| 1307 | + #define CMDQ_RESIZE_CQ_PG_SIZE_SFT 2 | |
| 1308 | + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K (0x0UL << 2) | |
| 1309 | + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K (0x1UL << 2) | |
| 1310 | + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K (0x2UL << 2) | |
| 1311 | + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M (0x3UL << 2) | |
| 1312 | + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M (0x4UL << 2) | |
| 1313 | + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G (0x5UL << 2) | |
| 1314 | + #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK 0x1fffe0UL | |
| 1315 | + #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5 | |
| 1316 | + __le64 new_pbl; | |
| 1317 | + __le32 new_cq_fco; | |
| 1318 | + __le32 unused_2; | |
| 1319 | +}; | |
| 1320 | + | |
| 1321 | +/* Allocate MRW command (32 bytes) */ | |
| 1322 | +struct cmdq_allocate_mrw { | |
| 1323 | + u8 opcode; | |
| 1324 | + #define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW 0xdUL | |
| 1325 | + u8 cmd_size; | |
| 1326 | + __le16 flags; | |
| 1327 | + __le16 cookie; | |
| 1328 | + u8 resp_size; | |
| 1329 | + u8 reserved8; | |
| 1330 | + __le64 resp_addr; | |
| 1331 | + __le64 mrw_handle; | |
| 1332 | + u8 mrw_flags; | |
| 1333 | + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK 0xfUL | |
| 1334 | + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT 0 | |
| 1335 | + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR 0x0UL | |
| 1336 | + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR 0x1UL | |
| 1337 | + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 0x2UL | |
| 1338 | + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A 0x3UL | |
| 1339 | + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B 0x4UL | |
| 1340 | + u8 access; | |
| 1341 | + #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_MASK 0x1fUL | |
| 1342 | + #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_SFT 0 | |
| 1343 | + #define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY 0x20UL | |
| 1344 | + __le16 unused_1; | |
| 1345 | + __le32 pd_id; | |
| 1346 | +}; | |
| 1347 | + | |
| 1348 | +/* De-allocate key command (24 bytes) */ | |
| 1349 | +struct cmdq_deallocate_key { | |
| 1350 | + u8 opcode; | |
| 1351 | + #define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY 0xeUL | |
| 1352 | + u8 cmd_size; | |
| 1353 | + __le16 flags; | |
| 1354 | + __le16 cookie; | |
| 1355 | + u8 resp_size; | |
| 1356 | + u8 reserved8; | |
| 1357 | + __le64 resp_addr; | |
| 1358 | + u8 mrw_flags; | |
| 1359 | + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK 0xfUL | |
| 1360 | + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT 0 | |
| 1361 | + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR 0x0UL | |
| 1362 | + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR 0x1UL | |
| 1363 | + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 0x2UL | |
| 1364 | + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A 0x3UL | |
| 1365 | + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B 0x4UL | |
| 1366 | + u8 unused_1[3]; | |
| 1367 | + __le32 key; | |
| 1368 | +}; | |
| 1369 | + | |
| 1370 | +/* Register MR command (48 bytes) */ | |
| 1371 | +struct cmdq_register_mr { | |
| 1372 | + u8 opcode; | |
| 1373 | + #define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR 0xfUL | |
| 1374 | + u8 cmd_size; | |
| 1375 | + __le16 flags; | |
| 1376 | + __le16 cookie; | |
| 1377 | + u8 resp_size; | |
| 1378 | + u8 reserved8; | |
| 1379 | + __le64 resp_addr; | |
| 1380 | + u8 log2_pg_size_lvl; | |
| 1381 | + #define CMDQ_REGISTER_MR_LVL_MASK 0x3UL | |
| 1382 | + #define CMDQ_REGISTER_MR_LVL_SFT 0 | |
| 1383 | + #define CMDQ_REGISTER_MR_LVL_LVL_0 0x0UL | |
| 1384 | + #define CMDQ_REGISTER_MR_LVL_LVL_1 0x1UL | |
| 1385 | + #define CMDQ_REGISTER_MR_LVL_LVL_2 0x2UL | |
| 1386 | + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK 0x7cUL | |
| 1387 | + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT 2 | |
| 1388 | + u8 access; | |
| 1389 | + #define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE 0x1UL | |
| 1390 | + #define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ 0x2UL | |
| 1391 | + #define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE 0x4UL | |
| 1392 | + #define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC 0x8UL | |
| 1393 | + #define CMDQ_REGISTER_MR_ACCESS_MW_BIND 0x10UL | |
| 1394 | + #define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED 0x20UL | |
| 1395 | + __le16 unused_1; | |
| 1396 | + __le32 key; | |
| 1397 | + __le64 pbl; | |
| 1398 | + __le64 va; | |
| 1399 | + __le64 mr_size; | |
| 1400 | +}; | |
| 1401 | + | |
| 1402 | +/* Deregister MR command (24 bytes) */ | |
| 1403 | +struct cmdq_deregister_mr { | |
| 1404 | + u8 opcode; | |
| 1405 | + #define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR 0x10UL | |
| 1406 | + u8 cmd_size; | |
| 1407 | + __le16 flags; | |
| 1408 | + __le16 cookie; | |
| 1409 | + u8 resp_size; | |
| 1410 | + u8 reserved8; | |
| 1411 | + __le64 resp_addr; | |
| 1412 | + __le32 lkey; | |
| 1413 | + __le32 unused_0; | |
| 1414 | +}; | |
| 1415 | + | |
| 1416 | +/* Add GID command (48 bytes) */ | |
| 1417 | +struct cmdq_add_gid { | |
| 1418 | + u8 opcode; | |
| 1419 | + #define CMDQ_ADD_GID_OPCODE_ADD_GID 0x11UL | |
| 1420 | + u8 cmd_size; | |
| 1421 | + __le16 flags; | |
| 1422 | + __le16 cookie; | |
| 1423 | + u8 resp_size; | |
| 1424 | + u8 reserved8; | |
| 1425 | + __le64 resp_addr; | |
| 1426 | + __be32 gid[4]; | |
| 1427 | + __be16 src_mac[3]; | |
| 1428 | + __le16 vlan; | |
| 1429 | + #define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK 0xfffUL | |
| 1430 | + #define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT 0 | |
| 1431 | + #define CMDQ_ADD_GID_VLAN_TPID_MASK 0x7000UL | |
| 1432 | + #define CMDQ_ADD_GID_VLAN_TPID_SFT 12 | |
| 1433 | + #define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12) | |
| 1434 | + #define CMDQ_ADD_GID_VLAN_TPID_TPID_8100 (0x1UL << 12) | |
| 1435 | + #define CMDQ_ADD_GID_VLAN_TPID_TPID_9100 (0x2UL << 12) | |
| 1436 | + #define CMDQ_ADD_GID_VLAN_TPID_TPID_9200 (0x3UL << 12) | |
| 1437 | + #define CMDQ_ADD_GID_VLAN_TPID_TPID_9300 (0x4UL << 12) | |
| 1438 | + #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12) | |
| 1439 | + #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12) | |
| 1440 | + #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12) | |
| 1441 | + #define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 | |
| 1442 | + #define CMDQ_ADD_GID_VLAN_VLAN_EN 0x8000UL | |
| 1443 | + __le16 ipid; | |
| 1444 | + __le16 stats_ctx; | |
| 1445 | + #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL | |
| 1446 | + #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT 0 | |
| 1447 | + #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL | |
| 1448 | + __le32 unused_0; | |
| 1449 | +}; | |
| 1450 | + | |
| 1451 | +/* Delete GID command (24 bytes) */ | |
| 1452 | +struct cmdq_delete_gid { | |
| 1453 | + u8 opcode; | |
| 1454 | + #define CMDQ_DELETE_GID_OPCODE_DELETE_GID 0x12UL | |
| 1455 | + u8 cmd_size; | |
| 1456 | + __le16 flags; | |
| 1457 | + __le16 cookie; | |
| 1458 | + u8 resp_size; | |
| 1459 | + u8 reserved8; | |
| 1460 | + __le64 resp_addr; | |
| 1461 | + __le16 gid_index; | |
| 1462 | + __le16 unused_0; | |
| 1463 | + __le32 unused_1; | |
| 1464 | +}; | |
| 1465 | + | |
| 1466 | +/* Modify GID command (48 bytes) */ | |
| 1467 | +struct cmdq_modify_gid { | |
| 1468 | + u8 opcode; | |
| 1469 | + #define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID 0x17UL | |
| 1470 | + u8 cmd_size; | |
| 1471 | + __le16 flags; | |
| 1472 | + __le16 cookie; | |
| 1473 | + u8 resp_size; | |
| 1474 | + u8 reserved8; | |
| 1475 | + __le64 resp_addr; | |
| 1476 | + __le32 gid[4]; | |
| 1477 | + __le16 src_mac[3]; | |
| 1478 | + __le16 vlan; | |
| 1479 | + #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK 0xfffUL | |
| 1480 | + #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT 0 | |
| 1481 | + #define CMDQ_MODIFY_GID_VLAN_TPID_MASK 0x7000UL | |
| 1482 | + #define CMDQ_MODIFY_GID_VLAN_TPID_SFT 12 | |
| 1483 | + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12) | |
| 1484 | + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 (0x1UL << 12) | |
| 1485 | + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 (0x2UL << 12) | |
| 1486 | + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 (0x3UL << 12) | |
| 1487 | + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 (0x4UL << 12) | |
| 1488 | + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12) | |
| 1489 | + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12) | |
| 1490 | + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12) | |
| 1491 | + #define CMDQ_MODIFY_GID_VLAN_TPID_LAST \ | |
| 1492 | + CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 | |
| 1493 | + #define CMDQ_MODIFY_GID_VLAN_VLAN_EN 0x8000UL | |
| 1494 | + __le16 ipid; | |
| 1495 | + __le16 gid_index; | |
| 1496 | + __le16 stats_ctx; | |
| 1497 | + #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL | |
| 1498 | + #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT 0 | |
| 1499 | + #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL | |
| 1500 | + __le16 unused_0; | |
| 1501 | +}; | |
| 1502 | + | |
| 1503 | +/* Query GID command (24 bytes) */ | |
| 1504 | +struct cmdq_query_gid { | |
| 1505 | + u8 opcode; | |
| 1506 | + #define CMDQ_QUERY_GID_OPCODE_QUERY_GID 0x18UL | |
| 1507 | + u8 cmd_size; | |
| 1508 | + __le16 flags; | |
| 1509 | + __le16 cookie; | |
| 1510 | + u8 resp_size; | |
| 1511 | + u8 reserved8; | |
| 1512 | + __le64 resp_addr; | |
| 1513 | + __le16 gid_index; | |
| 1514 | + __le16 unused_0; | |
| 1515 | + __le32 unused_1; | |
| 1516 | +}; | |
| 1517 | + | |
| 1518 | +/* Create QP1 command (80 bytes) */ | |
| 1519 | +struct cmdq_create_qp1 { | |
| 1520 | + u8 opcode; | |
| 1521 | + #define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 0x13UL | |
| 1522 | + u8 cmd_size; | |
| 1523 | + __le16 flags; | |
| 1524 | + __le16 cookie; | |
| 1525 | + u8 resp_size; | |
| 1526 | + u8 reserved8; | |
| 1527 | + __le64 resp_addr; | |
| 1528 | + __le64 qp_handle; | |
| 1529 | + __le32 qp_flags; | |
| 1530 | + #define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED 0x1UL | |
| 1531 | + #define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION 0x2UL | |
| 1532 | + #define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL | |
| 1533 | + u8 type; | |
| 1534 | + #define CMDQ_CREATE_QP1_TYPE_GSI 0x1UL | |
| 1535 | + u8 sq_pg_size_sq_lvl; | |
| 1536 | + #define CMDQ_CREATE_QP1_SQ_LVL_MASK 0xfUL | |
| 1537 | + #define CMDQ_CREATE_QP1_SQ_LVL_SFT 0 | |
| 1538 | + #define CMDQ_CREATE_QP1_SQ_LVL_LVL_0 0x0UL | |
| 1539 | + #define CMDQ_CREATE_QP1_SQ_LVL_LVL_1 0x1UL | |
| 1540 | + #define CMDQ_CREATE_QP1_SQ_LVL_LVL_2 0x2UL | |
| 1541 | + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK 0xf0UL | |
| 1542 | + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT 4 | |
| 1543 | + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1544 | + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1545 | + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1546 | + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1547 | + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1548 | + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1549 | + u8 rq_pg_size_rq_lvl; | |
| 1550 | + #define CMDQ_CREATE_QP1_RQ_LVL_MASK 0xfUL | |
| 1551 | + #define CMDQ_CREATE_QP1_RQ_LVL_SFT 0 | |
| 1552 | + #define CMDQ_CREATE_QP1_RQ_LVL_LVL_0 0x0UL | |
| 1553 | + #define CMDQ_CREATE_QP1_RQ_LVL_LVL_1 0x1UL | |
| 1554 | + #define CMDQ_CREATE_QP1_RQ_LVL_LVL_2 0x2UL | |
| 1555 | + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK 0xf0UL | |
| 1556 | + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT 4 | |
| 1557 | + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1558 | + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1559 | + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1560 | + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1561 | + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1562 | + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1563 | + u8 unused_0; | |
| 1564 | + __le32 dpi; | |
| 1565 | + __le32 sq_size; | |
| 1566 | + __le32 rq_size; | |
| 1567 | + __le16 sq_fwo_sq_sge; | |
| 1568 | + #define CMDQ_CREATE_QP1_SQ_SGE_MASK 0xfUL | |
| 1569 | + #define CMDQ_CREATE_QP1_SQ_SGE_SFT 0 | |
| 1570 | + #define CMDQ_CREATE_QP1_SQ_FWO_MASK 0xfff0UL | |
| 1571 | + #define CMDQ_CREATE_QP1_SQ_FWO_SFT 4 | |
| 1572 | + __le16 rq_fwo_rq_sge; | |
| 1573 | + #define CMDQ_CREATE_QP1_RQ_SGE_MASK 0xfUL | |
| 1574 | + #define CMDQ_CREATE_QP1_RQ_SGE_SFT 0 | |
| 1575 | + #define CMDQ_CREATE_QP1_RQ_FWO_MASK 0xfff0UL | |
| 1576 | + #define CMDQ_CREATE_QP1_RQ_FWO_SFT 4 | |
| 1577 | + __le32 scq_cid; | |
| 1578 | + __le32 rcq_cid; | |
| 1579 | + __le32 srq_cid; | |
| 1580 | + __le32 pd_id; | |
| 1581 | + __le64 sq_pbl; | |
| 1582 | + __le64 rq_pbl; | |
| 1583 | +}; | |
| 1584 | + | |
| 1585 | +/* Destroy QP1 command (24 bytes) */ | |
| 1586 | +struct cmdq_destroy_qp1 { | |
| 1587 | + u8 opcode; | |
| 1588 | + #define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 0x14UL | |
| 1589 | + u8 cmd_size; | |
| 1590 | + __le16 flags; | |
| 1591 | + __le16 cookie; | |
| 1592 | + u8 resp_size; | |
| 1593 | + u8 reserved8; | |
| 1594 | + __le64 resp_addr; | |
| 1595 | + __le32 qp1_cid; | |
| 1596 | + __le32 unused_0; | |
| 1597 | +}; | |
| 1598 | + | |
| 1599 | +/* Create AH command (64 bytes) */ | |
| 1600 | +struct cmdq_create_ah { | |
| 1601 | + u8 opcode; | |
| 1602 | + #define CMDQ_CREATE_AH_OPCODE_CREATE_AH 0x15UL | |
| 1603 | + u8 cmd_size; | |
| 1604 | + __le16 flags; | |
| 1605 | + __le16 cookie; | |
| 1606 | + u8 resp_size; | |
| 1607 | + u8 reserved8; | |
| 1608 | + __le64 resp_addr; | |
| 1609 | + __le64 ah_handle; | |
| 1610 | + __le32 dgid[4]; | |
| 1611 | + u8 type; | |
| 1612 | + #define CMDQ_CREATE_AH_TYPE_V1 0x0UL | |
| 1613 | + #define CMDQ_CREATE_AH_TYPE_V2IPV4 0x2UL | |
| 1614 | + #define CMDQ_CREATE_AH_TYPE_V2IPV6 0x3UL | |
| 1615 | + u8 hop_limit; | |
| 1616 | + __le16 sgid_index; | |
| 1617 | + __le32 dest_vlan_id_flow_label; | |
| 1618 | + #define CMDQ_CREATE_AH_FLOW_LABEL_MASK 0xfffffUL | |
| 1619 | + #define CMDQ_CREATE_AH_FLOW_LABEL_SFT 0 | |
| 1620 | + #define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK 0xfff00000UL | |
| 1621 | + #define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20 | |
| 1622 | + __le32 pd_id; | |
| 1623 | + __le32 unused_0; | |
| 1624 | + __le16 dest_mac[3]; | |
| 1625 | + u8 traffic_class; | |
| 1626 | + u8 unused_1; | |
| 1627 | +}; | |
| 1628 | + | |
| 1629 | +/* Destroy AH command (24 bytes) */ | |
| 1630 | +struct cmdq_destroy_ah { | |
| 1631 | + u8 opcode; | |
| 1632 | + #define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH 0x16UL | |
| 1633 | + u8 cmd_size; | |
| 1634 | + __le16 flags; | |
| 1635 | + __le16 cookie; | |
| 1636 | + u8 resp_size; | |
| 1637 | + u8 reserved8; | |
| 1638 | + __le64 resp_addr; | |
| 1639 | + __le32 ah_cid; | |
| 1640 | + __le32 unused_0; | |
| 1641 | +}; | |
| 1642 | + | |
| 1643 | +/* Initialize Firmware command (112 bytes) */ | |
| 1644 | +struct cmdq_initialize_fw { | |
| 1645 | + u8 opcode; | |
| 1646 | + #define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW 0x80UL | |
| 1647 | + u8 cmd_size; | |
| 1648 | + __le16 flags; | |
| 1649 | + __le16 cookie; | |
| 1650 | + u8 resp_size; | |
| 1651 | + u8 reserved8; | |
| 1652 | + __le64 resp_addr; | |
| 1653 | + u8 qpc_pg_size_qpc_lvl; | |
| 1654 | + #define CMDQ_INITIALIZE_FW_QPC_LVL_MASK 0xfUL | |
| 1655 | + #define CMDQ_INITIALIZE_FW_QPC_LVL_SFT 0 | |
| 1656 | + #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 0x0UL | |
| 1657 | + #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 0x1UL | |
| 1658 | + #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 0x2UL | |
| 1659 | + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK 0xf0UL | |
| 1660 | + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT 4 | |
| 1661 | + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1662 | + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1663 | + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1664 | + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1665 | + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1666 | + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1667 | + u8 mrw_pg_size_mrw_lvl; | |
| 1668 | + #define CMDQ_INITIALIZE_FW_MRW_LVL_MASK 0xfUL | |
| 1669 | + #define CMDQ_INITIALIZE_FW_MRW_LVL_SFT 0 | |
| 1670 | + #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 0x0UL | |
| 1671 | + #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 0x1UL | |
| 1672 | + #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 0x2UL | |
| 1673 | + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK 0xf0UL | |
| 1674 | + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT 4 | |
| 1675 | + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1676 | + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1677 | + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1678 | + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1679 | + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1680 | + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1681 | + u8 srq_pg_size_srq_lvl; | |
| 1682 | + #define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK 0xfUL | |
| 1683 | + #define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT 0 | |
| 1684 | + #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 0x0UL | |
| 1685 | + #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 0x1UL | |
| 1686 | + #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 0x2UL | |
| 1687 | + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK 0xf0UL | |
| 1688 | + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT 4 | |
| 1689 | + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1690 | + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1691 | + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1692 | + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1693 | + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1694 | + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1695 | + u8 cq_pg_size_cq_lvl; | |
| 1696 | + #define CMDQ_INITIALIZE_FW_CQ_LVL_MASK 0xfUL | |
| 1697 | + #define CMDQ_INITIALIZE_FW_CQ_LVL_SFT 0 | |
| 1698 | + #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 0x0UL | |
| 1699 | + #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 0x1UL | |
| 1700 | + #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 0x2UL | |
| 1701 | + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK 0xf0UL | |
| 1702 | + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT 4 | |
| 1703 | + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1704 | + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1705 | + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1706 | + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1707 | + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1708 | + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1709 | + u8 tqm_pg_size_tqm_lvl; | |
| 1710 | + #define CMDQ_INITIALIZE_FW_TQM_LVL_MASK 0xfUL | |
| 1711 | + #define CMDQ_INITIALIZE_FW_TQM_LVL_SFT 0 | |
| 1712 | + #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 0x0UL | |
| 1713 | + #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 0x1UL | |
| 1714 | + #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 0x2UL | |
| 1715 | + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK 0xf0UL | |
| 1716 | + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT 4 | |
| 1717 | + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1718 | + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1719 | + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1720 | + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1721 | + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1722 | + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1723 | + u8 tim_pg_size_tim_lvl; | |
| 1724 | + #define CMDQ_INITIALIZE_FW_TIM_LVL_MASK 0xfUL | |
| 1725 | + #define CMDQ_INITIALIZE_FW_TIM_LVL_SFT 0 | |
| 1726 | + #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 0x0UL | |
| 1727 | + #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 0x1UL | |
| 1728 | + #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 0x2UL | |
| 1729 | + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK 0xf0UL | |
| 1730 | + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT 4 | |
| 1731 | + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K (0x0UL << 4) | |
| 1732 | + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K (0x1UL << 4) | |
| 1733 | + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K (0x2UL << 4) | |
| 1734 | + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M (0x3UL << 4) | |
| 1735 | + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M (0x4UL << 4) | |
| 1736 | + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G (0x5UL << 4) | |
| 1737 | + __le16 reserved16; | |
| 1738 | + __le64 qpc_page_dir; | |
| 1739 | + __le64 mrw_page_dir; | |
| 1740 | + __le64 srq_page_dir; | |
| 1741 | + __le64 cq_page_dir; | |
| 1742 | + __le64 tqm_page_dir; | |
| 1743 | + __le64 tim_page_dir; | |
| 1744 | + __le32 number_of_qp; | |
| 1745 | + __le32 number_of_mrw; | |
| 1746 | + __le32 number_of_srq; | |
| 1747 | + __le32 number_of_cq; | |
| 1748 | + __le32 max_qp_per_vf; | |
| 1749 | + __le32 max_mrw_per_vf; | |
| 1750 | + __le32 max_srq_per_vf; | |
| 1751 | + __le32 max_cq_per_vf; | |
| 1752 | + __le32 max_gid_per_vf; | |
| 1753 | + __le32 stat_ctx_id; | |
| 1754 | +}; | |
| 1755 | + | |
| 1756 | +/* De-initialize Firmware command (16 bytes) */ | |
| 1757 | +struct cmdq_deinitialize_fw { | |
| 1758 | + u8 opcode; | |
| 1759 | + #define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW 0x81UL | |
| 1760 | + u8 cmd_size; | |
| 1761 | + __le16 flags; | |
| 1762 | + __le16 cookie; | |
| 1763 | + u8 resp_size; | |
| 1764 | + u8 reserved8; | |
| 1765 | + __le64 resp_addr; | |
| 1766 | +}; | |
| 1767 | + | |
| 1768 | +/* Stop function command (16 bytes) */ | |
| 1769 | +struct cmdq_stop_func { | |
| 1770 | + u8 opcode; | |
| 1771 | + #define CMDQ_STOP_FUNC_OPCODE_STOP_FUNC 0x82UL | |
| 1772 | + u8 cmd_size; | |
| 1773 | + __le16 flags; | |
| 1774 | + __le16 cookie; | |
| 1775 | + u8 resp_size; | |
| 1776 | + u8 reserved8; | |
| 1777 | + __le64 resp_addr; | |
| 1778 | +}; | |
| 1779 | + | |
| 1780 | +/* Query function command (16 bytes) */ | |
| 1781 | +struct cmdq_query_func { | |
| 1782 | + u8 opcode; | |
| 1783 | + #define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC 0x83UL | |
| 1784 | + u8 cmd_size; | |
| 1785 | + __le16 flags; | |
| 1786 | + __le16 cookie; | |
| 1787 | + u8 resp_size; | |
| 1788 | + u8 reserved8; | |
| 1789 | + __le64 resp_addr; | |
| 1790 | +}; | |
| 1791 | + | |
| 1792 | +/* Set function resources command (16 bytes) */ | |
| 1793 | +struct cmdq_set_func_resources { | |
| 1794 | + u8 opcode; | |
| 1795 | + #define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL | |
| 1796 | + u8 cmd_size; | |
| 1797 | + __le16 flags; | |
| 1798 | + __le16 cookie; | |
| 1799 | + u8 resp_size; | |
| 1800 | + u8 reserved8; | |
| 1801 | + __le64 resp_addr; | |
| 1802 | +}; | |
| 1803 | + | |
| 1804 | +/* Read hardware resource context command (24 bytes) */ | |
| 1805 | +struct cmdq_read_context { | |
| 1806 | + u8 opcode; | |
| 1807 | + #define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT 0x85UL | |
| 1808 | + u8 cmd_size; | |
| 1809 | + __le16 flags; | |
| 1810 | + __le16 cookie; | |
| 1811 | + u8 resp_size; | |
| 1812 | + u8 reserved8; | |
| 1813 | + __le64 resp_addr; | |
| 1814 | + __le32 type_xid; | |
| 1815 | + #define CMDQ_READ_CONTEXT_XID_MASK 0xffffffUL | |
| 1816 | + #define CMDQ_READ_CONTEXT_XID_SFT 0 | |
| 1817 | + #define CMDQ_READ_CONTEXT_TYPE_MASK 0xff000000UL | |
| 1818 | + #define CMDQ_READ_CONTEXT_TYPE_SFT 24 | |
| 1819 | + #define CMDQ_READ_CONTEXT_TYPE_QPC (0x0UL << 24) | |
| 1820 | + #define CMDQ_READ_CONTEXT_TYPE_CQ (0x1UL << 24) | |
| 1821 | + #define CMDQ_READ_CONTEXT_TYPE_MRW (0x2UL << 24) | |
| 1822 | + #define CMDQ_READ_CONTEXT_TYPE_SRQ (0x3UL << 24) | |
| 1823 | + __le32 unused_0; | |
| 1824 | +}; | |
| 1825 | + | |
| 1826 | +/* Map TC to COS. Can only be issued from a PF (24 bytes) */ | |
| 1827 | +struct cmdq_map_tc_to_cos { | |
| 1828 | + u8 opcode; | |
| 1829 | + #define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS 0x8aUL | |
| 1830 | + u8 cmd_size; | |
| 1831 | + __le16 flags; | |
| 1832 | + __le16 cookie; | |
| 1833 | + u8 resp_size; | |
| 1834 | + u8 reserved8; | |
| 1835 | + __le64 resp_addr; | |
| 1836 | + __le16 cos0; | |
| 1837 | + #define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE 0xffffUL | |
| 1838 | + __le16 cos1; | |
| 1839 | + #define CMDQ_MAP_TC_TO_COS_COS1_DISABLE 0x8000UL | |
| 1840 | + #define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE 0xffffUL | |
| 1841 | + __le32 unused_0; | |
| 1842 | +}; | |
| 1843 | + | |
| 1844 | +/* Query version command (16 bytes) */ | |
| 1845 | +struct cmdq_query_version { | |
| 1846 | + u8 opcode; | |
| 1847 | + #define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION 0x8bUL | |
| 1848 | + u8 cmd_size; | |
| 1849 | + __le16 flags; | |
| 1850 | + __le16 cookie; | |
| 1851 | + u8 resp_size; | |
| 1852 | + u8 reserved8; | |
| 1853 | + __le64 resp_addr; | |
| 1854 | +}; | |
| 1855 | + | |
| 1856 | +/* Command-Response Event Queue (CREQ) Structures */ | |
| 1857 | +/* Base CREQ Record (16 bytes) */ | |
| 1858 | +struct creq_base { | |
| 1859 | + u8 type; | |
| 1860 | + #define CREQ_BASE_TYPE_MASK 0x3fUL | |
| 1861 | + #define CREQ_BASE_TYPE_SFT 0 | |
| 1862 | + #define CREQ_BASE_TYPE_QP_EVENT 0x38UL | |
| 1863 | + #define CREQ_BASE_TYPE_FUNC_EVENT 0x3aUL | |
| 1864 | + #define CREQ_BASE_RESERVED2_MASK 0xc0UL | |
| 1865 | + #define CREQ_BASE_RESERVED2_SFT 6 | |
| 1866 | + u8 reserved56[7]; | |
| 1867 | + u8 v; | |
| 1868 | + #define CREQ_BASE_V 0x1UL | |
| 1869 | + #define CREQ_BASE_RESERVED7_MASK 0xfeUL | |
| 1870 | + #define CREQ_BASE_RESERVED7_SFT 1 | |
| 1871 | + u8 event; | |
| 1872 | + __le16 reserved48[3]; | |
| 1873 | +}; | |
| 1874 | + | |
| 1875 | +/* RoCE Function Async Event Notification (16 bytes) */ | |
| 1876 | +struct creq_func_event { | |
| 1877 | + u8 type; | |
| 1878 | + #define CREQ_FUNC_EVENT_TYPE_MASK 0x3fUL | |
| 1879 | + #define CREQ_FUNC_EVENT_TYPE_SFT 0 | |
| 1880 | + #define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT 0x3aUL | |
| 1881 | + #define CREQ_FUNC_EVENT_RESERVED2_MASK 0xc0UL | |
| 1882 | + #define CREQ_FUNC_EVENT_RESERVED2_SFT 6 | |
| 1883 | + u8 reserved56[7]; | |
| 1884 | + u8 v; | |
| 1885 | + #define CREQ_FUNC_EVENT_V 0x1UL | |
| 1886 | + #define CREQ_FUNC_EVENT_RESERVED7_MASK 0xfeUL | |
| 1887 | + #define CREQ_FUNC_EVENT_RESERVED7_SFT 1 | |
| 1888 | + u8 event; | |
| 1889 | + #define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR 0x1UL | |
| 1890 | + #define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR 0x2UL | |
| 1891 | + #define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR 0x3UL | |
| 1892 | + #define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR 0x4UL | |
| 1893 | + #define CREQ_FUNC_EVENT_EVENT_CQ_ERROR 0x5UL | |
| 1894 | + #define CREQ_FUNC_EVENT_EVENT_TQM_ERROR 0x6UL | |
| 1895 | + #define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR 0x7UL | |
| 1896 | + #define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR 0x8UL | |
| 1897 | + #define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR 0x9UL | |
| 1898 | + #define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR 0xaUL | |
| 1899 | + #define CREQ_FUNC_EVENT_EVENT_TIM_ERROR 0xbUL | |
| 1900 | + #define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST 0x80UL | |
| 1901 | + #define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED 0x81UL | |
| 1902 | + __le16 reserved48[3]; | |
| 1903 | +}; | |
| 1904 | + | |
| 1905 | +/* RoCE Slowpath Command Completion (16 bytes) */ | |
| 1906 | +struct creq_qp_event { | |
| 1907 | + u8 type; | |
| 1908 | + #define CREQ_QP_EVENT_TYPE_MASK 0x3fUL | |
| 1909 | + #define CREQ_QP_EVENT_TYPE_SFT 0 | |
| 1910 | + #define CREQ_QP_EVENT_TYPE_QP_EVENT 0x38UL | |
| 1911 | + #define CREQ_QP_EVENT_RESERVED2_MASK 0xc0UL | |
| 1912 | + #define CREQ_QP_EVENT_RESERVED2_SFT 6 | |
| 1913 | + u8 status; | |
| 1914 | + __le16 cookie; | |
| 1915 | + __le32 reserved32; | |
| 1916 | + u8 v; | |
| 1917 | + #define CREQ_QP_EVENT_V 0x1UL | |
| 1918 | + #define CREQ_QP_EVENT_RESERVED7_MASK 0xfeUL | |
| 1919 | + #define CREQ_QP_EVENT_RESERVED7_SFT 1 | |
| 1920 | + u8 event; | |
| 1921 | + #define CREQ_QP_EVENT_EVENT_CREATE_QP 0x1UL | |
| 1922 | + #define CREQ_QP_EVENT_EVENT_DESTROY_QP 0x2UL | |
| 1923 | + #define CREQ_QP_EVENT_EVENT_MODIFY_QP 0x3UL | |
| 1924 | + #define CREQ_QP_EVENT_EVENT_QUERY_QP 0x4UL | |
| 1925 | + #define CREQ_QP_EVENT_EVENT_CREATE_SRQ 0x5UL | |
| 1926 | + #define CREQ_QP_EVENT_EVENT_DESTROY_SRQ 0x6UL | |
| 1927 | + #define CREQ_QP_EVENT_EVENT_QUERY_SRQ 0x8UL | |
| 1928 | + #define CREQ_QP_EVENT_EVENT_CREATE_CQ 0x9UL | |
| 1929 | + #define CREQ_QP_EVENT_EVENT_DESTROY_CQ 0xaUL | |
| 1930 | + #define CREQ_QP_EVENT_EVENT_RESIZE_CQ 0xcUL | |
| 1931 | + #define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW 0xdUL | |
| 1932 | + #define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY 0xeUL | |
| 1933 | + #define CREQ_QP_EVENT_EVENT_REGISTER_MR 0xfUL | |
| 1934 | + #define CREQ_QP_EVENT_EVENT_DEREGISTER_MR 0x10UL | |
| 1935 | + #define CREQ_QP_EVENT_EVENT_ADD_GID 0x11UL | |
| 1936 | + #define CREQ_QP_EVENT_EVENT_DELETE_GID 0x12UL | |
| 1937 | + #define CREQ_QP_EVENT_EVENT_MODIFY_GID 0x17UL | |
| 1938 | + #define CREQ_QP_EVENT_EVENT_QUERY_GID 0x18UL | |
| 1939 | + #define CREQ_QP_EVENT_EVENT_CREATE_QP1 0x13UL | |
| 1940 | + #define CREQ_QP_EVENT_EVENT_DESTROY_QP1 0x14UL | |
| 1941 | + #define CREQ_QP_EVENT_EVENT_CREATE_AH 0x15UL | |
| 1942 | + #define CREQ_QP_EVENT_EVENT_DESTROY_AH 0x16UL | |
| 1943 | + #define CREQ_QP_EVENT_EVENT_INITIALIZE_FW 0x80UL | |
| 1944 | + #define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW 0x81UL | |
| 1945 | + #define CREQ_QP_EVENT_EVENT_STOP_FUNC 0x82UL | |
| 1946 | + #define CREQ_QP_EVENT_EVENT_QUERY_FUNC 0x83UL | |
| 1947 | + #define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES 0x84UL | |
| 1948 | + #define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS 0x8aUL | |
| 1949 | + #define CREQ_QP_EVENT_EVENT_QUERY_VERSION 0x8bUL | |
| 1950 | + #define CREQ_QP_EVENT_EVENT_MODIFY_CC 0x8cUL | |
| 1951 | + #define CREQ_QP_EVENT_EVENT_QUERY_CC 0x8dUL | |
| 1952 | + #define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION 0xc0UL | |
| 1953 | + __le16 reserved48[3]; | |
| 1954 | +}; | |
| 1955 | + | |
| 1956 | +/* Create QP command response (16 bytes) */ | |
| 1957 | +struct creq_create_qp_resp { | |
| 1958 | + u8 type; | |
| 1959 | + #define CREQ_CREATE_QP_RESP_TYPE_MASK 0x3fUL | |
| 1960 | + #define CREQ_CREATE_QP_RESP_TYPE_SFT 0 | |
| 1961 | + #define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT 0x38UL | |
| 1962 | + #define CREQ_CREATE_QP_RESP_RESERVED2_MASK 0xc0UL | |
| 1963 | + #define CREQ_CREATE_QP_RESP_RESERVED2_SFT 6 | |
| 1964 | + u8 status; | |
| 1965 | + __le16 cookie; | |
| 1966 | + __le32 xid; | |
| 1967 | + u8 v; | |
| 1968 | + #define CREQ_CREATE_QP_RESP_V 0x1UL | |
| 1969 | + #define CREQ_CREATE_QP_RESP_RESERVED7_MASK 0xfeUL | |
| 1970 | + #define CREQ_CREATE_QP_RESP_RESERVED7_SFT 1 | |
| 1971 | + u8 event; | |
| 1972 | + #define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP 0x1UL | |
| 1973 | + __le16 reserved48[3]; | |
| 1974 | +}; | |
| 1975 | + | |
| 1976 | +/* Destroy QP command response (16 bytes) */ | |
| 1977 | +struct creq_destroy_qp_resp { | |
| 1978 | + u8 type; | |
| 1979 | + #define CREQ_DESTROY_QP_RESP_TYPE_MASK 0x3fUL | |
| 1980 | + #define CREQ_DESTROY_QP_RESP_TYPE_SFT 0 | |
| 1981 | + #define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT 0x38UL | |
| 1982 | + #define CREQ_DESTROY_QP_RESP_RESERVED2_MASK 0xc0UL | |
| 1983 | + #define CREQ_DESTROY_QP_RESP_RESERVED2_SFT 6 | |
| 1984 | + u8 status; | |
| 1985 | + __le16 cookie; | |
| 1986 | + __le32 xid; | |
| 1987 | + u8 v; | |
| 1988 | + #define CREQ_DESTROY_QP_RESP_V 0x1UL | |
| 1989 | + #define CREQ_DESTROY_QP_RESP_RESERVED7_MASK 0xfeUL | |
| 1990 | + #define CREQ_DESTROY_QP_RESP_RESERVED7_SFT 1 | |
| 1991 | + u8 event; | |
| 1992 | + #define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP 0x2UL | |
| 1993 | + __le16 reserved48[3]; | |
| 1994 | +}; | |
| 1995 | + | |
| 1996 | +/* Modify QP command response (16 bytes) */ | |
| 1997 | +struct creq_modify_qp_resp { | |
| 1998 | + u8 type; | |
| 1999 | + #define CREQ_MODIFY_QP_RESP_TYPE_MASK 0x3fUL | |
| 2000 | + #define CREQ_MODIFY_QP_RESP_TYPE_SFT 0 | |
| 2001 | + #define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT 0x38UL | |
| 2002 | + #define CREQ_MODIFY_QP_RESP_RESERVED2_MASK 0xc0UL | |
| 2003 | + #define CREQ_MODIFY_QP_RESP_RESERVED2_SFT 6 | |
| 2004 | + u8 status; | |
| 2005 | + __le16 cookie; | |
| 2006 | + __le32 xid; | |
| 2007 | + u8 v; | |
| 2008 | + #define CREQ_MODIFY_QP_RESP_V 0x1UL | |
| 2009 | + #define CREQ_MODIFY_QP_RESP_RESERVED7_MASK 0xfeUL | |
| 2010 | + #define CREQ_MODIFY_QP_RESP_RESERVED7_SFT 1 | |
| 2011 | + u8 event; | |
| 2012 | + #define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP 0x3UL | |
| 2013 | + __le16 reserved48[3]; | |
| 2014 | +}; | |
| 2015 | + | |
| 2016 | +/* Query QP command response (16 bytes) */ | |
| 2017 | +struct creq_query_qp_resp { | |
| 2018 | + u8 type; | |
| 2019 | + #define CREQ_QUERY_QP_RESP_TYPE_MASK 0x3fUL | |
| 2020 | + #define CREQ_QUERY_QP_RESP_TYPE_SFT 0 | |
| 2021 | + #define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT 0x38UL | |
| 2022 | + #define CREQ_QUERY_QP_RESP_RESERVED2_MASK 0xc0UL | |
| 2023 | + #define CREQ_QUERY_QP_RESP_RESERVED2_SFT 6 | |
| 2024 | + u8 status; | |
| 2025 | + __le16 cookie; | |
| 2026 | + __le32 size; | |
| 2027 | + u8 v; | |
| 2028 | + #define CREQ_QUERY_QP_RESP_V 0x1UL | |
| 2029 | + #define CREQ_QUERY_QP_RESP_RESERVED7_MASK 0xfeUL | |
| 2030 | + #define CREQ_QUERY_QP_RESP_RESERVED7_SFT 1 | |
| 2031 | + u8 event; | |
| 2032 | + #define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP 0x4UL | |
| 2033 | + __le16 reserved48[3]; | |
| 2034 | +}; | |
| 2035 | + | |
| 2036 | +/* Query QP command response side buffer structure (104 bytes) */ | |
| 2037 | +struct creq_query_qp_resp_sb { | |
| 2038 | + u8 opcode; | |
| 2039 | + #define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP 0x4UL | |
| 2040 | + u8 status; | |
| 2041 | + __le16 cookie; | |
| 2042 | + __le16 flags; | |
| 2043 | + u8 resp_size; | |
| 2044 | + u8 reserved8; | |
| 2045 | + __le32 xid; | |
| 2046 | + u8 en_sqd_async_notify_state; | |
| 2047 | + #define CREQ_QUERY_QP_RESP_SB_STATE_MASK 0xfUL | |
| 2048 | + #define CREQ_QUERY_QP_RESP_SB_STATE_SFT 0 | |
| 2049 | + #define CREQ_QUERY_QP_RESP_SB_STATE_RESET 0x0UL | |
| 2050 | + #define CREQ_QUERY_QP_RESP_SB_STATE_INIT 0x1UL | |
| 2051 | + #define CREQ_QUERY_QP_RESP_SB_STATE_RTR 0x2UL | |
| 2052 | + #define CREQ_QUERY_QP_RESP_SB_STATE_RTS 0x3UL | |
| 2053 | + #define CREQ_QUERY_QP_RESP_SB_STATE_SQD 0x4UL | |
| 2054 | + #define CREQ_QUERY_QP_RESP_SB_STATE_SQE 0x5UL | |
| 2055 | + #define CREQ_QUERY_QP_RESP_SB_STATE_ERR 0x6UL | |
| 2056 | + #define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY 0x10UL | |
| 2057 | + u8 access; | |
| 2058 | + #define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE 0x1UL | |
| 2059 | + #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE 0x2UL | |
| 2060 | + #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ 0x4UL | |
| 2061 | + #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC 0x8UL | |
| 2062 | + __le16 pkey; | |
| 2063 | + __le32 qkey; | |
| 2064 | + __le32 reserved32; | |
| 2065 | + __le32 dgid[4]; | |
| 2066 | + __le32 flow_label; | |
| 2067 | + __le16 sgid_index; | |
| 2068 | + u8 hop_limit; | |
| 2069 | + u8 traffic_class; | |
| 2070 | + __le16 dest_mac[3]; | |
| 2071 | + __le16 path_mtu_dest_vlan_id; | |
| 2072 | + #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK 0xfffUL | |
| 2073 | + #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0 | |
| 2074 | + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK 0xf000UL | |
| 2075 | + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT 12 | |
| 2076 | + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 (0x0UL << 12) | |
| 2077 | + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 (0x1UL << 12) | |
| 2078 | + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 (0x2UL << 12) | |
| 2079 | + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 (0x3UL << 12) | |
| 2080 | + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 (0x4UL << 12) | |
| 2081 | + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 (0x5UL << 12) | |
| 2082 | + u8 timeout; | |
| 2083 | + u8 retry_cnt; | |
| 2084 | + u8 rnr_retry; | |
| 2085 | + u8 min_rnr_timer; | |
| 2086 | + __le32 rq_psn; | |
| 2087 | + __le32 sq_psn; | |
| 2088 | + u8 max_rd_atomic; | |
| 2089 | + u8 max_dest_rd_atomic; | |
| 2090 | + u8 tos_dscp_tos_ecn; | |
| 2091 | + #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK 0x3UL | |
| 2092 | + #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT 0 | |
| 2093 | + #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK 0xfcUL | |
| 2094 | + #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2 | |
| 2095 | + u8 enable_cc; | |
| 2096 | + #define CREQ_QUERY_QP_RESP_SB_ENABLE_CC 0x1UL | |
| 2097 | + #define CREQ_QUERY_QP_RESP_SB_RESERVED7_MASK 0xfeUL | |
| 2098 | + #define CREQ_QUERY_QP_RESP_SB_RESERVED7_SFT 1 | |
| 2099 | + __le32 sq_size; | |
| 2100 | + __le32 rq_size; | |
| 2101 | + __le16 sq_sge; | |
| 2102 | + __le16 rq_sge; | |
| 2103 | + __le32 max_inline_data; | |
| 2104 | + __le32 dest_qp_id; | |
| 2105 | + __le32 unused_1; | |
| 2106 | + __le16 src_mac[3]; | |
| 2107 | + __le16 vlan_pcp_vlan_dei_vlan_id; | |
| 2108 | + #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK 0xfffUL | |
| 2109 | + #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT 0 | |
| 2110 | + #define CREQ_QUERY_QP_RESP_SB_VLAN_DEI 0x1000UL | |
| 2111 | + #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK 0xe000UL | |
| 2112 | + #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13 | |
| 2113 | +}; | |
| 2114 | + | |
| 2115 | +/* Create SRQ command response (16 bytes) */ | |
| 2116 | +struct creq_create_srq_resp { | |
| 2117 | + u8 type; | |
| 2118 | + #define CREQ_CREATE_SRQ_RESP_TYPE_MASK 0x3fUL | |
| 2119 | + #define CREQ_CREATE_SRQ_RESP_TYPE_SFT 0 | |
| 2120 | + #define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT 0x38UL | |
| 2121 | + #define CREQ_CREATE_SRQ_RESP_RESERVED2_MASK 0xc0UL | |
| 2122 | + #define CREQ_CREATE_SRQ_RESP_RESERVED2_SFT 6 | |
| 2123 | + u8 status; | |
| 2124 | + __le16 cookie; | |
| 2125 | + __le32 xid; | |
| 2126 | + u8 v; | |
| 2127 | + #define CREQ_CREATE_SRQ_RESP_V 0x1UL | |
| 2128 | + #define CREQ_CREATE_SRQ_RESP_RESERVED7_MASK 0xfeUL | |
| 2129 | + #define CREQ_CREATE_SRQ_RESP_RESERVED7_SFT 1 | |
| 2130 | + u8 event; | |
| 2131 | + #define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ 0x5UL | |
| 2132 | + __le16 reserved48[3]; | |
| 2133 | +}; | |
| 2134 | + | |
| 2135 | +/* Destroy SRQ command response (16 bytes) */ | |
| 2136 | +struct creq_destroy_srq_resp { | |
| 2137 | + u8 type; | |
| 2138 | + #define CREQ_DESTROY_SRQ_RESP_TYPE_MASK 0x3fUL | |
| 2139 | + #define CREQ_DESTROY_SRQ_RESP_TYPE_SFT 0 | |
| 2140 | + #define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT 0x38UL | |
| 2141 | + #define CREQ_DESTROY_SRQ_RESP_RESERVED2_MASK 0xc0UL | |
| 2142 | + #define CREQ_DESTROY_SRQ_RESP_RESERVED2_SFT 6 | |
| 2143 | + u8 status; | |
| 2144 | + __le16 cookie; | |
| 2145 | + __le32 xid; | |
| 2146 | + u8 v; | |
| 2147 | + #define CREQ_DESTROY_SRQ_RESP_V 0x1UL | |
| 2148 | + #define CREQ_DESTROY_SRQ_RESP_RESERVED7_MASK 0xfeUL | |
| 2149 | + #define CREQ_DESTROY_SRQ_RESP_RESERVED7_SFT 1 | |
| 2150 | + u8 event; | |
| 2151 | + #define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ 0x6UL | |
| 2152 | + __le16 enable_for_arm[3]; | |
| 2153 | + #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK 0x30000UL | |
| 2154 | + #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16 | |
| 2155 | + #define CREQ_DESTROY_SRQ_RESP_RESERVED46_MASK 0xfffc0000UL | |
| 2156 | + #define CREQ_DESTROY_SRQ_RESP_RESERVED46_SFT 18 | |
| 2157 | +}; | |
| 2158 | + | |
| 2159 | +/* Query SRQ command response (16 bytes) */ | |
| 2160 | +struct creq_query_srq_resp { | |
| 2161 | + u8 type; | |
| 2162 | + #define CREQ_QUERY_SRQ_RESP_TYPE_MASK 0x3fUL | |
| 2163 | + #define CREQ_QUERY_SRQ_RESP_TYPE_SFT 0 | |
| 2164 | + #define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT 0x38UL | |
| 2165 | + #define CREQ_QUERY_SRQ_RESP_RESERVED2_MASK 0xc0UL | |
| 2166 | + #define CREQ_QUERY_SRQ_RESP_RESERVED2_SFT 6 | |
| 2167 | + u8 status; | |
| 2168 | + __le16 cookie; | |
| 2169 | + __le32 size; | |
| 2170 | + u8 v; | |
| 2171 | + #define CREQ_QUERY_SRQ_RESP_V 0x1UL | |
| 2172 | + #define CREQ_QUERY_SRQ_RESP_RESERVED7_MASK 0xfeUL | |
| 2173 | + #define CREQ_QUERY_SRQ_RESP_RESERVED7_SFT 1 | |
| 2174 | + u8 event; | |
| 2175 | + #define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ 0x8UL | |
| 2176 | + __le16 reserved48[3]; | |
| 2177 | +}; | |
| 2178 | + | |
| 2179 | +/* Query SRQ command response side buffer structure (24 bytes) */ | |
| 2180 | +struct creq_query_srq_resp_sb { | |
| 2181 | + u8 opcode; | |
| 2182 | + #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ 0x8UL | |
| 2183 | + u8 status; | |
| 2184 | + __le16 cookie; | |
| 2185 | + __le16 flags; | |
| 2186 | + u8 resp_size; | |
| 2187 | + u8 reserved8; | |
| 2188 | + __le32 xid; | |
| 2189 | + __le16 srq_limit; | |
| 2190 | + __le16 reserved16; | |
| 2191 | + __le32 data[4]; | |
| 2192 | +}; | |
| 2193 | + | |
| 2194 | +/* Create CQ command Response (16 bytes) */ | |
| 2195 | +struct creq_create_cq_resp { | |
| 2196 | + u8 type; | |
| 2197 | + #define CREQ_CREATE_CQ_RESP_TYPE_MASK 0x3fUL | |
| 2198 | + #define CREQ_CREATE_CQ_RESP_TYPE_SFT 0 | |
| 2199 | + #define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT 0x38UL | |
| 2200 | + #define CREQ_CREATE_CQ_RESP_RESERVED2_MASK 0xc0UL | |
| 2201 | + #define CREQ_CREATE_CQ_RESP_RESERVED2_SFT 6 | |
| 2202 | + u8 status; | |
| 2203 | + __le16 cookie; | |
| 2204 | + __le32 xid; | |
| 2205 | + u8 v; | |
| 2206 | + #define CREQ_CREATE_CQ_RESP_V 0x1UL | |
| 2207 | + #define CREQ_CREATE_CQ_RESP_RESERVED7_MASK 0xfeUL | |
| 2208 | + #define CREQ_CREATE_CQ_RESP_RESERVED7_SFT 1 | |
| 2209 | + u8 event; | |
| 2210 | + #define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ 0x9UL | |
| 2211 | + __le16 reserved48[3]; | |
| 2212 | +}; | |
| 2213 | + | |
| 2214 | +/* Destroy CQ command response (16 bytes) */ | |
| 2215 | +struct creq_destroy_cq_resp { | |
| 2216 | + u8 type; | |
| 2217 | + #define CREQ_DESTROY_CQ_RESP_TYPE_MASK 0x3fUL | |
| 2218 | + #define CREQ_DESTROY_CQ_RESP_TYPE_SFT 0 | |
| 2219 | + #define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT 0x38UL | |
| 2220 | + #define CREQ_DESTROY_CQ_RESP_RESERVED2_MASK 0xc0UL | |
| 2221 | + #define CREQ_DESTROY_CQ_RESP_RESERVED2_SFT 6 | |
| 2222 | + u8 status; | |
| 2223 | + __le16 cookie; | |
| 2224 | + __le32 xid; | |
| 2225 | + u8 v; | |
| 2226 | + #define CREQ_DESTROY_CQ_RESP_V 0x1UL | |
| 2227 | + #define CREQ_DESTROY_CQ_RESP_RESERVED7_MASK 0xfeUL | |
| 2228 | + #define CREQ_DESTROY_CQ_RESP_RESERVED7_SFT 1 | |
| 2229 | + u8 event; | |
| 2230 | + #define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ 0xaUL | |
| 2231 | + __le16 cq_arm_lvl; | |
| 2232 | + #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK 0x3UL | |
| 2233 | + #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0 | |
| 2234 | + #define CREQ_DESTROY_CQ_RESP_RESERVED14_MASK 0xfffcUL | |
| 2235 | + #define CREQ_DESTROY_CQ_RESP_RESERVED14_SFT 2 | |
| 2236 | + __le16 total_cnq_events; | |
| 2237 | + __le16 reserved16; | |
| 2238 | +}; | |
| 2239 | + | |
| 2240 | +/* Resize CQ command response (16 bytes) */ | |
| 2241 | +struct creq_resize_cq_resp { | |
| 2242 | + u8 type; | |
| 2243 | + #define CREQ_RESIZE_CQ_RESP_TYPE_MASK 0x3fUL | |
| 2244 | + #define CREQ_RESIZE_CQ_RESP_TYPE_SFT 0 | |
| 2245 | + #define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT 0x38UL | |
| 2246 | + #define CREQ_RESIZE_CQ_RESP_RESERVED2_MASK 0xc0UL | |
| 2247 | + #define CREQ_RESIZE_CQ_RESP_RESERVED2_SFT 6 | |
| 2248 | + u8 status; | |
| 2249 | + __le16 cookie; | |
| 2250 | + __le32 xid; | |
| 2251 | + u8 v; | |
| 2252 | + #define CREQ_RESIZE_CQ_RESP_V 0x1UL | |
| 2253 | + #define CREQ_RESIZE_CQ_RESP_RESERVED7_MASK 0xfeUL | |
| 2254 | + #define CREQ_RESIZE_CQ_RESP_RESERVED7_SFT 1 | |
| 2255 | + u8 event; | |
| 2256 | + #define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ 0xcUL | |
| 2257 | + __le16 reserved48[3]; | |
| 2258 | +}; | |
| 2259 | + | |
| 2260 | +/* Allocate MRW command response (16 bytes) */ | |
| 2261 | +struct creq_allocate_mrw_resp { | |
| 2262 | + u8 type; | |
| 2263 | + #define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK 0x3fUL | |
| 2264 | + #define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT 0 | |
| 2265 | + #define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT 0x38UL | |
| 2266 | + #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_MASK 0xc0UL | |
| 2267 | + #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_SFT 6 | |
| 2268 | + u8 status; | |
| 2269 | + __le16 cookie; | |
| 2270 | + __le32 xid; | |
| 2271 | + u8 v; | |
| 2272 | + #define CREQ_ALLOCATE_MRW_RESP_V 0x1UL | |
| 2273 | + #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_MASK 0xfeUL | |
| 2274 | + #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_SFT 1 | |
| 2275 | + u8 event; | |
| 2276 | + #define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW 0xdUL | |
| 2277 | + __le16 reserved48[3]; | |
| 2278 | +}; | |
| 2279 | + | |
| 2280 | +/* De-allocate key command response (16 bytes) */ | |
| 2281 | +struct creq_deallocate_key_resp { | |
| 2282 | + u8 type; | |
| 2283 | + #define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK 0x3fUL | |
| 2284 | + #define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT 0 | |
| 2285 | + #define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT 0x38UL | |
| 2286 | + #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_MASK 0xc0UL | |
| 2287 | + #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_SFT 6 | |
| 2288 | + u8 status; | |
| 2289 | + __le16 cookie; | |
| 2290 | + __le32 xid; | |
| 2291 | + u8 v; | |
| 2292 | + #define CREQ_DEALLOCATE_KEY_RESP_V 0x1UL | |
| 2293 | + #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_MASK 0xfeUL | |
| 2294 | + #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_SFT 1 | |
| 2295 | + u8 event; | |
| 2296 | + #define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY 0xeUL | |
| 2297 | + __le16 reserved16; | |
| 2298 | + __le32 bound_window_info; | |
| 2299 | +}; | |
| 2300 | + | |
| 2301 | +/* Register MR command response (16 bytes) */ | |
| 2302 | +struct creq_register_mr_resp { | |
| 2303 | + u8 type; | |
| 2304 | + #define CREQ_REGISTER_MR_RESP_TYPE_MASK 0x3fUL | |
| 2305 | + #define CREQ_REGISTER_MR_RESP_TYPE_SFT 0 | |
| 2306 | + #define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL | |
| 2307 | + #define CREQ_REGISTER_MR_RESP_RESERVED2_MASK 0xc0UL | |
| 2308 | + #define CREQ_REGISTER_MR_RESP_RESERVED2_SFT 6 | |
| 2309 | + u8 status; | |
| 2310 | + __le16 cookie; | |
| 2311 | + __le32 xid; | |
| 2312 | + u8 v; | |
| 2313 | + #define CREQ_REGISTER_MR_RESP_V 0x1UL | |
| 2314 | + #define CREQ_REGISTER_MR_RESP_RESERVED7_MASK 0xfeUL | |
| 2315 | + #define CREQ_REGISTER_MR_RESP_RESERVED7_SFT 1 | |
| 2316 | + u8 event; | |
| 2317 | + #define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR 0xfUL | |
| 2318 | + __le16 reserved48[3]; | |
| 2319 | +}; | |
| 2320 | + | |
| 2321 | +/* Deregister MR command response (16 bytes) */ | |
| 2322 | +struct creq_deregister_mr_resp { | |
| 2323 | + u8 type; | |
| 2324 | + #define CREQ_DEREGISTER_MR_RESP_TYPE_MASK 0x3fUL | |
| 2325 | + #define CREQ_DEREGISTER_MR_RESP_TYPE_SFT 0 | |
| 2326 | + #define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL | |
| 2327 | + #define CREQ_DEREGISTER_MR_RESP_RESERVED2_MASK 0xc0UL | |
| 2328 | + #define CREQ_DEREGISTER_MR_RESP_RESERVED2_SFT 6 | |
| 2329 | + u8 status; | |
| 2330 | + __le16 cookie; | |
| 2331 | + __le32 xid; | |
| 2332 | + u8 v; | |
| 2333 | + #define CREQ_DEREGISTER_MR_RESP_V 0x1UL | |
| 2334 | + #define CREQ_DEREGISTER_MR_RESP_RESERVED7_MASK 0xfeUL | |
| 2335 | + #define CREQ_DEREGISTER_MR_RESP_RESERVED7_SFT 1 | |
| 2336 | + u8 event; | |
| 2337 | + #define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR 0x10UL | |
| 2338 | + __le16 reserved16; | |
| 2339 | + __le32 bound_windows; | |
| 2340 | +}; | |
| 2341 | + | |
| 2342 | +/* Add GID command response (16 bytes) */ | |
| 2343 | +struct creq_add_gid_resp { | |
| 2344 | + u8 type; | |
| 2345 | + #define CREQ_ADD_GID_RESP_TYPE_MASK 0x3fUL | |
| 2346 | + #define CREQ_ADD_GID_RESP_TYPE_SFT 0 | |
| 2347 | + #define CREQ_ADD_GID_RESP_TYPE_QP_EVENT 0x38UL | |
| 2348 | + #define CREQ_ADD_GID_RESP_RESERVED2_MASK 0xc0UL | |
| 2349 | + #define CREQ_ADD_GID_RESP_RESERVED2_SFT 6 | |
| 2350 | + u8 status; | |
| 2351 | + __le16 cookie; | |
| 2352 | + __le32 xid; | |
| 2353 | + u8 v; | |
| 2354 | + #define CREQ_ADD_GID_RESP_V 0x1UL | |
| 2355 | + #define CREQ_ADD_GID_RESP_RESERVED7_MASK 0xfeUL | |
| 2356 | + #define CREQ_ADD_GID_RESP_RESERVED7_SFT 1 | |
| 2357 | + u8 event; | |
| 2358 | + #define CREQ_ADD_GID_RESP_EVENT_ADD_GID 0x11UL | |
| 2359 | + __le16 reserved48[3]; | |
| 2360 | +}; | |
| 2361 | + | |
| 2362 | +/* Delete GID command response (16 bytes) */ | |
| 2363 | +struct creq_delete_gid_resp { | |
| 2364 | + u8 type; | |
| 2365 | + #define CREQ_DELETE_GID_RESP_TYPE_MASK 0x3fUL | |
| 2366 | + #define CREQ_DELETE_GID_RESP_TYPE_SFT 0 | |
| 2367 | + #define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT 0x38UL | |
| 2368 | + #define CREQ_DELETE_GID_RESP_RESERVED2_MASK 0xc0UL | |
| 2369 | + #define CREQ_DELETE_GID_RESP_RESERVED2_SFT 6 | |
| 2370 | + u8 status; | |
| 2371 | + __le16 cookie; | |
| 2372 | + __le32 xid; | |
| 2373 | + u8 v; | |
| 2374 | + #define CREQ_DELETE_GID_RESP_V 0x1UL | |
| 2375 | + #define CREQ_DELETE_GID_RESP_RESERVED7_MASK 0xfeUL | |
| 2376 | + #define CREQ_DELETE_GID_RESP_RESERVED7_SFT 1 | |
| 2377 | + u8 event; | |
| 2378 | + #define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID 0x12UL | |
| 2379 | + __le16 reserved48[3]; | |
| 2380 | +}; | |
| 2381 | + | |
| 2382 | +/* Modify GID command response (16 bytes) */ | |
| 2383 | +struct creq_modify_gid_resp { | |
| 2384 | + u8 type; | |
| 2385 | + #define CREQ_MODIFY_GID_RESP_TYPE_MASK 0x3fUL | |
| 2386 | + #define CREQ_MODIFY_GID_RESP_TYPE_SFT 0 | |
| 2387 | + #define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT 0x38UL | |
| 2388 | + #define CREQ_MODIFY_GID_RESP_RESERVED2_MASK 0xc0UL | |
| 2389 | + #define CREQ_MODIFY_GID_RESP_RESERVED2_SFT 6 | |
| 2390 | + u8 status; | |
| 2391 | + __le16 cookie; | |
| 2392 | + __le32 xid; | |
| 2393 | + u8 v; | |
| 2394 | + #define CREQ_MODIFY_GID_RESP_V 0x1UL | |
| 2395 | + #define CREQ_MODIFY_GID_RESP_RESERVED7_MASK 0xfeUL | |
| 2396 | + #define CREQ_MODIFY_GID_RESP_RESERVED7_SFT 1 | |
| 2397 | + u8 event; | |
| 2398 | + #define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID 0x11UL | |
| 2399 | + __le16 reserved48[3]; | |
| 2400 | +}; | |
| 2401 | + | |
| 2402 | +/* Query GID command response (16 bytes) */ | |
| 2403 | +struct creq_query_gid_resp { | |
| 2404 | + u8 type; | |
| 2405 | + #define CREQ_QUERY_GID_RESP_TYPE_MASK 0x3fUL | |
| 2406 | + #define CREQ_QUERY_GID_RESP_TYPE_SFT 0 | |
| 2407 | + #define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT 0x38UL | |
| 2408 | + #define CREQ_QUERY_GID_RESP_RESERVED2_MASK 0xc0UL | |
| 2409 | + #define CREQ_QUERY_GID_RESP_RESERVED2_SFT 6 | |
| 2410 | + u8 status; | |
| 2411 | + __le16 cookie; | |
| 2412 | + __le32 size; | |
| 2413 | + u8 v; | |
| 2414 | + #define CREQ_QUERY_GID_RESP_V 0x1UL | |
| 2415 | + #define CREQ_QUERY_GID_RESP_RESERVED7_MASK 0xfeUL | |
| 2416 | + #define CREQ_QUERY_GID_RESP_RESERVED7_SFT 1 | |
| 2417 | + u8 event; | |
| 2418 | + #define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID 0x18UL | |
| 2419 | + __le16 reserved48[3]; | |
| 2420 | +}; | |
| 2421 | + | |
| 2422 | +/* Query GID command response side buffer structure (40 bytes) */ | |
| 2423 | +struct creq_query_gid_resp_sb { | |
| 2424 | + u8 opcode; | |
| 2425 | + #define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID 0x18UL | |
| 2426 | + u8 status; | |
| 2427 | + __le16 cookie; | |
| 2428 | + __le16 flags; | |
| 2429 | + u8 resp_size; | |
| 2430 | + u8 reserved8; | |
| 2431 | + __le32 gid[4]; | |
| 2432 | + __le16 src_mac[3]; | |
| 2433 | + __le16 vlan; | |
| 2434 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK 0xfffUL | |
| 2435 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT 0 | |
| 2436 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK 0x7000UL | |
| 2437 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT 12 | |
| 2438 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 (0x0UL << 12) | |
| 2439 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 (0x1UL << 12) | |
| 2440 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 (0x2UL << 12) | |
| 2441 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 (0x3UL << 12) | |
| 2442 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 (0x4UL << 12) | |
| 2443 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 (0x5UL << 12) | |
| 2444 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 (0x6UL << 12) | |
| 2445 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 (0x7UL << 12) | |
| 2446 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST \ | |
| 2447 | + CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 | |
| 2448 | + #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN 0x8000UL | |
| 2449 | + __le16 ipid; | |
| 2450 | + __le16 gid_index; | |
| 2451 | + __le32 unused_0; | |
| 2452 | +}; | |
| 2453 | + | |
| 2454 | +/* Create QP1 command response (16 bytes) */ | |
| 2455 | +struct creq_create_qp1_resp { | |
| 2456 | + u8 type; | |
| 2457 | + #define CREQ_CREATE_QP1_RESP_TYPE_MASK 0x3fUL | |
| 2458 | + #define CREQ_CREATE_QP1_RESP_TYPE_SFT 0 | |
| 2459 | + #define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT 0x38UL | |
| 2460 | + #define CREQ_CREATE_QP1_RESP_RESERVED2_MASK 0xc0UL | |
| 2461 | + #define CREQ_CREATE_QP1_RESP_RESERVED2_SFT 6 | |
| 2462 | + u8 status; | |
| 2463 | + __le16 cookie; | |
| 2464 | + __le32 xid; | |
| 2465 | + u8 v; | |
| 2466 | + #define CREQ_CREATE_QP1_RESP_V 0x1UL | |
| 2467 | + #define CREQ_CREATE_QP1_RESP_RESERVED7_MASK 0xfeUL | |
| 2468 | + #define CREQ_CREATE_QP1_RESP_RESERVED7_SFT 1 | |
| 2469 | + u8 event; | |
| 2470 | + #define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 0x13UL | |
| 2471 | + __le16 reserved48[3]; | |
| 2472 | +}; | |
| 2473 | + | |
| 2474 | +/* Destroy QP1 command response (16 bytes) */ | |
| 2475 | +struct creq_destroy_qp1_resp { | |
| 2476 | + u8 type; | |
| 2477 | + #define CREQ_DESTROY_QP1_RESP_TYPE_MASK 0x3fUL | |
| 2478 | + #define CREQ_DESTROY_QP1_RESP_TYPE_SFT 0 | |
| 2479 | + #define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT 0x38UL | |
| 2480 | + #define CREQ_DESTROY_QP1_RESP_RESERVED2_MASK 0xc0UL | |
| 2481 | + #define CREQ_DESTROY_QP1_RESP_RESERVED2_SFT 6 | |
| 2482 | + u8 status; | |
| 2483 | + __le16 cookie; | |
| 2484 | + __le32 xid; | |
| 2485 | + u8 v; | |
| 2486 | + #define CREQ_DESTROY_QP1_RESP_V 0x1UL | |
| 2487 | + #define CREQ_DESTROY_QP1_RESP_RESERVED7_MASK 0xfeUL | |
| 2488 | + #define CREQ_DESTROY_QP1_RESP_RESERVED7_SFT 1 | |
| 2489 | + u8 event; | |
| 2490 | + #define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 0x14UL | |
| 2491 | + __le16 reserved48[3]; | |
| 2492 | +}; | |
| 2493 | + | |
| 2494 | +/* Create AH command response (16 bytes) */ | |
| 2495 | +struct creq_create_ah_resp { | |
| 2496 | + u8 type; | |
| 2497 | + #define CREQ_CREATE_AH_RESP_TYPE_MASK 0x3fUL | |
| 2498 | + #define CREQ_CREATE_AH_RESP_TYPE_SFT 0 | |
| 2499 | + #define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT 0x38UL | |
| 2500 | + #define CREQ_CREATE_AH_RESP_RESERVED2_MASK 0xc0UL | |
| 2501 | + #define CREQ_CREATE_AH_RESP_RESERVED2_SFT 6 | |
| 2502 | + u8 status; | |
| 2503 | + __le16 cookie; | |
| 2504 | + __le32 xid; | |
| 2505 | + u8 v; | |
| 2506 | + #define CREQ_CREATE_AH_RESP_V 0x1UL | |
| 2507 | + #define CREQ_CREATE_AH_RESP_RESERVED7_MASK 0xfeUL | |
| 2508 | + #define CREQ_CREATE_AH_RESP_RESERVED7_SFT 1 | |
| 2509 | + u8 event; | |
| 2510 | + #define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH 0x15UL | |
| 2511 | + __le16 reserved48[3]; | |
| 2512 | +}; | |
| 2513 | + | |
| 2514 | +/* Destroy AH command response (16 bytes) */ | |
| 2515 | +struct creq_destroy_ah_resp { | |
| 2516 | + u8 type; | |
| 2517 | + #define CREQ_DESTROY_AH_RESP_TYPE_MASK 0x3fUL | |
| 2518 | + #define CREQ_DESTROY_AH_RESP_TYPE_SFT 0 | |
| 2519 | + #define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT 0x38UL | |
| 2520 | + #define CREQ_DESTROY_AH_RESP_RESERVED2_MASK 0xc0UL | |
| 2521 | + #define CREQ_DESTROY_AH_RESP_RESERVED2_SFT 6 | |
| 2522 | + u8 status; | |
| 2523 | + __le16 cookie; | |
| 2524 | + __le32 xid; | |
| 2525 | + u8 v; | |
| 2526 | + #define CREQ_DESTROY_AH_RESP_V 0x1UL | |
| 2527 | + #define CREQ_DESTROY_AH_RESP_RESERVED7_MASK 0xfeUL | |
| 2528 | + #define CREQ_DESTROY_AH_RESP_RESERVED7_SFT 1 | |
| 2529 | + u8 event; | |
| 2530 | + #define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH 0x16UL | |
| 2531 | + __le16 reserved48[3]; | |
| 2532 | +}; | |
| 2533 | + | |
| 2534 | +/* Initialize Firmware command response (16 bytes) */ | |
| 2535 | +struct creq_initialize_fw_resp { | |
| 2536 | + u8 type; | |
| 2537 | + #define CREQ_INITIALIZE_FW_RESP_TYPE_MASK 0x3fUL | |
| 2538 | + #define CREQ_INITIALIZE_FW_RESP_TYPE_SFT 0 | |
| 2539 | + #define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL | |
| 2540 | + #define CREQ_INITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL | |
| 2541 | + #define CREQ_INITIALIZE_FW_RESP_RESERVED2_SFT 6 | |
| 2542 | + u8 status; | |
| 2543 | + __le16 cookie; | |
| 2544 | + __le32 reserved32; | |
| 2545 | + u8 v; | |
| 2546 | + #define CREQ_INITIALIZE_FW_RESP_V 0x1UL | |
| 2547 | + #define CREQ_INITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL | |
| 2548 | + #define CREQ_INITIALIZE_FW_RESP_RESERVED7_SFT 1 | |
| 2549 | + u8 event; | |
| 2550 | + #define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW 0x80UL | |
| 2551 | + __le16 reserved48[3]; | |
| 2552 | +}; | |
| 2553 | + | |
| 2554 | +/* De-initialize Firmware command response (16 bytes) */ | |
| 2555 | +struct creq_deinitialize_fw_resp { | |
| 2556 | + u8 type; | |
| 2557 | + #define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK 0x3fUL | |
| 2558 | + #define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT 0 | |
| 2559 | + #define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL | |
| 2560 | + #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL | |
| 2561 | + #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_SFT 6 | |
| 2562 | + u8 status; | |
| 2563 | + __le16 cookie; | |
| 2564 | + __le32 reserved32; | |
| 2565 | + u8 v; | |
| 2566 | + #define CREQ_DEINITIALIZE_FW_RESP_V 0x1UL | |
| 2567 | + #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL | |
| 2568 | + #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_SFT 1 | |
| 2569 | + u8 event; | |
| 2570 | + #define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW 0x81UL | |
| 2571 | + __le16 reserved48[3]; | |
| 2572 | +}; | |
| 2573 | + | |
| 2574 | +/* Stop function command response (16 bytes) */ | |
| 2575 | +struct creq_stop_func_resp { | |
| 2576 | + u8 type; | |
| 2577 | + #define CREQ_STOP_FUNC_RESP_TYPE_MASK 0x3fUL | |
| 2578 | + #define CREQ_STOP_FUNC_RESP_TYPE_SFT 0 | |
| 2579 | + #define CREQ_STOP_FUNC_RESP_TYPE_QP_EVENT 0x38UL | |
| 2580 | + #define CREQ_STOP_FUNC_RESP_RESERVED2_MASK 0xc0UL | |
| 2581 | + #define CREQ_STOP_FUNC_RESP_RESERVED2_SFT 6 | |
| 2582 | + u8 status; | |
| 2583 | + __le16 cookie; | |
| 2584 | + __le32 reserved32; | |
| 2585 | + u8 v; | |
| 2586 | + #define CREQ_STOP_FUNC_RESP_V 0x1UL | |
| 2587 | + #define CREQ_STOP_FUNC_RESP_RESERVED7_MASK 0xfeUL | |
| 2588 | + #define CREQ_STOP_FUNC_RESP_RESERVED7_SFT 1 | |
| 2589 | + u8 event; | |
| 2590 | + #define CREQ_STOP_FUNC_RESP_EVENT_STOP_FUNC 0x82UL | |
| 2591 | + __le16 reserved48[3]; | |
| 2592 | +}; | |
| 2593 | + | |
| 2594 | +/* Query function command response (16 bytes) */ | |
| 2595 | +struct creq_query_func_resp { | |
| 2596 | + u8 type; | |
| 2597 | + #define CREQ_QUERY_FUNC_RESP_TYPE_MASK 0x3fUL | |
| 2598 | + #define CREQ_QUERY_FUNC_RESP_TYPE_SFT 0 | |
| 2599 | + #define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT 0x38UL | |
| 2600 | + #define CREQ_QUERY_FUNC_RESP_RESERVED2_MASK 0xc0UL | |
| 2601 | + #define CREQ_QUERY_FUNC_RESP_RESERVED2_SFT 6 | |
| 2602 | + u8 status; | |
| 2603 | + __le16 cookie; | |
| 2604 | + __le32 size; | |
| 2605 | + u8 v; | |
| 2606 | + #define CREQ_QUERY_FUNC_RESP_V 0x1UL | |
| 2607 | + #define CREQ_QUERY_FUNC_RESP_RESERVED7_MASK 0xfeUL | |
| 2608 | + #define CREQ_QUERY_FUNC_RESP_RESERVED7_SFT 1 | |
| 2609 | + u8 event; | |
| 2610 | + #define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC 0x83UL | |
| 2611 | + __le16 reserved48[3]; | |
| 2612 | +}; | |
| 2613 | + | |
| 2614 | +/* Query function command response side buffer structure (88 bytes) */ | |
| 2615 | +struct creq_query_func_resp_sb { | |
| 2616 | + u8 opcode; | |
| 2617 | + #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC 0x83UL | |
| 2618 | + u8 status; | |
| 2619 | + __le16 cookie; | |
| 2620 | + __le16 flags; | |
| 2621 | + u8 resp_size; | |
| 2622 | + u8 reserved8; | |
| 2623 | + __le64 max_mr_size; | |
| 2624 | + __le32 max_qp; | |
| 2625 | + __le16 max_qp_wr; | |
| 2626 | + __le16 dev_cap_flags; | |
| 2627 | + #define CREQ_QUERY_FUNC_RESP_SB_DEV_CAP_FLAGS_RESIZE_QP 0x1UL | |
| 2628 | + __le32 max_cq; | |
| 2629 | + __le32 max_cqe; | |
| 2630 | + __le32 max_pd; | |
| 2631 | + u8 max_sge; | |
| 2632 | + u8 max_srq_sge; | |
| 2633 | + u8 max_qp_rd_atom; | |
| 2634 | + u8 max_qp_init_rd_atom; | |
| 2635 | + __le32 max_mr; | |
| 2636 | + __le32 max_mw; | |
| 2637 | + __le32 max_raw_eth_qp; | |
| 2638 | + __le32 max_ah; | |
| 2639 | + __le32 max_fmr; | |
| 2640 | + __le32 max_srq_wr; | |
| 2641 | + __le32 max_pkeys; | |
| 2642 | + __le32 max_inline_data; | |
| 2643 | + u8 max_map_per_fmr; | |
| 2644 | + u8 l2_db_space_size; | |
| 2645 | + __le16 max_srq; | |
| 2646 | + __le32 max_gid; | |
| 2647 | + __le32 tqm_alloc_reqs[8]; | |
| 2648 | +}; | |
| 2649 | + | |
| 2650 | +/* Set resources command response (16 bytes) */ | |
| 2651 | +struct creq_set_func_resources_resp { | |
| 2652 | + u8 type; | |
| 2653 | + #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK 0x3fUL | |
| 2654 | + #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT 0 | |
| 2655 | + #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT 0x38UL | |
| 2656 | + #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_MASK 0xc0UL | |
| 2657 | + #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_SFT 6 | |
| 2658 | + u8 status; | |
| 2659 | + __le16 cookie; | |
| 2660 | + __le32 reserved32; | |
| 2661 | + u8 v; | |
| 2662 | + #define CREQ_SET_FUNC_RESOURCES_RESP_V 0x1UL | |
| 2663 | + #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_MASK 0xfeUL | |
| 2664 | + #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_SFT 1 | |
| 2665 | + u8 event; | |
| 2666 | + #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL | |
| 2667 | + __le16 reserved48[3]; | |
| 2668 | +}; | |
| 2669 | + | |
| 2670 | +/* Map TC to COS response (16 bytes) */ | |
| 2671 | +struct creq_map_tc_to_cos_resp { | |
| 2672 | + u8 type; | |
| 2673 | + #define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK 0x3fUL | |
| 2674 | + #define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT 0 | |
| 2675 | + #define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT 0x38UL | |
| 2676 | + #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_MASK 0xc0UL | |
| 2677 | + #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_SFT 6 | |
| 2678 | + u8 status; | |
| 2679 | + __le16 cookie; | |
| 2680 | + __le32 reserved32; | |
| 2681 | + u8 v; | |
| 2682 | + #define CREQ_MAP_TC_TO_COS_RESP_V 0x1UL | |
| 2683 | + #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_MASK 0xfeUL | |
| 2684 | + #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_SFT 1 | |
| 2685 | + u8 event; | |
| 2686 | + #define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS 0x8aUL | |
| 2687 | + __le16 reserved48[3]; | |
| 2688 | +}; | |
| 2689 | + | |
| 2690 | +/* Query version response (16 bytes) */ | |
| 2691 | +struct creq_query_version_resp { | |
| 2692 | + u8 type; | |
| 2693 | + #define CREQ_QUERY_VERSION_RESP_TYPE_MASK 0x3fUL | |
| 2694 | + #define CREQ_QUERY_VERSION_RESP_TYPE_SFT 0 | |
| 2695 | + #define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT 0x38UL | |
| 2696 | + #define CREQ_QUERY_VERSION_RESP_RESERVED2_MASK 0xc0UL | |
| 2697 | + #define CREQ_QUERY_VERSION_RESP_RESERVED2_SFT 6 | |
| 2698 | + u8 status; | |
| 2699 | + __le16 cookie; | |
| 2700 | + u8 fw_maj; | |
| 2701 | + u8 fw_minor; | |
| 2702 | + u8 fw_bld; | |
| 2703 | + u8 fw_rsvd; | |
| 2704 | + u8 v; | |
| 2705 | + #define CREQ_QUERY_VERSION_RESP_V 0x1UL | |
| 2706 | + #define CREQ_QUERY_VERSION_RESP_RESERVED7_MASK 0xfeUL | |
| 2707 | + #define CREQ_QUERY_VERSION_RESP_RESERVED7_SFT 1 | |
| 2708 | + u8 event; | |
| 2709 | + #define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION 0x8bUL | |
| 2710 | + __le16 reserved16; | |
| 2711 | + u8 intf_maj; | |
| 2712 | + u8 intf_minor; | |
| 2713 | + u8 intf_bld; | |
| 2714 | + u8 intf_rsvd; | |
| 2715 | +}; | |
| 2716 | + | |
| 2717 | +/* Modify congestion control command response (16 bytes) */ | |
| 2718 | +struct creq_modify_cc_resp { | |
| 2719 | + u8 type; | |
| 2720 | + #define CREQ_MODIFY_CC_RESP_TYPE_MASK 0x3fUL | |
| 2721 | + #define CREQ_MODIFY_CC_RESP_TYPE_SFT 0 | |
| 2722 | + #define CREQ_MODIFY_CC_RESP_TYPE_QP_EVENT 0x38UL | |
| 2723 | + #define CREQ_MODIFY_CC_RESP_RESERVED2_MASK 0xc0UL | |
| 2724 | + #define CREQ_MODIFY_CC_RESP_RESERVED2_SFT 6 | |
| 2725 | + u8 status; | |
| 2726 | + __le16 cookie; | |
| 2727 | + __le32 reserved32; | |
| 2728 | + u8 v; | |
| 2729 | + #define CREQ_MODIFY_CC_RESP_V 0x1UL | |
| 2730 | + #define CREQ_MODIFY_CC_RESP_RESERVED7_MASK 0xfeUL | |
| 2731 | + #define CREQ_MODIFY_CC_RESP_RESERVED7_SFT 1 | |
| 2732 | + u8 event; | |
| 2733 | + #define CREQ_MODIFY_CC_RESP_EVENT_MODIFY_CC 0x8cUL | |
| 2734 | + __le16 reserved48[3]; | |
| 2735 | +}; | |
| 2736 | + | |
| 2737 | +/* Query congestion control command response (16 bytes) */ | |
| 2738 | +struct creq_query_cc_resp { | |
| 2739 | + u8 type; | |
| 2740 | + #define CREQ_QUERY_CC_RESP_TYPE_MASK 0x3fUL | |
| 2741 | + #define CREQ_QUERY_CC_RESP_TYPE_SFT 0 | |
| 2742 | + #define CREQ_QUERY_CC_RESP_TYPE_QP_EVENT 0x38UL | |
| 2743 | + #define CREQ_QUERY_CC_RESP_RESERVED2_MASK 0xc0UL | |
| 2744 | + #define CREQ_QUERY_CC_RESP_RESERVED2_SFT 6 | |
| 2745 | + u8 status; | |
| 2746 | + __le16 cookie; | |
| 2747 | + __le32 size; | |
| 2748 | + u8 v; | |
| 2749 | + #define CREQ_QUERY_CC_RESP_V 0x1UL | |
| 2750 | + #define CREQ_QUERY_CC_RESP_RESERVED7_MASK 0xfeUL | |
| 2751 | + #define CREQ_QUERY_CC_RESP_RESERVED7_SFT 1 | |
| 2752 | + u8 event; | |
| 2753 | + #define CREQ_QUERY_CC_RESP_EVENT_QUERY_CC 0x8dUL | |
| 2754 | + __le16 reserved48[3]; | |
| 2755 | +}; | |
| 2756 | + | |
| 2757 | +/* Query congestion control command response side buffer structure (32 bytes) */ | |
| 2758 | +struct creq_query_cc_resp_sb { | |
| 2759 | + u8 opcode; | |
| 2760 | + #define CREQ_QUERY_CC_RESP_SB_OPCODE_QUERY_CC 0x8dUL | |
| 2761 | + u8 status; | |
| 2762 | + __le16 cookie; | |
| 2763 | + __le16 flags; | |
| 2764 | + u8 resp_size; | |
| 2765 | + u8 reserved8; | |
| 2766 | + u8 enable_cc; | |
| 2767 | + #define CREQ_QUERY_CC_RESP_SB_ENABLE_CC 0x1UL | |
| 2768 | + u8 g; | |
| 2769 | + #define CREQ_QUERY_CC_RESP_SB_G_MASK 0x7UL | |
| 2770 | + #define CREQ_QUERY_CC_RESP_SB_G_SFT 0 | |
| 2771 | + u8 num_phases_per_state; | |
| 2772 | + __le16 init_cr; | |
| 2773 | + u8 unused_2; | |
| 2774 | + __le16 unused_3; | |
| 2775 | + u8 unused_4; | |
| 2776 | + __le16 init_tr; | |
| 2777 | + u8 tos_dscp_tos_ecn; | |
| 2778 | + #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_MASK 0x3UL | |
| 2779 | + #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_SFT 0 | |
| 2780 | + #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_MASK 0xfcUL | |
| 2781 | + #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_SFT 2 | |
| 2782 | + __le64 reserved64; | |
| 2783 | + __le64 reserved64_1; | |
| 2784 | +}; | |
| 2785 | + | |
| 2786 | +/* QP error notification event (16 bytes) */ | |
| 2787 | +struct creq_qp_error_notification { | |
| 2788 | + u8 type; | |
| 2789 | + #define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK 0x3fUL | |
| 2790 | + #define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT 0 | |
| 2791 | + #define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT 0x38UL | |
| 2792 | + #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_MASK 0xc0UL | |
| 2793 | + #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_SFT 6 | |
| 2794 | + u8 status; | |
| 2795 | + u8 req_slow_path_state; | |
| 2796 | + u8 req_err_state_reason; | |
| 2797 | + __le32 xid; | |
| 2798 | + u8 v; | |
| 2799 | + #define CREQ_QP_ERROR_NOTIFICATION_V 0x1UL | |
| 2800 | + #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_MASK 0xfeUL | |
| 2801 | + #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_SFT 1 | |
| 2802 | + u8 event; | |
| 2803 | + #define CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION 0xc0UL | |
| 2804 | + u8 res_slow_path_state; | |
| 2805 | + u8 res_err_state_reason; | |
| 2806 | + __le16 sq_cons_idx; | |
| 2807 | + __le16 rq_cons_idx; | |
| 2808 | +}; | |
| 2809 | + | |
| 2810 | +/* RoCE Slowpath HSI Specification 1.6.0 */ | |
| 2811 | +#define ROCE_SP_HSI_VERSION_MAJOR 1 | |
| 2812 | +#define ROCE_SP_HSI_VERSION_MINOR 6 | |
| 2813 | +#define ROCE_SP_HSI_VERSION_UPDATE 0 | |
| 2814 | + | |
| 2815 | +#define ROCE_SP_HSI_VERSION_STR "1.6.0" | |
| 2816 | +/* | |
| 2817 | + * Following is the signature for ROCE_SP_HSI message field that indicates not | |
| 2818 | + * applicable (All F's). Need to cast it the size of the field if needed. | |
| 2819 | + */ | |
| 2820 | +#define ROCE_SP_HSI_NA_SIGNATURE ((__le32)(-1)) | |
| 2821 | +#endif /* __BNXT_RE_HSI_H__ */ | |
| @@ -0,0 +1,89 @@ | ||
| 1 | +/* | |
| 2 | + * Broadcom NetXtreme-E RoCE driver. | |
| 3 | + * | |
| 4 | + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
| 5 | + * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
| 6 | + * | |
| 7 | + * This software is available to you under a choice of one of two | |
| 8 | + * licenses. You may choose to be licensed under the terms of the GNU | |
| 9 | + * General Public License (GPL) Version 2, available from the file | |
| 10 | + * COPYING in the main directory of this source tree, or the | |
| 11 | + * BSD license below: | |
| 12 | + * | |
| 13 | + * Redistribution and use in source and binary forms, with or without | |
| 14 | + * modification, are permitted provided that the following conditions | |
| 15 | + * are met: | |
| 16 | + * | |
| 17 | + * 1. Redistributions of source code must retain the above copyright | |
| 18 | + * notice, this list of conditions and the following disclaimer. | |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright | |
| 20 | + * notice, this list of conditions and the following disclaimer in | |
| 21 | + * the documentation and/or other materials provided with the | |
| 22 | + * distribution. | |
| 23 | + * | |
| 24 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
| 25 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 26 | + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 27 | + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
| 28 | + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 29 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 30 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
| 31 | + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
| 32 | + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
| 33 | + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
| 34 | + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 35 | + * | |
| 36 | + * Description: Uverbs ABI header file | |
| 37 | + */ | |
| 38 | + | |
| 39 | +#ifndef __BNXT_RE_UVERBS_ABI_H__ | |
| 40 | +#define __BNXT_RE_UVERBS_ABI_H__ | |
| 41 | + | |
| 42 | +#define BNXT_RE_ABI_VERSION 1 | |
| 43 | + | |
| 44 | +struct bnxt_re_uctx_resp { | |
| 45 | + __u32 dev_id; | |
| 46 | + __u32 max_qp; | |
| 47 | + __u32 pg_size; | |
| 48 | + __u32 cqe_sz; | |
| 49 | + __u32 max_cqd; | |
| 50 | + __u32 rsvd; | |
| 51 | +}; | |
| 52 | + | |
| 53 | +struct bnxt_re_pd_resp { | |
| 54 | + __u32 pdid; | |
| 55 | + __u32 dpi; | |
| 56 | + __u64 dbr; | |
| 57 | +}; | |
| 58 | + | |
| 59 | +struct bnxt_re_cq_req { | |
| 60 | + __u64 cq_va; | |
| 61 | + __u64 cq_handle; | |
| 62 | +}; | |
| 63 | + | |
| 64 | +struct bnxt_re_cq_resp { | |
| 65 | + __u32 cqid; | |
| 66 | + __u32 tail; | |
| 67 | + __u32 phase; | |
| 68 | + __u32 rsvd; | |
| 69 | +}; | |
| 70 | + | |
| 71 | +struct bnxt_re_qp_req { | |
| 72 | + __u64 qpsva; | |
| 73 | + __u64 qprva; | |
| 74 | + __u64 qp_handle; | |
| 75 | +}; | |
| 76 | + | |
| 77 | +struct bnxt_re_qp_resp { | |
| 78 | + __u32 qpid; | |
| 79 | + __u32 rsvd; | |
| 80 | +}; | |
| 81 | + | |
| 82 | +enum bnxt_re_shpg_offt { | |
| 83 | + BNXT_RE_BEG_RESV_OFFT = 0x00, | |
| 84 | + BNXT_RE_AVID_OFFT = 0x10, | |
| 85 | + BNXT_RE_AVID_SIZE = 0x04, | |
| 86 | + BNXT_RE_END_RESV_OFFT = 0xFF0 | |
| 87 | +}; | |
| 88 | + | |
| 89 | +#endif /* __BNXT_RE_UVERBS_ABI_H__*/ | |