This patch introduces the RoCE driver for the Broadcom NetXtreme-E 10/25/40/50G RoCE HCAs. The RoCE driver is a two part driver that relies on the parent bnxt_en NIC driver to operate. The changes needed in the bnxt_en driver have already been incorporated via Dave Miller's net tree into the mainline kernel. The vendor official git repository for this driver is available on github as: https://github.com/Broadcom/linux-rdma-nxt/
Unified · Split
| @@ -0,0 +1,146 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: Slow Path Operators (header) | ||
| 37 | * | ||
| 38 | */ | ||
| 39 | |||
| 40 | #ifndef __BNXT_RE_H__ | ||
| 41 | #define __BNXT_RE_H__ | ||
| 42 | #define ROCE_DRV_MODULE_NAME "bnxt_re" | ||
| 43 | #define ROCE_DRV_MODULE_VERSION "1.0.0" | ||
| 44 | |||
| 45 | #define BNXT_RE_DESC "Broadcom NetXtreme-C/E RoCE Driver" | ||
| 46 | |||
| 47 | #define BNXT_RE_PAGE_SIZE_4K BIT(12) | ||
| 48 | #define BNXT_RE_PAGE_SIZE_8K BIT(13) | ||
| 49 | #define BNXT_RE_PAGE_SIZE_64K BIT(16) | ||
| 50 | #define BNXT_RE_PAGE_SIZE_2M BIT(21) | ||
| 51 | #define BNXT_RE_PAGE_SIZE_8M BIT(23) | ||
| 52 | #define BNXT_RE_PAGE_SIZE_1G BIT(30) | ||
| 53 | |||
| 54 | #define BNXT_RE_MAX_QPC_COUNT (64 * 1024) | ||
| 55 | #define BNXT_RE_MAX_MRW_COUNT (64 * 1024) | ||
| 56 | #define BNXT_RE_MAX_SRQC_COUNT (64 * 1024) | ||
| 57 | #define BNXT_RE_MAX_CQ_COUNT (64 * 1024) | ||
| 58 | |||
| 59 | struct bnxt_re_work { | ||
| 60 | struct work_struct work; | ||
| 61 | unsigned long event; | ||
| 62 | struct bnxt_re_dev *rdev; | ||
| 63 | struct net_device *vlan_dev; | ||
| 64 | }; | ||
| 65 | |||
| 66 | struct bnxt_re_sqp_entries { | ||
| 67 | struct bnxt_qplib_sge sge; | ||
| 68 | u64 wrid; | ||
| 69 | /* For storing the actual qp1 cqe */ | ||
| 70 | struct bnxt_qplib_cqe cqe; | ||
| 71 | struct bnxt_re_qp *qp1_qp; | ||
| 72 | }; | ||
| 73 | |||
| 74 | #define BNXT_RE_MIN_MSIX 2 | ||
| 75 | #define BNXT_RE_MAX_MSIX 16 | ||
| 76 | #define BNXT_RE_AEQ_IDX 0 | ||
| 77 | #define BNXT_RE_NQ_IDX 1 | ||
| 78 | |||
| 79 | struct bnxt_re_dev { | ||
| 80 | struct ib_device ibdev; | ||
| 81 | struct list_head list; | ||
| 82 | unsigned long flags; | ||
| 83 | #define BNXT_RE_FLAG_NETDEV_REGISTERED 0 | ||
| 84 | #define BNXT_RE_FLAG_IBDEV_REGISTERED 1 | ||
| 85 | #define BNXT_RE_FLAG_GOT_MSIX 2 | ||
| 86 | #define BNXT_RE_FLAG_RCFW_CHANNEL_EN 8 | ||
| 87 | #define BNXT_RE_FLAG_QOS_WORK_REG 16 | ||
| 88 | struct net_device *netdev; | ||
| 89 | unsigned int version, major, minor; | ||
| 90 | struct bnxt_en_dev *en_dev; | ||
| 91 | struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX]; | ||
| 92 | int num_msix; | ||
| 93 | |||
| 94 | int id; | ||
| 95 | |||
| 96 | struct delayed_work worker; | ||
| 97 | u8 cur_prio_map; | ||
| 98 | |||
| 99 | /* FP Notification Queue (CQ & SRQ) */ | ||
| 100 | struct tasklet_struct nq_task; | ||
| 101 | |||
| 102 | /* RCFW Channel */ | ||
| 103 | struct bnxt_qplib_rcfw rcfw; | ||
| 104 | |||
| 105 | /* NQ */ | ||
| 106 | struct bnxt_qplib_nq nq; | ||
| 107 | |||
| 108 | /* Device Resources */ | ||
| 109 | struct bnxt_qplib_dev_attr dev_attr; | ||
| 110 | struct bnxt_qplib_ctx qplib_ctx; | ||
| 111 | struct bnxt_qplib_res qplib_res; | ||
| 112 | struct bnxt_qplib_dpi dpi_privileged; | ||
| 113 | |||
| 114 | atomic_t qp_count; | ||
| 115 | struct mutex qp_lock; /* protect qp list */ | ||
| 116 | struct list_head qp_list; | ||
| 117 | |||
| 118 | atomic_t cq_count; | ||
| 119 | atomic_t srq_count; | ||
| 120 | atomic_t mr_count; | ||
| 121 | atomic_t mw_count; | ||
| 122 | /* Max of 2 lossless traffic class supported per port */ | ||
| 123 | u16 cosq[2]; | ||
| 124 | |||
| 125 | /* QP for for handling QP1 packets */ | ||
| 126 | u32 sqp_id; | ||
| 127 | struct bnxt_re_qp *qp1_sqp; | ||
| 128 | struct bnxt_re_ah *sqp_ah; | ||
| 129 | struct bnxt_re_sqp_entries sqp_tbl[1024]; | ||
| 130 | }; | ||
| 131 | |||
| 132 | #define to_bnxt_re_dev(ptr, member) \ | ||
| 133 | container_of((ptr), struct bnxt_re_dev, member) | ||
| 134 | |||
| 135 | #define BNXT_RE_ROCE_V1_PACKET 0 | ||
| 136 | #define BNXT_RE_ROCEV2_IPV4_PACKET 2 | ||
| 137 | #define BNXT_RE_ROCEV2_IPV6_PACKET 3 | ||
| 138 | |||
| 139 | static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev) | ||
| 140 | { | ||
| 141 | if (rdev) | ||
| 142 | return &rdev->ibdev.dev; | ||
| 143 | return NULL; | ||
| 144 | } | ||
| 145 | |||
| 146 | #endif | ||
| @@ -0,0 +1,3202 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: IB Verbs interpreter | ||
| 37 | */ | ||
| 38 | |||
| 39 | #include <linux/interrupt.h> | ||
| 40 | #include <linux/types.h> | ||
| 41 | #include <linux/pci.h> | ||
| 42 | #include <linux/netdevice.h> | ||
| 43 | #include <linux/if_ether.h> | ||
| 44 | |||
| 45 | #include <rdma/ib_verbs.h> | ||
| 46 | #include <rdma/ib_user_verbs.h> | ||
| 47 | #include <rdma/ib_umem.h> | ||
| 48 | #include <rdma/ib_addr.h> | ||
| 49 | #include <rdma/ib_mad.h> | ||
| 50 | #include <rdma/ib_cache.h> | ||
| 51 | |||
| 52 | #include "bnxt_ulp.h" | ||
| 53 | |||
| 54 | #include "roce_hsi.h" | ||
| 55 | #include "qplib_res.h" | ||
| 56 | #include "qplib_sp.h" | ||
| 57 | #include "qplib_fp.h" | ||
| 58 | #include "qplib_rcfw.h" | ||
| 59 | |||
| 60 | #include "bnxt_re.h" | ||
| 61 | #include "ib_verbs.h" | ||
| 62 | #include <rdma/bnxt_re-abi.h> | ||
| 63 | |||
| 64 | static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, | ||
| 65 | struct bnxt_qplib_sge *sg_list, int num) | ||
| 66 | { | ||
| 67 | int i, total = 0; | ||
| 68 | |||
| 69 | for (i = 0; i < num; i++) { | ||
| 70 | sg_list[i].addr = ib_sg_list[i].addr; | ||
| 71 | sg_list[i].lkey = ib_sg_list[i].lkey; | ||
| 72 | sg_list[i].size = ib_sg_list[i].length; | ||
| 73 | total += sg_list[i].size; | ||
| 74 | } | ||
| 75 | return total; | ||
| 76 | } | ||
| 77 | |||
| 78 | /* Device */ | ||
| 79 | struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num) | ||
| 80 | { | ||
| 81 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 82 | struct net_device *netdev = NULL; | ||
| 83 | |||
| 84 | rcu_read_lock(); | ||
| 85 | if (rdev) | ||
| 86 | netdev = rdev->netdev; | ||
| 87 | if (netdev) | ||
| 88 | dev_hold(netdev); | ||
| 89 | |||
| 90 | rcu_read_unlock(); | ||
| 91 | return netdev; | ||
| 92 | } | ||
| 93 | |||
| 94 | int bnxt_re_query_device(struct ib_device *ibdev, | ||
| 95 | struct ib_device_attr *ib_attr, | ||
| 96 | struct ib_udata *udata) | ||
| 97 | { | ||
| 98 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 99 | struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | ||
| 100 | |||
| 101 | memset(ib_attr, 0, sizeof(*ib_attr)); | ||
| 102 | |||
| 103 | ib_attr->fw_ver = (u64)(unsigned long)(dev_attr->fw_ver); | ||
| 104 | bnxt_qplib_get_guid(rdev->netdev->dev_addr, | ||
| 105 | (u8 *)&ib_attr->sys_image_guid); | ||
| 106 | ib_attr->max_mr_size = ~0ull; | ||
| 107 | ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_8K | | ||
| 108 | BNXT_RE_PAGE_SIZE_64K | BNXT_RE_PAGE_SIZE_2M | | ||
| 109 | BNXT_RE_PAGE_SIZE_8M | BNXT_RE_PAGE_SIZE_1G; | ||
| 110 | |||
| 111 | ib_attr->vendor_id = rdev->en_dev->pdev->vendor; | ||
| 112 | ib_attr->vendor_part_id = rdev->en_dev->pdev->device; | ||
| 113 | ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; | ||
| 114 | ib_attr->max_qp = dev_attr->max_qp; | ||
| 115 | ib_attr->max_qp_wr = dev_attr->max_qp_wqes; | ||
| 116 | ib_attr->device_cap_flags = | ||
| 117 | IB_DEVICE_CURR_QP_STATE_MOD | ||
| 118 | | IB_DEVICE_RC_RNR_NAK_GEN | ||
| 119 | | IB_DEVICE_SHUTDOWN_PORT | ||
| 120 | | IB_DEVICE_SYS_IMAGE_GUID | ||
| 121 | | IB_DEVICE_LOCAL_DMA_LKEY | ||
| 122 | | IB_DEVICE_RESIZE_MAX_WR | ||
| 123 | | IB_DEVICE_PORT_ACTIVE_EVENT | ||
| 124 | | IB_DEVICE_N_NOTIFY_CQ | ||
| 125 | | IB_DEVICE_MEM_WINDOW | ||
| 126 | | IB_DEVICE_MEM_WINDOW_TYPE_2B | ||
| 127 | | IB_DEVICE_MEM_MGT_EXTENSIONS; | ||
| 128 | ib_attr->max_sge = dev_attr->max_qp_sges; | ||
| 129 | ib_attr->max_sge_rd = dev_attr->max_qp_sges; | ||
| 130 | ib_attr->max_cq = dev_attr->max_cq; | ||
| 131 | ib_attr->max_cqe = dev_attr->max_cq_wqes; | ||
| 132 | ib_attr->max_mr = dev_attr->max_mr; | ||
| 133 | ib_attr->max_pd = dev_attr->max_pd; | ||
| 134 | ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; | ||
| 135 | ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_rd_atom; | ||
| 136 | ib_attr->atomic_cap = IB_ATOMIC_HCA; | ||
| 137 | ib_attr->masked_atomic_cap = IB_ATOMIC_HCA; | ||
| 138 | |||
| 139 | ib_attr->max_ee_rd_atom = 0; | ||
| 140 | ib_attr->max_res_rd_atom = 0; | ||
| 141 | ib_attr->max_ee_init_rd_atom = 0; | ||
| 142 | ib_attr->max_ee = 0; | ||
| 143 | ib_attr->max_rdd = 0; | ||
| 144 | ib_attr->max_mw = dev_attr->max_mw; | ||
| 145 | ib_attr->max_raw_ipv6_qp = 0; | ||
| 146 | ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; | ||
| 147 | ib_attr->max_mcast_grp = 0; | ||
| 148 | ib_attr->max_mcast_qp_attach = 0; | ||
| 149 | ib_attr->max_total_mcast_qp_attach = 0; | ||
| 150 | ib_attr->max_ah = dev_attr->max_ah; | ||
| 151 | |||
| 152 | ib_attr->max_fmr = dev_attr->max_fmr; | ||
| 153 | ib_attr->max_map_per_fmr = 1; /* ? */ | ||
| 154 | |||
| 155 | ib_attr->max_srq = dev_attr->max_srq; | ||
| 156 | ib_attr->max_srq_wr = dev_attr->max_srq_wqes; | ||
| 157 | ib_attr->max_srq_sge = dev_attr->max_srq_sges; | ||
| 158 | |||
| 159 | ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; | ||
| 160 | |||
| 161 | ib_attr->max_pkeys = 1; | ||
| 162 | ib_attr->local_ca_ack_delay = 0; | ||
| 163 | return 0; | ||
| 164 | } | ||
| 165 | |||
| 166 | int bnxt_re_modify_device(struct ib_device *ibdev, | ||
| 167 | int device_modify_mask, | ||
| 168 | struct ib_device_modify *device_modify) | ||
| 169 | { | ||
| 170 | switch (device_modify_mask) { | ||
| 171 | case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: | ||
| 172 | /* Modify the GUID requires the modification of the GID table */ | ||
| 173 | /* GUID should be made as READ-ONLY */ | ||
| 174 | break; | ||
| 175 | case IB_DEVICE_MODIFY_NODE_DESC: | ||
| 176 | /* Node Desc should be made as READ-ONLY */ | ||
| 177 | break; | ||
| 178 | default: | ||
| 179 | break; | ||
| 180 | } | ||
| 181 | return 0; | ||
| 182 | } | ||
| 183 | |||
| 184 | static void __to_ib_speed_width(struct net_device *netdev, u8 *speed, u8 *width) | ||
| 185 | { | ||
| 186 | struct ethtool_link_ksettings lksettings; | ||
| 187 | u32 espeed; | ||
| 188 | |||
| 189 | if (netdev->ethtool_ops && netdev->ethtool_ops->get_link_ksettings) { | ||
| 190 | memset(&lksettings, 0, sizeof(lksettings)); | ||
| 191 | rtnl_lock(); | ||
| 192 | netdev->ethtool_ops->get_link_ksettings(netdev, &lksettings); | ||
| 193 | rtnl_unlock(); | ||
| 194 | espeed = lksettings.base.speed; | ||
| 195 | } else { | ||
| 196 | espeed = SPEED_UNKNOWN; | ||
| 197 | } | ||
| 198 | switch (espeed) { | ||
| 199 | case SPEED_1000: | ||
| 200 | *speed = IB_SPEED_SDR; | ||
| 201 | *width = IB_WIDTH_1X; | ||
| 202 | break; | ||
| 203 | case SPEED_10000: | ||
| 204 | *speed = IB_SPEED_QDR; | ||
| 205 | *width = IB_WIDTH_1X; | ||
| 206 | break; | ||
| 207 | case SPEED_20000: | ||
| 208 | *speed = IB_SPEED_DDR; | ||
| 209 | *width = IB_WIDTH_4X; | ||
| 210 | break; | ||
| 211 | case SPEED_25000: | ||
| 212 | *speed = IB_SPEED_EDR; | ||
| 213 | *width = IB_WIDTH_1X; | ||
| 214 | break; | ||
| 215 | case SPEED_40000: | ||
| 216 | *speed = IB_SPEED_QDR; | ||
| 217 | *width = IB_WIDTH_4X; | ||
| 218 | break; | ||
| 219 | case SPEED_50000: | ||
| 220 | break; | ||
| 221 | default: | ||
| 222 | *speed = IB_SPEED_SDR; | ||
| 223 | *width = IB_WIDTH_1X; | ||
| 224 | break; | ||
| 225 | } | ||
| 226 | } | ||
| 227 | |||
| 228 | /* Port */ | ||
| 229 | int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, | ||
| 230 | struct ib_port_attr *port_attr) | ||
| 231 | { | ||
| 232 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 233 | struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | ||
| 234 | |||
| 235 | memset(port_attr, 0, sizeof(*port_attr)); | ||
| 236 | |||
| 237 | if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { | ||
| 238 | port_attr->state = IB_PORT_ACTIVE; | ||
| 239 | port_attr->phys_state = 5; | ||
| 240 | } else { | ||
| 241 | port_attr->state = IB_PORT_DOWN; | ||
| 242 | port_attr->phys_state = 3; | ||
| 243 | } | ||
| 244 | port_attr->max_mtu = IB_MTU_4096; | ||
| 245 | port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); | ||
| 246 | port_attr->gid_tbl_len = dev_attr->max_sgid; | ||
| 247 | port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | | ||
| 248 | IB_PORT_DEVICE_MGMT_SUP | | ||
| 249 | IB_PORT_VENDOR_CLASS_SUP | | ||
| 250 | IB_PORT_IP_BASED_GIDS; | ||
| 251 | |||
| 252 | /* Max MSG size set to 2G for now */ | ||
| 253 | port_attr->max_msg_sz = 0x80000000; | ||
| 254 | port_attr->bad_pkey_cntr = 0; | ||
| 255 | port_attr->qkey_viol_cntr = 0; | ||
| 256 | port_attr->pkey_tbl_len = dev_attr->max_pkey; | ||
| 257 | port_attr->lid = 0; | ||
| 258 | port_attr->sm_lid = 0; | ||
| 259 | port_attr->lmc = 0; | ||
| 260 | port_attr->max_vl_num = 4; | ||
| 261 | port_attr->sm_sl = 0; | ||
| 262 | port_attr->subnet_timeout = 0; | ||
| 263 | port_attr->init_type_reply = 0; | ||
| 264 | /* call the underlying netdev's ethtool hooks to query speed settings | ||
| 265 | * for which we acquire rtnl_lock _only_ if it's registered with | ||
| 266 | * IB stack to avoid race in the NETDEV_UNREG path | ||
| 267 | */ | ||
| 268 | if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) | ||
| 269 | __to_ib_speed_width(rdev->netdev, &port_attr->active_speed, | ||
| 270 | &port_attr->active_width); | ||
| 271 | return 0; | ||
| 272 | } | ||
| 273 | |||
| 274 | int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, | ||
| 275 | int port_modify_mask, | ||
| 276 | struct ib_port_modify *port_modify) | ||
| 277 | { | ||
| 278 | switch (port_modify_mask) { | ||
| 279 | case IB_PORT_SHUTDOWN: | ||
| 280 | break; | ||
| 281 | case IB_PORT_INIT_TYPE: | ||
| 282 | break; | ||
| 283 | case IB_PORT_RESET_QKEY_CNTR: | ||
| 284 | break; | ||
| 285 | default: | ||
| 286 | break; | ||
| 287 | } | ||
| 288 | return 0; | ||
| 289 | } | ||
| 290 | |||
| 291 | int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, | ||
| 292 | struct ib_port_immutable *immutable) | ||
| 293 | { | ||
| 294 | struct ib_port_attr port_attr; | ||
| 295 | |||
| 296 | if (bnxt_re_query_port(ibdev, port_num, &port_attr)) | ||
| 297 | return -EINVAL; | ||
| 298 | |||
| 299 | immutable->pkey_tbl_len = port_attr.pkey_tbl_len; | ||
| 300 | immutable->gid_tbl_len = port_attr.gid_tbl_len; | ||
| 301 | immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; | ||
| 302 | immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; | ||
| 303 | immutable->max_mad_size = IB_MGMT_MAD_SIZE; | ||
| 304 | return 0; | ||
| 305 | } | ||
| 306 | |||
| 307 | int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, | ||
| 308 | u16 index, u16 *pkey) | ||
| 309 | { | ||
| 310 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 311 | |||
| 312 | /* Ignore port_num */ | ||
| 313 | |||
| 314 | memset(pkey, 0, sizeof(*pkey)); | ||
| 315 | return bnxt_qplib_get_pkey(&rdev->qplib_res, | ||
| 316 | &rdev->qplib_res.pkey_tbl, index, pkey); | ||
| 317 | } | ||
| 318 | |||
| 319 | int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, | ||
| 320 | int index, union ib_gid *gid) | ||
| 321 | { | ||
| 322 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 323 | int rc = 0; | ||
| 324 | |||
| 325 | /* Ignore port_num */ | ||
| 326 | memset(gid, 0, sizeof(*gid)); | ||
| 327 | rc = bnxt_qplib_get_sgid(&rdev->qplib_res, | ||
| 328 | &rdev->qplib_res.sgid_tbl, index, | ||
| 329 | (struct bnxt_qplib_gid *)gid); | ||
| 330 | return rc; | ||
| 331 | } | ||
| 332 | |||
| 333 | int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, | ||
| 334 | unsigned int index, void **context) | ||
| 335 | { | ||
| 336 | int rc = 0; | ||
| 337 | struct bnxt_re_gid_ctx *ctx, **ctx_tbl; | ||
| 338 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 339 | struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; | ||
| 340 | |||
| 341 | /* Delete the entry from the hardware */ | ||
| 342 | ctx = *context; | ||
| 343 | if (!ctx) | ||
| 344 | return -EINVAL; | ||
| 345 | |||
| 346 | if (sgid_tbl && sgid_tbl->active) { | ||
| 347 | if (ctx->idx >= sgid_tbl->max) | ||
| 348 | return -EINVAL; | ||
| 349 | ctx->refcnt--; | ||
| 350 | if (!ctx->refcnt) { | ||
| 351 | rc = bnxt_qplib_del_sgid | ||
| 352 | (sgid_tbl, | ||
| 353 | &sgid_tbl->tbl[ctx->idx], true); | ||
| 354 | if (rc) | ||
| 355 | dev_err(rdev_to_dev(rdev), | ||
| 356 | "Failed to remove GID: %#x", rc); | ||
| 357 | ctx_tbl = sgid_tbl->ctx; | ||
| 358 | ctx_tbl[ctx->idx] = NULL; | ||
| 359 | kfree(ctx); | ||
| 360 | } | ||
| 361 | } else { | ||
| 362 | return -EINVAL; | ||
| 363 | } | ||
| 364 | return rc; | ||
| 365 | } | ||
| 366 | |||
| 367 | int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, | ||
| 368 | unsigned int index, const union ib_gid *gid, | ||
| 369 | const struct ib_gid_attr *attr, void **context) | ||
| 370 | { | ||
| 371 | int rc; | ||
| 372 | u32 tbl_idx = 0; | ||
| 373 | u16 vlan_id = 0xFFFF; | ||
| 374 | struct bnxt_re_gid_ctx *ctx, **ctx_tbl; | ||
| 375 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 376 | struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; | ||
| 377 | |||
| 378 | if ((attr->ndev) && is_vlan_dev(attr->ndev)) | ||
| 379 | vlan_id = vlan_dev_vlan_id(attr->ndev); | ||
| 380 | |||
| 381 | rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)gid, | ||
| 382 | rdev->qplib_res.netdev->dev_addr, | ||
| 383 | vlan_id, true, &tbl_idx); | ||
| 384 | if (rc == -EALREADY) { | ||
| 385 | ctx_tbl = sgid_tbl->ctx; | ||
| 386 | ctx_tbl[tbl_idx]->refcnt++; | ||
| 387 | *context = ctx_tbl[tbl_idx]; | ||
| 388 | return 0; | ||
| 389 | } | ||
| 390 | |||
| 391 | if (rc < 0) { | ||
| 392 | dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc); | ||
| 393 | return rc; | ||
| 394 | } | ||
| 395 | |||
| 396 | ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); | ||
| 397 | if (!ctx) | ||
| 398 | return -ENOMEM; | ||
| 399 | ctx_tbl = sgid_tbl->ctx; | ||
| 400 | ctx->idx = tbl_idx; | ||
| 401 | ctx->refcnt = 1; | ||
| 402 | ctx_tbl[tbl_idx] = ctx; | ||
| 403 | |||
| 404 | return rc; | ||
| 405 | } | ||
| 406 | |||
| 407 | enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, | ||
| 408 | u8 port_num) | ||
| 409 | { | ||
| 410 | return IB_LINK_LAYER_ETHERNET; | ||
| 411 | } | ||
| 412 | |||
| 413 | /* Protection Domains */ | ||
| 414 | int bnxt_re_dealloc_pd(struct ib_pd *ib_pd) | ||
| 415 | { | ||
| 416 | struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | ||
| 417 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 418 | int rc; | ||
| 419 | |||
| 420 | if (ib_pd->uobject && pd->dpi.dbr) { | ||
| 421 | struct ib_ucontext *ib_uctx = ib_pd->uobject->context; | ||
| 422 | struct bnxt_re_ucontext *ucntx; | ||
| 423 | |||
| 424 | /* Free DPI only if this is the first PD allocated by the | ||
| 425 | * application and mark the context dpi as NULL | ||
| 426 | */ | ||
| 427 | ucntx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); | ||
| 428 | |||
| 429 | rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, | ||
| 430 | &rdev->qplib_res.dpi_tbl, | ||
| 431 | &pd->dpi); | ||
| 432 | if (rc) | ||
| 433 | dev_err(rdev_to_dev(rdev), "Failed to deallocate HW DPI"); | ||
| 434 | /* Don't fail, continue*/ | ||
| 435 | ucntx->dpi = NULL; | ||
| 436 | } | ||
| 437 | |||
| 438 | rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, | ||
| 439 | &rdev->qplib_res.pd_tbl, | ||
| 440 | &pd->qplib_pd); | ||
| 441 | if (rc) { | ||
| 442 | dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD"); | ||
| 443 | return rc; | ||
| 444 | } | ||
| 445 | |||
| 446 | kfree(pd); | ||
| 447 | return 0; | ||
| 448 | } | ||
| 449 | |||
| 450 | struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev, | ||
| 451 | struct ib_ucontext *ucontext, | ||
| 452 | struct ib_udata *udata) | ||
| 453 | { | ||
| 454 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 455 | struct bnxt_re_ucontext *ucntx = container_of(ucontext, | ||
| 456 | struct bnxt_re_ucontext, | ||
| 457 | ib_uctx); | ||
| 458 | struct bnxt_re_pd *pd; | ||
| 459 | int rc; | ||
| 460 | |||
| 461 | pd = kzalloc(sizeof(*pd), GFP_KERNEL); | ||
| 462 | if (!pd) | ||
| 463 | return ERR_PTR(-ENOMEM); | ||
| 464 | |||
| 465 | pd->rdev = rdev; | ||
| 466 | if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { | ||
| 467 | dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD"); | ||
| 468 | rc = -ENOMEM; | ||
| 469 | goto fail; | ||
| 470 | } | ||
| 471 | |||
| 472 | if (udata) { | ||
| 473 | struct bnxt_re_pd_resp resp; | ||
| 474 | |||
| 475 | if (!ucntx->dpi) { | ||
| 476 | /* Allocate DPI in alloc_pd to avoid failing of | ||
| 477 | * ibv_devinfo and family of application when DPIs | ||
| 478 | * are depleted. | ||
| 479 | */ | ||
| 480 | if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, | ||
| 481 | &pd->dpi, ucntx)) { | ||
| 482 | rc = -ENOMEM; | ||
| 483 | goto dbfail; | ||
| 484 | } | ||
| 485 | ucntx->dpi = &pd->dpi; | ||
| 486 | } | ||
| 487 | |||
| 488 | resp.pdid = pd->qplib_pd.id; | ||
| 489 | /* Still allow mapping this DBR to the new user PD. */ | ||
| 490 | resp.dpi = ucntx->dpi->dpi; | ||
| 491 | resp.dbr = (u64)ucntx->dpi->umdbr; | ||
| 492 | |||
| 493 | rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); | ||
| 494 | if (rc) { | ||
| 495 | dev_err(rdev_to_dev(rdev), | ||
| 496 | "Failed to copy user response\n"); | ||
| 497 | goto dbfail; | ||
| 498 | } | ||
| 499 | } | ||
| 500 | |||
| 501 | return &pd->ib_pd; | ||
| 502 | dbfail: | ||
| 503 | (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, | ||
| 504 | &pd->qplib_pd); | ||
| 505 | fail: | ||
| 506 | kfree(pd); | ||
| 507 | return ERR_PTR(rc); | ||
| 508 | } | ||
| 509 | |||
| 510 | /* Address Handles */ | ||
| 511 | int bnxt_re_destroy_ah(struct ib_ah *ib_ah) | ||
| 512 | { | ||
| 513 | struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); | ||
| 514 | struct bnxt_re_dev *rdev = ah->rdev; | ||
| 515 | int rc; | ||
| 516 | |||
| 517 | rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah); | ||
| 518 | if (rc) { | ||
| 519 | dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH"); | ||
| 520 | return rc; | ||
| 521 | } | ||
| 522 | kfree(ah); | ||
| 523 | return 0; | ||
| 524 | } | ||
| 525 | |||
| 526 | struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd, | ||
| 527 | struct ib_ah_attr *ah_attr, | ||
| 528 | struct ib_udata *udata) | ||
| 529 | { | ||
| 530 | struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | ||
| 531 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 532 | struct bnxt_re_ah *ah; | ||
| 533 | int rc; | ||
| 534 | u16 vlan_tag; | ||
| 535 | u8 nw_type; | ||
| 536 | |||
| 537 | struct ib_gid_attr sgid_attr; | ||
| 538 | |||
| 539 | if (!(ah_attr->ah_flags & IB_AH_GRH)) { | ||
| 540 | dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set"); | ||
| 541 | return ERR_PTR(-EINVAL); | ||
| 542 | } | ||
| 543 | ah = kzalloc(sizeof(*ah), GFP_ATOMIC); | ||
| 544 | if (!ah) | ||
| 545 | return ERR_PTR(-ENOMEM); | ||
| 546 | |||
| 547 | ah->rdev = rdev; | ||
| 548 | ah->qplib_ah.pd = &pd->qplib_pd; | ||
| 549 | |||
| 550 | /* Supply the configuration for the HW */ | ||
| 551 | memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw, | ||
| 552 | sizeof(union ib_gid)); | ||
| 553 | /* | ||
| 554 | * If RoCE V2 is enabled, stack will have two entries for | ||
| 555 | * each GID entry. Avoiding this duplicte entry in HW. Dividing | ||
| 556 | * the GID index by 2 for RoCE V2 | ||
| 557 | */ | ||
| 558 | ah->qplib_ah.sgid_index = ah_attr->grh.sgid_index / 2; | ||
| 559 | ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index; | ||
| 560 | ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class; | ||
| 561 | ah->qplib_ah.flow_label = ah_attr->grh.flow_label; | ||
| 562 | ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit; | ||
| 563 | ah->qplib_ah.sl = ah_attr->sl; | ||
| 564 | if (ib_pd->uobject && | ||
| 565 | !rdma_is_multicast_addr((struct in6_addr *) | ||
| 566 | ah_attr->grh.dgid.raw) && | ||
| 567 | !rdma_link_local_addr((struct in6_addr *) | ||
| 568 | ah_attr->grh.dgid.raw)) { | ||
| 569 | union ib_gid sgid; | ||
| 570 | |||
| 571 | rc = ib_get_cached_gid(&rdev->ibdev, 1, | ||
| 572 | ah_attr->grh.sgid_index, &sgid, | ||
| 573 | &sgid_attr); | ||
| 574 | if (rc) { | ||
| 575 | dev_err(rdev_to_dev(rdev), | ||
| 576 | "Failed to query gid at index %d", | ||
| 577 | ah_attr->grh.sgid_index); | ||
| 578 | goto fail; | ||
| 579 | } | ||
| 580 | if (sgid_attr.ndev) { | ||
| 581 | if (is_vlan_dev(sgid_attr.ndev)) | ||
| 582 | vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev); | ||
| 583 | dev_put(sgid_attr.ndev); | ||
| 584 | } | ||
| 585 | /* Get network header type for this GID */ | ||
| 586 | nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); | ||
| 587 | switch (nw_type) { | ||
| 588 | case RDMA_NETWORK_IPV4: | ||
| 589 | ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; | ||
| 590 | break; | ||
| 591 | case RDMA_NETWORK_IPV6: | ||
| 592 | ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; | ||
| 593 | break; | ||
| 594 | default: | ||
| 595 | ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1; | ||
| 596 | break; | ||
| 597 | } | ||
| 598 | rc = rdma_addr_find_l2_eth_by_grh(&sgid, &ah_attr->grh.dgid, | ||
| 599 | ah_attr->dmac, &vlan_tag, | ||
| 600 | &sgid_attr.ndev->ifindex, | ||
| 601 | NULL); | ||
| 602 | if (rc) { | ||
| 603 | dev_err(rdev_to_dev(rdev), "Failed to get dmac\n"); | ||
| 604 | goto fail; | ||
| 605 | } | ||
| 606 | } | ||
| 607 | |||
| 608 | memcpy(ah->qplib_ah.dmac, ah_attr->dmac, ETH_ALEN); | ||
| 609 | rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah); | ||
| 610 | if (rc) { | ||
| 611 | dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH"); | ||
| 612 | goto fail; | ||
| 613 | } | ||
| 614 | |||
| 615 | /* Write AVID to shared page. */ | ||
| 616 | if (ib_pd->uobject) { | ||
| 617 | struct ib_ucontext *ib_uctx = ib_pd->uobject->context; | ||
| 618 | struct bnxt_re_ucontext *uctx; | ||
| 619 | unsigned long flag; | ||
| 620 | u32 *wrptr; | ||
| 621 | |||
| 622 | uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); | ||
| 623 | spin_lock_irqsave(&uctx->sh_lock, flag); | ||
| 624 | wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); | ||
| 625 | *wrptr = ah->qplib_ah.id; | ||
| 626 | wmb(); /* make sure cache is updated. */ | ||
| 627 | spin_unlock_irqrestore(&uctx->sh_lock, flag); | ||
| 628 | } | ||
| 629 | |||
| 630 | return &ah->ib_ah; | ||
| 631 | |||
| 632 | fail: | ||
| 633 | kfree(ah); | ||
| 634 | return ERR_PTR(rc); | ||
| 635 | } | ||
| 636 | |||
| 637 | int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) | ||
| 638 | { | ||
| 639 | return 0; | ||
| 640 | } | ||
| 641 | |||
| 642 | int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) | ||
| 643 | { | ||
| 644 | struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); | ||
| 645 | |||
| 646 | memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data, | ||
| 647 | sizeof(union ib_gid)); | ||
| 648 | ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index; | ||
| 649 | ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class; | ||
| 650 | ah_attr->sl = ah->qplib_ah.sl; | ||
| 651 | memcpy(ah_attr->dmac, ah->qplib_ah.dmac, ETH_ALEN); | ||
| 652 | ah_attr->ah_flags = IB_AH_GRH; | ||
| 653 | ah_attr->port_num = 1; | ||
| 654 | ah_attr->static_rate = 0; | ||
| 655 | return 0; | ||
| 656 | } | ||
| 657 | |||
| 658 | /* Queue Pairs */ | ||
| 659 | int bnxt_re_destroy_qp(struct ib_qp *ib_qp) | ||
| 660 | { | ||
| 661 | struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | ||
| 662 | struct bnxt_re_dev *rdev = qp->rdev; | ||
| 663 | int rc; | ||
| 664 | |||
| 665 | rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); | ||
| 666 | if (rc) { | ||
| 667 | dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP"); | ||
| 668 | return rc; | ||
| 669 | } | ||
| 670 | if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) { | ||
| 671 | rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, | ||
| 672 | &rdev->sqp_ah->qplib_ah); | ||
| 673 | if (rc) { | ||
| 674 | dev_err(rdev_to_dev(rdev), | ||
| 675 | "Failed to destroy HW AH for shadow QP"); | ||
| 676 | return rc; | ||
| 677 | } | ||
| 678 | |||
| 679 | rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, | ||
| 680 | &rdev->qp1_sqp->qplib_qp); | ||
| 681 | if (rc) { | ||
| 682 | dev_err(rdev_to_dev(rdev), | ||
| 683 | "Failed to destroy Shadow QP"); | ||
| 684 | return rc; | ||
| 685 | } | ||
| 686 | mutex_lock(&rdev->qp_lock); | ||
| 687 | list_del(&rdev->qp1_sqp->list); | ||
| 688 | atomic_dec(&rdev->qp_count); | ||
| 689 | mutex_unlock(&rdev->qp_lock); | ||
| 690 | |||
| 691 | kfree(rdev->sqp_ah); | ||
| 692 | kfree(rdev->qp1_sqp); | ||
| 693 | } | ||
| 694 | |||
| 695 | if (qp->rumem && !IS_ERR(qp->rumem)) | ||
| 696 | ib_umem_release(qp->rumem); | ||
| 697 | if (qp->sumem && !IS_ERR(qp->sumem)) | ||
| 698 | ib_umem_release(qp->sumem); | ||
| 699 | |||
| 700 | mutex_lock(&rdev->qp_lock); | ||
| 701 | list_del(&qp->list); | ||
| 702 | atomic_dec(&rdev->qp_count); | ||
| 703 | mutex_unlock(&rdev->qp_lock); | ||
| 704 | kfree(qp); | ||
| 705 | return 0; | ||
| 706 | } | ||
| 707 | |||
| 708 | static u8 __from_ib_qp_type(enum ib_qp_type type) | ||
| 709 | { | ||
| 710 | switch (type) { | ||
| 711 | case IB_QPT_GSI: | ||
| 712 | return CMDQ_CREATE_QP1_TYPE_GSI; | ||
| 713 | case IB_QPT_RC: | ||
| 714 | return CMDQ_CREATE_QP_TYPE_RC; | ||
| 715 | case IB_QPT_UD: | ||
| 716 | return CMDQ_CREATE_QP_TYPE_UD; | ||
| 717 | default: | ||
| 718 | return IB_QPT_MAX; | ||
| 719 | } | ||
| 720 | } | ||
| 721 | |||
| 722 | static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, | ||
| 723 | struct bnxt_re_qp *qp, struct ib_udata *udata) | ||
| 724 | { | ||
| 725 | struct bnxt_re_qp_req ureq; | ||
| 726 | struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; | ||
| 727 | struct ib_umem *umem; | ||
| 728 | int bytes = 0; | ||
| 729 | struct ib_ucontext *context = pd->ib_pd.uobject->context; | ||
| 730 | struct bnxt_re_ucontext *cntx = container_of(context, | ||
| 731 | struct bnxt_re_ucontext, | ||
| 732 | ib_uctx); | ||
| 733 | if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) | ||
| 734 | return -EFAULT; | ||
| 735 | |||
| 736 | bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); | ||
| 737 | /* Consider mapping PSN search memory only for RC QPs. */ | ||
| 738 | if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) | ||
| 739 | bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search)); | ||
| 740 | bytes = PAGE_ALIGN(bytes); | ||
| 741 | umem = ib_umem_get(context, ureq.qpsva, bytes, | ||
| 742 | IB_ACCESS_LOCAL_WRITE, 1); | ||
| 743 | if (IS_ERR(umem)) | ||
| 744 | return PTR_ERR(umem); | ||
| 745 | |||
| 746 | qp->sumem = umem; | ||
| 747 | qplib_qp->sq.sglist = umem->sg_head.sgl; | ||
| 748 | qplib_qp->sq.nmap = umem->nmap; | ||
| 749 | qplib_qp->qp_handle = ureq.qp_handle; | ||
| 750 | |||
| 751 | if (!qp->qplib_qp.srq) { | ||
| 752 | bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); | ||
| 753 | bytes = PAGE_ALIGN(bytes); | ||
| 754 | umem = ib_umem_get(context, ureq.qprva, bytes, | ||
| 755 | IB_ACCESS_LOCAL_WRITE, 1); | ||
| 756 | if (IS_ERR(umem)) | ||
| 757 | goto rqfail; | ||
| 758 | qp->rumem = umem; | ||
| 759 | qplib_qp->rq.sglist = umem->sg_head.sgl; | ||
| 760 | qplib_qp->rq.nmap = umem->nmap; | ||
| 761 | } | ||
| 762 | |||
| 763 | qplib_qp->dpi = cntx->dpi; | ||
| 764 | return 0; | ||
| 765 | rqfail: | ||
| 766 | ib_umem_release(qp->sumem); | ||
| 767 | qp->sumem = NULL; | ||
| 768 | qplib_qp->sq.sglist = NULL; | ||
| 769 | qplib_qp->sq.nmap = 0; | ||
| 770 | |||
| 771 | return PTR_ERR(umem); | ||
| 772 | } | ||
| 773 | |||
| 774 | static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah | ||
| 775 | (struct bnxt_re_pd *pd, | ||
| 776 | struct bnxt_qplib_res *qp1_res, | ||
| 777 | struct bnxt_qplib_qp *qp1_qp) | ||
| 778 | { | ||
| 779 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 780 | struct bnxt_re_ah *ah; | ||
| 781 | union ib_gid sgid; | ||
| 782 | int rc; | ||
| 783 | |||
| 784 | ah = kzalloc(sizeof(*ah), GFP_KERNEL); | ||
| 785 | if (!ah) | ||
| 786 | return NULL; | ||
| 787 | |||
| 788 | memset(ah, 0, sizeof(*ah)); | ||
| 789 | ah->rdev = rdev; | ||
| 790 | ah->qplib_ah.pd = &pd->qplib_pd; | ||
| 791 | |||
| 792 | rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); | ||
| 793 | if (rc) | ||
| 794 | goto fail; | ||
| 795 | |||
| 796 | /* supply the dgid data same as sgid */ | ||
| 797 | memcpy(ah->qplib_ah.dgid.data, &sgid.raw, | ||
| 798 | sizeof(union ib_gid)); | ||
| 799 | ah->qplib_ah.sgid_index = 0; | ||
| 800 | |||
| 801 | ah->qplib_ah.traffic_class = 0; | ||
| 802 | ah->qplib_ah.flow_label = 0; | ||
| 803 | ah->qplib_ah.hop_limit = 1; | ||
| 804 | ah->qplib_ah.sl = 0; | ||
| 805 | /* Have DMAC same as SMAC */ | ||
| 806 | ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); | ||
| 807 | |||
| 808 | rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah); | ||
| 809 | if (rc) { | ||
| 810 | dev_err(rdev_to_dev(rdev), | ||
| 811 | "Failed to allocate HW AH for Shadow QP"); | ||
| 812 | goto fail; | ||
| 813 | } | ||
| 814 | |||
| 815 | return ah; | ||
| 816 | |||
| 817 | fail: | ||
| 818 | kfree(ah); | ||
| 819 | return NULL; | ||
| 820 | } | ||
| 821 | |||
| 822 | static struct bnxt_re_qp *bnxt_re_create_shadow_qp | ||
| 823 | (struct bnxt_re_pd *pd, | ||
| 824 | struct bnxt_qplib_res *qp1_res, | ||
| 825 | struct bnxt_qplib_qp *qp1_qp) | ||
| 826 | { | ||
| 827 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 828 | struct bnxt_re_qp *qp; | ||
| 829 | int rc; | ||
| 830 | |||
| 831 | qp = kzalloc(sizeof(*qp), GFP_KERNEL); | ||
| 832 | if (!qp) | ||
| 833 | return NULL; | ||
| 834 | |||
| 835 | memset(qp, 0, sizeof(*qp)); | ||
| 836 | qp->rdev = rdev; | ||
| 837 | |||
| 838 | /* Initialize the shadow QP structure from the QP1 values */ | ||
| 839 | ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); | ||
| 840 | |||
| 841 | qp->qplib_qp.pd = &pd->qplib_pd; | ||
| 842 | qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); | ||
| 843 | qp->qplib_qp.type = IB_QPT_UD; | ||
| 844 | |||
| 845 | qp->qplib_qp.max_inline_data = 0; | ||
| 846 | qp->qplib_qp.sig_type = true; | ||
| 847 | |||
| 848 | /* Shadow QP SQ depth should be same as QP1 RQ depth */ | ||
| 849 | qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; | ||
| 850 | qp->qplib_qp.sq.max_sge = 2; | ||
| 851 | |||
| 852 | qp->qplib_qp.scq = qp1_qp->scq; | ||
| 853 | qp->qplib_qp.rcq = qp1_qp->rcq; | ||
| 854 | |||
| 855 | qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; | ||
| 856 | qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; | ||
| 857 | |||
| 858 | qp->qplib_qp.mtu = qp1_qp->mtu; | ||
| 859 | |||
| 860 | qp->qplib_qp.sq_hdr_buf_size = 0; | ||
| 861 | qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; | ||
| 862 | qp->qplib_qp.dpi = &rdev->dpi_privileged; | ||
| 863 | |||
| 864 | rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); | ||
| 865 | if (rc) | ||
| 866 | goto fail; | ||
| 867 | |||
| 868 | rdev->sqp_id = qp->qplib_qp.id; | ||
| 869 | |||
| 870 | spin_lock_init(&qp->sq_lock); | ||
| 871 | INIT_LIST_HEAD(&qp->list); | ||
| 872 | mutex_lock(&rdev->qp_lock); | ||
| 873 | list_add_tail(&qp->list, &rdev->qp_list); | ||
| 874 | atomic_inc(&rdev->qp_count); | ||
| 875 | mutex_unlock(&rdev->qp_lock); | ||
| 876 | return qp; | ||
| 877 | fail: | ||
| 878 | kfree(qp); | ||
| 879 | return NULL; | ||
| 880 | } | ||
| 881 | |||
| 882 | struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, | ||
| 883 | struct ib_qp_init_attr *qp_init_attr, | ||
| 884 | struct ib_udata *udata) | ||
| 885 | { | ||
| 886 | struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | ||
| 887 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 888 | struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | ||
| 889 | struct bnxt_re_qp *qp; | ||
| 890 | struct bnxt_re_cq *cq; | ||
| 891 | int rc, entries; | ||
| 892 | |||
| 893 | if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || | ||
| 894 | (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || | ||
| 895 | (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || | ||
| 896 | (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || | ||
| 897 | (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data)) | ||
| 898 | return ERR_PTR(-EINVAL); | ||
| 899 | |||
| 900 | qp = kzalloc(sizeof(*qp), GFP_KERNEL); | ||
| 901 | if (!qp) | ||
| 902 | return ERR_PTR(-ENOMEM); | ||
| 903 | |||
| 904 | qp->rdev = rdev; | ||
| 905 | ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); | ||
| 906 | qp->qplib_qp.pd = &pd->qplib_pd; | ||
| 907 | qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); | ||
| 908 | qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type); | ||
| 909 | if (qp->qplib_qp.type == IB_QPT_MAX) { | ||
| 910 | dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported", | ||
| 911 | qp->qplib_qp.type); | ||
| 912 | rc = -EINVAL; | ||
| 913 | goto fail; | ||
| 914 | } | ||
| 915 | qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data; | ||
| 916 | qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type == | ||
| 917 | IB_SIGNAL_ALL_WR) ? true : false); | ||
| 918 | |||
| 919 | entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1); | ||
| 920 | qp->qplib_qp.sq.max_wqe = min_t(u32, entries, | ||
| 921 | dev_attr->max_qp_wqes + 1); | ||
| 922 | |||
| 923 | qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge; | ||
| 924 | if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) | ||
| 925 | qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; | ||
| 926 | |||
| 927 | if (qp_init_attr->send_cq) { | ||
| 928 | cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq, | ||
| 929 | ib_cq); | ||
| 930 | if (!cq) { | ||
| 931 | dev_err(rdev_to_dev(rdev), "Send CQ not found"); | ||
| 932 | rc = -EINVAL; | ||
| 933 | goto fail; | ||
| 934 | } | ||
| 935 | qp->qplib_qp.scq = &cq->qplib_cq; | ||
| 936 | } | ||
| 937 | |||
| 938 | if (qp_init_attr->recv_cq) { | ||
| 939 | cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq, | ||
| 940 | ib_cq); | ||
| 941 | if (!cq) { | ||
| 942 | dev_err(rdev_to_dev(rdev), "Receive CQ not found"); | ||
| 943 | rc = -EINVAL; | ||
| 944 | goto fail; | ||
| 945 | } | ||
| 946 | qp->qplib_qp.rcq = &cq->qplib_cq; | ||
| 947 | } | ||
| 948 | |||
| 949 | if (qp_init_attr->srq) { | ||
| 950 | dev_err(rdev_to_dev(rdev), "SRQ not supported"); | ||
| 951 | rc = -ENOTSUPP; | ||
| 952 | goto fail; | ||
| 953 | } else { | ||
| 954 | /* Allocate 1 more than what's provided so posting max doesn't | ||
| 955 | * mean empty | ||
| 956 | */ | ||
| 957 | entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1); | ||
| 958 | qp->qplib_qp.rq.max_wqe = min_t(u32, entries, | ||
| 959 | dev_attr->max_qp_wqes + 1); | ||
| 960 | |||
| 961 | qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge; | ||
| 962 | if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) | ||
| 963 | qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; | ||
| 964 | } | ||
| 965 | |||
| 966 | qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); | ||
| 967 | |||
| 968 | if (qp_init_attr->qp_type == IB_QPT_GSI) { | ||
| 969 | qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; | ||
| 970 | if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) | ||
| 971 | qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; | ||
| 972 | qp->qplib_qp.sq.max_sge++; | ||
| 973 | if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) | ||
| 974 | qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; | ||
| 975 | |||
| 976 | qp->qplib_qp.rq_hdr_buf_size = | ||
| 977 | BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; | ||
| 978 | |||
| 979 | qp->qplib_qp.sq_hdr_buf_size = | ||
| 980 | BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; | ||
| 981 | qp->qplib_qp.dpi = &rdev->dpi_privileged; | ||
| 982 | rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp); | ||
| 983 | if (rc) { | ||
| 984 | dev_err(rdev_to_dev(rdev), "Failed to create HW QP1"); | ||
| 985 | goto fail; | ||
| 986 | } | ||
| 987 | /* Create a shadow QP to handle the QP1 traffic */ | ||
| 988 | rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, | ||
| 989 | &qp->qplib_qp); | ||
| 990 | if (!rdev->qp1_sqp) { | ||
| 991 | rc = -EINVAL; | ||
| 992 | dev_err(rdev_to_dev(rdev), | ||
| 993 | "Failed to create Shadow QP for QP1"); | ||
| 994 | goto qp_destroy; | ||
| 995 | } | ||
| 996 | rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, | ||
| 997 | &qp->qplib_qp); | ||
| 998 | if (!rdev->sqp_ah) { | ||
| 999 | bnxt_qplib_destroy_qp(&rdev->qplib_res, | ||
| 1000 | &rdev->qp1_sqp->qplib_qp); | ||
| 1001 | rc = -EINVAL; | ||
| 1002 | dev_err(rdev_to_dev(rdev), | ||
| 1003 | "Failed to create AH entry for ShadowQP"); | ||
| 1004 | goto qp_destroy; | ||
| 1005 | } | ||
| 1006 | |||
| 1007 | } else { | ||
| 1008 | qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom; | ||
| 1009 | qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; | ||
| 1010 | if (udata) { | ||
| 1011 | rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); | ||
| 1012 | if (rc) | ||
| 1013 | goto fail; | ||
| 1014 | } else { | ||
| 1015 | qp->qplib_qp.dpi = &rdev->dpi_privileged; | ||
| 1016 | } | ||
| 1017 | |||
| 1018 | rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); | ||
| 1019 | if (rc) { | ||
| 1020 | dev_err(rdev_to_dev(rdev), "Failed to create HW QP"); | ||
| 1021 | goto fail; | ||
| 1022 | } | ||
| 1023 | } | ||
| 1024 | |||
| 1025 | qp->ib_qp.qp_num = qp->qplib_qp.id; | ||
| 1026 | spin_lock_init(&qp->sq_lock); | ||
| 1027 | |||
| 1028 | if (udata) { | ||
| 1029 | struct bnxt_re_qp_resp resp; | ||
| 1030 | |||
| 1031 | resp.qpid = qp->ib_qp.qp_num; | ||
| 1032 | resp.rsvd = 0; | ||
| 1033 | rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); | ||
| 1034 | if (rc) { | ||
| 1035 | dev_err(rdev_to_dev(rdev), "Failed to copy QP udata"); | ||
| 1036 | goto qp_destroy; | ||
| 1037 | } | ||
| 1038 | } | ||
| 1039 | INIT_LIST_HEAD(&qp->list); | ||
| 1040 | mutex_lock(&rdev->qp_lock); | ||
| 1041 | list_add_tail(&qp->list, &rdev->qp_list); | ||
| 1042 | atomic_inc(&rdev->qp_count); | ||
| 1043 | mutex_unlock(&rdev->qp_lock); | ||
| 1044 | |||
| 1045 | return &qp->ib_qp; | ||
| 1046 | qp_destroy: | ||
| 1047 | bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); | ||
| 1048 | fail: | ||
| 1049 | kfree(qp); | ||
| 1050 | return ERR_PTR(rc); | ||
| 1051 | } | ||
| 1052 | |||
| 1053 | static u8 __from_ib_qp_state(enum ib_qp_state state) | ||
| 1054 | { | ||
| 1055 | switch (state) { | ||
| 1056 | case IB_QPS_RESET: | ||
| 1057 | return CMDQ_MODIFY_QP_NEW_STATE_RESET; | ||
| 1058 | case IB_QPS_INIT: | ||
| 1059 | return CMDQ_MODIFY_QP_NEW_STATE_INIT; | ||
| 1060 | case IB_QPS_RTR: | ||
| 1061 | return CMDQ_MODIFY_QP_NEW_STATE_RTR; | ||
| 1062 | case IB_QPS_RTS: | ||
| 1063 | return CMDQ_MODIFY_QP_NEW_STATE_RTS; | ||
| 1064 | case IB_QPS_SQD: | ||
| 1065 | return CMDQ_MODIFY_QP_NEW_STATE_SQD; | ||
| 1066 | case IB_QPS_SQE: | ||
| 1067 | return CMDQ_MODIFY_QP_NEW_STATE_SQE; | ||
| 1068 | case IB_QPS_ERR: | ||
| 1069 | default: | ||
| 1070 | return CMDQ_MODIFY_QP_NEW_STATE_ERR; | ||
| 1071 | } | ||
| 1072 | } | ||
| 1073 | |||
| 1074 | static enum ib_qp_state __to_ib_qp_state(u8 state) | ||
| 1075 | { | ||
| 1076 | switch (state) { | ||
| 1077 | case CMDQ_MODIFY_QP_NEW_STATE_RESET: | ||
| 1078 | return IB_QPS_RESET; | ||
| 1079 | case CMDQ_MODIFY_QP_NEW_STATE_INIT: | ||
| 1080 | return IB_QPS_INIT; | ||
| 1081 | case CMDQ_MODIFY_QP_NEW_STATE_RTR: | ||
| 1082 | return IB_QPS_RTR; | ||
| 1083 | case CMDQ_MODIFY_QP_NEW_STATE_RTS: | ||
| 1084 | return IB_QPS_RTS; | ||
| 1085 | case CMDQ_MODIFY_QP_NEW_STATE_SQD: | ||
| 1086 | return IB_QPS_SQD; | ||
| 1087 | case CMDQ_MODIFY_QP_NEW_STATE_SQE: | ||
| 1088 | return IB_QPS_SQE; | ||
| 1089 | case CMDQ_MODIFY_QP_NEW_STATE_ERR: | ||
| 1090 | default: | ||
| 1091 | return IB_QPS_ERR; | ||
| 1092 | } | ||
| 1093 | } | ||
| 1094 | |||
| 1095 | static u32 __from_ib_mtu(enum ib_mtu mtu) | ||
| 1096 | { | ||
| 1097 | switch (mtu) { | ||
| 1098 | case IB_MTU_256: | ||
| 1099 | return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; | ||
| 1100 | case IB_MTU_512: | ||
| 1101 | return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; | ||
| 1102 | case IB_MTU_1024: | ||
| 1103 | return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; | ||
| 1104 | case IB_MTU_2048: | ||
| 1105 | return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; | ||
| 1106 | case IB_MTU_4096: | ||
| 1107 | return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; | ||
| 1108 | default: | ||
| 1109 | return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; | ||
| 1110 | } | ||
| 1111 | } | ||
| 1112 | |||
| 1113 | static enum ib_mtu __to_ib_mtu(u32 mtu) | ||
| 1114 | { | ||
| 1115 | switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { | ||
| 1116 | case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: | ||
| 1117 | return IB_MTU_256; | ||
| 1118 | case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: | ||
| 1119 | return IB_MTU_512; | ||
| 1120 | case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: | ||
| 1121 | return IB_MTU_1024; | ||
| 1122 | case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: | ||
| 1123 | return IB_MTU_2048; | ||
| 1124 | case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: | ||
| 1125 | return IB_MTU_4096; | ||
| 1126 | default: | ||
| 1127 | return IB_MTU_2048; | ||
| 1128 | } | ||
| 1129 | } | ||
| 1130 | |||
| 1131 | static int __from_ib_access_flags(int iflags) | ||
| 1132 | { | ||
| 1133 | int qflags = 0; | ||
| 1134 | |||
| 1135 | if (iflags & IB_ACCESS_LOCAL_WRITE) | ||
| 1136 | qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; | ||
| 1137 | if (iflags & IB_ACCESS_REMOTE_READ) | ||
| 1138 | qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; | ||
| 1139 | if (iflags & IB_ACCESS_REMOTE_WRITE) | ||
| 1140 | qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; | ||
| 1141 | if (iflags & IB_ACCESS_REMOTE_ATOMIC) | ||
| 1142 | qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; | ||
| 1143 | if (iflags & IB_ACCESS_MW_BIND) | ||
| 1144 | qflags |= BNXT_QPLIB_ACCESS_MW_BIND; | ||
| 1145 | if (iflags & IB_ZERO_BASED) | ||
| 1146 | qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; | ||
| 1147 | if (iflags & IB_ACCESS_ON_DEMAND) | ||
| 1148 | qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; | ||
| 1149 | return qflags; | ||
| 1150 | }; | ||
| 1151 | |||
| 1152 | static enum ib_access_flags __to_ib_access_flags(int qflags) | ||
| 1153 | { | ||
| 1154 | enum ib_access_flags iflags = 0; | ||
| 1155 | |||
| 1156 | if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) | ||
| 1157 | iflags |= IB_ACCESS_LOCAL_WRITE; | ||
| 1158 | if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) | ||
| 1159 | iflags |= IB_ACCESS_REMOTE_WRITE; | ||
| 1160 | if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) | ||
| 1161 | iflags |= IB_ACCESS_REMOTE_READ; | ||
| 1162 | if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) | ||
| 1163 | iflags |= IB_ACCESS_REMOTE_ATOMIC; | ||
| 1164 | if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) | ||
| 1165 | iflags |= IB_ACCESS_MW_BIND; | ||
| 1166 | if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) | ||
| 1167 | iflags |= IB_ZERO_BASED; | ||
| 1168 | if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) | ||
| 1169 | iflags |= IB_ACCESS_ON_DEMAND; | ||
| 1170 | return iflags; | ||
| 1171 | }; | ||
| 1172 | |||
| 1173 | static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, | ||
| 1174 | struct bnxt_re_qp *qp1_qp, | ||
| 1175 | int qp_attr_mask) | ||
| 1176 | { | ||
| 1177 | struct bnxt_re_qp *qp = rdev->qp1_sqp; | ||
| 1178 | int rc = 0; | ||
| 1179 | |||
| 1180 | if (qp_attr_mask & IB_QP_STATE) { | ||
| 1181 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; | ||
| 1182 | qp->qplib_qp.state = qp1_qp->qplib_qp.state; | ||
| 1183 | } | ||
| 1184 | if (qp_attr_mask & IB_QP_PKEY_INDEX) { | ||
| 1185 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; | ||
| 1186 | qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; | ||
| 1187 | } | ||
| 1188 | |||
| 1189 | if (qp_attr_mask & IB_QP_QKEY) { | ||
| 1190 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; | ||
| 1191 | /* Using a Random QKEY */ | ||
| 1192 | qp->qplib_qp.qkey = 0x81818181; | ||
| 1193 | } | ||
| 1194 | if (qp_attr_mask & IB_QP_SQ_PSN) { | ||
| 1195 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; | ||
| 1196 | qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; | ||
| 1197 | } | ||
| 1198 | |||
| 1199 | rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); | ||
| 1200 | if (rc) | ||
| 1201 | dev_err(rdev_to_dev(rdev), | ||
| 1202 | "Failed to modify Shadow QP for QP1"); | ||
| 1203 | return rc; | ||
| 1204 | } | ||
| 1205 | |||
| 1206 | int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, | ||
| 1207 | int qp_attr_mask, struct ib_udata *udata) | ||
| 1208 | { | ||
| 1209 | struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | ||
| 1210 | struct bnxt_re_dev *rdev = qp->rdev; | ||
| 1211 | struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | ||
| 1212 | enum ib_qp_state curr_qp_state, new_qp_state; | ||
| 1213 | int rc, entries; | ||
| 1214 | int status; | ||
| 1215 | union ib_gid sgid; | ||
| 1216 | struct ib_gid_attr sgid_attr; | ||
| 1217 | u8 nw_type; | ||
| 1218 | |||
| 1219 | qp->qplib_qp.modify_flags = 0; | ||
| 1220 | if (qp_attr_mask & IB_QP_STATE) { | ||
| 1221 | curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); | ||
| 1222 | new_qp_state = qp_attr->qp_state; | ||
| 1223 | if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, | ||
| 1224 | ib_qp->qp_type, qp_attr_mask, | ||
| 1225 | IB_LINK_LAYER_ETHERNET)) { | ||
| 1226 | dev_err(rdev_to_dev(rdev), | ||
| 1227 | "Invalid attribute mask: %#x specified ", | ||
| 1228 | qp_attr_mask); | ||
| 1229 | dev_err(rdev_to_dev(rdev), | ||
| 1230 | "for qpn: %#x type: %#x", | ||
| 1231 | ib_qp->qp_num, ib_qp->qp_type); | ||
| 1232 | dev_err(rdev_to_dev(rdev), | ||
| 1233 | "curr_qp_state=0x%x, new_qp_state=0x%x\n", | ||
| 1234 | curr_qp_state, new_qp_state); | ||
| 1235 | return -EINVAL; | ||
| 1236 | } | ||
| 1237 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; | ||
| 1238 | qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); | ||
| 1239 | } | ||
| 1240 | if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { | ||
| 1241 | qp->qplib_qp.modify_flags |= | ||
| 1242 | CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; | ||
| 1243 | qp->qplib_qp.en_sqd_async_notify = true; | ||
| 1244 | } | ||
| 1245 | if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { | ||
| 1246 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; | ||
| 1247 | qp->qplib_qp.access = | ||
| 1248 | __from_ib_access_flags(qp_attr->qp_access_flags); | ||
| 1249 | /* LOCAL_WRITE access must be set to allow RC receive */ | ||
| 1250 | qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; | ||
| 1251 | } | ||
| 1252 | if (qp_attr_mask & IB_QP_PKEY_INDEX) { | ||
| 1253 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; | ||
| 1254 | qp->qplib_qp.pkey_index = qp_attr->pkey_index; | ||
| 1255 | } | ||
| 1256 | if (qp_attr_mask & IB_QP_QKEY) { | ||
| 1257 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; | ||
| 1258 | qp->qplib_qp.qkey = qp_attr->qkey; | ||
| 1259 | } | ||
| 1260 | if (qp_attr_mask & IB_QP_AV) { | ||
| 1261 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | | ||
| 1262 | CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | | ||
| 1263 | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | | ||
| 1264 | CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | | ||
| 1265 | CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | | ||
| 1266 | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | | ||
| 1267 | CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; | ||
| 1268 | memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw, | ||
| 1269 | sizeof(qp->qplib_qp.ah.dgid.data)); | ||
| 1270 | qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label; | ||
| 1271 | /* If RoCE V2 is enabled, stack will have two entries for | ||
| 1272 | * each GID entry. Avoiding this duplicte entry in HW. Dividing | ||
| 1273 | * the GID index by 2 for RoCE V2 | ||
| 1274 | */ | ||
| 1275 | qp->qplib_qp.ah.sgid_index = | ||
| 1276 | qp_attr->ah_attr.grh.sgid_index / 2; | ||
| 1277 | qp->qplib_qp.ah.host_sgid_index = | ||
| 1278 | qp_attr->ah_attr.grh.sgid_index; | ||
| 1279 | qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit; | ||
| 1280 | qp->qplib_qp.ah.traffic_class = | ||
| 1281 | qp_attr->ah_attr.grh.traffic_class; | ||
| 1282 | qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl; | ||
| 1283 | ether_addr_copy(qp->qplib_qp.ah.dmac, qp_attr->ah_attr.dmac); | ||
| 1284 | |||
| 1285 | status = ib_get_cached_gid(&rdev->ibdev, 1, | ||
| 1286 | qp_attr->ah_attr.grh.sgid_index, | ||
| 1287 | &sgid, &sgid_attr); | ||
| 1288 | if (!status && sgid_attr.ndev) { | ||
| 1289 | memcpy(qp->qplib_qp.smac, sgid_attr.ndev->dev_addr, | ||
| 1290 | ETH_ALEN); | ||
| 1291 | dev_put(sgid_attr.ndev); | ||
| 1292 | nw_type = ib_gid_to_network_type(sgid_attr.gid_type, | ||
| 1293 | &sgid); | ||
| 1294 | switch (nw_type) { | ||
| 1295 | case RDMA_NETWORK_IPV4: | ||
| 1296 | qp->qplib_qp.nw_type = | ||
| 1297 | CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; | ||
| 1298 | break; | ||
| 1299 | case RDMA_NETWORK_IPV6: | ||
| 1300 | qp->qplib_qp.nw_type = | ||
| 1301 | CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; | ||
| 1302 | break; | ||
| 1303 | default: | ||
| 1304 | qp->qplib_qp.nw_type = | ||
| 1305 | CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; | ||
| 1306 | break; | ||
| 1307 | } | ||
| 1308 | } | ||
| 1309 | } | ||
| 1310 | |||
| 1311 | if (qp_attr_mask & IB_QP_PATH_MTU) { | ||
| 1312 | qp->qplib_qp.modify_flags |= | ||
| 1313 | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; | ||
| 1314 | qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); | ||
| 1315 | } else if (qp_attr->qp_state == IB_QPS_RTR) { | ||
| 1316 | qp->qplib_qp.modify_flags |= | ||
| 1317 | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; | ||
| 1318 | qp->qplib_qp.path_mtu = | ||
| 1319 | __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); | ||
| 1320 | } | ||
| 1321 | |||
| 1322 | if (qp_attr_mask & IB_QP_TIMEOUT) { | ||
| 1323 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; | ||
| 1324 | qp->qplib_qp.timeout = qp_attr->timeout; | ||
| 1325 | } | ||
| 1326 | if (qp_attr_mask & IB_QP_RETRY_CNT) { | ||
| 1327 | qp->qplib_qp.modify_flags |= | ||
| 1328 | CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; | ||
| 1329 | qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; | ||
| 1330 | } | ||
| 1331 | if (qp_attr_mask & IB_QP_RNR_RETRY) { | ||
| 1332 | qp->qplib_qp.modify_flags |= | ||
| 1333 | CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; | ||
| 1334 | qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; | ||
| 1335 | } | ||
| 1336 | if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { | ||
| 1337 | qp->qplib_qp.modify_flags |= | ||
| 1338 | CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; | ||
| 1339 | qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; | ||
| 1340 | } | ||
| 1341 | if (qp_attr_mask & IB_QP_RQ_PSN) { | ||
| 1342 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; | ||
| 1343 | qp->qplib_qp.rq.psn = qp_attr->rq_psn; | ||
| 1344 | } | ||
| 1345 | if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { | ||
| 1346 | qp->qplib_qp.modify_flags |= | ||
| 1347 | CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; | ||
| 1348 | qp->qplib_qp.max_rd_atomic = qp_attr->max_rd_atomic; | ||
| 1349 | } | ||
| 1350 | if (qp_attr_mask & IB_QP_SQ_PSN) { | ||
| 1351 | qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; | ||
| 1352 | qp->qplib_qp.sq.psn = qp_attr->sq_psn; | ||
| 1353 | } | ||
| 1354 | if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { | ||
| 1355 | qp->qplib_qp.modify_flags |= | ||
| 1356 | CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; | ||
| 1357 | qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; | ||
| 1358 | } | ||
| 1359 | if (qp_attr_mask & IB_QP_CAP) { | ||
| 1360 | qp->qplib_qp.modify_flags |= | ||
| 1361 | CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | | ||
| 1362 | CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | | ||
| 1363 | CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | | ||
| 1364 | CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | | ||
| 1365 | CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; | ||
| 1366 | if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || | ||
| 1367 | (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || | ||
| 1368 | (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || | ||
| 1369 | (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || | ||
| 1370 | (qp_attr->cap.max_inline_data >= | ||
| 1371 | dev_attr->max_inline_data)) { | ||
| 1372 | dev_err(rdev_to_dev(rdev), | ||
| 1373 | "Create QP failed - max exceeded"); | ||
| 1374 | return -EINVAL; | ||
| 1375 | } | ||
| 1376 | entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); | ||
| 1377 | qp->qplib_qp.sq.max_wqe = min_t(u32, entries, | ||
| 1378 | dev_attr->max_qp_wqes + 1); | ||
| 1379 | qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; | ||
| 1380 | if (qp->qplib_qp.rq.max_wqe) { | ||
| 1381 | entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); | ||
| 1382 | qp->qplib_qp.rq.max_wqe = | ||
| 1383 | min_t(u32, entries, dev_attr->max_qp_wqes + 1); | ||
| 1384 | qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; | ||
| 1385 | } else { | ||
| 1386 | /* SRQ was used prior, just ignore the RQ caps */ | ||
| 1387 | } | ||
| 1388 | } | ||
| 1389 | if (qp_attr_mask & IB_QP_DEST_QPN) { | ||
| 1390 | qp->qplib_qp.modify_flags |= | ||
| 1391 | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; | ||
| 1392 | qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; | ||
| 1393 | } | ||
| 1394 | rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); | ||
| 1395 | if (rc) { | ||
| 1396 | dev_err(rdev_to_dev(rdev), "Failed to modify HW QP"); | ||
| 1397 | return rc; | ||
| 1398 | } | ||
| 1399 | if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) | ||
| 1400 | rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); | ||
| 1401 | return rc; | ||
| 1402 | } | ||
| 1403 | |||
| 1404 | int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, | ||
| 1405 | int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) | ||
| 1406 | { | ||
| 1407 | struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | ||
| 1408 | struct bnxt_re_dev *rdev = qp->rdev; | ||
| 1409 | struct bnxt_qplib_qp qplib_qp; | ||
| 1410 | int rc; | ||
| 1411 | |||
| 1412 | memset(&qplib_qp, 0, sizeof(struct bnxt_qplib_qp)); | ||
| 1413 | qplib_qp.id = qp->qplib_qp.id; | ||
| 1414 | qplib_qp.ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; | ||
| 1415 | |||
| 1416 | rc = bnxt_qplib_query_qp(&rdev->qplib_res, &qplib_qp); | ||
| 1417 | if (rc) { | ||
| 1418 | dev_err(rdev_to_dev(rdev), "Failed to query HW QP"); | ||
| 1419 | return rc; | ||
| 1420 | } | ||
| 1421 | qp_attr->qp_state = __to_ib_qp_state(qplib_qp.state); | ||
| 1422 | qp_attr->en_sqd_async_notify = qplib_qp.en_sqd_async_notify ? 1 : 0; | ||
| 1423 | qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp.access); | ||
| 1424 | qp_attr->pkey_index = qplib_qp.pkey_index; | ||
| 1425 | qp_attr->qkey = qplib_qp.qkey; | ||
| 1426 | memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp.ah.dgid.data, | ||
| 1427 | sizeof(qplib_qp.ah.dgid.data)); | ||
| 1428 | qp_attr->ah_attr.grh.flow_label = qplib_qp.ah.flow_label; | ||
| 1429 | qp_attr->ah_attr.grh.sgid_index = qplib_qp.ah.host_sgid_index; | ||
| 1430 | qp_attr->ah_attr.grh.hop_limit = qplib_qp.ah.hop_limit; | ||
| 1431 | qp_attr->ah_attr.grh.traffic_class = qplib_qp.ah.traffic_class; | ||
| 1432 | qp_attr->ah_attr.sl = qplib_qp.ah.sl; | ||
| 1433 | ether_addr_copy(qp_attr->ah_attr.dmac, qplib_qp.ah.dmac); | ||
| 1434 | qp_attr->path_mtu = __to_ib_mtu(qplib_qp.path_mtu); | ||
| 1435 | qp_attr->timeout = qplib_qp.timeout; | ||
| 1436 | qp_attr->retry_cnt = qplib_qp.retry_cnt; | ||
| 1437 | qp_attr->rnr_retry = qplib_qp.rnr_retry; | ||
| 1438 | qp_attr->min_rnr_timer = qplib_qp.min_rnr_timer; | ||
| 1439 | qp_attr->rq_psn = qplib_qp.rq.psn; | ||
| 1440 | qp_attr->max_rd_atomic = qplib_qp.max_rd_atomic; | ||
| 1441 | qp_attr->sq_psn = qplib_qp.sq.psn; | ||
| 1442 | qp_attr->max_dest_rd_atomic = qplib_qp.max_dest_rd_atomic; | ||
| 1443 | qp_init_attr->sq_sig_type = qplib_qp.sig_type ? IB_SIGNAL_ALL_WR : | ||
| 1444 | IB_SIGNAL_REQ_WR; | ||
| 1445 | qp_attr->dest_qp_num = qplib_qp.dest_qpn; | ||
| 1446 | |||
| 1447 | qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; | ||
| 1448 | qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; | ||
| 1449 | qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; | ||
| 1450 | qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; | ||
| 1451 | qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; | ||
| 1452 | qp_init_attr->cap = qp_attr->cap; | ||
| 1453 | |||
| 1454 | return 0; | ||
| 1455 | } | ||
| 1456 | |||
| 1457 | /* Routine for sending QP1 packets for RoCE V1 an V2 | ||
| 1458 | */ | ||
| 1459 | static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, | ||
| 1460 | struct ib_send_wr *wr, | ||
| 1461 | struct bnxt_qplib_swqe *wqe, | ||
| 1462 | int payload_size) | ||
| 1463 | { | ||
| 1464 | struct ib_device *ibdev = &qp->rdev->ibdev; | ||
| 1465 | struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, | ||
| 1466 | ib_ah); | ||
| 1467 | struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; | ||
| 1468 | struct bnxt_qplib_sge sge; | ||
| 1469 | union ib_gid sgid; | ||
| 1470 | u8 nw_type; | ||
| 1471 | u16 ether_type; | ||
| 1472 | struct ib_gid_attr sgid_attr; | ||
| 1473 | union ib_gid dgid; | ||
| 1474 | bool is_eth = false; | ||
| 1475 | bool is_vlan = false; | ||
| 1476 | bool is_grh = false; | ||
| 1477 | bool is_udp = false; | ||
| 1478 | u8 ip_version = 0; | ||
| 1479 | u16 vlan_id = 0xFFFF; | ||
| 1480 | void *buf; | ||
| 1481 | int i, rc = 0, size; | ||
| 1482 | |||
| 1483 | memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); | ||
| 1484 | |||
| 1485 | rc = ib_get_cached_gid(ibdev, 1, | ||
| 1486 | qplib_ah->host_sgid_index, &sgid, | ||
| 1487 | &sgid_attr); | ||
| 1488 | if (rc) { | ||
| 1489 | dev_err(rdev_to_dev(qp->rdev), | ||
| 1490 | "Failed to query gid at index %d", | ||
| 1491 | qplib_ah->host_sgid_index); | ||
| 1492 | return rc; | ||
| 1493 | } | ||
| 1494 | if (sgid_attr.ndev) { | ||
| 1495 | if (is_vlan_dev(sgid_attr.ndev)) | ||
| 1496 | vlan_id = vlan_dev_vlan_id(sgid_attr.ndev); | ||
| 1497 | dev_put(sgid_attr.ndev); | ||
| 1498 | } | ||
| 1499 | /* Get network header type for this GID */ | ||
| 1500 | nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); | ||
| 1501 | switch (nw_type) { | ||
| 1502 | case RDMA_NETWORK_IPV4: | ||
| 1503 | nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; | ||
| 1504 | break; | ||
| 1505 | case RDMA_NETWORK_IPV6: | ||
| 1506 | nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; | ||
| 1507 | break; | ||
| 1508 | default: | ||
| 1509 | nw_type = BNXT_RE_ROCE_V1_PACKET; | ||
| 1510 | break; | ||
| 1511 | } | ||
| 1512 | memcpy(&dgid.raw, &qplib_ah->dgid, 16); | ||
| 1513 | is_udp = sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; | ||
| 1514 | if (is_udp) { | ||
| 1515 | if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) { | ||
| 1516 | ip_version = 4; | ||
| 1517 | ether_type = ETH_P_IP; | ||
| 1518 | } else { | ||
| 1519 | ip_version = 6; | ||
| 1520 | ether_type = ETH_P_IPV6; | ||
| 1521 | } | ||
| 1522 | is_grh = false; | ||
| 1523 | } else { | ||
| 1524 | ether_type = ETH_P_IBOE; | ||
| 1525 | is_grh = true; | ||
| 1526 | } | ||
| 1527 | |||
| 1528 | is_eth = true; | ||
| 1529 | is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; | ||
| 1530 | |||
| 1531 | ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, | ||
| 1532 | ip_version, is_udp, 0, &qp->qp1_hdr); | ||
| 1533 | |||
| 1534 | /* ETH */ | ||
| 1535 | ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); | ||
| 1536 | ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); | ||
| 1537 | |||
| 1538 | /* For vlan, check the sgid for vlan existence */ | ||
| 1539 | |||
| 1540 | if (!is_vlan) { | ||
| 1541 | qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); | ||
| 1542 | } else { | ||
| 1543 | qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); | ||
| 1544 | qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); | ||
| 1545 | } | ||
| 1546 | |||
| 1547 | if (is_grh || (ip_version == 6)) { | ||
| 1548 | memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); | ||
| 1549 | memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, | ||
| 1550 | sizeof(sgid)); | ||
| 1551 | qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; | ||
| 1552 | } | ||
| 1553 | |||
| 1554 | if (ip_version == 4) { | ||
| 1555 | qp->qp1_hdr.ip4.tos = 0; | ||
| 1556 | qp->qp1_hdr.ip4.id = 0; | ||
| 1557 | qp->qp1_hdr.ip4.frag_off = htons(IP_DF); | ||
| 1558 | qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; | ||
| 1559 | |||
| 1560 | memcpy(&qp->qp1_hdr.ip4.saddr, sgid.raw + 12, 4); | ||
| 1561 | memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); | ||
| 1562 | qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); | ||
| 1563 | } | ||
| 1564 | |||
| 1565 | if (is_udp) { | ||
| 1566 | qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); | ||
| 1567 | qp->qp1_hdr.udp.sport = htons(0x8CD1); | ||
| 1568 | qp->qp1_hdr.udp.csum = 0; | ||
| 1569 | } | ||
| 1570 | |||
| 1571 | /* BTH */ | ||
| 1572 | if (wr->opcode == IB_WR_SEND_WITH_IMM) { | ||
| 1573 | qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; | ||
| 1574 | qp->qp1_hdr.immediate_present = 1; | ||
| 1575 | } else { | ||
| 1576 | qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; | ||
| 1577 | } | ||
| 1578 | if (wr->send_flags & IB_SEND_SOLICITED) | ||
| 1579 | qp->qp1_hdr.bth.solicited_event = 1; | ||
| 1580 | /* pad_count */ | ||
| 1581 | qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; | ||
| 1582 | |||
| 1583 | /* P_key for QP1 is for all members */ | ||
| 1584 | qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); | ||
| 1585 | qp->qp1_hdr.bth.destination_qpn = IB_QP1; | ||
| 1586 | qp->qp1_hdr.bth.ack_req = 0; | ||
| 1587 | qp->send_psn++; | ||
| 1588 | qp->send_psn &= BTH_PSN_MASK; | ||
| 1589 | qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); | ||
| 1590 | /* DETH */ | ||
| 1591 | /* Use the priviledged Q_Key for QP1 */ | ||
| 1592 | qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); | ||
| 1593 | qp->qp1_hdr.deth.source_qpn = IB_QP1; | ||
| 1594 | |||
| 1595 | /* Pack the QP1 to the transmit buffer */ | ||
| 1596 | buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); | ||
| 1597 | if (buf) { | ||
| 1598 | size = ib_ud_header_pack(&qp->qp1_hdr, buf); | ||
| 1599 | for (i = wqe->num_sge; i; i--) { | ||
| 1600 | wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; | ||
| 1601 | wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; | ||
| 1602 | wqe->sg_list[i].size = wqe->sg_list[i - 1].size; | ||
| 1603 | } | ||
| 1604 | |||
| 1605 | /* | ||
| 1606 | * Max Header buf size for IPV6 RoCE V2 is 86, | ||
| 1607 | * which is same as the QP1 SQ header buffer. | ||
| 1608 | * Header buf size for IPV4 RoCE V2 can be 66. | ||
| 1609 | * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). | ||
| 1610 | * Subtract 20 bytes from QP1 SQ header buf size | ||
| 1611 | */ | ||
| 1612 | if (is_udp && ip_version == 4) | ||
| 1613 | sge.size -= 20; | ||
| 1614 | /* | ||
| 1615 | * Max Header buf size for RoCE V1 is 78. | ||
| 1616 | * ETH(14) + VLAN(4) + GRH(40) + BTH(20). | ||
| 1617 | * Subtract 8 bytes from QP1 SQ header buf size | ||
| 1618 | */ | ||
| 1619 | if (!is_udp) | ||
| 1620 | sge.size -= 8; | ||
| 1621 | |||
| 1622 | /* Subtract 4 bytes for non vlan packets */ | ||
| 1623 | if (!is_vlan) | ||
| 1624 | sge.size -= 4; | ||
| 1625 | |||
| 1626 | wqe->sg_list[0].addr = sge.addr; | ||
| 1627 | wqe->sg_list[0].lkey = sge.lkey; | ||
| 1628 | wqe->sg_list[0].size = sge.size; | ||
| 1629 | wqe->num_sge++; | ||
| 1630 | |||
| 1631 | } else { | ||
| 1632 | dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!"); | ||
| 1633 | rc = -ENOMEM; | ||
| 1634 | } | ||
| 1635 | return rc; | ||
| 1636 | } | ||
| 1637 | |||
| 1638 | /* For the MAD layer, it only provides the recv SGE the size of | ||
| 1639 | * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, | ||
| 1640 | * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire | ||
| 1641 | * receive packet (334 bytes) with no VLAN and then copy the GRH | ||
| 1642 | * and the MAD datagram out to the provided SGE. | ||
| 1643 | */ | ||
| 1644 | static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, | ||
| 1645 | struct ib_recv_wr *wr, | ||
| 1646 | struct bnxt_qplib_swqe *wqe, | ||
| 1647 | int payload_size) | ||
| 1648 | { | ||
| 1649 | struct bnxt_qplib_sge ref, sge; | ||
| 1650 | u32 rq_prod_index; | ||
| 1651 | struct bnxt_re_sqp_entries *sqp_entry; | ||
| 1652 | |||
| 1653 | rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); | ||
| 1654 | |||
| 1655 | if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) | ||
| 1656 | return -ENOMEM; | ||
| 1657 | |||
| 1658 | /* Create 1 SGE to receive the entire | ||
| 1659 | * ethernet packet | ||
| 1660 | */ | ||
| 1661 | /* Save the reference from ULP */ | ||
| 1662 | ref.addr = wqe->sg_list[0].addr; | ||
| 1663 | ref.lkey = wqe->sg_list[0].lkey; | ||
| 1664 | ref.size = wqe->sg_list[0].size; | ||
| 1665 | |||
| 1666 | sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index]; | ||
| 1667 | |||
| 1668 | /* SGE 1 */ | ||
| 1669 | wqe->sg_list[0].addr = sge.addr; | ||
| 1670 | wqe->sg_list[0].lkey = sge.lkey; | ||
| 1671 | wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; | ||
| 1672 | sge.size -= wqe->sg_list[0].size; | ||
| 1673 | |||
| 1674 | sqp_entry->sge.addr = ref.addr; | ||
| 1675 | sqp_entry->sge.lkey = ref.lkey; | ||
| 1676 | sqp_entry->sge.size = ref.size; | ||
| 1677 | /* Store the wrid for reporting completion */ | ||
| 1678 | sqp_entry->wrid = wqe->wr_id; | ||
| 1679 | /* change the wqe->wrid to table index */ | ||
| 1680 | wqe->wr_id = rq_prod_index; | ||
| 1681 | return 0; | ||
| 1682 | } | ||
| 1683 | |||
| 1684 | static int is_ud_qp(struct bnxt_re_qp *qp) | ||
| 1685 | { | ||
| 1686 | return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD; | ||
| 1687 | } | ||
| 1688 | |||
| 1689 | static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, | ||
| 1690 | struct ib_send_wr *wr, | ||
| 1691 | struct bnxt_qplib_swqe *wqe) | ||
| 1692 | { | ||
| 1693 | struct bnxt_re_ah *ah = NULL; | ||
| 1694 | |||
| 1695 | if (is_ud_qp(qp)) { | ||
| 1696 | ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); | ||
| 1697 | wqe->send.q_key = ud_wr(wr)->remote_qkey; | ||
| 1698 | wqe->send.dst_qp = ud_wr(wr)->remote_qpn; | ||
| 1699 | wqe->send.avid = ah->qplib_ah.id; | ||
| 1700 | } | ||
| 1701 | switch (wr->opcode) { | ||
| 1702 | case IB_WR_SEND: | ||
| 1703 | wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; | ||
| 1704 | break; | ||
| 1705 | case IB_WR_SEND_WITH_IMM: | ||
| 1706 | wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; | ||
| 1707 | wqe->send.imm_data = wr->ex.imm_data; | ||
| 1708 | break; | ||
| 1709 | case IB_WR_SEND_WITH_INV: | ||
| 1710 | wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; | ||
| 1711 | wqe->send.inv_key = wr->ex.invalidate_rkey; | ||
| 1712 | break; | ||
| 1713 | default: | ||
| 1714 | return -EINVAL; | ||
| 1715 | } | ||
| 1716 | if (wr->send_flags & IB_SEND_SIGNALED) | ||
| 1717 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | ||
| 1718 | if (wr->send_flags & IB_SEND_FENCE) | ||
| 1719 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | ||
| 1720 | if (wr->send_flags & IB_SEND_SOLICITED) | ||
| 1721 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; | ||
| 1722 | if (wr->send_flags & IB_SEND_INLINE) | ||
| 1723 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; | ||
| 1724 | |||
| 1725 | return 0; | ||
| 1726 | } | ||
| 1727 | |||
| 1728 | static int bnxt_re_build_rdma_wqe(struct ib_send_wr *wr, | ||
| 1729 | struct bnxt_qplib_swqe *wqe) | ||
| 1730 | { | ||
| 1731 | switch (wr->opcode) { | ||
| 1732 | case IB_WR_RDMA_WRITE: | ||
| 1733 | wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; | ||
| 1734 | break; | ||
| 1735 | case IB_WR_RDMA_WRITE_WITH_IMM: | ||
| 1736 | wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; | ||
| 1737 | wqe->rdma.imm_data = wr->ex.imm_data; | ||
| 1738 | break; | ||
| 1739 | case IB_WR_RDMA_READ: | ||
| 1740 | wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; | ||
| 1741 | wqe->rdma.inv_key = wr->ex.invalidate_rkey; | ||
| 1742 | break; | ||
| 1743 | default: | ||
| 1744 | return -EINVAL; | ||
| 1745 | } | ||
| 1746 | wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; | ||
| 1747 | wqe->rdma.r_key = rdma_wr(wr)->rkey; | ||
| 1748 | if (wr->send_flags & IB_SEND_SIGNALED) | ||
| 1749 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | ||
| 1750 | if (wr->send_flags & IB_SEND_FENCE) | ||
| 1751 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | ||
| 1752 | if (wr->send_flags & IB_SEND_SOLICITED) | ||
| 1753 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; | ||
| 1754 | if (wr->send_flags & IB_SEND_INLINE) | ||
| 1755 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; | ||
| 1756 | |||
| 1757 | return 0; | ||
| 1758 | } | ||
| 1759 | |||
| 1760 | static int bnxt_re_build_atomic_wqe(struct ib_send_wr *wr, | ||
| 1761 | struct bnxt_qplib_swqe *wqe) | ||
| 1762 | { | ||
| 1763 | switch (wr->opcode) { | ||
| 1764 | case IB_WR_ATOMIC_CMP_AND_SWP: | ||
| 1765 | wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; | ||
| 1766 | wqe->atomic.swap_data = atomic_wr(wr)->swap; | ||
| 1767 | break; | ||
| 1768 | case IB_WR_ATOMIC_FETCH_AND_ADD: | ||
| 1769 | wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; | ||
| 1770 | wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; | ||
| 1771 | break; | ||
| 1772 | default: | ||
| 1773 | return -EINVAL; | ||
| 1774 | } | ||
| 1775 | wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; | ||
| 1776 | wqe->atomic.r_key = atomic_wr(wr)->rkey; | ||
| 1777 | if (wr->send_flags & IB_SEND_SIGNALED) | ||
| 1778 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | ||
| 1779 | if (wr->send_flags & IB_SEND_FENCE) | ||
| 1780 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | ||
| 1781 | if (wr->send_flags & IB_SEND_SOLICITED) | ||
| 1782 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; | ||
| 1783 | return 0; | ||
| 1784 | } | ||
| 1785 | |||
| 1786 | static int bnxt_re_build_inv_wqe(struct ib_send_wr *wr, | ||
| 1787 | struct bnxt_qplib_swqe *wqe) | ||
| 1788 | { | ||
| 1789 | wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; | ||
| 1790 | wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; | ||
| 1791 | |||
| 1792 | if (wr->send_flags & IB_SEND_SIGNALED) | ||
| 1793 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | ||
| 1794 | if (wr->send_flags & IB_SEND_FENCE) | ||
| 1795 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | ||
| 1796 | if (wr->send_flags & IB_SEND_SOLICITED) | ||
| 1797 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; | ||
| 1798 | |||
| 1799 | return 0; | ||
| 1800 | } | ||
| 1801 | |||
| 1802 | static int bnxt_re_build_reg_wqe(struct ib_reg_wr *wr, | ||
| 1803 | struct bnxt_qplib_swqe *wqe) | ||
| 1804 | { | ||
| 1805 | struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); | ||
| 1806 | struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; | ||
| 1807 | int access = wr->access; | ||
| 1808 | |||
| 1809 | wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; | ||
| 1810 | wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; | ||
| 1811 | wqe->frmr.page_list = mr->pages; | ||
| 1812 | wqe->frmr.page_list_len = mr->npages; | ||
| 1813 | wqe->frmr.levels = qplib_frpl->hwq.level + 1; | ||
| 1814 | wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; | ||
| 1815 | |||
| 1816 | if (wr->wr.send_flags & IB_SEND_FENCE) | ||
| 1817 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; | ||
| 1818 | if (wr->wr.send_flags & IB_SEND_SIGNALED) | ||
| 1819 | wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; | ||
| 1820 | |||
| 1821 | if (access & IB_ACCESS_LOCAL_WRITE) | ||
| 1822 | wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; | ||
| 1823 | if (access & IB_ACCESS_REMOTE_READ) | ||
| 1824 | wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; | ||
| 1825 | if (access & IB_ACCESS_REMOTE_WRITE) | ||
| 1826 | wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; | ||
| 1827 | if (access & IB_ACCESS_REMOTE_ATOMIC) | ||
| 1828 | wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; | ||
| 1829 | if (access & IB_ACCESS_MW_BIND) | ||
| 1830 | wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; | ||
| 1831 | |||
| 1832 | wqe->frmr.l_key = wr->key; | ||
| 1833 | wqe->frmr.length = wr->mr->length; | ||
| 1834 | wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1; | ||
| 1835 | wqe->frmr.va = wr->mr->iova; | ||
| 1836 | return 0; | ||
| 1837 | } | ||
| 1838 | |||
| 1839 | static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, | ||
| 1840 | struct ib_send_wr *wr, | ||
| 1841 | struct bnxt_qplib_swqe *wqe) | ||
| 1842 | { | ||
| 1843 | /* Copy the inline data to the data field */ | ||
| 1844 | u8 *in_data; | ||
| 1845 | u32 i, sge_len; | ||
| 1846 | void *sge_addr; | ||
| 1847 | |||
| 1848 | in_data = wqe->inline_data; | ||
| 1849 | for (i = 0; i < wr->num_sge; i++) { | ||
| 1850 | sge_addr = (void *)(unsigned long) | ||
| 1851 | wr->sg_list[i].addr; | ||
| 1852 | sge_len = wr->sg_list[i].length; | ||
| 1853 | |||
| 1854 | if ((sge_len + wqe->inline_len) > | ||
| 1855 | BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { | ||
| 1856 | dev_err(rdev_to_dev(rdev), | ||
| 1857 | "Inline data size requested > supported value"); | ||
| 1858 | return -EINVAL; | ||
| 1859 | } | ||
| 1860 | sge_len = wr->sg_list[i].length; | ||
| 1861 | |||
| 1862 | memcpy(in_data, sge_addr, sge_len); | ||
| 1863 | in_data += wr->sg_list[i].length; | ||
| 1864 | wqe->inline_len += wr->sg_list[i].length; | ||
| 1865 | } | ||
| 1866 | return wqe->inline_len; | ||
| 1867 | } | ||
| 1868 | |||
| 1869 | static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, | ||
| 1870 | struct ib_send_wr *wr, | ||
| 1871 | struct bnxt_qplib_swqe *wqe) | ||
| 1872 | { | ||
| 1873 | int payload_sz = 0; | ||
| 1874 | |||
| 1875 | if (wr->send_flags & IB_SEND_INLINE) | ||
| 1876 | payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); | ||
| 1877 | else | ||
| 1878 | payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, | ||
| 1879 | wqe->num_sge); | ||
| 1880 | |||
| 1881 | return payload_sz; | ||
| 1882 | } | ||
| 1883 | |||
| 1884 | static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, | ||
| 1885 | struct bnxt_re_qp *qp, | ||
| 1886 | struct ib_send_wr *wr) | ||
| 1887 | { | ||
| 1888 | struct bnxt_qplib_swqe wqe; | ||
| 1889 | int rc = 0, payload_sz = 0; | ||
| 1890 | unsigned long flags; | ||
| 1891 | |||
| 1892 | spin_lock_irqsave(&qp->sq_lock, flags); | ||
| 1893 | memset(&wqe, 0, sizeof(wqe)); | ||
| 1894 | while (wr) { | ||
| 1895 | /* House keeping */ | ||
| 1896 | memset(&wqe, 0, sizeof(wqe)); | ||
| 1897 | |||
| 1898 | /* Common */ | ||
| 1899 | wqe.num_sge = wr->num_sge; | ||
| 1900 | if (wr->num_sge > qp->qplib_qp.sq.max_sge) { | ||
| 1901 | dev_err(rdev_to_dev(rdev), | ||
| 1902 | "Limit exceeded for Send SGEs"); | ||
| 1903 | rc = -EINVAL; | ||
| 1904 | goto bad; | ||
| 1905 | } | ||
| 1906 | |||
| 1907 | payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); | ||
| 1908 | if (payload_sz < 0) { | ||
| 1909 | rc = -EINVAL; | ||
| 1910 | goto bad; | ||
| 1911 | } | ||
| 1912 | wqe.wr_id = wr->wr_id; | ||
| 1913 | |||
| 1914 | wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; | ||
| 1915 | |||
| 1916 | rc = bnxt_re_build_send_wqe(qp, wr, &wqe); | ||
| 1917 | if (!rc) | ||
| 1918 | rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); | ||
| 1919 | bad: | ||
| 1920 | if (rc) { | ||
| 1921 | dev_err(rdev_to_dev(rdev), | ||
| 1922 | "Post send failed opcode = %#x rc = %d", | ||
| 1923 | wr->opcode, rc); | ||
| 1924 | break; | ||
| 1925 | } | ||
| 1926 | wr = wr->next; | ||
| 1927 | } | ||
| 1928 | bnxt_qplib_post_send_db(&qp->qplib_qp); | ||
| 1929 | spin_unlock_irqrestore(&qp->sq_lock, flags); | ||
| 1930 | return rc; | ||
| 1931 | } | ||
| 1932 | |||
| 1933 | int bnxt_re_post_send(struct ib_qp *ib_qp, struct ib_send_wr *wr, | ||
| 1934 | struct ib_send_wr **bad_wr) | ||
| 1935 | { | ||
| 1936 | struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | ||
| 1937 | struct bnxt_qplib_swqe wqe; | ||
| 1938 | int rc = 0, payload_sz = 0; | ||
| 1939 | unsigned long flags; | ||
| 1940 | |||
| 1941 | spin_lock_irqsave(&qp->sq_lock, flags); | ||
| 1942 | while (wr) { | ||
| 1943 | /* House keeping */ | ||
| 1944 | memset(&wqe, 0, sizeof(wqe)); | ||
| 1945 | |||
| 1946 | /* Common */ | ||
| 1947 | wqe.num_sge = wr->num_sge; | ||
| 1948 | if (wr->num_sge > qp->qplib_qp.sq.max_sge) { | ||
| 1949 | dev_err(rdev_to_dev(qp->rdev), | ||
| 1950 | "Limit exceeded for Send SGEs"); | ||
| 1951 | rc = -EINVAL; | ||
| 1952 | goto bad; | ||
| 1953 | } | ||
| 1954 | |||
| 1955 | payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); | ||
| 1956 | if (payload_sz < 0) { | ||
| 1957 | rc = -EINVAL; | ||
| 1958 | goto bad; | ||
| 1959 | } | ||
| 1960 | wqe.wr_id = wr->wr_id; | ||
| 1961 | |||
| 1962 | switch (wr->opcode) { | ||
| 1963 | case IB_WR_SEND: | ||
| 1964 | case IB_WR_SEND_WITH_IMM: | ||
| 1965 | if (ib_qp->qp_type == IB_QPT_GSI) { | ||
| 1966 | rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, | ||
| 1967 | payload_sz); | ||
| 1968 | if (rc) | ||
| 1969 | goto bad; | ||
| 1970 | wqe.rawqp1.lflags |= | ||
| 1971 | SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; | ||
| 1972 | } | ||
| 1973 | switch (wr->send_flags) { | ||
| 1974 | case IB_SEND_IP_CSUM: | ||
| 1975 | wqe.rawqp1.lflags |= | ||
| 1976 | SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; | ||
| 1977 | break; | ||
| 1978 | default: | ||
| 1979 | break; | ||
| 1980 | } | ||
| 1981 | /* Fall thru to build the wqe */ | ||
| 1982 | case IB_WR_SEND_WITH_INV: | ||
| 1983 | rc = bnxt_re_build_send_wqe(qp, wr, &wqe); | ||
| 1984 | break; | ||
| 1985 | case IB_WR_RDMA_WRITE: | ||
| 1986 | case IB_WR_RDMA_WRITE_WITH_IMM: | ||
| 1987 | case IB_WR_RDMA_READ: | ||
| 1988 | rc = bnxt_re_build_rdma_wqe(wr, &wqe); | ||
| 1989 | break; | ||
| 1990 | case IB_WR_ATOMIC_CMP_AND_SWP: | ||
| 1991 | case IB_WR_ATOMIC_FETCH_AND_ADD: | ||
| 1992 | rc = bnxt_re_build_atomic_wqe(wr, &wqe); | ||
| 1993 | break; | ||
| 1994 | case IB_WR_RDMA_READ_WITH_INV: | ||
| 1995 | dev_err(rdev_to_dev(qp->rdev), | ||
| 1996 | "RDMA Read with Invalidate is not supported"); | ||
| 1997 | rc = -EINVAL; | ||
| 1998 | goto bad; | ||
| 1999 | case IB_WR_LOCAL_INV: | ||
| 2000 | rc = bnxt_re_build_inv_wqe(wr, &wqe); | ||
| 2001 | break; | ||
| 2002 | case IB_WR_REG_MR: | ||
| 2003 | rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); | ||
| 2004 | break; | ||
| 2005 | default: | ||
| 2006 | /* Unsupported WRs */ | ||
| 2007 | dev_err(rdev_to_dev(qp->rdev), | ||
| 2008 | "WR (%#x) is not supported", wr->opcode); | ||
| 2009 | rc = -EINVAL; | ||
| 2010 | goto bad; | ||
| 2011 | } | ||
| 2012 | if (!rc) | ||
| 2013 | rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); | ||
| 2014 | bad: | ||
| 2015 | if (rc) { | ||
| 2016 | dev_err(rdev_to_dev(qp->rdev), | ||
| 2017 | "post_send failed op:%#x qps = %#x rc = %d\n", | ||
| 2018 | wr->opcode, qp->qplib_qp.state, rc); | ||
| 2019 | *bad_wr = wr; | ||
| 2020 | break; | ||
| 2021 | } | ||
| 2022 | wr = wr->next; | ||
| 2023 | } | ||
| 2024 | bnxt_qplib_post_send_db(&qp->qplib_qp); | ||
| 2025 | spin_unlock_irqrestore(&qp->sq_lock, flags); | ||
| 2026 | |||
| 2027 | return rc; | ||
| 2028 | } | ||
| 2029 | |||
| 2030 | static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, | ||
| 2031 | struct bnxt_re_qp *qp, | ||
| 2032 | struct ib_recv_wr *wr) | ||
| 2033 | { | ||
| 2034 | struct bnxt_qplib_swqe wqe; | ||
| 2035 | int rc = 0, payload_sz = 0; | ||
| 2036 | |||
| 2037 | memset(&wqe, 0, sizeof(wqe)); | ||
| 2038 | while (wr) { | ||
| 2039 | /* House keeping */ | ||
| 2040 | memset(&wqe, 0, sizeof(wqe)); | ||
| 2041 | |||
| 2042 | /* Common */ | ||
| 2043 | wqe.num_sge = wr->num_sge; | ||
| 2044 | if (wr->num_sge > qp->qplib_qp.rq.max_sge) { | ||
| 2045 | dev_err(rdev_to_dev(rdev), | ||
| 2046 | "Limit exceeded for Receive SGEs"); | ||
| 2047 | rc = -EINVAL; | ||
| 2048 | break; | ||
| 2049 | } | ||
| 2050 | payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, | ||
| 2051 | wr->num_sge); | ||
| 2052 | wqe.wr_id = wr->wr_id; | ||
| 2053 | wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; | ||
| 2054 | |||
| 2055 | rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); | ||
| 2056 | if (rc) | ||
| 2057 | break; | ||
| 2058 | |||
| 2059 | wr = wr->next; | ||
| 2060 | } | ||
| 2061 | if (!rc) | ||
| 2062 | bnxt_qplib_post_recv_db(&qp->qplib_qp); | ||
| 2063 | return rc; | ||
| 2064 | } | ||
| 2065 | |||
| 2066 | int bnxt_re_post_recv(struct ib_qp *ib_qp, struct ib_recv_wr *wr, | ||
| 2067 | struct ib_recv_wr **bad_wr) | ||
| 2068 | { | ||
| 2069 | struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); | ||
| 2070 | struct bnxt_qplib_swqe wqe; | ||
| 2071 | int rc = 0, payload_sz = 0; | ||
| 2072 | |||
| 2073 | while (wr) { | ||
| 2074 | /* House keeping */ | ||
| 2075 | memset(&wqe, 0, sizeof(wqe)); | ||
| 2076 | |||
| 2077 | /* Common */ | ||
| 2078 | wqe.num_sge = wr->num_sge; | ||
| 2079 | if (wr->num_sge > qp->qplib_qp.rq.max_sge) { | ||
| 2080 | dev_err(rdev_to_dev(qp->rdev), | ||
| 2081 | "Limit exceeded for Receive SGEs"); | ||
| 2082 | rc = -EINVAL; | ||
| 2083 | *bad_wr = wr; | ||
| 2084 | break; | ||
| 2085 | } | ||
| 2086 | |||
| 2087 | payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, | ||
| 2088 | wr->num_sge); | ||
| 2089 | wqe.wr_id = wr->wr_id; | ||
| 2090 | wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; | ||
| 2091 | |||
| 2092 | if (ib_qp->qp_type == IB_QPT_GSI) | ||
| 2093 | rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, | ||
| 2094 | payload_sz); | ||
| 2095 | if (!rc) | ||
| 2096 | rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); | ||
| 2097 | if (rc) { | ||
| 2098 | *bad_wr = wr; | ||
| 2099 | break; | ||
| 2100 | } | ||
| 2101 | wr = wr->next; | ||
| 2102 | } | ||
| 2103 | bnxt_qplib_post_recv_db(&qp->qplib_qp); | ||
| 2104 | return rc; | ||
| 2105 | } | ||
| 2106 | |||
| 2107 | /* Completion Queues */ | ||
| 2108 | int bnxt_re_destroy_cq(struct ib_cq *ib_cq) | ||
| 2109 | { | ||
| 2110 | struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); | ||
| 2111 | struct bnxt_re_dev *rdev = cq->rdev; | ||
| 2112 | int rc; | ||
| 2113 | |||
| 2114 | rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); | ||
| 2115 | if (rc) { | ||
| 2116 | dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ"); | ||
| 2117 | return rc; | ||
| 2118 | } | ||
| 2119 | if (cq->umem && !IS_ERR(cq->umem)) | ||
| 2120 | ib_umem_release(cq->umem); | ||
| 2121 | |||
| 2122 | if (cq) { | ||
| 2123 | kfree(cq->cql); | ||
| 2124 | kfree(cq); | ||
| 2125 | } | ||
| 2126 | atomic_dec(&rdev->cq_count); | ||
| 2127 | rdev->nq.budget--; | ||
| 2128 | return 0; | ||
| 2129 | } | ||
| 2130 | |||
| 2131 | struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, | ||
| 2132 | const struct ib_cq_init_attr *attr, | ||
| 2133 | struct ib_ucontext *context, | ||
| 2134 | struct ib_udata *udata) | ||
| 2135 | { | ||
| 2136 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 2137 | struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | ||
| 2138 | struct bnxt_re_cq *cq = NULL; | ||
| 2139 | int rc, entries; | ||
| 2140 | int cqe = attr->cqe; | ||
| 2141 | |||
| 2142 | /* Validate CQ fields */ | ||
| 2143 | if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { | ||
| 2144 | dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded"); | ||
| 2145 | return ERR_PTR(-EINVAL); | ||
| 2146 | } | ||
| 2147 | cq = kzalloc(sizeof(*cq), GFP_KERNEL); | ||
| 2148 | if (!cq) | ||
| 2149 | return ERR_PTR(-ENOMEM); | ||
| 2150 | |||
| 2151 | cq->rdev = rdev; | ||
| 2152 | cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); | ||
| 2153 | |||
| 2154 | entries = roundup_pow_of_two(cqe + 1); | ||
| 2155 | if (entries > dev_attr->max_cq_wqes + 1) | ||
| 2156 | entries = dev_attr->max_cq_wqes + 1; | ||
| 2157 | |||
| 2158 | if (context) { | ||
| 2159 | struct bnxt_re_cq_req req; | ||
| 2160 | struct bnxt_re_ucontext *uctx = container_of | ||
| 2161 | (context, | ||
| 2162 | struct bnxt_re_ucontext, | ||
| 2163 | ib_uctx); | ||
| 2164 | if (ib_copy_from_udata(&req, udata, sizeof(req))) { | ||
| 2165 | rc = -EFAULT; | ||
| 2166 | goto fail; | ||
| 2167 | } | ||
| 2168 | |||
| 2169 | cq->umem = ib_umem_get(context, req.cq_va, | ||
| 2170 | entries * sizeof(struct cq_base), | ||
| 2171 | IB_ACCESS_LOCAL_WRITE, 1); | ||
| 2172 | if (IS_ERR(cq->umem)) { | ||
| 2173 | rc = PTR_ERR(cq->umem); | ||
| 2174 | goto fail; | ||
| 2175 | } | ||
| 2176 | cq->qplib_cq.sghead = cq->umem->sg_head.sgl; | ||
| 2177 | cq->qplib_cq.nmap = cq->umem->nmap; | ||
| 2178 | cq->qplib_cq.dpi = uctx->dpi; | ||
| 2179 | } else { | ||
| 2180 | cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); | ||
| 2181 | cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), | ||
| 2182 | GFP_KERNEL); | ||
| 2183 | if (!cq->cql) { | ||
| 2184 | rc = -ENOMEM; | ||
| 2185 | goto fail; | ||
| 2186 | } | ||
| 2187 | |||
| 2188 | cq->qplib_cq.dpi = &rdev->dpi_privileged; | ||
| 2189 | cq->qplib_cq.sghead = NULL; | ||
| 2190 | cq->qplib_cq.nmap = 0; | ||
| 2191 | } | ||
| 2192 | cq->qplib_cq.max_wqe = entries; | ||
| 2193 | cq->qplib_cq.cnq_hw_ring_id = rdev->nq.ring_id; | ||
| 2194 | |||
| 2195 | rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); | ||
| 2196 | if (rc) { | ||
| 2197 | dev_err(rdev_to_dev(rdev), "Failed to create HW CQ"); | ||
| 2198 | goto fail; | ||
| 2199 | } | ||
| 2200 | |||
| 2201 | cq->ib_cq.cqe = entries; | ||
| 2202 | cq->cq_period = cq->qplib_cq.period; | ||
| 2203 | rdev->nq.budget++; | ||
| 2204 | |||
| 2205 | atomic_inc(&rdev->cq_count); | ||
| 2206 | |||
| 2207 | if (context) { | ||
| 2208 | struct bnxt_re_cq_resp resp; | ||
| 2209 | |||
| 2210 | resp.cqid = cq->qplib_cq.id; | ||
| 2211 | resp.tail = cq->qplib_cq.hwq.cons; | ||
| 2212 | resp.phase = cq->qplib_cq.period; | ||
| 2213 | resp.rsvd = 0; | ||
| 2214 | rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); | ||
| 2215 | if (rc) { | ||
| 2216 | dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata"); | ||
| 2217 | bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); | ||
| 2218 | goto c2fail; | ||
| 2219 | } | ||
| 2220 | } | ||
| 2221 | |||
| 2222 | return &cq->ib_cq; | ||
| 2223 | |||
| 2224 | c2fail: | ||
| 2225 | if (context) | ||
| 2226 | ib_umem_release(cq->umem); | ||
| 2227 | fail: | ||
| 2228 | kfree(cq->cql); | ||
| 2229 | kfree(cq); | ||
| 2230 | return ERR_PTR(rc); | ||
| 2231 | } | ||
| 2232 | |||
| 2233 | static u8 __req_to_ib_wc_status(u8 qstatus) | ||
| 2234 | { | ||
| 2235 | switch (qstatus) { | ||
| 2236 | case CQ_REQ_STATUS_OK: | ||
| 2237 | return IB_WC_SUCCESS; | ||
| 2238 | case CQ_REQ_STATUS_BAD_RESPONSE_ERR: | ||
| 2239 | return IB_WC_BAD_RESP_ERR; | ||
| 2240 | case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: | ||
| 2241 | return IB_WC_LOC_LEN_ERR; | ||
| 2242 | case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: | ||
| 2243 | return IB_WC_LOC_QP_OP_ERR; | ||
| 2244 | case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: | ||
| 2245 | return IB_WC_LOC_PROT_ERR; | ||
| 2246 | case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: | ||
| 2247 | return IB_WC_GENERAL_ERR; | ||
| 2248 | case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: | ||
| 2249 | return IB_WC_REM_INV_REQ_ERR; | ||
| 2250 | case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: | ||
| 2251 | return IB_WC_REM_ACCESS_ERR; | ||
| 2252 | case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: | ||
| 2253 | return IB_WC_REM_OP_ERR; | ||
| 2254 | case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: | ||
| 2255 | return IB_WC_RNR_RETRY_EXC_ERR; | ||
| 2256 | case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: | ||
| 2257 | return IB_WC_RETRY_EXC_ERR; | ||
| 2258 | case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: | ||
| 2259 | return IB_WC_WR_FLUSH_ERR; | ||
| 2260 | default: | ||
| 2261 | return IB_WC_GENERAL_ERR; | ||
| 2262 | } | ||
| 2263 | return 0; | ||
| 2264 | } | ||
| 2265 | |||
| 2266 | static u8 __rawqp1_to_ib_wc_status(u8 qstatus) | ||
| 2267 | { | ||
| 2268 | switch (qstatus) { | ||
| 2269 | case CQ_RES_RAWETH_QP1_STATUS_OK: | ||
| 2270 | return IB_WC_SUCCESS; | ||
| 2271 | case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: | ||
| 2272 | return IB_WC_LOC_ACCESS_ERR; | ||
| 2273 | case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: | ||
| 2274 | return IB_WC_LOC_LEN_ERR; | ||
| 2275 | case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: | ||
| 2276 | return IB_WC_LOC_PROT_ERR; | ||
| 2277 | case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: | ||
| 2278 | return IB_WC_LOC_QP_OP_ERR; | ||
| 2279 | case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: | ||
| 2280 | return IB_WC_GENERAL_ERR; | ||
| 2281 | case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: | ||
| 2282 | return IB_WC_WR_FLUSH_ERR; | ||
| 2283 | case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: | ||
| 2284 | return IB_WC_WR_FLUSH_ERR; | ||
| 2285 | default: | ||
| 2286 | return IB_WC_GENERAL_ERR; | ||
| 2287 | } | ||
| 2288 | } | ||
| 2289 | |||
| 2290 | static u8 __rc_to_ib_wc_status(u8 qstatus) | ||
| 2291 | { | ||
| 2292 | switch (qstatus) { | ||
| 2293 | case CQ_RES_RC_STATUS_OK: | ||
| 2294 | return IB_WC_SUCCESS; | ||
| 2295 | case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: | ||
| 2296 | return IB_WC_LOC_ACCESS_ERR; | ||
| 2297 | case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: | ||
| 2298 | return IB_WC_LOC_LEN_ERR; | ||
| 2299 | case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: | ||
| 2300 | return IB_WC_LOC_PROT_ERR; | ||
| 2301 | case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: | ||
| 2302 | return IB_WC_LOC_QP_OP_ERR; | ||
| 2303 | case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: | ||
| 2304 | return IB_WC_GENERAL_ERR; | ||
| 2305 | case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: | ||
| 2306 | return IB_WC_REM_INV_REQ_ERR; | ||
| 2307 | case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: | ||
| 2308 | return IB_WC_WR_FLUSH_ERR; | ||
| 2309 | case CQ_RES_RC_STATUS_HW_FLUSH_ERR: | ||
| 2310 | return IB_WC_WR_FLUSH_ERR; | ||
| 2311 | default: | ||
| 2312 | return IB_WC_GENERAL_ERR; | ||
| 2313 | } | ||
| 2314 | } | ||
| 2315 | |||
| 2316 | static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) | ||
| 2317 | { | ||
| 2318 | switch (cqe->type) { | ||
| 2319 | case BNXT_QPLIB_SWQE_TYPE_SEND: | ||
| 2320 | wc->opcode = IB_WC_SEND; | ||
| 2321 | break; | ||
| 2322 | case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: | ||
| 2323 | wc->opcode = IB_WC_SEND; | ||
| 2324 | wc->wc_flags |= IB_WC_WITH_IMM; | ||
| 2325 | break; | ||
| 2326 | case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: | ||
| 2327 | wc->opcode = IB_WC_SEND; | ||
| 2328 | wc->wc_flags |= IB_WC_WITH_INVALIDATE; | ||
| 2329 | break; | ||
| 2330 | case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: | ||
| 2331 | wc->opcode = IB_WC_RDMA_WRITE; | ||
| 2332 | break; | ||
| 2333 | case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: | ||
| 2334 | wc->opcode = IB_WC_RDMA_WRITE; | ||
| 2335 | wc->wc_flags |= IB_WC_WITH_IMM; | ||
| 2336 | break; | ||
| 2337 | case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: | ||
| 2338 | wc->opcode = IB_WC_RDMA_READ; | ||
| 2339 | break; | ||
| 2340 | case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: | ||
| 2341 | wc->opcode = IB_WC_COMP_SWAP; | ||
| 2342 | break; | ||
| 2343 | case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: | ||
| 2344 | wc->opcode = IB_WC_FETCH_ADD; | ||
| 2345 | break; | ||
| 2346 | case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: | ||
| 2347 | wc->opcode = IB_WC_LOCAL_INV; | ||
| 2348 | break; | ||
| 2349 | case BNXT_QPLIB_SWQE_TYPE_REG_MR: | ||
| 2350 | wc->opcode = IB_WC_REG_MR; | ||
| 2351 | break; | ||
| 2352 | default: | ||
| 2353 | wc->opcode = IB_WC_SEND; | ||
| 2354 | break; | ||
| 2355 | } | ||
| 2356 | |||
| 2357 | wc->status = __req_to_ib_wc_status(cqe->status); | ||
| 2358 | } | ||
| 2359 | |||
| 2360 | static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, | ||
| 2361 | u16 raweth_qp1_flags2) | ||
| 2362 | { | ||
| 2363 | bool is_udp = false, is_ipv6 = false, is_ipv4 = false; | ||
| 2364 | |||
| 2365 | /* raweth_qp1_flags Bit 9-6 indicates itype */ | ||
| 2366 | if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) | ||
| 2367 | != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) | ||
| 2368 | return -1; | ||
| 2369 | |||
| 2370 | if (raweth_qp1_flags2 & | ||
| 2371 | CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && | ||
| 2372 | raweth_qp1_flags2 & | ||
| 2373 | CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { | ||
| 2374 | is_udp = true; | ||
| 2375 | /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ | ||
| 2376 | (raweth_qp1_flags2 & | ||
| 2377 | CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? | ||
| 2378 | (is_ipv6 = true) : (is_ipv4 = true); | ||
| 2379 | return ((is_ipv6) ? | ||
| 2380 | BNXT_RE_ROCEV2_IPV6_PACKET : | ||
| 2381 | BNXT_RE_ROCEV2_IPV4_PACKET); | ||
| 2382 | } else { | ||
| 2383 | return BNXT_RE_ROCE_V1_PACKET; | ||
| 2384 | } | ||
| 2385 | } | ||
| 2386 | |||
| 2387 | static int bnxt_re_to_ib_nw_type(int nw_type) | ||
| 2388 | { | ||
| 2389 | u8 nw_hdr_type = 0xFF; | ||
| 2390 | |||
| 2391 | switch (nw_type) { | ||
| 2392 | case BNXT_RE_ROCE_V1_PACKET: | ||
| 2393 | nw_hdr_type = RDMA_NETWORK_ROCE_V1; | ||
| 2394 | break; | ||
| 2395 | case BNXT_RE_ROCEV2_IPV4_PACKET: | ||
| 2396 | nw_hdr_type = RDMA_NETWORK_IPV4; | ||
| 2397 | break; | ||
| 2398 | case BNXT_RE_ROCEV2_IPV6_PACKET: | ||
| 2399 | nw_hdr_type = RDMA_NETWORK_IPV6; | ||
| 2400 | break; | ||
| 2401 | } | ||
| 2402 | return nw_hdr_type; | ||
| 2403 | } | ||
| 2404 | |||
| 2405 | static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, | ||
| 2406 | void *rq_hdr_buf) | ||
| 2407 | { | ||
| 2408 | u8 *tmp_buf = NULL; | ||
| 2409 | struct ethhdr *eth_hdr; | ||
| 2410 | u16 eth_type; | ||
| 2411 | bool rc = false; | ||
| 2412 | |||
| 2413 | tmp_buf = (u8 *)rq_hdr_buf; | ||
| 2414 | /* | ||
| 2415 | * If dest mac is not same as I/F mac, this could be a | ||
| 2416 | * loopback address or multicast address, check whether | ||
| 2417 | * it is a loopback packet | ||
| 2418 | */ | ||
| 2419 | if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { | ||
| 2420 | tmp_buf += 4; | ||
| 2421 | /* Check the ether type */ | ||
| 2422 | eth_hdr = (struct ethhdr *)tmp_buf; | ||
| 2423 | eth_type = ntohs(eth_hdr->h_proto); | ||
| 2424 | switch (eth_type) { | ||
| 2425 | case ETH_P_IBOE: | ||
| 2426 | rc = true; | ||
| 2427 | break; | ||
| 2428 | case ETH_P_IP: | ||
| 2429 | case ETH_P_IPV6: { | ||
| 2430 | u32 len; | ||
| 2431 | struct udphdr *udp_hdr; | ||
| 2432 | |||
| 2433 | len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : | ||
| 2434 | sizeof(struct ipv6hdr)); | ||
| 2435 | tmp_buf += sizeof(struct ethhdr) + len; | ||
| 2436 | udp_hdr = (struct udphdr *)tmp_buf; | ||
| 2437 | if (ntohs(udp_hdr->dest) == | ||
| 2438 | ROCE_V2_UDP_DPORT) | ||
| 2439 | rc = true; | ||
| 2440 | break; | ||
| 2441 | } | ||
| 2442 | default: | ||
| 2443 | break; | ||
| 2444 | } | ||
| 2445 | } | ||
| 2446 | |||
| 2447 | return rc; | ||
| 2448 | } | ||
| 2449 | |||
| 2450 | static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp, | ||
| 2451 | struct bnxt_qplib_cqe *cqe) | ||
| 2452 | { | ||
| 2453 | struct bnxt_re_dev *rdev = qp1_qp->rdev; | ||
| 2454 | struct bnxt_re_sqp_entries *sqp_entry = NULL; | ||
| 2455 | struct bnxt_re_qp *qp = rdev->qp1_sqp; | ||
| 2456 | struct ib_send_wr *swr; | ||
| 2457 | struct ib_ud_wr udwr; | ||
| 2458 | struct ib_recv_wr rwr; | ||
| 2459 | int pkt_type = 0; | ||
| 2460 | u32 tbl_idx; | ||
| 2461 | void *rq_hdr_buf; | ||
| 2462 | dma_addr_t rq_hdr_buf_map; | ||
| 2463 | dma_addr_t shrq_hdr_buf_map; | ||
| 2464 | u32 offset = 0; | ||
| 2465 | u32 skip_bytes = 0; | ||
| 2466 | struct ib_sge s_sge[2]; | ||
| 2467 | struct ib_sge r_sge[2]; | ||
| 2468 | int rc; | ||
| 2469 | |||
| 2470 | memset(&udwr, 0, sizeof(udwr)); | ||
| 2471 | memset(&rwr, 0, sizeof(rwr)); | ||
| 2472 | memset(&s_sge, 0, sizeof(s_sge)); | ||
| 2473 | memset(&r_sge, 0, sizeof(r_sge)); | ||
| 2474 | |||
| 2475 | swr = &udwr.wr; | ||
| 2476 | tbl_idx = cqe->wr_id; | ||
| 2477 | |||
| 2478 | rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf + | ||
| 2479 | (tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size); | ||
| 2480 | rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp, | ||
| 2481 | tbl_idx); | ||
| 2482 | |||
| 2483 | /* Shadow QP header buffer */ | ||
| 2484 | shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp, | ||
| 2485 | tbl_idx); | ||
| 2486 | sqp_entry = &rdev->sqp_tbl[tbl_idx]; | ||
| 2487 | |||
| 2488 | /* Store this cqe */ | ||
| 2489 | memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); | ||
| 2490 | sqp_entry->qp1_qp = qp1_qp; | ||
| 2491 | |||
| 2492 | /* Find packet type from the cqe */ | ||
| 2493 | |||
| 2494 | pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, | ||
| 2495 | cqe->raweth_qp1_flags2); | ||
| 2496 | if (pkt_type < 0) { | ||
| 2497 | dev_err(rdev_to_dev(rdev), "Invalid packet\n"); | ||
| 2498 | return -EINVAL; | ||
| 2499 | } | ||
| 2500 | |||
| 2501 | /* Adjust the offset for the user buffer and post in the rq */ | ||
| 2502 | |||
| 2503 | if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) | ||
| 2504 | offset = 20; | ||
| 2505 | |||
| 2506 | /* | ||
| 2507 | * QP1 loopback packet has 4 bytes of internal header before | ||
| 2508 | * ether header. Skip these four bytes. | ||
| 2509 | */ | ||
| 2510 | if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) | ||
| 2511 | skip_bytes = 4; | ||
| 2512 | |||
| 2513 | /* First send SGE . Skip the ether header*/ | ||
| 2514 | s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE | ||
| 2515 | + skip_bytes; | ||
| 2516 | s_sge[0].lkey = 0xFFFFFFFF; | ||
| 2517 | s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : | ||
| 2518 | BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; | ||
| 2519 | |||
| 2520 | /* Second Send SGE */ | ||
| 2521 | s_sge[1].addr = s_sge[0].addr + s_sge[0].length + | ||
| 2522 | BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; | ||
| 2523 | if (pkt_type != BNXT_RE_ROCE_V1_PACKET) | ||
| 2524 | s_sge[1].addr += 8; | ||
| 2525 | s_sge[1].lkey = 0xFFFFFFFF; | ||
| 2526 | s_sge[1].length = 256; | ||
| 2527 | |||
| 2528 | /* First recv SGE */ | ||
| 2529 | |||
| 2530 | r_sge[0].addr = shrq_hdr_buf_map; | ||
| 2531 | r_sge[0].lkey = 0xFFFFFFFF; | ||
| 2532 | r_sge[0].length = 40; | ||
| 2533 | |||
| 2534 | r_sge[1].addr = sqp_entry->sge.addr + offset; | ||
| 2535 | r_sge[1].lkey = sqp_entry->sge.lkey; | ||
| 2536 | r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; | ||
| 2537 | |||
| 2538 | /* Create receive work request */ | ||
| 2539 | rwr.num_sge = 2; | ||
| 2540 | rwr.sg_list = r_sge; | ||
| 2541 | rwr.wr_id = tbl_idx; | ||
| 2542 | rwr.next = NULL; | ||
| 2543 | |||
| 2544 | rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr); | ||
| 2545 | if (rc) { | ||
| 2546 | dev_err(rdev_to_dev(rdev), | ||
| 2547 | "Failed to post Rx buffers to shadow QP"); | ||
| 2548 | return -ENOMEM; | ||
| 2549 | } | ||
| 2550 | |||
| 2551 | swr->num_sge = 2; | ||
| 2552 | swr->sg_list = s_sge; | ||
| 2553 | swr->wr_id = tbl_idx; | ||
| 2554 | swr->opcode = IB_WR_SEND; | ||
| 2555 | swr->next = NULL; | ||
| 2556 | |||
| 2557 | udwr.ah = &rdev->sqp_ah->ib_ah; | ||
| 2558 | udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id; | ||
| 2559 | udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey; | ||
| 2560 | |||
| 2561 | /* post data received in the send queue */ | ||
| 2562 | rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr); | ||
| 2563 | |||
| 2564 | return 0; | ||
| 2565 | } | ||
| 2566 | |||
| 2567 | static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, | ||
| 2568 | struct bnxt_qplib_cqe *cqe) | ||
| 2569 | { | ||
| 2570 | wc->opcode = IB_WC_RECV; | ||
| 2571 | wc->status = __rawqp1_to_ib_wc_status(cqe->status); | ||
| 2572 | wc->wc_flags |= IB_WC_GRH; | ||
| 2573 | } | ||
| 2574 | |||
| 2575 | static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, | ||
| 2576 | struct bnxt_qplib_cqe *cqe) | ||
| 2577 | { | ||
| 2578 | wc->opcode = IB_WC_RECV; | ||
| 2579 | wc->status = __rc_to_ib_wc_status(cqe->status); | ||
| 2580 | |||
| 2581 | if (cqe->flags & CQ_RES_RC_FLAGS_IMM) | ||
| 2582 | wc->wc_flags |= IB_WC_WITH_IMM; | ||
| 2583 | if (cqe->flags & CQ_RES_RC_FLAGS_INV) | ||
| 2584 | wc->wc_flags |= IB_WC_WITH_INVALIDATE; | ||
| 2585 | if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == | ||
| 2586 | (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) | ||
| 2587 | wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; | ||
| 2588 | } | ||
| 2589 | |||
| 2590 | static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp, | ||
| 2591 | struct ib_wc *wc, | ||
| 2592 | struct bnxt_qplib_cqe *cqe) | ||
| 2593 | { | ||
| 2594 | u32 tbl_idx; | ||
| 2595 | struct bnxt_re_dev *rdev = qp->rdev; | ||
| 2596 | struct bnxt_re_qp *qp1_qp = NULL; | ||
| 2597 | struct bnxt_qplib_cqe *orig_cqe = NULL; | ||
| 2598 | struct bnxt_re_sqp_entries *sqp_entry = NULL; | ||
| 2599 | int nw_type; | ||
| 2600 | |||
| 2601 | tbl_idx = cqe->wr_id; | ||
| 2602 | |||
| 2603 | sqp_entry = &rdev->sqp_tbl[tbl_idx]; | ||
| 2604 | qp1_qp = sqp_entry->qp1_qp; | ||
| 2605 | orig_cqe = &sqp_entry->cqe; | ||
| 2606 | |||
| 2607 | wc->wr_id = sqp_entry->wrid; | ||
| 2608 | wc->byte_len = orig_cqe->length; | ||
| 2609 | wc->qp = &qp1_qp->ib_qp; | ||
| 2610 | |||
| 2611 | wc->ex.imm_data = orig_cqe->immdata; | ||
| 2612 | wc->src_qp = orig_cqe->src_qp; | ||
| 2613 | memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); | ||
| 2614 | wc->port_num = 1; | ||
| 2615 | wc->vendor_err = orig_cqe->status; | ||
| 2616 | |||
| 2617 | wc->opcode = IB_WC_RECV; | ||
| 2618 | wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); | ||
| 2619 | wc->wc_flags |= IB_WC_GRH; | ||
| 2620 | |||
| 2621 | nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, | ||
| 2622 | orig_cqe->raweth_qp1_flags2); | ||
| 2623 | if (nw_type >= 0) { | ||
| 2624 | wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); | ||
| 2625 | wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; | ||
| 2626 | } | ||
| 2627 | } | ||
| 2628 | |||
| 2629 | static void bnxt_re_process_res_ud_wc(struct ib_wc *wc, | ||
| 2630 | struct bnxt_qplib_cqe *cqe) | ||
| 2631 | { | ||
| 2632 | wc->opcode = IB_WC_RECV; | ||
| 2633 | wc->status = __rc_to_ib_wc_status(cqe->status); | ||
| 2634 | |||
| 2635 | if (cqe->flags & CQ_RES_RC_FLAGS_IMM) | ||
| 2636 | wc->wc_flags |= IB_WC_WITH_IMM; | ||
| 2637 | if (cqe->flags & CQ_RES_RC_FLAGS_INV) | ||
| 2638 | wc->wc_flags |= IB_WC_WITH_INVALIDATE; | ||
| 2639 | if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == | ||
| 2640 | (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) | ||
| 2641 | wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; | ||
| 2642 | } | ||
| 2643 | |||
| 2644 | int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) | ||
| 2645 | { | ||
| 2646 | struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); | ||
| 2647 | struct bnxt_re_qp *qp; | ||
| 2648 | struct bnxt_qplib_cqe *cqe; | ||
| 2649 | int i, ncqe, budget; | ||
| 2650 | u32 tbl_idx; | ||
| 2651 | struct bnxt_re_sqp_entries *sqp_entry = NULL; | ||
| 2652 | unsigned long flags; | ||
| 2653 | |||
| 2654 | spin_lock_irqsave(&cq->cq_lock, flags); | ||
| 2655 | budget = min_t(u32, num_entries, cq->max_cql); | ||
| 2656 | if (!cq->cql) { | ||
| 2657 | dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use"); | ||
| 2658 | goto exit; | ||
| 2659 | } | ||
| 2660 | cqe = &cq->cql[0]; | ||
| 2661 | while (budget) { | ||
| 2662 | ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget); | ||
| 2663 | if (!ncqe) | ||
| 2664 | break; | ||
| 2665 | |||
| 2666 | for (i = 0; i < ncqe; i++, cqe++) { | ||
| 2667 | /* Transcribe each qplib_wqe back to ib_wc */ | ||
| 2668 | memset(wc, 0, sizeof(*wc)); | ||
| 2669 | |||
| 2670 | wc->wr_id = cqe->wr_id; | ||
| 2671 | wc->byte_len = cqe->length; | ||
| 2672 | qp = container_of | ||
| 2673 | ((struct bnxt_qplib_qp *) | ||
| 2674 | (unsigned long)(cqe->qp_handle), | ||
| 2675 | struct bnxt_re_qp, qplib_qp); | ||
| 2676 | if (!qp) { | ||
| 2677 | dev_err(rdev_to_dev(cq->rdev), | ||
| 2678 | "POLL CQ : bad QP handle"); | ||
| 2679 | continue; | ||
| 2680 | } | ||
| 2681 | wc->qp = &qp->ib_qp; | ||
| 2682 | wc->ex.imm_data = cqe->immdata; | ||
| 2683 | wc->src_qp = cqe->src_qp; | ||
| 2684 | memcpy(wc->smac, cqe->smac, ETH_ALEN); | ||
| 2685 | wc->port_num = 1; | ||
| 2686 | wc->vendor_err = cqe->status; | ||
| 2687 | |||
| 2688 | switch (cqe->opcode) { | ||
| 2689 | case CQ_BASE_CQE_TYPE_REQ: | ||
| 2690 | if (qp->qplib_qp.id == | ||
| 2691 | qp->rdev->qp1_sqp->qplib_qp.id) { | ||
| 2692 | /* Handle this completion with | ||
| 2693 | * the stored completion | ||
| 2694 | */ | ||
| 2695 | memset(wc, 0, sizeof(*wc)); | ||
| 2696 | continue; | ||
| 2697 | } | ||
| 2698 | bnxt_re_process_req_wc(wc, cqe); | ||
| 2699 | break; | ||
| 2700 | case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: | ||
| 2701 | if (!cqe->status) { | ||
| 2702 | int rc = 0; | ||
| 2703 | |||
| 2704 | rc = bnxt_re_process_raw_qp_pkt_rx | ||
| 2705 | (qp, cqe); | ||
| 2706 | if (!rc) { | ||
| 2707 | memset(wc, 0, sizeof(*wc)); | ||
| 2708 | continue; | ||
| 2709 | } | ||
| 2710 | cqe->status = -1; | ||
| 2711 | } | ||
| 2712 | /* Errors need not be looped back. | ||
| 2713 | * But change the wr_id to the one | ||
| 2714 | * stored in the table | ||
| 2715 | */ | ||
| 2716 | tbl_idx = cqe->wr_id; | ||
| 2717 | sqp_entry = &cq->rdev->sqp_tbl[tbl_idx]; | ||
| 2718 | wc->wr_id = sqp_entry->wrid; | ||
| 2719 | bnxt_re_process_res_rawqp1_wc(wc, cqe); | ||
| 2720 | break; | ||
| 2721 | case CQ_BASE_CQE_TYPE_RES_RC: | ||
| 2722 | bnxt_re_process_res_rc_wc(wc, cqe); | ||
| 2723 | break; | ||
| 2724 | case CQ_BASE_CQE_TYPE_RES_UD: | ||
| 2725 | if (qp->qplib_qp.id == | ||
| 2726 | qp->rdev->qp1_sqp->qplib_qp.id) { | ||
| 2727 | /* Handle this completion with | ||
| 2728 | * the stored completion | ||
| 2729 | */ | ||
| 2730 | if (cqe->status) { | ||
| 2731 | continue; | ||
| 2732 | } else { | ||
| 2733 | bnxt_re_process_res_shadow_qp_wc | ||
| 2734 | (qp, wc, cqe); | ||
| 2735 | break; | ||
| 2736 | } | ||
| 2737 | } | ||
| 2738 | bnxt_re_process_res_ud_wc(wc, cqe); | ||
| 2739 | break; | ||
| 2740 | default: | ||
| 2741 | dev_err(rdev_to_dev(cq->rdev), | ||
| 2742 | "POLL CQ : type 0x%x not handled", | ||
| 2743 | cqe->opcode); | ||
| 2744 | continue; | ||
| 2745 | } | ||
| 2746 | wc++; | ||
| 2747 | budget--; | ||
| 2748 | } | ||
| 2749 | } | ||
| 2750 | exit: | ||
| 2751 | spin_unlock_irqrestore(&cq->cq_lock, flags); | ||
| 2752 | return num_entries - budget; | ||
| 2753 | } | ||
| 2754 | |||
| 2755 | int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, | ||
| 2756 | enum ib_cq_notify_flags ib_cqn_flags) | ||
| 2757 | { | ||
| 2758 | struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); | ||
| 2759 | int type = 0; | ||
| 2760 | |||
| 2761 | /* Trigger on the very next completion */ | ||
| 2762 | if (ib_cqn_flags & IB_CQ_NEXT_COMP) | ||
| 2763 | type = DBR_DBR_TYPE_CQ_ARMALL; | ||
| 2764 | /* Trigger on the next solicited completion */ | ||
| 2765 | else if (ib_cqn_flags & IB_CQ_SOLICITED) | ||
| 2766 | type = DBR_DBR_TYPE_CQ_ARMSE; | ||
| 2767 | |||
| 2768 | bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); | ||
| 2769 | |||
| 2770 | return 0; | ||
| 2771 | } | ||
| 2772 | |||
| 2773 | /* Memory Regions */ | ||
| 2774 | struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) | ||
| 2775 | { | ||
| 2776 | struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | ||
| 2777 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 2778 | struct bnxt_re_mr *mr; | ||
| 2779 | u64 pbl = 0; | ||
| 2780 | int rc; | ||
| 2781 | |||
| 2782 | mr = kzalloc(sizeof(*mr), GFP_KERNEL); | ||
| 2783 | if (!mr) | ||
| 2784 | return ERR_PTR(-ENOMEM); | ||
| 2785 | |||
| 2786 | mr->rdev = rdev; | ||
| 2787 | mr->qplib_mr.pd = &pd->qplib_pd; | ||
| 2788 | mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); | ||
| 2789 | mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; | ||
| 2790 | |||
| 2791 | /* Allocate and register 0 as the address */ | ||
| 2792 | rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); | ||
| 2793 | if (rc) | ||
| 2794 | goto fail; | ||
| 2795 | |||
| 2796 | mr->qplib_mr.hwq.level = PBL_LVL_MAX; | ||
| 2797 | mr->qplib_mr.total_size = -1; /* Infinte length */ | ||
| 2798 | rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false); | ||
| 2799 | if (rc) | ||
| 2800 | goto fail_mr; | ||
| 2801 | |||
| 2802 | mr->ib_mr.lkey = mr->qplib_mr.lkey; | ||
| 2803 | if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | | ||
| 2804 | IB_ACCESS_REMOTE_ATOMIC)) | ||
| 2805 | mr->ib_mr.rkey = mr->ib_mr.lkey; | ||
| 2806 | atomic_inc(&rdev->mr_count); | ||
| 2807 | |||
| 2808 | return &mr->ib_mr; | ||
| 2809 | |||
| 2810 | fail_mr: | ||
| 2811 | bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); | ||
| 2812 | fail: | ||
| 2813 | kfree(mr); | ||
| 2814 | return ERR_PTR(rc); | ||
| 2815 | } | ||
| 2816 | |||
| 2817 | int bnxt_re_dereg_mr(struct ib_mr *ib_mr) | ||
| 2818 | { | ||
| 2819 | struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); | ||
| 2820 | struct bnxt_re_dev *rdev = mr->rdev; | ||
| 2821 | int rc = 0; | ||
| 2822 | |||
| 2823 | if (mr->npages && mr->pages) { | ||
| 2824 | rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, | ||
| 2825 | &mr->qplib_frpl); | ||
| 2826 | kfree(mr->pages); | ||
| 2827 | mr->npages = 0; | ||
| 2828 | mr->pages = NULL; | ||
| 2829 | } | ||
| 2830 | rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); | ||
| 2831 | |||
| 2832 | if (!IS_ERR(mr->ib_umem) && mr->ib_umem) | ||
| 2833 | ib_umem_release(mr->ib_umem); | ||
| 2834 | |||
| 2835 | kfree(mr); | ||
| 2836 | atomic_dec(&rdev->mr_count); | ||
| 2837 | return rc; | ||
| 2838 | } | ||
| 2839 | |||
| 2840 | static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) | ||
| 2841 | { | ||
| 2842 | struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); | ||
| 2843 | |||
| 2844 | if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) | ||
| 2845 | return -ENOMEM; | ||
| 2846 | |||
| 2847 | mr->pages[mr->npages++] = addr; | ||
| 2848 | return 0; | ||
| 2849 | } | ||
| 2850 | |||
| 2851 | int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, | ||
| 2852 | unsigned int *sg_offset) | ||
| 2853 | { | ||
| 2854 | struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); | ||
| 2855 | |||
| 2856 | mr->npages = 0; | ||
| 2857 | return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); | ||
| 2858 | } | ||
| 2859 | |||
| 2860 | struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, | ||
| 2861 | u32 max_num_sg) | ||
| 2862 | { | ||
| 2863 | struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | ||
| 2864 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 2865 | struct bnxt_re_mr *mr = NULL; | ||
| 2866 | int rc; | ||
| 2867 | |||
| 2868 | if (type != IB_MR_TYPE_MEM_REG) { | ||
| 2869 | dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type); | ||
| 2870 | return ERR_PTR(-EINVAL); | ||
| 2871 | } | ||
| 2872 | if (max_num_sg > MAX_PBL_LVL_1_PGS) | ||
| 2873 | return ERR_PTR(-EINVAL); | ||
| 2874 | |||
| 2875 | mr = kzalloc(sizeof(*mr), GFP_KERNEL); | ||
| 2876 | if (!mr) | ||
| 2877 | return ERR_PTR(-ENOMEM); | ||
| 2878 | |||
| 2879 | mr->rdev = rdev; | ||
| 2880 | mr->qplib_mr.pd = &pd->qplib_pd; | ||
| 2881 | mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; | ||
| 2882 | mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; | ||
| 2883 | |||
| 2884 | rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); | ||
| 2885 | if (rc) | ||
| 2886 | goto fail; | ||
| 2887 | |||
| 2888 | mr->ib_mr.lkey = mr->qplib_mr.lkey; | ||
| 2889 | mr->ib_mr.rkey = mr->ib_mr.lkey; | ||
| 2890 | |||
| 2891 | mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); | ||
| 2892 | if (!mr->pages) { | ||
| 2893 | rc = -ENOMEM; | ||
| 2894 | goto fail; | ||
| 2895 | } | ||
| 2896 | rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, | ||
| 2897 | &mr->qplib_frpl, max_num_sg); | ||
| 2898 | if (rc) { | ||
| 2899 | dev_err(rdev_to_dev(rdev), | ||
| 2900 | "Failed to allocate HW FR page list"); | ||
| 2901 | goto fail_mr; | ||
| 2902 | } | ||
| 2903 | |||
| 2904 | atomic_inc(&rdev->mr_count); | ||
| 2905 | return &mr->ib_mr; | ||
| 2906 | |||
| 2907 | fail_mr: | ||
| 2908 | bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); | ||
| 2909 | fail: | ||
| 2910 | kfree(mr->pages); | ||
| 2911 | kfree(mr); | ||
| 2912 | return ERR_PTR(rc); | ||
| 2913 | } | ||
| 2914 | |||
| 2915 | /* Fast Memory Regions */ | ||
| 2916 | struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *ib_pd, int mr_access_flags, | ||
| 2917 | struct ib_fmr_attr *fmr_attr) | ||
| 2918 | { | ||
| 2919 | struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | ||
| 2920 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 2921 | struct bnxt_re_fmr *fmr; | ||
| 2922 | int rc; | ||
| 2923 | |||
| 2924 | if (fmr_attr->max_pages > MAX_PBL_LVL_2_PGS || | ||
| 2925 | fmr_attr->max_maps > rdev->dev_attr.max_map_per_fmr) { | ||
| 2926 | dev_err(rdev_to_dev(rdev), "Allocate FMR exceeded Max limit"); | ||
| 2927 | return ERR_PTR(-ENOMEM); | ||
| 2928 | } | ||
| 2929 | fmr = kzalloc(sizeof(*fmr), GFP_KERNEL); | ||
| 2930 | if (!fmr) | ||
| 2931 | return ERR_PTR(-ENOMEM); | ||
| 2932 | |||
| 2933 | fmr->rdev = rdev; | ||
| 2934 | fmr->qplib_fmr.pd = &pd->qplib_pd; | ||
| 2935 | fmr->qplib_fmr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; | ||
| 2936 | |||
| 2937 | rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &fmr->qplib_fmr); | ||
| 2938 | if (rc) | ||
| 2939 | goto fail; | ||
| 2940 | |||
| 2941 | fmr->qplib_fmr.flags = __from_ib_access_flags(mr_access_flags); | ||
| 2942 | fmr->ib_fmr.lkey = fmr->qplib_fmr.lkey; | ||
| 2943 | fmr->ib_fmr.rkey = fmr->ib_fmr.lkey; | ||
| 2944 | |||
| 2945 | atomic_inc(&rdev->mr_count); | ||
| 2946 | return &fmr->ib_fmr; | ||
| 2947 | fail: | ||
| 2948 | kfree(fmr); | ||
| 2949 | return ERR_PTR(rc); | ||
| 2950 | } | ||
| 2951 | |||
| 2952 | int bnxt_re_map_phys_fmr(struct ib_fmr *ib_fmr, u64 *page_list, int list_len, | ||
| 2953 | u64 iova) | ||
| 2954 | { | ||
| 2955 | struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr, | ||
| 2956 | ib_fmr); | ||
| 2957 | struct bnxt_re_dev *rdev = fmr->rdev; | ||
| 2958 | int rc; | ||
| 2959 | |||
| 2960 | fmr->qplib_fmr.va = iova; | ||
| 2961 | fmr->qplib_fmr.total_size = list_len * PAGE_SIZE; | ||
| 2962 | |||
| 2963 | rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &fmr->qplib_fmr, page_list, | ||
| 2964 | list_len, true); | ||
| 2965 | if (rc) | ||
| 2966 | dev_err(rdev_to_dev(rdev), "Failed to map FMR for lkey = 0x%x!", | ||
| 2967 | fmr->ib_fmr.lkey); | ||
| 2968 | return rc; | ||
| 2969 | } | ||
| 2970 | |||
| 2971 | int bnxt_re_unmap_fmr(struct list_head *fmr_list) | ||
| 2972 | { | ||
| 2973 | struct bnxt_re_dev *rdev; | ||
| 2974 | struct bnxt_re_fmr *fmr; | ||
| 2975 | struct ib_fmr *ib_fmr; | ||
| 2976 | int rc = 0; | ||
| 2977 | |||
| 2978 | /* Validate each FMRs inside the fmr_list */ | ||
| 2979 | list_for_each_entry(ib_fmr, fmr_list, list) { | ||
| 2980 | fmr = container_of(ib_fmr, struct bnxt_re_fmr, ib_fmr); | ||
| 2981 | rdev = fmr->rdev; | ||
| 2982 | |||
| 2983 | if (rdev) { | ||
| 2984 | rc = bnxt_qplib_dereg_mrw(&rdev->qplib_res, | ||
| 2985 | &fmr->qplib_fmr, true); | ||
| 2986 | if (rc) | ||
| 2987 | break; | ||
| 2988 | } | ||
| 2989 | } | ||
| 2990 | return rc; | ||
| 2991 | } | ||
| 2992 | |||
| 2993 | int bnxt_re_dealloc_fmr(struct ib_fmr *ib_fmr) | ||
| 2994 | { | ||
| 2995 | struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr, | ||
| 2996 | ib_fmr); | ||
| 2997 | struct bnxt_re_dev *rdev = fmr->rdev; | ||
| 2998 | int rc; | ||
| 2999 | |||
| 3000 | rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &fmr->qplib_fmr); | ||
| 3001 | if (rc) | ||
| 3002 | dev_err(rdev_to_dev(rdev), "Failed to free FMR"); | ||
| 3003 | |||
| 3004 | kfree(fmr); | ||
| 3005 | atomic_dec(&rdev->mr_count); | ||
| 3006 | return rc; | ||
| 3007 | } | ||
| 3008 | |||
| 3009 | /* uverbs */ | ||
| 3010 | struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, | ||
| 3011 | u64 virt_addr, int mr_access_flags, | ||
| 3012 | struct ib_udata *udata) | ||
| 3013 | { | ||
| 3014 | struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); | ||
| 3015 | struct bnxt_re_dev *rdev = pd->rdev; | ||
| 3016 | struct bnxt_re_mr *mr; | ||
| 3017 | struct ib_umem *umem; | ||
| 3018 | u64 *pbl_tbl, *pbl_tbl_orig; | ||
| 3019 | int i, umem_pgs, pages, page_shift, rc; | ||
| 3020 | struct scatterlist *sg; | ||
| 3021 | int entry; | ||
| 3022 | |||
| 3023 | mr = kzalloc(sizeof(*mr), GFP_KERNEL); | ||
| 3024 | if (!mr) | ||
| 3025 | return ERR_PTR(-ENOMEM); | ||
| 3026 | |||
| 3027 | mr->rdev = rdev; | ||
| 3028 | mr->qplib_mr.pd = &pd->qplib_pd; | ||
| 3029 | mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); | ||
| 3030 | mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; | ||
| 3031 | |||
| 3032 | umem = ib_umem_get(ib_pd->uobject->context, start, length, | ||
| 3033 | mr_access_flags, 0); | ||
| 3034 | if (IS_ERR(umem)) { | ||
| 3035 | dev_err(rdev_to_dev(rdev), "Failed to get umem"); | ||
| 3036 | rc = -EFAULT; | ||
| 3037 | goto free_mr; | ||
| 3038 | } | ||
| 3039 | mr->ib_umem = umem; | ||
| 3040 | |||
| 3041 | rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); | ||
| 3042 | if (rc) { | ||
| 3043 | dev_err(rdev_to_dev(rdev), "Failed to allocate MR"); | ||
| 3044 | goto release_umem; | ||
| 3045 | } | ||
| 3046 | /* The fixed portion of the rkey is the same as the lkey */ | ||
| 3047 | mr->ib_mr.rkey = mr->qplib_mr.rkey; | ||
| 3048 | |||
| 3049 | mr->qplib_mr.va = virt_addr; | ||
| 3050 | umem_pgs = ib_umem_page_count(umem); | ||
| 3051 | if (!umem_pgs) { | ||
| 3052 | dev_err(rdev_to_dev(rdev), "umem is invalid!"); | ||
| 3053 | rc = -EINVAL; | ||
| 3054 | goto free_mrw; | ||
| 3055 | } | ||
| 3056 | mr->qplib_mr.total_size = length; | ||
| 3057 | |||
| 3058 | pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL); | ||
| 3059 | if (!pbl_tbl) { | ||
| 3060 | rc = -EINVAL; | ||
| 3061 | goto free_mrw; | ||
| 3062 | } | ||
| 3063 | pbl_tbl_orig = pbl_tbl; | ||
| 3064 | |||
| 3065 | page_shift = ilog2(umem->page_size); | ||
| 3066 | if (umem->hugetlb) { | ||
| 3067 | dev_err(rdev_to_dev(rdev), "umem hugetlb not supported!"); | ||
| 3068 | rc = -EFAULT; | ||
| 3069 | goto fail; | ||
| 3070 | } | ||
| 3071 | if (umem->page_size != PAGE_SIZE) { | ||
| 3072 | dev_err(rdev_to_dev(rdev), "umem page size unsupported!"); | ||
| 3073 | rc = -EFAULT; | ||
| 3074 | goto fail; | ||
| 3075 | } | ||
| 3076 | /* Map umem buf ptrs to the PBL */ | ||
| 3077 | for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { | ||
| 3078 | pages = sg_dma_len(sg) >> page_shift; | ||
| 3079 | for (i = 0; i < pages; i++, pbl_tbl++) | ||
| 3080 | *pbl_tbl = sg_dma_address(sg) + (i << page_shift); | ||
| 3081 | } | ||
| 3082 | rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl_orig, | ||
| 3083 | umem_pgs, false); | ||
| 3084 | if (rc) { | ||
| 3085 | dev_err(rdev_to_dev(rdev), "Failed to register user MR"); | ||
| 3086 | goto fail; | ||
| 3087 | } | ||
| 3088 | |||
| 3089 | kfree(pbl_tbl_orig); | ||
| 3090 | |||
| 3091 | mr->ib_mr.lkey = mr->qplib_mr.lkey; | ||
| 3092 | mr->ib_mr.rkey = mr->qplib_mr.lkey; | ||
| 3093 | atomic_inc(&rdev->mr_count); | ||
| 3094 | |||
| 3095 | return &mr->ib_mr; | ||
| 3096 | fail: | ||
| 3097 | kfree(pbl_tbl_orig); | ||
| 3098 | free_mrw: | ||
| 3099 | bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); | ||
| 3100 | release_umem: | ||
| 3101 | ib_umem_release(umem); | ||
| 3102 | free_mr: | ||
| 3103 | kfree(mr); | ||
| 3104 | return ERR_PTR(rc); | ||
| 3105 | } | ||
| 3106 | |||
| 3107 | struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev, | ||
| 3108 | struct ib_udata *udata) | ||
| 3109 | { | ||
| 3110 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); | ||
| 3111 | struct bnxt_re_uctx_resp resp; | ||
| 3112 | struct bnxt_re_ucontext *uctx; | ||
| 3113 | struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; | ||
| 3114 | int rc; | ||
| 3115 | |||
| 3116 | dev_dbg(rdev_to_dev(rdev), "ABI version requested %d", | ||
| 3117 | ibdev->uverbs_abi_ver); | ||
| 3118 | |||
| 3119 | if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) { | ||
| 3120 | dev_dbg(rdev_to_dev(rdev), " is different from the device %d ", | ||
| 3121 | BNXT_RE_ABI_VERSION); | ||
| 3122 | return ERR_PTR(-EPERM); | ||
| 3123 | } | ||
| 3124 | |||
| 3125 | uctx = kzalloc(sizeof(*uctx), GFP_KERNEL); | ||
| 3126 | if (!uctx) | ||
| 3127 | return ERR_PTR(-ENOMEM); | ||
| 3128 | |||
| 3129 | uctx->rdev = rdev; | ||
| 3130 | |||
| 3131 | uctx->shpg = (void *)__get_free_page(GFP_KERNEL); | ||
| 3132 | if (!uctx->shpg) { | ||
| 3133 | rc = -ENOMEM; | ||
| 3134 | goto fail; | ||
| 3135 | } | ||
| 3136 | spin_lock_init(&uctx->sh_lock); | ||
| 3137 | |||
| 3138 | resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/ | ||
| 3139 | resp.max_qp = rdev->qplib_ctx.qpc_count; | ||
| 3140 | resp.pg_size = PAGE_SIZE; | ||
| 3141 | resp.cqe_sz = sizeof(struct cq_base); | ||
| 3142 | resp.max_cqd = dev_attr->max_cq_wqes; | ||
| 3143 | resp.rsvd = 0; | ||
| 3144 | |||
| 3145 | rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); | ||
| 3146 | if (rc) { | ||
| 3147 | dev_err(rdev_to_dev(rdev), "Failed to copy user context"); | ||
| 3148 | rc = -EFAULT; | ||
| 3149 | goto cfail; | ||
| 3150 | } | ||
| 3151 | |||
| 3152 | return &uctx->ib_uctx; | ||
| 3153 | cfail: | ||
| 3154 | free_page((unsigned long)uctx->shpg); | ||
| 3155 | uctx->shpg = NULL; | ||
| 3156 | fail: | ||
| 3157 | kfree(uctx); | ||
| 3158 | return ERR_PTR(rc); | ||
| 3159 | } | ||
| 3160 | |||
| 3161 | int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) | ||
| 3162 | { | ||
| 3163 | struct bnxt_re_ucontext *uctx = container_of(ib_uctx, | ||
| 3164 | struct bnxt_re_ucontext, | ||
| 3165 | ib_uctx); | ||
| 3166 | if (uctx->shpg) | ||
| 3167 | free_page((unsigned long)uctx->shpg); | ||
| 3168 | kfree(uctx); | ||
| 3169 | return 0; | ||
| 3170 | } | ||
| 3171 | |||
| 3172 | /* Helper function to mmap the virtual memory from user app */ | ||
| 3173 | int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) | ||
| 3174 | { | ||
| 3175 | struct bnxt_re_ucontext *uctx = container_of(ib_uctx, | ||
| 3176 | struct bnxt_re_ucontext, | ||
| 3177 | ib_uctx); | ||
| 3178 | struct bnxt_re_dev *rdev = uctx->rdev; | ||
| 3179 | u64 pfn; | ||
| 3180 | |||
| 3181 | if (vma->vm_end - vma->vm_start != PAGE_SIZE) | ||
| 3182 | return -EINVAL; | ||
| 3183 | |||
| 3184 | if (vma->vm_pgoff) { | ||
| 3185 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); | ||
| 3186 | if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, | ||
| 3187 | PAGE_SIZE, vma->vm_page_prot)) { | ||
| 3188 | dev_err(rdev_to_dev(rdev), "Failed to map DPI"); | ||
| 3189 | return -EAGAIN; | ||
| 3190 | } | ||
| 3191 | } else { | ||
| 3192 | pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT; | ||
| 3193 | if (remap_pfn_range(vma, vma->vm_start, | ||
| 3194 | pfn, PAGE_SIZE, vma->vm_page_prot)) { | ||
| 3195 | dev_err(rdev_to_dev(rdev), | ||
| 3196 | "Failed to map shared page"); | ||
| 3197 | return -EAGAIN; | ||
| 3198 | } | ||
| 3199 | } | ||
| 3200 | |||
| 3201 | return 0; | ||
| 3202 | } | ||
| @@ -0,0 +1,197 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: IB Verbs interpreter (header) | ||
| 37 | */ | ||
| 38 | |||
| 39 | #ifndef __BNXT_RE_IB_VERBS_H__ | ||
| 40 | #define __BNXT_RE_IB_VERBS_H__ | ||
| 41 | |||
| 42 | struct bnxt_re_gid_ctx { | ||
| 43 | u32 idx; | ||
| 44 | u32 refcnt; | ||
| 45 | }; | ||
| 46 | |||
| 47 | struct bnxt_re_pd { | ||
| 48 | struct bnxt_re_dev *rdev; | ||
| 49 | struct ib_pd ib_pd; | ||
| 50 | struct bnxt_qplib_pd qplib_pd; | ||
| 51 | struct bnxt_qplib_dpi dpi; | ||
| 52 | }; | ||
| 53 | |||
| 54 | struct bnxt_re_ah { | ||
| 55 | struct bnxt_re_dev *rdev; | ||
| 56 | struct ib_ah ib_ah; | ||
| 57 | struct bnxt_qplib_ah qplib_ah; | ||
| 58 | }; | ||
| 59 | |||
| 60 | struct bnxt_re_qp { | ||
| 61 | struct list_head list; | ||
| 62 | struct bnxt_re_dev *rdev; | ||
| 63 | struct ib_qp ib_qp; | ||
| 64 | spinlock_t sq_lock; /* protect sq */ | ||
| 65 | struct bnxt_qplib_qp qplib_qp; | ||
| 66 | struct ib_umem *sumem; | ||
| 67 | struct ib_umem *rumem; | ||
| 68 | /* QP1 */ | ||
| 69 | u32 send_psn; | ||
| 70 | struct ib_ud_header qp1_hdr; | ||
| 71 | }; | ||
| 72 | |||
| 73 | struct bnxt_re_cq { | ||
| 74 | struct bnxt_re_dev *rdev; | ||
| 75 | spinlock_t cq_lock; /* protect cq */ | ||
| 76 | u16 cq_count; | ||
| 77 | u16 cq_period; | ||
| 78 | struct ib_cq ib_cq; | ||
| 79 | struct bnxt_qplib_cq qplib_cq; | ||
| 80 | struct bnxt_qplib_cqe *cql; | ||
| 81 | #define MAX_CQL_PER_POLL 1024 | ||
| 82 | u32 max_cql; | ||
| 83 | struct ib_umem *umem; | ||
| 84 | }; | ||
| 85 | |||
| 86 | struct bnxt_re_mr { | ||
| 87 | struct bnxt_re_dev *rdev; | ||
| 88 | struct ib_mr ib_mr; | ||
| 89 | struct ib_umem *ib_umem; | ||
| 90 | struct bnxt_qplib_mrw qplib_mr; | ||
| 91 | u32 npages; | ||
| 92 | u64 *pages; | ||
| 93 | struct bnxt_qplib_frpl qplib_frpl; | ||
| 94 | }; | ||
| 95 | |||
| 96 | struct bnxt_re_frpl { | ||
| 97 | struct bnxt_re_dev *rdev; | ||
| 98 | struct bnxt_qplib_frpl qplib_frpl; | ||
| 99 | u64 *page_list; | ||
| 100 | }; | ||
| 101 | |||
| 102 | struct bnxt_re_fmr { | ||
| 103 | struct bnxt_re_dev *rdev; | ||
| 104 | struct ib_fmr ib_fmr; | ||
| 105 | struct bnxt_qplib_mrw qplib_fmr; | ||
| 106 | }; | ||
| 107 | |||
| 108 | struct bnxt_re_mw { | ||
| 109 | struct bnxt_re_dev *rdev; | ||
| 110 | struct ib_mw ib_mw; | ||
| 111 | struct bnxt_qplib_mrw qplib_mw; | ||
| 112 | }; | ||
| 113 | |||
| 114 | struct bnxt_re_ucontext { | ||
| 115 | struct bnxt_re_dev *rdev; | ||
| 116 | struct ib_ucontext ib_uctx; | ||
| 117 | struct bnxt_qplib_dpi *dpi; | ||
| 118 | void *shpg; | ||
| 119 | spinlock_t sh_lock; /* protect shpg */ | ||
| 120 | }; | ||
| 121 | |||
| 122 | struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num); | ||
| 123 | |||
| 124 | int bnxt_re_query_device(struct ib_device *ibdev, | ||
| 125 | struct ib_device_attr *ib_attr, | ||
| 126 | struct ib_udata *udata); | ||
| 127 | int bnxt_re_modify_device(struct ib_device *ibdev, | ||
| 128 | int device_modify_mask, | ||
| 129 | struct ib_device_modify *device_modify); | ||
| 130 | int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, | ||
| 131 | struct ib_port_attr *port_attr); | ||
| 132 | int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, | ||
| 133 | int port_modify_mask, | ||
| 134 | struct ib_port_modify *port_modify); | ||
| 135 | int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, | ||
| 136 | struct ib_port_immutable *immutable); | ||
| 137 | int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, | ||
| 138 | u16 index, u16 *pkey); | ||
| 139 | int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, | ||
| 140 | unsigned int index, void **context); | ||
| 141 | int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, | ||
| 142 | unsigned int index, const union ib_gid *gid, | ||
| 143 | const struct ib_gid_attr *attr, void **context); | ||
| 144 | int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, | ||
| 145 | int index, union ib_gid *gid); | ||
| 146 | enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, | ||
| 147 | u8 port_num); | ||
| 148 | struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev, | ||
| 149 | struct ib_ucontext *context, | ||
| 150 | struct ib_udata *udata); | ||
| 151 | int bnxt_re_dealloc_pd(struct ib_pd *pd); | ||
| 152 | struct ib_ah *bnxt_re_create_ah(struct ib_pd *pd, | ||
| 153 | struct ib_ah_attr *ah_attr, | ||
| 154 | struct ib_udata *udata); | ||
| 155 | int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); | ||
| 156 | int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); | ||
| 157 | int bnxt_re_destroy_ah(struct ib_ah *ah); | ||
| 158 | struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd, | ||
| 159 | struct ib_qp_init_attr *qp_init_attr, | ||
| 160 | struct ib_udata *udata); | ||
| 161 | int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, | ||
| 162 | int qp_attr_mask, struct ib_udata *udata); | ||
| 163 | int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, | ||
| 164 | int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); | ||
| 165 | int bnxt_re_destroy_qp(struct ib_qp *qp); | ||
| 166 | int bnxt_re_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr, | ||
| 167 | struct ib_send_wr **bad_send_wr); | ||
| 168 | int bnxt_re_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr, | ||
| 169 | struct ib_recv_wr **bad_recv_wr); | ||
| 170 | struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, | ||
| 171 | const struct ib_cq_init_attr *attr, | ||
| 172 | struct ib_ucontext *context, | ||
| 173 | struct ib_udata *udata); | ||
| 174 | int bnxt_re_destroy_cq(struct ib_cq *cq); | ||
| 175 | int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); | ||
| 176 | int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); | ||
| 177 | struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); | ||
| 178 | |||
| 179 | int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, | ||
| 180 | unsigned int *sg_offset); | ||
| 181 | struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type, | ||
| 182 | u32 max_num_sg); | ||
| 183 | int bnxt_re_dereg_mr(struct ib_mr *mr); | ||
| 184 | struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *pd, int mr_access_flags, | ||
| 185 | struct ib_fmr_attr *fmr_attr); | ||
| 186 | int bnxt_re_map_phys_fmr(struct ib_fmr *fmr, u64 *page_list, int list_len, | ||
| 187 | u64 iova); | ||
| 188 | int bnxt_re_unmap_fmr(struct list_head *fmr_list); | ||
| 189 | int bnxt_re_dealloc_fmr(struct ib_fmr *fmr); | ||
| 190 | struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, | ||
| 191 | u64 virt_addr, int mr_access_flags, | ||
| 192 | struct ib_udata *udata); | ||
| 193 | struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev, | ||
| 194 | struct ib_udata *udata); | ||
| 195 | int bnxt_re_dealloc_ucontext(struct ib_ucontext *context); | ||
| 196 | int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); | ||
| 197 | #endif /* __BNXT_RE_IB_VERBS_H__ */ | ||
| @@ -0,0 +1,1315 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: Main component of the bnxt_re driver | ||
| 37 | */ | ||
| 38 | |||
| 39 | #include <linux/module.h> | ||
| 40 | #include <linux/netdevice.h> | ||
| 41 | #include <linux/ethtool.h> | ||
| 42 | #include <linux/mutex.h> | ||
| 43 | #include <linux/list.h> | ||
| 44 | #include <linux/rculist.h> | ||
| 45 | #include <linux/spinlock.h> | ||
| 46 | #include <linux/pci.h> | ||
| 47 | #include <net/dcbnl.h> | ||
| 48 | #include <net/ipv6.h> | ||
| 49 | #include <net/addrconf.h> | ||
| 50 | #include <linux/if_ether.h> | ||
| 51 | |||
| 52 | #include <rdma/ib_verbs.h> | ||
| 53 | #include <rdma/ib_user_verbs.h> | ||
| 54 | #include <rdma/ib_umem.h> | ||
| 55 | #include <rdma/ib_addr.h> | ||
| 56 | |||
| 57 | #include "bnxt_ulp.h" | ||
| 58 | #include "roce_hsi.h" | ||
| 59 | #include "qplib_res.h" | ||
| 60 | #include "qplib_sp.h" | ||
| 61 | #include "qplib_fp.h" | ||
| 62 | #include "qplib_rcfw.h" | ||
| 63 | #include "bnxt_re.h" | ||
| 64 | #include "ib_verbs.h" | ||
| 65 | #include <rdma/bnxt_re-abi.h> | ||
| 66 | #include "bnxt.h" | ||
| 67 | static char version[] = | ||
| 68 | BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n"; | ||
| 69 | |||
| 70 | MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>"); | ||
| 71 | MODULE_DESCRIPTION(BNXT_RE_DESC " Driver"); | ||
| 72 | MODULE_LICENSE("Dual BSD/GPL"); | ||
| 73 | MODULE_VERSION(ROCE_DRV_MODULE_VERSION); | ||
| 74 | |||
| 75 | /* globals */ | ||
| 76 | static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list); | ||
| 77 | /* Mutex to protect the list of bnxt_re devices added */ | ||
| 78 | static DEFINE_MUTEX(bnxt_re_dev_lock); | ||
| 79 | static struct workqueue_struct *bnxt_re_wq; | ||
| 80 | |||
| 81 | /* for handling bnxt_en callbacks later */ | ||
| 82 | static void bnxt_re_stop(void *p) | ||
| 83 | { | ||
| 84 | } | ||
| 85 | |||
| 86 | static void bnxt_re_start(void *p) | ||
| 87 | { | ||
| 88 | } | ||
| 89 | |||
| 90 | static void bnxt_re_sriov_config(void *p, int num_vfs) | ||
| 91 | { | ||
| 92 | } | ||
| 93 | |||
| 94 | static struct bnxt_ulp_ops bnxt_re_ulp_ops = { | ||
| 95 | .ulp_async_notifier = NULL, | ||
| 96 | .ulp_stop = bnxt_re_stop, | ||
| 97 | .ulp_start = bnxt_re_start, | ||
| 98 | .ulp_sriov_config = bnxt_re_sriov_config | ||
| 99 | }; | ||
| 100 | |||
| 101 | /* RoCE -> Net driver */ | ||
| 102 | |||
| 103 | /* Driver registration routines used to let the networking driver (bnxt_en) | ||
| 104 | * to know that the RoCE driver is now installed | ||
| 105 | */ | ||
| 106 | static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev, bool lock_wait) | ||
| 107 | { | ||
| 108 | struct bnxt_en_dev *en_dev; | ||
| 109 | int rc; | ||
| 110 | |||
| 111 | if (!rdev) | ||
| 112 | return -EINVAL; | ||
| 113 | |||
| 114 | en_dev = rdev->en_dev; | ||
| 115 | /* Acquire rtnl lock if it is not invokded from netdev event */ | ||
| 116 | if (lock_wait) | ||
| 117 | rtnl_lock(); | ||
| 118 | |||
| 119 | rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, | ||
| 120 | BNXT_ROCE_ULP); | ||
| 121 | if (lock_wait) | ||
| 122 | rtnl_unlock(); | ||
| 123 | return rc; | ||
| 124 | } | ||
| 125 | |||
| 126 | static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) | ||
| 127 | { | ||
| 128 | struct bnxt_en_dev *en_dev; | ||
| 129 | int rc = 0; | ||
| 130 | |||
| 131 | if (!rdev) | ||
| 132 | return -EINVAL; | ||
| 133 | |||
| 134 | en_dev = rdev->en_dev; | ||
| 135 | |||
| 136 | rtnl_lock(); | ||
| 137 | rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP, | ||
| 138 | &bnxt_re_ulp_ops, rdev); | ||
| 139 | rtnl_unlock(); | ||
| 140 | return rc; | ||
| 141 | } | ||
| 142 | |||
| 143 | static int bnxt_re_free_msix(struct bnxt_re_dev *rdev, bool lock_wait) | ||
| 144 | { | ||
| 145 | struct bnxt_en_dev *en_dev; | ||
| 146 | int rc; | ||
| 147 | |||
| 148 | if (!rdev) | ||
| 149 | return -EINVAL; | ||
| 150 | |||
| 151 | en_dev = rdev->en_dev; | ||
| 152 | |||
| 153 | if (lock_wait) | ||
| 154 | rtnl_lock(); | ||
| 155 | |||
| 156 | rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); | ||
| 157 | |||
| 158 | if (lock_wait) | ||
| 159 | rtnl_unlock(); | ||
| 160 | return rc; | ||
| 161 | } | ||
| 162 | |||
| 163 | static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) | ||
| 164 | { | ||
| 165 | int rc = 0, num_msix_want = BNXT_RE_MIN_MSIX, num_msix_got; | ||
| 166 | struct bnxt_en_dev *en_dev; | ||
| 167 | |||
| 168 | if (!rdev) | ||
| 169 | return -EINVAL; | ||
| 170 | |||
| 171 | en_dev = rdev->en_dev; | ||
| 172 | |||
| 173 | rtnl_lock(); | ||
| 174 | num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, | ||
| 175 | rdev->msix_entries, | ||
| 176 | num_msix_want); | ||
| 177 | if (num_msix_got < BNXT_RE_MIN_MSIX) { | ||
| 178 | rc = -EINVAL; | ||
| 179 | goto done; | ||
| 180 | } | ||
| 181 | if (num_msix_got != num_msix_want) { | ||
| 182 | dev_warn(rdev_to_dev(rdev), | ||
| 183 | "Requested %d MSI-X vectors, got %d\n", | ||
| 184 | num_msix_want, num_msix_got); | ||
| 185 | } | ||
| 186 | rdev->num_msix = num_msix_got; | ||
| 187 | done: | ||
| 188 | rtnl_unlock(); | ||
| 189 | return rc; | ||
| 190 | } | ||
| 191 | |||
| 192 | static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr, | ||
| 193 | u16 opcd, u16 crid, u16 trid) | ||
| 194 | { | ||
| 195 | hdr->req_type = cpu_to_le16(opcd); | ||
| 196 | hdr->cmpl_ring = cpu_to_le16(crid); | ||
| 197 | hdr->target_id = cpu_to_le16(trid); | ||
| 198 | } | ||
| 199 | |||
| 200 | static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg, | ||
| 201 | int msg_len, void *resp, int resp_max_len, | ||
| 202 | int timeout) | ||
| 203 | { | ||
| 204 | fw_msg->msg = msg; | ||
| 205 | fw_msg->msg_len = msg_len; | ||
| 206 | fw_msg->resp = resp; | ||
| 207 | fw_msg->resp_max_len = resp_max_len; | ||
| 208 | fw_msg->timeout = timeout; | ||
| 209 | } | ||
| 210 | |||
| 211 | static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id, | ||
| 212 | bool lock_wait) | ||
| 213 | { | ||
| 214 | struct bnxt_en_dev *en_dev = rdev->en_dev; | ||
| 215 | struct hwrm_ring_free_input req = {0}; | ||
| 216 | struct hwrm_ring_free_output resp; | ||
| 217 | struct bnxt_fw_msg fw_msg; | ||
| 218 | bool do_unlock = false; | ||
| 219 | int rc = -EINVAL; | ||
| 220 | |||
| 221 | if (!en_dev) | ||
| 222 | return rc; | ||
| 223 | |||
| 224 | memset(&fw_msg, 0, sizeof(fw_msg)); | ||
| 225 | if (lock_wait) { | ||
| 226 | rtnl_lock(); | ||
| 227 | do_unlock = true; | ||
| 228 | } | ||
| 229 | |||
| 230 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); | ||
| 231 | req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL; | ||
| 232 | req.ring_id = cpu_to_le16(fw_ring_id); | ||
| 233 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | ||
| 234 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | ||
| 235 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | ||
| 236 | if (rc) | ||
| 237 | dev_err(rdev_to_dev(rdev), | ||
| 238 | "Failed to free HW ring:%d :%#x", req.ring_id, rc); | ||
| 239 | if (do_unlock) | ||
| 240 | rtnl_unlock(); | ||
| 241 | return rc; | ||
| 242 | } | ||
| 243 | |||
| 244 | static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr, | ||
| 245 | int pages, int type, u32 ring_mask, | ||
| 246 | u32 map_index, u16 *fw_ring_id) | ||
| 247 | { | ||
| 248 | struct bnxt_en_dev *en_dev = rdev->en_dev; | ||
| 249 | struct hwrm_ring_alloc_input req = {0}; | ||
| 250 | struct hwrm_ring_alloc_output resp; | ||
| 251 | struct bnxt_fw_msg fw_msg; | ||
| 252 | int rc = -EINVAL; | ||
| 253 | |||
| 254 | if (!en_dev) | ||
| 255 | return rc; | ||
| 256 | |||
| 257 | memset(&fw_msg, 0, sizeof(fw_msg)); | ||
| 258 | rtnl_lock(); | ||
| 259 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); | ||
| 260 | req.enables = 0; | ||
| 261 | req.page_tbl_addr = cpu_to_le64(dma_arr[0]); | ||
| 262 | if (pages > 1) { | ||
| 263 | /* Page size is in log2 units */ | ||
| 264 | req.page_size = BNXT_PAGE_SHIFT; | ||
| 265 | req.page_tbl_depth = 1; | ||
| 266 | } | ||
| 267 | req.fbo = 0; | ||
| 268 | /* Association of ring index with doorbell index and MSIX number */ | ||
| 269 | req.logical_id = cpu_to_le16(map_index); | ||
| 270 | req.length = cpu_to_le32(ring_mask + 1); | ||
| 271 | req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL; | ||
| 272 | req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; | ||
| 273 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | ||
| 274 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | ||
| 275 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | ||
| 276 | if (!rc) | ||
| 277 | *fw_ring_id = le16_to_cpu(resp.ring_id); | ||
| 278 | |||
| 279 | rtnl_unlock(); | ||
| 280 | return rc; | ||
| 281 | } | ||
| 282 | |||
| 283 | static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, | ||
| 284 | u32 fw_stats_ctx_id, bool lock_wait) | ||
| 285 | { | ||
| 286 | struct bnxt_en_dev *en_dev = rdev->en_dev; | ||
| 287 | struct hwrm_stat_ctx_free_input req = {0}; | ||
| 288 | struct bnxt_fw_msg fw_msg; | ||
| 289 | bool do_unlock = false; | ||
| 290 | int rc = -EINVAL; | ||
| 291 | |||
| 292 | if (!en_dev) | ||
| 293 | return rc; | ||
| 294 | |||
| 295 | memset(&fw_msg, 0, sizeof(fw_msg)); | ||
| 296 | if (lock_wait) { | ||
| 297 | rtnl_lock(); | ||
| 298 | do_unlock = true; | ||
| 299 | } | ||
| 300 | |||
| 301 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1); | ||
| 302 | req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); | ||
| 303 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&req, | ||
| 304 | sizeof(req), DFLT_HWRM_CMD_TIMEOUT); | ||
| 305 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | ||
| 306 | if (rc) | ||
| 307 | dev_err(rdev_to_dev(rdev), | ||
| 308 | "Failed to free HW stats context %#x", rc); | ||
| 309 | |||
| 310 | if (do_unlock) | ||
| 311 | rtnl_unlock(); | ||
| 312 | return rc; | ||
| 313 | } | ||
| 314 | |||
| 315 | static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, | ||
| 316 | dma_addr_t dma_map, | ||
| 317 | u32 *fw_stats_ctx_id) | ||
| 318 | { | ||
| 319 | struct hwrm_stat_ctx_alloc_output resp = {0}; | ||
| 320 | struct hwrm_stat_ctx_alloc_input req = {0}; | ||
| 321 | struct bnxt_en_dev *en_dev = rdev->en_dev; | ||
| 322 | struct bnxt_fw_msg fw_msg; | ||
| 323 | int rc = -EINVAL; | ||
| 324 | |||
| 325 | *fw_stats_ctx_id = INVALID_STATS_CTX_ID; | ||
| 326 | |||
| 327 | if (!en_dev) | ||
| 328 | return rc; | ||
| 329 | |||
| 330 | memset(&fw_msg, 0, sizeof(fw_msg)); | ||
| 331 | rtnl_lock(); | ||
| 332 | |||
| 333 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1); | ||
| 334 | req.update_period_ms = cpu_to_le32(1000); | ||
| 335 | req.stats_dma_addr = cpu_to_le64(dma_map); | ||
| 336 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | ||
| 337 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | ||
| 338 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | ||
| 339 | if (!rc) | ||
| 340 | *fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id); | ||
| 341 | |||
| 342 | rtnl_unlock(); | ||
| 343 | return rc; | ||
| 344 | } | ||
| 345 | |||
| 346 | /* Device */ | ||
| 347 | |||
| 348 | static bool is_bnxt_re_dev(struct net_device *netdev) | ||
| 349 | { | ||
| 350 | struct ethtool_drvinfo drvinfo; | ||
| 351 | |||
| 352 | if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) { | ||
| 353 | memset(&drvinfo, 0, sizeof(drvinfo)); | ||
| 354 | netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo); | ||
| 355 | |||
| 356 | if (strcmp(drvinfo.driver, "bnxt_en")) | ||
| 357 | return false; | ||
| 358 | return true; | ||
| 359 | } | ||
| 360 | return false; | ||
| 361 | } | ||
| 362 | |||
| 363 | static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) | ||
| 364 | { | ||
| 365 | struct bnxt_re_dev *rdev; | ||
| 366 | |||
| 367 | rcu_read_lock(); | ||
| 368 | list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { | ||
| 369 | if (rdev->netdev == netdev) { | ||
| 370 | rcu_read_unlock(); | ||
| 371 | return rdev; | ||
| 372 | } | ||
| 373 | } | ||
| 374 | rcu_read_unlock(); | ||
| 375 | return NULL; | ||
| 376 | } | ||
| 377 | |||
| 378 | static void bnxt_re_dev_unprobe(struct net_device *netdev, | ||
| 379 | struct bnxt_en_dev *en_dev) | ||
| 380 | { | ||
| 381 | dev_put(netdev); | ||
| 382 | module_put(en_dev->pdev->driver->driver.owner); | ||
| 383 | } | ||
| 384 | |||
| 385 | static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev) | ||
| 386 | { | ||
| 387 | struct bnxt *bp = netdev_priv(netdev); | ||
| 388 | struct bnxt_en_dev *en_dev; | ||
| 389 | struct pci_dev *pdev; | ||
| 390 | |||
| 391 | /* Call bnxt_en's RoCE probe via indirect API */ | ||
| 392 | if (!bp->ulp_probe) | ||
| 393 | return ERR_PTR(-EINVAL); | ||
| 394 | |||
| 395 | en_dev = bp->ulp_probe(netdev); | ||
| 396 | if (IS_ERR(en_dev)) | ||
| 397 | return en_dev; | ||
| 398 | |||
| 399 | pdev = en_dev->pdev; | ||
| 400 | if (!pdev) | ||
| 401 | return ERR_PTR(-EINVAL); | ||
| 402 | |||
| 403 | if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) { | ||
| 404 | dev_dbg(&pdev->dev, | ||
| 405 | "%s: probe error: RoCE is not supported on this device", | ||
| 406 | ROCE_DRV_MODULE_NAME); | ||
| 407 | return ERR_PTR(-ENODEV); | ||
| 408 | } | ||
| 409 | |||
| 410 | /* Bump net device reference count */ | ||
| 411 | if (!try_module_get(pdev->driver->driver.owner)) | ||
| 412 | return ERR_PTR(-ENODEV); | ||
| 413 | |||
| 414 | dev_hold(netdev); | ||
| 415 | |||
| 416 | return en_dev; | ||
| 417 | } | ||
| 418 | |||
| 419 | static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) | ||
| 420 | { | ||
| 421 | ib_unregister_device(&rdev->ibdev); | ||
| 422 | } | ||
| 423 | |||
| 424 | static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) | ||
| 425 | { | ||
| 426 | struct ib_device *ibdev = &rdev->ibdev; | ||
| 427 | |||
| 428 | /* ib device init */ | ||
| 429 | ibdev->owner = THIS_MODULE; | ||
| 430 | ibdev->node_type = RDMA_NODE_IB_CA; | ||
| 431 | strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX); | ||
| 432 | strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", | ||
| 433 | strlen(BNXT_RE_DESC) + 5); | ||
| 434 | ibdev->phys_port_cnt = 1; | ||
| 435 | |||
| 436 | bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid); | ||
| 437 | |||
| 438 | ibdev->num_comp_vectors = 1; | ||
| 439 | ibdev->dma_device = &rdev->en_dev->pdev->dev; | ||
| 440 | ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; | ||
| 441 | |||
| 442 | /* User space */ | ||
| 443 | ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; | ||
| 444 | ibdev->uverbs_cmd_mask = | ||
| 445 | (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | | ||
| 446 | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | | ||
| 447 | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | | ||
| 448 | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | | ||
| 449 | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | | ||
| 450 | (1ull << IB_USER_VERBS_CMD_REG_MR) | | ||
| 451 | (1ull << IB_USER_VERBS_CMD_REREG_MR) | | ||
| 452 | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | | ||
| 453 | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | | ||
| 454 | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | | ||
| 455 | (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | | ||
| 456 | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | | ||
| 457 | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | | ||
| 458 | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | | ||
| 459 | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | | ||
| 460 | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | | ||
| 461 | (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | | ||
| 462 | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | | ||
| 463 | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | | ||
| 464 | (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | | ||
| 465 | (1ull << IB_USER_VERBS_CMD_CREATE_AH) | | ||
| 466 | (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | | ||
| 467 | (1ull << IB_USER_VERBS_CMD_QUERY_AH) | | ||
| 468 | (1ull << IB_USER_VERBS_CMD_DESTROY_AH); | ||
| 469 | /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */ | ||
| 470 | |||
| 471 | /* Kernel verbs */ | ||
| 472 | ibdev->query_device = bnxt_re_query_device; | ||
| 473 | ibdev->modify_device = bnxt_re_modify_device; | ||
| 474 | |||
| 475 | ibdev->query_port = bnxt_re_query_port; | ||
| 476 | ibdev->modify_port = bnxt_re_modify_port; | ||
| 477 | ibdev->get_port_immutable = bnxt_re_get_port_immutable; | ||
| 478 | ibdev->query_pkey = bnxt_re_query_pkey; | ||
| 479 | ibdev->query_gid = bnxt_re_query_gid; | ||
| 480 | ibdev->get_netdev = bnxt_re_get_netdev; | ||
| 481 | ibdev->add_gid = bnxt_re_add_gid; | ||
| 482 | ibdev->del_gid = bnxt_re_del_gid; | ||
| 483 | ibdev->get_link_layer = bnxt_re_get_link_layer; | ||
| 484 | |||
| 485 | ibdev->alloc_pd = bnxt_re_alloc_pd; | ||
| 486 | ibdev->dealloc_pd = bnxt_re_dealloc_pd; | ||
| 487 | |||
| 488 | ibdev->create_ah = bnxt_re_create_ah; | ||
| 489 | ibdev->modify_ah = bnxt_re_modify_ah; | ||
| 490 | ibdev->query_ah = bnxt_re_query_ah; | ||
| 491 | ibdev->destroy_ah = bnxt_re_destroy_ah; | ||
| 492 | |||
| 493 | ibdev->create_qp = bnxt_re_create_qp; | ||
| 494 | ibdev->modify_qp = bnxt_re_modify_qp; | ||
| 495 | ibdev->query_qp = bnxt_re_query_qp; | ||
| 496 | ibdev->destroy_qp = bnxt_re_destroy_qp; | ||
| 497 | |||
| 498 | ibdev->post_send = bnxt_re_post_send; | ||
| 499 | ibdev->post_recv = bnxt_re_post_recv; | ||
| 500 | |||
| 501 | ibdev->create_cq = bnxt_re_create_cq; | ||
| 502 | ibdev->destroy_cq = bnxt_re_destroy_cq; | ||
| 503 | ibdev->poll_cq = bnxt_re_poll_cq; | ||
| 504 | ibdev->req_notify_cq = bnxt_re_req_notify_cq; | ||
| 505 | |||
| 506 | ibdev->get_dma_mr = bnxt_re_get_dma_mr; | ||
| 507 | ibdev->dereg_mr = bnxt_re_dereg_mr; | ||
| 508 | ibdev->alloc_mr = bnxt_re_alloc_mr; | ||
| 509 | ibdev->map_mr_sg = bnxt_re_map_mr_sg; | ||
| 510 | ibdev->alloc_fmr = bnxt_re_alloc_fmr; | ||
| 511 | ibdev->map_phys_fmr = bnxt_re_map_phys_fmr; | ||
| 512 | ibdev->unmap_fmr = bnxt_re_unmap_fmr; | ||
| 513 | ibdev->dealloc_fmr = bnxt_re_dealloc_fmr; | ||
| 514 | |||
| 515 | ibdev->reg_user_mr = bnxt_re_reg_user_mr; | ||
| 516 | ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; | ||
| 517 | ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; | ||
| 518 | ibdev->mmap = bnxt_re_mmap; | ||
| 519 | |||
| 520 | return ib_register_device(ibdev, NULL); | ||
| 521 | } | ||
| 522 | |||
| 523 | static ssize_t show_rev(struct device *device, struct device_attribute *attr, | ||
| 524 | char *buf) | ||
| 525 | { | ||
| 526 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | ||
| 527 | |||
| 528 | return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); | ||
| 529 | } | ||
| 530 | |||
| 531 | static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, | ||
| 532 | char *buf) | ||
| 533 | { | ||
| 534 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | ||
| 535 | |||
| 536 | return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->dev_attr.fw_ver); | ||
| 537 | } | ||
| 538 | |||
| 539 | static ssize_t show_hca(struct device *device, struct device_attribute *attr, | ||
| 540 | char *buf) | ||
| 541 | { | ||
| 542 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | ||
| 543 | |||
| 544 | return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); | ||
| 545 | } | ||
| 546 | |||
| 547 | static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); | ||
| 548 | static DEVICE_ATTR(fw_rev, 0444, show_fw_ver, NULL); | ||
| 549 | static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); | ||
| 550 | |||
| 551 | static struct device_attribute *bnxt_re_attributes[] = { | ||
| 552 | &dev_attr_hw_rev, | ||
| 553 | &dev_attr_fw_rev, | ||
| 554 | &dev_attr_hca_type | ||
| 555 | }; | ||
| 556 | |||
| 557 | static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) | ||
| 558 | { | ||
| 559 | dev_put(rdev->netdev); | ||
| 560 | rdev->netdev = NULL; | ||
| 561 | |||
| 562 | mutex_lock(&bnxt_re_dev_lock); | ||
| 563 | list_del_rcu(&rdev->list); | ||
| 564 | mutex_unlock(&bnxt_re_dev_lock); | ||
| 565 | |||
| 566 | synchronize_rcu(); | ||
| 567 | flush_workqueue(bnxt_re_wq); | ||
| 568 | |||
| 569 | ib_dealloc_device(&rdev->ibdev); | ||
| 570 | /* rdev is gone */ | ||
| 571 | } | ||
| 572 | |||
| 573 | static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev, | ||
| 574 | struct bnxt_en_dev *en_dev) | ||
| 575 | { | ||
| 576 | struct bnxt_re_dev *rdev; | ||
| 577 | |||
| 578 | /* Allocate bnxt_re_dev instance here */ | ||
| 579 | rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev)); | ||
| 580 | if (!rdev) { | ||
| 581 | dev_err(NULL, "%s: bnxt_re_dev allocation failure!", | ||
| 582 | ROCE_DRV_MODULE_NAME); | ||
| 583 | return NULL; | ||
| 584 | } | ||
| 585 | /* Default values */ | ||
| 586 | rdev->netdev = netdev; | ||
| 587 | dev_hold(rdev->netdev); | ||
| 588 | rdev->en_dev = en_dev; | ||
| 589 | rdev->id = rdev->en_dev->pdev->devfn; | ||
| 590 | INIT_LIST_HEAD(&rdev->qp_list); | ||
| 591 | mutex_init(&rdev->qp_lock); | ||
| 592 | atomic_set(&rdev->qp_count, 0); | ||
| 593 | atomic_set(&rdev->cq_count, 0); | ||
| 594 | atomic_set(&rdev->srq_count, 0); | ||
| 595 | atomic_set(&rdev->mr_count, 0); | ||
| 596 | atomic_set(&rdev->mw_count, 0); | ||
| 597 | rdev->cosq[0] = 0xFFFF; | ||
| 598 | rdev->cosq[1] = 0xFFFF; | ||
| 599 | |||
| 600 | mutex_lock(&bnxt_re_dev_lock); | ||
| 601 | list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); | ||
| 602 | mutex_unlock(&bnxt_re_dev_lock); | ||
| 603 | return rdev; | ||
| 604 | } | ||
| 605 | |||
| 606 | static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, | ||
| 607 | struct creq_func_event *aeqe) | ||
| 608 | { | ||
| 609 | switch (aeqe->event) { | ||
| 610 | case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: | ||
| 611 | break; | ||
| 612 | case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: | ||
| 613 | break; | ||
| 614 | case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: | ||
| 615 | break; | ||
| 616 | case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: | ||
| 617 | break; | ||
| 618 | case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: | ||
| 619 | break; | ||
| 620 | case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: | ||
| 621 | break; | ||
| 622 | case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: | ||
| 623 | break; | ||
| 624 | case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: | ||
| 625 | break; | ||
| 626 | case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: | ||
| 627 | break; | ||
| 628 | case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: | ||
| 629 | break; | ||
| 630 | case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: | ||
| 631 | break; | ||
| 632 | default: | ||
| 633 | return -EINVAL; | ||
| 634 | } | ||
| 635 | return 0; | ||
| 636 | } | ||
| 637 | |||
| 638 | static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, | ||
| 639 | struct bnxt_qplib_cq *handle) | ||
| 640 | { | ||
| 641 | struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq, | ||
| 642 | qplib_cq); | ||
| 643 | |||
| 644 | if (!cq) { | ||
| 645 | dev_err(NULL, "%s: CQ is NULL, CQN not handled", | ||
| 646 | ROCE_DRV_MODULE_NAME); | ||
| 647 | return -EINVAL; | ||
| 648 | } | ||
| 649 | if (cq->ib_cq.comp_handler) { | ||
| 650 | /* Lock comp_handler? */ | ||
| 651 | (*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context); | ||
| 652 | } | ||
| 653 | |||
| 654 | return 0; | ||
| 655 | } | ||
| 656 | |||
| 657 | static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev) | ||
| 658 | { | ||
| 659 | if (rdev->nq.hwq.max_elements) | ||
| 660 | bnxt_qplib_disable_nq(&rdev->nq); | ||
| 661 | |||
| 662 | if (rdev->qplib_res.rcfw) | ||
| 663 | bnxt_qplib_cleanup_res(&rdev->qplib_res); | ||
| 664 | } | ||
| 665 | |||
| 666 | static int bnxt_re_init_res(struct bnxt_re_dev *rdev) | ||
| 667 | { | ||
| 668 | int rc = 0; | ||
| 669 | |||
| 670 | bnxt_qplib_init_res(&rdev->qplib_res); | ||
| 671 | |||
| 672 | if (rdev->msix_entries[BNXT_RE_NQ_IDX].vector <= 0) | ||
| 673 | return -EINVAL; | ||
| 674 | |||
| 675 | rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq, | ||
| 676 | rdev->msix_entries[BNXT_RE_NQ_IDX].vector, | ||
| 677 | rdev->msix_entries[BNXT_RE_NQ_IDX].db_offset, | ||
| 678 | &bnxt_re_cqn_handler, | ||
| 679 | NULL); | ||
| 680 | |||
| 681 | if (rc) | ||
| 682 | dev_err(rdev_to_dev(rdev), "Failed to enable NQ: %#x", rc); | ||
| 683 | |||
| 684 | return rc; | ||
| 685 | } | ||
| 686 | |||
| 687 | static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait) | ||
| 688 | { | ||
| 689 | if (rdev->nq.hwq.max_elements) { | ||
| 690 | bnxt_re_net_ring_free(rdev, rdev->nq.ring_id, lock_wait); | ||
| 691 | bnxt_qplib_free_nq(&rdev->nq); | ||
| 692 | } | ||
| 693 | if (rdev->qplib_res.dpi_tbl.max) { | ||
| 694 | bnxt_qplib_dealloc_dpi(&rdev->qplib_res, | ||
| 695 | &rdev->qplib_res.dpi_tbl, | ||
| 696 | &rdev->dpi_privileged); | ||
| 697 | } | ||
| 698 | if (rdev->qplib_res.rcfw) { | ||
| 699 | bnxt_qplib_free_res(&rdev->qplib_res); | ||
| 700 | rdev->qplib_res.rcfw = NULL; | ||
| 701 | } | ||
| 702 | } | ||
| 703 | |||
| 704 | static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) | ||
| 705 | { | ||
| 706 | int rc = 0; | ||
| 707 | |||
| 708 | /* Configure and allocate resources for qplib */ | ||
| 709 | rdev->qplib_res.rcfw = &rdev->rcfw; | ||
| 710 | rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); | ||
| 711 | if (rc) | ||
| 712 | goto fail; | ||
| 713 | |||
| 714 | rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev, | ||
| 715 | rdev->netdev, &rdev->dev_attr); | ||
| 716 | if (rc) | ||
| 717 | goto fail; | ||
| 718 | |||
| 719 | rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, | ||
| 720 | &rdev->dpi_privileged, | ||
| 721 | rdev); | ||
| 722 | if (rc) | ||
| 723 | goto fail; | ||
| 724 | |||
| 725 | rdev->nq.hwq.max_elements = BNXT_RE_MAX_CQ_COUNT + | ||
| 726 | BNXT_RE_MAX_SRQC_COUNT + 2; | ||
| 727 | rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq); | ||
| 728 | if (rc) { | ||
| 729 | dev_err(rdev_to_dev(rdev), | ||
| 730 | "Failed to allocate NQ memory: %#x", rc); | ||
| 731 | goto fail; | ||
| 732 | } | ||
| 733 | rc = bnxt_re_net_ring_alloc | ||
| 734 | (rdev, rdev->nq.hwq.pbl[PBL_LVL_0].pg_map_arr, | ||
| 735 | rdev->nq.hwq.pbl[rdev->nq.hwq.level].pg_count, | ||
| 736 | HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_NQE_MAX_CNT - 1, | ||
| 737 | rdev->msix_entries[BNXT_RE_NQ_IDX].ring_idx, | ||
| 738 | &rdev->nq.ring_id); | ||
| 739 | if (rc) { | ||
| 740 | dev_err(rdev_to_dev(rdev), | ||
| 741 | "Failed to allocate NQ ring: %#x", rc); | ||
| 742 | goto free_nq; | ||
| 743 | } | ||
| 744 | return 0; | ||
| 745 | free_nq: | ||
| 746 | bnxt_qplib_free_nq(&rdev->nq); | ||
| 747 | fail: | ||
| 748 | rdev->qplib_res.rcfw = NULL; | ||
| 749 | return rc; | ||
| 750 | } | ||
| 751 | |||
| 752 | static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, | ||
| 753 | u8 port_num, enum ib_event_type event) | ||
| 754 | { | ||
| 755 | struct ib_event ib_event; | ||
| 756 | |||
| 757 | ib_event.device = ibdev; | ||
| 758 | if (qp) | ||
| 759 | ib_event.element.qp = qp; | ||
| 760 | else | ||
| 761 | ib_event.element.port_num = port_num; | ||
| 762 | ib_event.event = event; | ||
| 763 | ib_dispatch_event(&ib_event); | ||
| 764 | } | ||
| 765 | |||
| 766 | #define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02 | ||
| 767 | static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir, | ||
| 768 | u64 *cid_map) | ||
| 769 | { | ||
| 770 | struct hwrm_queue_pri2cos_qcfg_input req = {0}; | ||
| 771 | struct bnxt *bp = netdev_priv(rdev->netdev); | ||
| 772 | struct hwrm_queue_pri2cos_qcfg_output resp; | ||
| 773 | struct bnxt_en_dev *en_dev = rdev->en_dev; | ||
| 774 | struct bnxt_fw_msg fw_msg; | ||
| 775 | u32 flags = 0; | ||
| 776 | u8 *qcfgmap, *tmp_map; | ||
| 777 | int rc = 0, i; | ||
| 778 | |||
| 779 | if (!cid_map) | ||
| 780 | return -EINVAL; | ||
| 781 | |||
| 782 | memset(&fw_msg, 0, sizeof(fw_msg)); | ||
| 783 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, | ||
| 784 | HWRM_QUEUE_PRI2COS_QCFG, -1, -1); | ||
| 785 | flags |= (dir & 0x01); | ||
| 786 | flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN; | ||
| 787 | req.flags = cpu_to_le32(flags); | ||
| 788 | req.port_id = bp->pf.port_id; | ||
| 789 | |||
| 790 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | ||
| 791 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | ||
| 792 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | ||
| 793 | if (rc) | ||
| 794 | return rc; | ||
| 795 | |||
| 796 | if (resp.queue_cfg_info) { | ||
| 797 | dev_warn(rdev_to_dev(rdev), | ||
| 798 | "Asymmetric cos queue configuration detected"); | ||
| 799 | dev_warn(rdev_to_dev(rdev), | ||
| 800 | " on device, QoS may not be fully functional\n"); | ||
| 801 | } | ||
| 802 | qcfgmap = &resp.pri0_cos_queue_id; | ||
| 803 | tmp_map = (u8 *)cid_map; | ||
| 804 | for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) | ||
| 805 | tmp_map[i] = qcfgmap[i]; | ||
| 806 | |||
| 807 | return rc; | ||
| 808 | } | ||
| 809 | |||
| 810 | static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, | ||
| 811 | struct bnxt_re_qp *qp) | ||
| 812 | { | ||
| 813 | return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp); | ||
| 814 | } | ||
| 815 | |||
| 816 | static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev) | ||
| 817 | { | ||
| 818 | int mask = IB_QP_STATE; | ||
| 819 | struct ib_qp_attr qp_attr; | ||
| 820 | struct bnxt_re_qp *qp; | ||
| 821 | |||
| 822 | qp_attr.qp_state = IB_QPS_ERR; | ||
| 823 | mutex_lock(&rdev->qp_lock); | ||
| 824 | list_for_each_entry(qp, &rdev->qp_list, list) { | ||
| 825 | /* Modify the state of all QPs except QP1/Shadow QP */ | ||
| 826 | if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { | ||
| 827 | if (qp->qplib_qp.state != | ||
| 828 | CMDQ_MODIFY_QP_NEW_STATE_RESET && | ||
| 829 | qp->qplib_qp.state != | ||
| 830 | CMDQ_MODIFY_QP_NEW_STATE_ERR) { | ||
| 831 | bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, | ||
| 832 | 1, IB_EVENT_QP_FATAL); | ||
| 833 | bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, | ||
| 834 | NULL); | ||
| 835 | } | ||
| 836 | } | ||
| 837 | } | ||
| 838 | mutex_unlock(&rdev->qp_lock); | ||
| 839 | } | ||
| 840 | |||
| 841 | static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev) | ||
| 842 | { | ||
| 843 | u32 prio_map = 0, tmp_map = 0; | ||
| 844 | struct net_device *netdev; | ||
| 845 | struct dcb_app app; | ||
| 846 | |||
| 847 | netdev = rdev->netdev; | ||
| 848 | |||
| 849 | memset(&app, 0, sizeof(app)); | ||
| 850 | app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE; | ||
| 851 | app.protocol = ETH_P_IBOE; | ||
| 852 | tmp_map = dcb_ieee_getapp_mask(netdev, &app); | ||
| 853 | prio_map = tmp_map; | ||
| 854 | |||
| 855 | app.selector = IEEE_8021QAZ_APP_SEL_DGRAM; | ||
| 856 | app.protocol = ROCE_V2_UDP_DPORT; | ||
| 857 | tmp_map = dcb_ieee_getapp_mask(netdev, &app); | ||
| 858 | prio_map |= tmp_map; | ||
| 859 | |||
| 860 | if (!prio_map) | ||
| 861 | prio_map = -EFAULT; | ||
| 862 | return prio_map; | ||
| 863 | } | ||
| 864 | |||
| 865 | static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq) | ||
| 866 | { | ||
| 867 | u16 prio; | ||
| 868 | u8 id; | ||
| 869 | |||
| 870 | for (prio = 0, id = 0; prio < 8; prio++) { | ||
| 871 | if (prio_map & (1 << prio)) { | ||
| 872 | cosq[id] = cid_map[prio]; | ||
| 873 | id++; | ||
| 874 | if (id == 2) /* Max 2 tcs supported */ | ||
| 875 | break; | ||
| 876 | } | ||
| 877 | } | ||
| 878 | } | ||
| 879 | |||
| 880 | static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev) | ||
| 881 | { | ||
| 882 | u8 prio_map = 0; | ||
| 883 | u64 cid_map; | ||
| 884 | int rc; | ||
| 885 | |||
| 886 | /* Get priority for roce */ | ||
| 887 | rc = bnxt_re_get_priority_mask(rdev); | ||
| 888 | if (rc < 0) | ||
| 889 | return rc; | ||
| 890 | prio_map = (u8)rc; | ||
| 891 | |||
| 892 | if (prio_map == rdev->cur_prio_map) | ||
| 893 | return 0; | ||
| 894 | rdev->cur_prio_map = prio_map; | ||
| 895 | /* Get cosq id for this priority */ | ||
| 896 | rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map); | ||
| 897 | if (rc) { | ||
| 898 | dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map); | ||
| 899 | return rc; | ||
| 900 | } | ||
| 901 | /* Parse CoS IDs for app priority */ | ||
| 902 | bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq); | ||
| 903 | |||
| 904 | /* Config BONO. */ | ||
| 905 | rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq); | ||
| 906 | if (rc) { | ||
| 907 | dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n", | ||
| 908 | rdev->cosq[0], rdev->cosq[1]); | ||
| 909 | return rc; | ||
| 910 | } | ||
| 911 | |||
| 912 | return 0; | ||
| 913 | } | ||
| 914 | |||
| 915 | static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait) | ||
| 916 | { | ||
| 917 | int i, rc; | ||
| 918 | |||
| 919 | if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { | ||
| 920 | for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) | ||
| 921 | device_remove_file(&rdev->ibdev.dev, | ||
| 922 | bnxt_re_attributes[i]); | ||
| 923 | /* Cleanup ib dev */ | ||
| 924 | bnxt_re_unregister_ib(rdev); | ||
| 925 | } | ||
| 926 | if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags)) | ||
| 927 | cancel_delayed_work(&rdev->worker); | ||
| 928 | |||
| 929 | bnxt_re_cleanup_res(rdev); | ||
| 930 | bnxt_re_free_res(rdev, lock_wait); | ||
| 931 | |||
| 932 | if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) { | ||
| 933 | rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); | ||
| 934 | if (rc) | ||
| 935 | dev_warn(rdev_to_dev(rdev), | ||
| 936 | "Failed to deinitialize RCFW: %#x", rc); | ||
| 937 | bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, | ||
| 938 | lock_wait); | ||
| 939 | bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); | ||
| 940 | bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); | ||
| 941 | bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, lock_wait); | ||
| 942 | bnxt_qplib_free_rcfw_channel(&rdev->rcfw); | ||
| 943 | } | ||
| 944 | if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) { | ||
| 945 | rc = bnxt_re_free_msix(rdev, lock_wait); | ||
| 946 | if (rc) | ||
| 947 | dev_warn(rdev_to_dev(rdev), | ||
| 948 | "Failed to free MSI-X vectors: %#x", rc); | ||
| 949 | } | ||
| 950 | if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { | ||
| 951 | rc = bnxt_re_unregister_netdev(rdev, lock_wait); | ||
| 952 | if (rc) | ||
| 953 | dev_warn(rdev_to_dev(rdev), | ||
| 954 | "Failed to unregister with netdev: %#x", rc); | ||
| 955 | } | ||
| 956 | } | ||
| 957 | |||
| 958 | static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) | ||
| 959 | { | ||
| 960 | u32 i; | ||
| 961 | |||
| 962 | rdev->qplib_ctx.qpc_count = BNXT_RE_MAX_QPC_COUNT; | ||
| 963 | rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT; | ||
| 964 | rdev->qplib_ctx.srqc_count = BNXT_RE_MAX_SRQC_COUNT; | ||
| 965 | rdev->qplib_ctx.cq_count = BNXT_RE_MAX_CQ_COUNT; | ||
| 966 | for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) | ||
| 967 | rdev->qplib_ctx.tqm_count[i] = | ||
| 968 | rdev->dev_attr.tqm_alloc_reqs[i]; | ||
| 969 | } | ||
| 970 | |||
| 971 | /* worker thread for polling periodic events. Now used for QoS programming*/ | ||
| 972 | static void bnxt_re_worker(struct work_struct *work) | ||
| 973 | { | ||
| 974 | struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, | ||
| 975 | worker.work); | ||
| 976 | |||
| 977 | bnxt_re_setup_qos(rdev); | ||
| 978 | schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); | ||
| 979 | } | ||
| 980 | |||
| 981 | static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev) | ||
| 982 | { | ||
| 983 | int i, j, rc; | ||
| 984 | |||
| 985 | /* Registered a new RoCE device instance to netdev */ | ||
| 986 | rc = bnxt_re_register_netdev(rdev); | ||
| 987 | if (rc) { | ||
| 988 | pr_err("Failed to register with netedev: %#x\n", rc); | ||
| 989 | return -EINVAL; | ||
| 990 | } | ||
| 991 | set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); | ||
| 992 | |||
| 993 | rc = bnxt_re_request_msix(rdev); | ||
| 994 | if (rc) { | ||
| 995 | pr_err("Failed to get MSI-X vectors: %#x\n", rc); | ||
| 996 | rc = -EINVAL; | ||
| 997 | goto fail; | ||
| 998 | } | ||
| 999 | set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); | ||
| 1000 | |||
| 1001 | /* Establish RCFW Communication Channel to initialize the context | ||
| 1002 | * memory for the function and all child VFs | ||
| 1003 | */ | ||
| 1004 | rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw); | ||
| 1005 | if (rc) | ||
| 1006 | goto fail; | ||
| 1007 | |||
| 1008 | rc = bnxt_re_net_ring_alloc | ||
| 1009 | (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr, | ||
| 1010 | rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count, | ||
| 1011 | HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1, | ||
| 1012 | rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx, | ||
| 1013 | &rdev->rcfw.creq_ring_id); | ||
| 1014 | if (rc) { | ||
| 1015 | pr_err("Failed to allocate CREQ: %#x\n", rc); | ||
| 1016 | goto free_rcfw; | ||
| 1017 | } | ||
| 1018 | rc = bnxt_qplib_enable_rcfw_channel | ||
| 1019 | (rdev->en_dev->pdev, &rdev->rcfw, | ||
| 1020 | rdev->msix_entries[BNXT_RE_AEQ_IDX].vector, | ||
| 1021 | rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset, | ||
| 1022 | 0, &bnxt_re_aeq_handler); | ||
| 1023 | if (rc) { | ||
| 1024 | pr_err("Failed to enable RCFW channel: %#x\n", rc); | ||
| 1025 | goto free_ring; | ||
| 1026 | } | ||
| 1027 | |||
| 1028 | rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); | ||
| 1029 | if (rc) | ||
| 1030 | goto disable_rcfw; | ||
| 1031 | bnxt_re_set_resource_limits(rdev); | ||
| 1032 | |||
| 1033 | rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0); | ||
| 1034 | if (rc) { | ||
| 1035 | pr_err("Failed to allocate QPLIB context: %#x\n", rc); | ||
| 1036 | goto disable_rcfw; | ||
| 1037 | } | ||
| 1038 | rc = bnxt_re_net_stats_ctx_alloc(rdev, | ||
| 1039 | rdev->qplib_ctx.stats.dma_map, | ||
| 1040 | &rdev->qplib_ctx.stats.fw_id); | ||
| 1041 | if (rc) { | ||
| 1042 | pr_err("Failed to allocate stats context: %#x\n", rc); | ||
| 1043 | goto free_ctx; | ||
| 1044 | } | ||
| 1045 | |||
| 1046 | rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, 0); | ||
| 1047 | if (rc) { | ||
| 1048 | pr_err("Failed to initialize RCFW: %#x\n", rc); | ||
| 1049 | goto free_sctx; | ||
| 1050 | } | ||
| 1051 | set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); | ||
| 1052 | |||
| 1053 | /* Resources based on the 'new' device caps */ | ||
| 1054 | rc = bnxt_re_alloc_res(rdev); | ||
| 1055 | if (rc) { | ||
| 1056 | pr_err("Failed to allocate resources: %#x\n", rc); | ||
| 1057 | goto fail; | ||
| 1058 | } | ||
| 1059 | rc = bnxt_re_init_res(rdev); | ||
| 1060 | if (rc) { | ||
| 1061 | pr_err("Failed to initialize resources: %#x\n", rc); | ||
| 1062 | goto fail; | ||
| 1063 | } | ||
| 1064 | |||
| 1065 | rc = bnxt_re_setup_qos(rdev); | ||
| 1066 | if (rc) | ||
| 1067 | pr_info("RoCE priority not yet configured\n"); | ||
| 1068 | |||
| 1069 | INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); | ||
| 1070 | set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags); | ||
| 1071 | schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); | ||
| 1072 | |||
| 1073 | /* Register ib dev */ | ||
| 1074 | rc = bnxt_re_register_ib(rdev); | ||
| 1075 | if (rc) { | ||
| 1076 | pr_err("Failed to register with IB: %#x\n", rc); | ||
| 1077 | goto fail; | ||
| 1078 | } | ||
| 1079 | dev_info(rdev_to_dev(rdev), "Device registered successfully"); | ||
| 1080 | for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { | ||
| 1081 | rc = device_create_file(&rdev->ibdev.dev, | ||
| 1082 | bnxt_re_attributes[i]); | ||
| 1083 | if (rc) { | ||
| 1084 | dev_err(rdev_to_dev(rdev), | ||
| 1085 | "Failed to create IB sysfs: %#x", rc); | ||
| 1086 | /* Must clean up all created device files */ | ||
| 1087 | for (j = 0; j < i; j++) | ||
| 1088 | device_remove_file(&rdev->ibdev.dev, | ||
| 1089 | bnxt_re_attributes[j]); | ||
| 1090 | bnxt_re_unregister_ib(rdev); | ||
| 1091 | goto fail; | ||
| 1092 | } | ||
| 1093 | } | ||
| 1094 | set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); | ||
| 1095 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); | ||
| 1096 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); | ||
| 1097 | |||
| 1098 | return 0; | ||
| 1099 | free_sctx: | ||
| 1100 | bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, true); | ||
| 1101 | free_ctx: | ||
| 1102 | bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); | ||
| 1103 | disable_rcfw: | ||
| 1104 | bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); | ||
| 1105 | free_ring: | ||
| 1106 | bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, true); | ||
| 1107 | free_rcfw: | ||
| 1108 | bnxt_qplib_free_rcfw_channel(&rdev->rcfw); | ||
| 1109 | fail: | ||
| 1110 | bnxt_re_ib_unreg(rdev, true); | ||
| 1111 | return rc; | ||
| 1112 | } | ||
| 1113 | |||
| 1114 | static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) | ||
| 1115 | { | ||
| 1116 | struct bnxt_en_dev *en_dev = rdev->en_dev; | ||
| 1117 | struct net_device *netdev = rdev->netdev; | ||
| 1118 | |||
| 1119 | bnxt_re_dev_remove(rdev); | ||
| 1120 | |||
| 1121 | if (netdev) | ||
| 1122 | bnxt_re_dev_unprobe(netdev, en_dev); | ||
| 1123 | } | ||
| 1124 | |||
| 1125 | static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev) | ||
| 1126 | { | ||
| 1127 | struct bnxt_en_dev *en_dev; | ||
| 1128 | int rc = 0; | ||
| 1129 | |||
| 1130 | if (!is_bnxt_re_dev(netdev)) | ||
| 1131 | return -ENODEV; | ||
| 1132 | |||
| 1133 | en_dev = bnxt_re_dev_probe(netdev); | ||
| 1134 | if (IS_ERR(en_dev)) { | ||
| 1135 | if (en_dev != ERR_PTR(-ENODEV)) | ||
| 1136 | pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME); | ||
| 1137 | rc = PTR_ERR(en_dev); | ||
| 1138 | goto exit; | ||
| 1139 | } | ||
| 1140 | *rdev = bnxt_re_dev_add(netdev, en_dev); | ||
| 1141 | if (!*rdev) { | ||
| 1142 | rc = -ENOMEM; | ||
| 1143 | bnxt_re_dev_unprobe(netdev, en_dev); | ||
| 1144 | goto exit; | ||
| 1145 | } | ||
| 1146 | exit: | ||
| 1147 | return rc; | ||
| 1148 | } | ||
| 1149 | |||
| 1150 | static void bnxt_re_remove_one(struct bnxt_re_dev *rdev) | ||
| 1151 | { | ||
| 1152 | pci_dev_put(rdev->en_dev->pdev); | ||
| 1153 | } | ||
| 1154 | |||
| 1155 | /* Handle all deferred netevents tasks */ | ||
| 1156 | static void bnxt_re_task(struct work_struct *work) | ||
| 1157 | { | ||
| 1158 | struct bnxt_re_work *re_work; | ||
| 1159 | struct bnxt_re_dev *rdev; | ||
| 1160 | int rc = 0; | ||
| 1161 | |||
| 1162 | re_work = container_of(work, struct bnxt_re_work, work); | ||
| 1163 | rdev = re_work->rdev; | ||
| 1164 | |||
| 1165 | if (re_work->event != NETDEV_REGISTER && | ||
| 1166 | !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) | ||
| 1167 | return; | ||
| 1168 | |||
| 1169 | switch (re_work->event) { | ||
| 1170 | case NETDEV_REGISTER: | ||
| 1171 | rc = bnxt_re_ib_reg(rdev); | ||
| 1172 | if (rc) | ||
| 1173 | dev_err(rdev_to_dev(rdev), | ||
| 1174 | "Failed to register with IB: %#x", rc); | ||
| 1175 | break; | ||
| 1176 | case NETDEV_UP: | ||
| 1177 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, | ||
| 1178 | IB_EVENT_PORT_ACTIVE); | ||
| 1179 | break; | ||
| 1180 | case NETDEV_DOWN: | ||
| 1181 | bnxt_re_dev_stop(rdev); | ||
| 1182 | break; | ||
| 1183 | case NETDEV_CHANGE: | ||
| 1184 | if (!netif_carrier_ok(rdev->netdev)) | ||
| 1185 | bnxt_re_dev_stop(rdev); | ||
| 1186 | else if (netif_carrier_ok(rdev->netdev)) | ||
| 1187 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, | ||
| 1188 | IB_EVENT_PORT_ACTIVE); | ||
| 1189 | break; | ||
| 1190 | default: | ||
| 1191 | break; | ||
| 1192 | } | ||
| 1193 | kfree(re_work); | ||
| 1194 | } | ||
| 1195 | |||
| 1196 | static void bnxt_re_init_one(struct bnxt_re_dev *rdev) | ||
| 1197 | { | ||
| 1198 | pci_dev_get(rdev->en_dev->pdev); | ||
| 1199 | } | ||
| 1200 | |||
| 1201 | /* | ||
| 1202 | * "Notifier chain callback can be invoked for the same chain from | ||
| 1203 | * different CPUs at the same time". | ||
| 1204 | * | ||
| 1205 | * For cases when the netdev is already present, our call to the | ||
| 1206 | * register_netdevice_notifier() will actually get the rtnl_lock() | ||
| 1207 | * before sending NETDEV_REGISTER and (if up) NETDEV_UP | ||
| 1208 | * events. | ||
| 1209 | * | ||
| 1210 | * But for cases when the netdev is not already present, the notifier | ||
| 1211 | * chain is subjected to be invoked from different CPUs simultaneously. | ||
| 1212 | * | ||
| 1213 | * This is protected by the netdev_mutex. | ||
| 1214 | */ | ||
| 1215 | static int bnxt_re_netdev_event(struct notifier_block *notifier, | ||
| 1216 | unsigned long event, void *ptr) | ||
| 1217 | { | ||
| 1218 | struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr); | ||
| 1219 | struct bnxt_re_work *re_work; | ||
| 1220 | struct bnxt_re_dev *rdev; | ||
| 1221 | int rc = 0; | ||
| 1222 | bool sch_work = false; | ||
| 1223 | |||
| 1224 | real_dev = rdma_vlan_dev_real_dev(netdev); | ||
| 1225 | if (!real_dev) | ||
| 1226 | real_dev = netdev; | ||
| 1227 | |||
| 1228 | rdev = bnxt_re_from_netdev(real_dev); | ||
| 1229 | if (!rdev && event != NETDEV_REGISTER) | ||
| 1230 | goto exit; | ||
| 1231 | if (real_dev != netdev) | ||
| 1232 | goto exit; | ||
| 1233 | |||
| 1234 | switch (event) { | ||
| 1235 | case NETDEV_REGISTER: | ||
| 1236 | if (rdev) | ||
| 1237 | break; | ||
| 1238 | rc = bnxt_re_dev_reg(&rdev, real_dev); | ||
| 1239 | if (rc == -ENODEV) | ||
| 1240 | break; | ||
| 1241 | if (rc) { | ||
| 1242 | pr_err("Failed to register with the device %s: %#x\n", | ||
| 1243 | real_dev->name, rc); | ||
| 1244 | break; | ||
| 1245 | } | ||
| 1246 | bnxt_re_init_one(rdev); | ||
| 1247 | sch_work = true; | ||
| 1248 | break; | ||
| 1249 | |||
| 1250 | case NETDEV_UNREGISTER: | ||
| 1251 | bnxt_re_ib_unreg(rdev, false); | ||
| 1252 | bnxt_re_remove_one(rdev); | ||
| 1253 | bnxt_re_dev_unreg(rdev); | ||
| 1254 | break; | ||
| 1255 | |||
| 1256 | default: | ||
| 1257 | sch_work = true; | ||
| 1258 | break; | ||
| 1259 | } | ||
| 1260 | if (sch_work) { | ||
| 1261 | /* Allocate for the deferred task */ | ||
| 1262 | re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC); | ||
| 1263 | if (re_work) { | ||
| 1264 | re_work->rdev = rdev; | ||
| 1265 | re_work->event = event; | ||
| 1266 | re_work->vlan_dev = (real_dev == netdev ? | ||
| 1267 | NULL : netdev); | ||
| 1268 | INIT_WORK(&re_work->work, bnxt_re_task); | ||
| 1269 | queue_work(bnxt_re_wq, &re_work->work); | ||
| 1270 | } | ||
| 1271 | } | ||
| 1272 | |||
| 1273 | exit: | ||
| 1274 | return NOTIFY_DONE; | ||
| 1275 | } | ||
| 1276 | |||
| 1277 | static struct notifier_block bnxt_re_netdev_notifier = { | ||
| 1278 | .notifier_call = bnxt_re_netdev_event | ||
| 1279 | }; | ||
| 1280 | |||
| 1281 | static int __init bnxt_re_mod_init(void) | ||
| 1282 | { | ||
| 1283 | int rc = 0; | ||
| 1284 | |||
| 1285 | pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version); | ||
| 1286 | |||
| 1287 | bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); | ||
| 1288 | if (!bnxt_re_wq) | ||
| 1289 | return -ENOMEM; | ||
| 1290 | |||
| 1291 | INIT_LIST_HEAD(&bnxt_re_dev_list); | ||
| 1292 | |||
| 1293 | rc = register_netdevice_notifier(&bnxt_re_netdev_notifier); | ||
| 1294 | if (rc) { | ||
| 1295 | pr_err("%s: Cannot register to netdevice_notifier", | ||
| 1296 | ROCE_DRV_MODULE_NAME); | ||
| 1297 | goto err_netdev; | ||
| 1298 | } | ||
| 1299 | return 0; | ||
| 1300 | |||
| 1301 | err_netdev: | ||
| 1302 | destroy_workqueue(bnxt_re_wq); | ||
| 1303 | |||
| 1304 | return rc; | ||
| 1305 | } | ||
| 1306 | |||
| 1307 | static void __exit bnxt_re_mod_exit(void) | ||
| 1308 | { | ||
| 1309 | unregister_netdevice_notifier(&bnxt_re_netdev_notifier); | ||
| 1310 | if (bnxt_re_wq) | ||
| 1311 | destroy_workqueue(bnxt_re_wq); | ||
| 1312 | } | ||
| 1313 | |||
| 1314 | module_init(bnxt_re_mod_init); | ||
| 1315 | module_exit(bnxt_re_mod_exit); | ||
| @@ -0,0 +1,2167 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: Fast Path Operators | ||
| 37 | */ | ||
| 38 | |||
| 39 | #include <linux/interrupt.h> | ||
| 40 | #include <linux/spinlock.h> | ||
| 41 | #include <linux/sched.h> | ||
| 42 | #include <linux/slab.h> | ||
| 43 | #include <linux/pci.h> | ||
| 44 | #include <linux/prefetch.h> | ||
| 45 | |||
| 46 | #include "roce_hsi.h" | ||
| 47 | |||
| 48 | #include "qplib_res.h" | ||
| 49 | #include "qplib_rcfw.h" | ||
| 50 | #include "qplib_sp.h" | ||
| 51 | #include "qplib_fp.h" | ||
| 52 | |||
| 53 | static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq); | ||
| 54 | |||
| 55 | static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, | ||
| 56 | struct bnxt_qplib_qp *qp) | ||
| 57 | { | ||
| 58 | struct bnxt_qplib_q *rq = &qp->rq; | ||
| 59 | struct bnxt_qplib_q *sq = &qp->sq; | ||
| 60 | |||
| 61 | if (qp->rq_hdr_buf) | ||
| 62 | dma_free_coherent(&res->pdev->dev, | ||
| 63 | rq->hwq.max_elements * qp->rq_hdr_buf_size, | ||
| 64 | qp->rq_hdr_buf, qp->rq_hdr_buf_map); | ||
| 65 | if (qp->sq_hdr_buf) | ||
| 66 | dma_free_coherent(&res->pdev->dev, | ||
| 67 | sq->hwq.max_elements * qp->sq_hdr_buf_size, | ||
| 68 | qp->sq_hdr_buf, qp->sq_hdr_buf_map); | ||
| 69 | qp->rq_hdr_buf = NULL; | ||
| 70 | qp->sq_hdr_buf = NULL; | ||
| 71 | qp->rq_hdr_buf_map = 0; | ||
| 72 | qp->sq_hdr_buf_map = 0; | ||
| 73 | qp->sq_hdr_buf_size = 0; | ||
| 74 | qp->rq_hdr_buf_size = 0; | ||
| 75 | } | ||
| 76 | |||
| 77 | static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, | ||
| 78 | struct bnxt_qplib_qp *qp) | ||
| 79 | { | ||
| 80 | struct bnxt_qplib_q *rq = &qp->rq; | ||
| 81 | struct bnxt_qplib_q *sq = &qp->rq; | ||
| 82 | int rc = 0; | ||
| 83 | |||
| 84 | if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { | ||
| 85 | qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, | ||
| 86 | sq->hwq.max_elements * | ||
| 87 | qp->sq_hdr_buf_size, | ||
| 88 | &qp->sq_hdr_buf_map, GFP_KERNEL); | ||
| 89 | if (!qp->sq_hdr_buf) { | ||
| 90 | rc = -ENOMEM; | ||
| 91 | dev_err(&res->pdev->dev, | ||
| 92 | "QPLIB: Failed to create sq_hdr_buf"); | ||
| 93 | goto fail; | ||
| 94 | } | ||
| 95 | } | ||
| 96 | |||
| 97 | if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { | ||
| 98 | qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, | ||
| 99 | rq->hwq.max_elements * | ||
| 100 | qp->rq_hdr_buf_size, | ||
| 101 | &qp->rq_hdr_buf_map, | ||
| 102 | GFP_KERNEL); | ||
| 103 | if (!qp->rq_hdr_buf) { | ||
| 104 | rc = -ENOMEM; | ||
| 105 | dev_err(&res->pdev->dev, | ||
| 106 | "QPLIB: Failed to create rq_hdr_buf"); | ||
| 107 | goto fail; | ||
| 108 | } | ||
| 109 | } | ||
| 110 | return 0; | ||
| 111 | |||
| 112 | fail: | ||
| 113 | bnxt_qplib_free_qp_hdr_buf(res, qp); | ||
| 114 | return rc; | ||
| 115 | } | ||
| 116 | |||
| 117 | static void bnxt_qplib_service_nq(unsigned long data) | ||
| 118 | { | ||
| 119 | struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; | ||
| 120 | struct bnxt_qplib_hwq *hwq = &nq->hwq; | ||
| 121 | struct nq_base *nqe, **nq_ptr; | ||
| 122 | int num_cqne_processed = 0; | ||
| 123 | u32 sw_cons, raw_cons; | ||
| 124 | u16 type; | ||
| 125 | int budget = nq->budget; | ||
| 126 | u64 q_handle; | ||
| 127 | |||
| 128 | /* Service the NQ until empty */ | ||
| 129 | raw_cons = hwq->cons; | ||
| 130 | while (budget--) { | ||
| 131 | sw_cons = HWQ_CMP(raw_cons, hwq); | ||
| 132 | nq_ptr = (struct nq_base **)hwq->pbl_ptr; | ||
| 133 | nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]; | ||
| 134 | if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) | ||
| 135 | break; | ||
| 136 | |||
| 137 | type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; | ||
| 138 | switch (type) { | ||
| 139 | case NQ_BASE_TYPE_CQ_NOTIFICATION: | ||
| 140 | { | ||
| 141 | struct nq_cn *nqcne = (struct nq_cn *)nqe; | ||
| 142 | |||
| 143 | q_handle = le32_to_cpu(nqcne->cq_handle_low); | ||
| 144 | q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) | ||
| 145 | << 32; | ||
| 146 | bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *) | ||
| 147 | ((unsigned long)q_handle)); | ||
| 148 | if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *) | ||
| 149 | ((unsigned long)q_handle))) | ||
| 150 | num_cqne_processed++; | ||
| 151 | else | ||
| 152 | dev_warn(&nq->pdev->dev, | ||
| 153 | "QPLIB: cqn - type 0x%x not handled", | ||
| 154 | type); | ||
| 155 | break; | ||
| 156 | } | ||
| 157 | case NQ_BASE_TYPE_DBQ_EVENT: | ||
| 158 | break; | ||
| 159 | default: | ||
| 160 | dev_warn(&nq->pdev->dev, | ||
| 161 | "QPLIB: nqe with type = 0x%x not handled", | ||
| 162 | type); | ||
| 163 | break; | ||
| 164 | } | ||
| 165 | raw_cons++; | ||
| 166 | } | ||
| 167 | if (hwq->cons != raw_cons) { | ||
| 168 | hwq->cons = raw_cons; | ||
| 169 | NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements); | ||
| 170 | } | ||
| 171 | } | ||
| 172 | |||
| 173 | static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) | ||
| 174 | { | ||
| 175 | struct bnxt_qplib_nq *nq = dev_instance; | ||
| 176 | struct bnxt_qplib_hwq *hwq = &nq->hwq; | ||
| 177 | struct nq_base **nq_ptr; | ||
| 178 | u32 sw_cons; | ||
| 179 | |||
| 180 | /* Prefetch the NQ element */ | ||
| 181 | sw_cons = HWQ_CMP(hwq->cons, hwq); | ||
| 182 | nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr; | ||
| 183 | prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]); | ||
| 184 | |||
| 185 | /* Fan out to CPU affinitized kthreads? */ | ||
| 186 | tasklet_schedule(&nq->worker); | ||
| 187 | |||
| 188 | return IRQ_HANDLED; | ||
| 189 | } | ||
| 190 | |||
| 191 | void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) | ||
| 192 | { | ||
| 193 | /* Make sure the HW is stopped! */ | ||
| 194 | synchronize_irq(nq->vector); | ||
| 195 | tasklet_disable(&nq->worker); | ||
| 196 | tasklet_kill(&nq->worker); | ||
| 197 | |||
| 198 | if (nq->requested) { | ||
| 199 | free_irq(nq->vector, nq); | ||
| 200 | nq->requested = false; | ||
| 201 | } | ||
| 202 | if (nq->bar_reg_iomem) | ||
| 203 | iounmap(nq->bar_reg_iomem); | ||
| 204 | nq->bar_reg_iomem = NULL; | ||
| 205 | |||
| 206 | nq->cqn_handler = NULL; | ||
| 207 | nq->srqn_handler = NULL; | ||
| 208 | nq->vector = 0; | ||
| 209 | } | ||
| 210 | |||
| 211 | int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, | ||
| 212 | int msix_vector, int bar_reg_offset, | ||
| 213 | int (*cqn_handler)(struct bnxt_qplib_nq *nq, | ||
| 214 | struct bnxt_qplib_cq *), | ||
| 215 | int (*srqn_handler)(struct bnxt_qplib_nq *nq, | ||
| 216 | void *, u8 event)) | ||
| 217 | { | ||
| 218 | resource_size_t nq_base; | ||
| 219 | int rc; | ||
| 220 | |||
| 221 | nq->pdev = pdev; | ||
| 222 | nq->vector = msix_vector; | ||
| 223 | |||
| 224 | nq->cqn_handler = cqn_handler; | ||
| 225 | |||
| 226 | nq->srqn_handler = srqn_handler; | ||
| 227 | |||
| 228 | tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq); | ||
| 229 | |||
| 230 | nq->requested = false; | ||
| 231 | rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq); | ||
| 232 | if (rc) { | ||
| 233 | dev_err(&nq->pdev->dev, | ||
| 234 | "Failed to request IRQ for NQ: %#x", rc); | ||
| 235 | bnxt_qplib_disable_nq(nq); | ||
| 236 | goto fail; | ||
| 237 | } | ||
| 238 | nq->requested = true; | ||
| 239 | nq->bar_reg = NQ_CONS_PCI_BAR_REGION; | ||
| 240 | nq->bar_reg_off = bar_reg_offset; | ||
| 241 | nq_base = pci_resource_start(pdev, nq->bar_reg); | ||
| 242 | if (!nq_base) { | ||
| 243 | rc = -ENOMEM; | ||
| 244 | goto fail; | ||
| 245 | } | ||
| 246 | nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4); | ||
| 247 | if (!nq->bar_reg_iomem) { | ||
| 248 | rc = -ENOMEM; | ||
| 249 | goto fail; | ||
| 250 | } | ||
| 251 | NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); | ||
| 252 | |||
| 253 | return 0; | ||
| 254 | fail: | ||
| 255 | bnxt_qplib_disable_nq(nq); | ||
| 256 | return rc; | ||
| 257 | } | ||
| 258 | |||
| 259 | void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq) | ||
| 260 | { | ||
| 261 | if (nq->hwq.max_elements) | ||
| 262 | bnxt_qplib_free_hwq(nq->pdev, &nq->hwq); | ||
| 263 | } | ||
| 264 | |||
| 265 | int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq) | ||
| 266 | { | ||
| 267 | nq->pdev = pdev; | ||
| 268 | if (!nq->hwq.max_elements || | ||
| 269 | nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) | ||
| 270 | nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; | ||
| 271 | |||
| 272 | if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0, | ||
| 273 | &nq->hwq.max_elements, | ||
| 274 | BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0, | ||
| 275 | PAGE_SIZE, HWQ_TYPE_L2_CMPL)) | ||
| 276 | return -ENOMEM; | ||
| 277 | |||
| 278 | nq->budget = 8; | ||
| 279 | return 0; | ||
| 280 | } | ||
| 281 | |||
| 282 | /* QP */ | ||
| 283 | int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | ||
| 284 | { | ||
| 285 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 286 | struct cmdq_create_qp1 req; | ||
| 287 | struct creq_create_qp1_resp *resp; | ||
| 288 | struct bnxt_qplib_pbl *pbl; | ||
| 289 | struct bnxt_qplib_q *sq = &qp->sq; | ||
| 290 | struct bnxt_qplib_q *rq = &qp->rq; | ||
| 291 | int rc; | ||
| 292 | u16 cmd_flags = 0; | ||
| 293 | u32 qp_flags = 0; | ||
| 294 | |||
| 295 | RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags); | ||
| 296 | |||
| 297 | /* General */ | ||
| 298 | req.type = qp->type; | ||
| 299 | req.dpi = cpu_to_le32(qp->dpi->dpi); | ||
| 300 | req.qp_handle = cpu_to_le64(qp->qp_handle); | ||
| 301 | |||
| 302 | /* SQ */ | ||
| 303 | sq->hwq.max_elements = sq->max_wqe; | ||
| 304 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0, | ||
| 305 | &sq->hwq.max_elements, | ||
| 306 | BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0, | ||
| 307 | PAGE_SIZE, HWQ_TYPE_QUEUE); | ||
| 308 | if (rc) | ||
| 309 | goto exit; | ||
| 310 | |||
| 311 | sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); | ||
| 312 | if (!sq->swq) { | ||
| 313 | rc = -ENOMEM; | ||
| 314 | goto fail_sq; | ||
| 315 | } | ||
| 316 | pbl = &sq->hwq.pbl[PBL_LVL_0]; | ||
| 317 | req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | ||
| 318 | req.sq_pg_size_sq_lvl = | ||
| 319 | ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) | ||
| 320 | << CMDQ_CREATE_QP1_SQ_LVL_SFT) | | ||
| 321 | (pbl->pg_size == ROCE_PG_SIZE_4K ? | ||
| 322 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K : | ||
| 323 | pbl->pg_size == ROCE_PG_SIZE_8K ? | ||
| 324 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K : | ||
| 325 | pbl->pg_size == ROCE_PG_SIZE_64K ? | ||
| 326 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K : | ||
| 327 | pbl->pg_size == ROCE_PG_SIZE_2M ? | ||
| 328 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M : | ||
| 329 | pbl->pg_size == ROCE_PG_SIZE_8M ? | ||
| 330 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M : | ||
| 331 | pbl->pg_size == ROCE_PG_SIZE_1G ? | ||
| 332 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G : | ||
| 333 | CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K); | ||
| 334 | |||
| 335 | if (qp->scq) | ||
| 336 | req.scq_cid = cpu_to_le32(qp->scq->id); | ||
| 337 | |||
| 338 | qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; | ||
| 339 | |||
| 340 | /* RQ */ | ||
| 341 | if (rq->max_wqe) { | ||
| 342 | rq->hwq.max_elements = qp->rq.max_wqe; | ||
| 343 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0, | ||
| 344 | &rq->hwq.max_elements, | ||
| 345 | BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, | ||
| 346 | PAGE_SIZE, HWQ_TYPE_QUEUE); | ||
| 347 | if (rc) | ||
| 348 | goto fail_sq; | ||
| 349 | |||
| 350 | rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), | ||
| 351 | GFP_KERNEL); | ||
| 352 | if (!rq->swq) { | ||
| 353 | rc = -ENOMEM; | ||
| 354 | goto fail_rq; | ||
| 355 | } | ||
| 356 | pbl = &rq->hwq.pbl[PBL_LVL_0]; | ||
| 357 | req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | ||
| 358 | req.rq_pg_size_rq_lvl = | ||
| 359 | ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << | ||
| 360 | CMDQ_CREATE_QP1_RQ_LVL_SFT) | | ||
| 361 | (pbl->pg_size == ROCE_PG_SIZE_4K ? | ||
| 362 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K : | ||
| 363 | pbl->pg_size == ROCE_PG_SIZE_8K ? | ||
| 364 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K : | ||
| 365 | pbl->pg_size == ROCE_PG_SIZE_64K ? | ||
| 366 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K : | ||
| 367 | pbl->pg_size == ROCE_PG_SIZE_2M ? | ||
| 368 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M : | ||
| 369 | pbl->pg_size == ROCE_PG_SIZE_8M ? | ||
| 370 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M : | ||
| 371 | pbl->pg_size == ROCE_PG_SIZE_1G ? | ||
| 372 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G : | ||
| 373 | CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K); | ||
| 374 | if (qp->rcq) | ||
| 375 | req.rcq_cid = cpu_to_le32(qp->rcq->id); | ||
| 376 | } | ||
| 377 | |||
| 378 | /* Header buffer - allow hdr_buf pass in */ | ||
| 379 | rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); | ||
| 380 | if (rc) { | ||
| 381 | rc = -ENOMEM; | ||
| 382 | goto fail; | ||
| 383 | } | ||
| 384 | req.qp_flags = cpu_to_le32(qp_flags); | ||
| 385 | req.sq_size = cpu_to_le32(sq->hwq.max_elements); | ||
| 386 | req.rq_size = cpu_to_le32(rq->hwq.max_elements); | ||
| 387 | |||
| 388 | req.sq_fwo_sq_sge = | ||
| 389 | cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << | ||
| 390 | CMDQ_CREATE_QP1_SQ_SGE_SFT); | ||
| 391 | req.rq_fwo_rq_sge = | ||
| 392 | cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << | ||
| 393 | CMDQ_CREATE_QP1_RQ_SGE_SFT); | ||
| 394 | |||
| 395 | req.pd_id = cpu_to_le32(qp->pd->id); | ||
| 396 | |||
| 397 | resp = (struct creq_create_qp1_resp *) | ||
| 398 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 399 | NULL, 0); | ||
| 400 | if (!resp) { | ||
| 401 | dev_err(&res->pdev->dev, "QPLIB: FP: CREATE_QP1 send failed"); | ||
| 402 | rc = -EINVAL; | ||
| 403 | goto fail; | ||
| 404 | } | ||
| 405 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 406 | /* Cmd timed out */ | ||
| 407 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 timed out"); | ||
| 408 | rc = -ETIMEDOUT; | ||
| 409 | goto fail; | ||
| 410 | } | ||
| 411 | if (resp->status || | ||
| 412 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 413 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 failed "); | ||
| 414 | dev_err(&rcfw->pdev->dev, | ||
| 415 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 416 | resp->status, le16_to_cpu(req.cookie), | ||
| 417 | le16_to_cpu(resp->cookie)); | ||
| 418 | rc = -EINVAL; | ||
| 419 | goto fail; | ||
| 420 | } | ||
| 421 | qp->id = le32_to_cpu(resp->xid); | ||
| 422 | qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; | ||
| 423 | sq->flush_in_progress = false; | ||
| 424 | rq->flush_in_progress = false; | ||
| 425 | |||
| 426 | return 0; | ||
| 427 | |||
| 428 | fail: | ||
| 429 | bnxt_qplib_free_qp_hdr_buf(res, qp); | ||
| 430 | fail_rq: | ||
| 431 | bnxt_qplib_free_hwq(res->pdev, &rq->hwq); | ||
| 432 | kfree(rq->swq); | ||
| 433 | fail_sq: | ||
| 434 | bnxt_qplib_free_hwq(res->pdev, &sq->hwq); | ||
| 435 | kfree(sq->swq); | ||
| 436 | exit: | ||
| 437 | return rc; | ||
| 438 | } | ||
| 439 | |||
| 440 | int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | ||
| 441 | { | ||
| 442 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 443 | struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; | ||
| 444 | struct cmdq_create_qp req; | ||
| 445 | struct creq_create_qp_resp *resp; | ||
| 446 | struct bnxt_qplib_pbl *pbl; | ||
| 447 | struct sq_psn_search **psn_search_ptr; | ||
| 448 | unsigned long int psn_search, poff = 0; | ||
| 449 | struct bnxt_qplib_q *sq = &qp->sq; | ||
| 450 | struct bnxt_qplib_q *rq = &qp->rq; | ||
| 451 | struct bnxt_qplib_hwq *xrrq; | ||
| 452 | int i, rc, req_size, psn_sz; | ||
| 453 | u16 cmd_flags = 0, max_ssge; | ||
| 454 | u32 sw_prod, qp_flags = 0; | ||
| 455 | |||
| 456 | RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); | ||
| 457 | |||
| 458 | /* General */ | ||
| 459 | req.type = qp->type; | ||
| 460 | req.dpi = cpu_to_le32(qp->dpi->dpi); | ||
| 461 | req.qp_handle = cpu_to_le64(qp->qp_handle); | ||
| 462 | |||
| 463 | /* SQ */ | ||
| 464 | psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ? | ||
| 465 | sizeof(struct sq_psn_search) : 0; | ||
| 466 | sq->hwq.max_elements = sq->max_wqe; | ||
| 467 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist, | ||
| 468 | sq->nmap, &sq->hwq.max_elements, | ||
| 469 | BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, | ||
| 470 | psn_sz, | ||
| 471 | PAGE_SIZE, HWQ_TYPE_QUEUE); | ||
| 472 | if (rc) | ||
| 473 | goto exit; | ||
| 474 | |||
| 475 | sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); | ||
| 476 | if (!sq->swq) { | ||
| 477 | rc = -ENOMEM; | ||
| 478 | goto fail_sq; | ||
| 479 | } | ||
| 480 | hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | ||
| 481 | if (psn_sz) { | ||
| 482 | psn_search_ptr = (struct sq_psn_search **) | ||
| 483 | &hw_sq_send_ptr[get_sqe_pg | ||
| 484 | (sq->hwq.max_elements)]; | ||
| 485 | psn_search = (unsigned long int) | ||
| 486 | &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)] | ||
| 487 | [get_sqe_idx(sq->hwq.max_elements)]; | ||
| 488 | if (psn_search & ~PAGE_MASK) { | ||
| 489 | /* If the psn_search does not start on a page boundary, | ||
| 490 | * then calculate the offset | ||
| 491 | */ | ||
| 492 | poff = (psn_search & ~PAGE_MASK) / | ||
| 493 | BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE; | ||
| 494 | } | ||
| 495 | for (i = 0; i < sq->hwq.max_elements; i++) | ||
| 496 | sq->swq[i].psn_search = | ||
| 497 | &psn_search_ptr[get_psne_pg(i + poff)] | ||
| 498 | [get_psne_idx(i + poff)]; | ||
| 499 | } | ||
| 500 | pbl = &sq->hwq.pbl[PBL_LVL_0]; | ||
| 501 | req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | ||
| 502 | req.sq_pg_size_sq_lvl = | ||
| 503 | ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) | ||
| 504 | << CMDQ_CREATE_QP_SQ_LVL_SFT) | | ||
| 505 | (pbl->pg_size == ROCE_PG_SIZE_4K ? | ||
| 506 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K : | ||
| 507 | pbl->pg_size == ROCE_PG_SIZE_8K ? | ||
| 508 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K : | ||
| 509 | pbl->pg_size == ROCE_PG_SIZE_64K ? | ||
| 510 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K : | ||
| 511 | pbl->pg_size == ROCE_PG_SIZE_2M ? | ||
| 512 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M : | ||
| 513 | pbl->pg_size == ROCE_PG_SIZE_8M ? | ||
| 514 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M : | ||
| 515 | pbl->pg_size == ROCE_PG_SIZE_1G ? | ||
| 516 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G : | ||
| 517 | CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K); | ||
| 518 | |||
| 519 | /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */ | ||
| 520 | hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | ||
| 521 | for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) { | ||
| 522 | hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] | ||
| 523 | [get_sqe_idx(sw_prod)]; | ||
| 524 | hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID; | ||
| 525 | } | ||
| 526 | |||
| 527 | if (qp->scq) | ||
| 528 | req.scq_cid = cpu_to_le32(qp->scq->id); | ||
| 529 | |||
| 530 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; | ||
| 531 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; | ||
| 532 | if (qp->sig_type) | ||
| 533 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; | ||
| 534 | |||
| 535 | /* RQ */ | ||
| 536 | if (rq->max_wqe) { | ||
| 537 | rq->hwq.max_elements = rq->max_wqe; | ||
| 538 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist, | ||
| 539 | rq->nmap, &rq->hwq.max_elements, | ||
| 540 | BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, | ||
| 541 | PAGE_SIZE, HWQ_TYPE_QUEUE); | ||
| 542 | if (rc) | ||
| 543 | goto fail_sq; | ||
| 544 | |||
| 545 | rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), | ||
| 546 | GFP_KERNEL); | ||
| 547 | if (!rq->swq) { | ||
| 548 | rc = -ENOMEM; | ||
| 549 | goto fail_rq; | ||
| 550 | } | ||
| 551 | pbl = &rq->hwq.pbl[PBL_LVL_0]; | ||
| 552 | req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); | ||
| 553 | req.rq_pg_size_rq_lvl = | ||
| 554 | ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << | ||
| 555 | CMDQ_CREATE_QP_RQ_LVL_SFT) | | ||
| 556 | (pbl->pg_size == ROCE_PG_SIZE_4K ? | ||
| 557 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K : | ||
| 558 | pbl->pg_size == ROCE_PG_SIZE_8K ? | ||
| 559 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K : | ||
| 560 | pbl->pg_size == ROCE_PG_SIZE_64K ? | ||
| 561 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K : | ||
| 562 | pbl->pg_size == ROCE_PG_SIZE_2M ? | ||
| 563 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M : | ||
| 564 | pbl->pg_size == ROCE_PG_SIZE_8M ? | ||
| 565 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M : | ||
| 566 | pbl->pg_size == ROCE_PG_SIZE_1G ? | ||
| 567 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G : | ||
| 568 | CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K); | ||
| 569 | } | ||
| 570 | |||
| 571 | if (qp->rcq) | ||
| 572 | req.rcq_cid = cpu_to_le32(qp->rcq->id); | ||
| 573 | req.qp_flags = cpu_to_le32(qp_flags); | ||
| 574 | req.sq_size = cpu_to_le32(sq->hwq.max_elements); | ||
| 575 | req.rq_size = cpu_to_le32(rq->hwq.max_elements); | ||
| 576 | qp->sq_hdr_buf = NULL; | ||
| 577 | qp->rq_hdr_buf = NULL; | ||
| 578 | |||
| 579 | rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); | ||
| 580 | if (rc) | ||
| 581 | goto fail_rq; | ||
| 582 | |||
| 583 | /* CTRL-22434: Irrespective of the requested SGE count on the SQ | ||
| 584 | * always create the QP with max send sges possible if the requested | ||
| 585 | * inline size is greater than 0. | ||
| 586 | */ | ||
| 587 | max_ssge = qp->max_inline_data ? 6 : sq->max_sge; | ||
| 588 | req.sq_fwo_sq_sge = cpu_to_le16( | ||
| 589 | ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) | ||
| 590 | << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); | ||
| 591 | req.rq_fwo_rq_sge = cpu_to_le16( | ||
| 592 | ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK) | ||
| 593 | << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); | ||
| 594 | /* ORRQ and IRRQ */ | ||
| 595 | if (psn_sz) { | ||
| 596 | xrrq = &qp->orrq; | ||
| 597 | xrrq->max_elements = | ||
| 598 | ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); | ||
| 599 | req_size = xrrq->max_elements * | ||
| 600 | BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; | ||
| 601 | req_size &= ~(PAGE_SIZE - 1); | ||
| 602 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, | ||
| 603 | &xrrq->max_elements, | ||
| 604 | BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE, | ||
| 605 | 0, req_size, HWQ_TYPE_CTX); | ||
| 606 | if (rc) | ||
| 607 | goto fail_buf_free; | ||
| 608 | pbl = &xrrq->pbl[PBL_LVL_0]; | ||
| 609 | req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); | ||
| 610 | |||
| 611 | xrrq = &qp->irrq; | ||
| 612 | xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( | ||
| 613 | qp->max_dest_rd_atomic); | ||
| 614 | req_size = xrrq->max_elements * | ||
| 615 | BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; | ||
| 616 | req_size &= ~(PAGE_SIZE - 1); | ||
| 617 | |||
| 618 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, | ||
| 619 | &xrrq->max_elements, | ||
| 620 | BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE, | ||
| 621 | 0, req_size, HWQ_TYPE_CTX); | ||
| 622 | if (rc) | ||
| 623 | goto fail_orrq; | ||
| 624 | |||
| 625 | pbl = &xrrq->pbl[PBL_LVL_0]; | ||
| 626 | req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); | ||
| 627 | } | ||
| 628 | req.pd_id = cpu_to_le32(qp->pd->id); | ||
| 629 | |||
| 630 | resp = (struct creq_create_qp_resp *) | ||
| 631 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 632 | NULL, 0); | ||
| 633 | if (!resp) { | ||
| 634 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP send failed"); | ||
| 635 | rc = -EINVAL; | ||
| 636 | goto fail; | ||
| 637 | } | ||
| 638 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 639 | /* Cmd timed out */ | ||
| 640 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP timed out"); | ||
| 641 | rc = -ETIMEDOUT; | ||
| 642 | goto fail; | ||
| 643 | } | ||
| 644 | if (resp->status || | ||
| 645 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 646 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP failed "); | ||
| 647 | dev_err(&rcfw->pdev->dev, | ||
| 648 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 649 | resp->status, le16_to_cpu(req.cookie), | ||
| 650 | le16_to_cpu(resp->cookie)); | ||
| 651 | rc = -EINVAL; | ||
| 652 | goto fail; | ||
| 653 | } | ||
| 654 | qp->id = le32_to_cpu(resp->xid); | ||
| 655 | qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; | ||
| 656 | sq->flush_in_progress = false; | ||
| 657 | rq->flush_in_progress = false; | ||
| 658 | |||
| 659 | return 0; | ||
| 660 | |||
| 661 | fail: | ||
| 662 | if (qp->irrq.max_elements) | ||
| 663 | bnxt_qplib_free_hwq(res->pdev, &qp->irrq); | ||
| 664 | fail_orrq: | ||
| 665 | if (qp->orrq.max_elements) | ||
| 666 | bnxt_qplib_free_hwq(res->pdev, &qp->orrq); | ||
| 667 | fail_buf_free: | ||
| 668 | bnxt_qplib_free_qp_hdr_buf(res, qp); | ||
| 669 | fail_rq: | ||
| 670 | bnxt_qplib_free_hwq(res->pdev, &rq->hwq); | ||
| 671 | kfree(rq->swq); | ||
| 672 | fail_sq: | ||
| 673 | bnxt_qplib_free_hwq(res->pdev, &sq->hwq); | ||
| 674 | kfree(sq->swq); | ||
| 675 | exit: | ||
| 676 | return rc; | ||
| 677 | } | ||
| 678 | |||
| 679 | static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp) | ||
| 680 | { | ||
| 681 | switch (qp->state) { | ||
| 682 | case CMDQ_MODIFY_QP_NEW_STATE_RTR: | ||
| 683 | /* INIT->RTR, configure the path_mtu to the default | ||
| 684 | * 2048 if not being requested | ||
| 685 | */ | ||
| 686 | if (!(qp->modify_flags & | ||
| 687 | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { | ||
| 688 | qp->modify_flags |= | ||
| 689 | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; | ||
| 690 | qp->path_mtu = | ||
| 691 | CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; | ||
| 692 | } | ||
| 693 | qp->modify_flags &= | ||
| 694 | ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; | ||
| 695 | /* Bono FW require the max_dest_rd_atomic to be >= 1 */ | ||
| 696 | if (qp->max_dest_rd_atomic < 1) | ||
| 697 | qp->max_dest_rd_atomic = 1; | ||
| 698 | qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; | ||
| 699 | /* Bono FW 20.6.5 requires SGID_INDEX configuration */ | ||
| 700 | if (!(qp->modify_flags & | ||
| 701 | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { | ||
| 702 | qp->modify_flags |= | ||
| 703 | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; | ||
| 704 | qp->ah.sgid_index = 0; | ||
| 705 | } | ||
| 706 | break; | ||
| 707 | default: | ||
| 708 | break; | ||
| 709 | } | ||
| 710 | } | ||
| 711 | |||
| 712 | static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp) | ||
| 713 | { | ||
| 714 | switch (qp->state) { | ||
| 715 | case CMDQ_MODIFY_QP_NEW_STATE_RTS: | ||
| 716 | /* Bono FW requires the max_rd_atomic to be >= 1 */ | ||
| 717 | if (qp->max_rd_atomic < 1) | ||
| 718 | qp->max_rd_atomic = 1; | ||
| 719 | /* Bono FW does not allow PKEY_INDEX, | ||
| 720 | * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT, | ||
| 721 | * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN, | ||
| 722 | * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID | ||
| 723 | * modification | ||
| 724 | */ | ||
| 725 | qp->modify_flags &= | ||
| 726 | ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | | ||
| 727 | CMDQ_MODIFY_QP_MODIFY_MASK_DGID | | ||
| 728 | CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | | ||
| 729 | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | | ||
| 730 | CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | | ||
| 731 | CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | | ||
| 732 | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | | ||
| 733 | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | | ||
| 734 | CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | | ||
| 735 | CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | | ||
| 736 | CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | | ||
| 737 | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); | ||
| 738 | break; | ||
| 739 | default: | ||
| 740 | break; | ||
| 741 | } | ||
| 742 | } | ||
| 743 | |||
| 744 | static void __filter_modify_flags(struct bnxt_qplib_qp *qp) | ||
| 745 | { | ||
| 746 | switch (qp->cur_qp_state) { | ||
| 747 | case CMDQ_MODIFY_QP_NEW_STATE_RESET: | ||
| 748 | break; | ||
| 749 | case CMDQ_MODIFY_QP_NEW_STATE_INIT: | ||
| 750 | __modify_flags_from_init_state(qp); | ||
| 751 | break; | ||
| 752 | case CMDQ_MODIFY_QP_NEW_STATE_RTR: | ||
| 753 | __modify_flags_from_rtr_state(qp); | ||
| 754 | break; | ||
| 755 | case CMDQ_MODIFY_QP_NEW_STATE_RTS: | ||
| 756 | break; | ||
| 757 | case CMDQ_MODIFY_QP_NEW_STATE_SQD: | ||
| 758 | break; | ||
| 759 | case CMDQ_MODIFY_QP_NEW_STATE_SQE: | ||
| 760 | break; | ||
| 761 | case CMDQ_MODIFY_QP_NEW_STATE_ERR: | ||
| 762 | break; | ||
| 763 | default: | ||
| 764 | break; | ||
| 765 | } | ||
| 766 | } | ||
| 767 | |||
| 768 | int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | ||
| 769 | { | ||
| 770 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 771 | struct cmdq_modify_qp req; | ||
| 772 | struct creq_modify_qp_resp *resp; | ||
| 773 | u16 cmd_flags = 0, pkey; | ||
| 774 | u32 temp32[4]; | ||
| 775 | u32 bmask; | ||
| 776 | |||
| 777 | RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags); | ||
| 778 | |||
| 779 | /* Filter out the qp_attr_mask based on the state->new transition */ | ||
| 780 | __filter_modify_flags(qp); | ||
| 781 | bmask = qp->modify_flags; | ||
| 782 | req.modify_mask = cpu_to_le32(qp->modify_flags); | ||
| 783 | req.qp_cid = cpu_to_le32(qp->id); | ||
| 784 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { | ||
| 785 | req.network_type_en_sqd_async_notify_new_state = | ||
| 786 | (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | | ||
| 787 | (qp->en_sqd_async_notify ? | ||
| 788 | CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); | ||
| 789 | } | ||
| 790 | req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; | ||
| 791 | |||
| 792 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) | ||
| 793 | req.access = qp->access; | ||
| 794 | |||
| 795 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) { | ||
| 796 | if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl, | ||
| 797 | qp->pkey_index, &pkey)) | ||
| 798 | req.pkey = cpu_to_le16(pkey); | ||
| 799 | } | ||
| 800 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) | ||
| 801 | req.qkey = cpu_to_le32(qp->qkey); | ||
| 802 | |||
| 803 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { | ||
| 804 | memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); | ||
| 805 | req.dgid[0] = cpu_to_le32(temp32[0]); | ||
| 806 | req.dgid[1] = cpu_to_le32(temp32[1]); | ||
| 807 | req.dgid[2] = cpu_to_le32(temp32[2]); | ||
| 808 | req.dgid[3] = cpu_to_le32(temp32[3]); | ||
| 809 | } | ||
| 810 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) | ||
| 811 | req.flow_label = cpu_to_le32(qp->ah.flow_label); | ||
| 812 | |||
| 813 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) | ||
| 814 | req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id | ||
| 815 | [qp->ah.sgid_index]); | ||
| 816 | |||
| 817 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) | ||
| 818 | req.hop_limit = qp->ah.hop_limit; | ||
| 819 | |||
| 820 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) | ||
| 821 | req.traffic_class = qp->ah.traffic_class; | ||
| 822 | |||
| 823 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) | ||
| 824 | memcpy(req.dest_mac, qp->ah.dmac, 6); | ||
| 825 | |||
| 826 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) | ||
| 827 | req.path_mtu = qp->path_mtu; | ||
| 828 | |||
| 829 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) | ||
| 830 | req.timeout = qp->timeout; | ||
| 831 | |||
| 832 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) | ||
| 833 | req.retry_cnt = qp->retry_cnt; | ||
| 834 | |||
| 835 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) | ||
| 836 | req.rnr_retry = qp->rnr_retry; | ||
| 837 | |||
| 838 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) | ||
| 839 | req.min_rnr_timer = qp->min_rnr_timer; | ||
| 840 | |||
| 841 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) | ||
| 842 | req.rq_psn = cpu_to_le32(qp->rq.psn); | ||
| 843 | |||
| 844 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) | ||
| 845 | req.sq_psn = cpu_to_le32(qp->sq.psn); | ||
| 846 | |||
| 847 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) | ||
| 848 | req.max_rd_atomic = | ||
| 849 | ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); | ||
| 850 | |||
| 851 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) | ||
| 852 | req.max_dest_rd_atomic = | ||
| 853 | IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); | ||
| 854 | |||
| 855 | req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); | ||
| 856 | req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); | ||
| 857 | req.sq_sge = cpu_to_le16(qp->sq.max_sge); | ||
| 858 | req.rq_sge = cpu_to_le16(qp->rq.max_sge); | ||
| 859 | req.max_inline_data = cpu_to_le32(qp->max_inline_data); | ||
| 860 | if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) | ||
| 861 | req.dest_qp_id = cpu_to_le32(qp->dest_qpn); | ||
| 862 | |||
| 863 | req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); | ||
| 864 | |||
| 865 | resp = (struct creq_modify_qp_resp *) | ||
| 866 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 867 | NULL, 0); | ||
| 868 | if (!resp) { | ||
| 869 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP send failed"); | ||
| 870 | return -EINVAL; | ||
| 871 | } | ||
| 872 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 873 | /* Cmd timed out */ | ||
| 874 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP timed out"); | ||
| 875 | return -ETIMEDOUT; | ||
| 876 | } | ||
| 877 | if (resp->status || | ||
| 878 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 879 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP failed "); | ||
| 880 | dev_err(&rcfw->pdev->dev, | ||
| 881 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 882 | resp->status, le16_to_cpu(req.cookie), | ||
| 883 | le16_to_cpu(resp->cookie)); | ||
| 884 | return -EINVAL; | ||
| 885 | } | ||
| 886 | qp->cur_qp_state = qp->state; | ||
| 887 | return 0; | ||
| 888 | } | ||
| 889 | |||
| 890 | int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) | ||
| 891 | { | ||
| 892 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 893 | struct cmdq_query_qp req; | ||
| 894 | struct creq_query_qp_resp *resp; | ||
| 895 | struct creq_query_qp_resp_sb *sb; | ||
| 896 | u16 cmd_flags = 0; | ||
| 897 | u32 temp32[4]; | ||
| 898 | int i; | ||
| 899 | |||
| 900 | RCFW_CMD_PREP(req, QUERY_QP, cmd_flags); | ||
| 901 | |||
| 902 | req.qp_cid = cpu_to_le32(qp->id); | ||
| 903 | req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; | ||
| 904 | resp = (struct creq_query_qp_resp *) | ||
| 905 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 906 | (void **)&sb, 0); | ||
| 907 | if (!resp) { | ||
| 908 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP send failed"); | ||
| 909 | return -EINVAL; | ||
| 910 | } | ||
| 911 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 912 | /* Cmd timed out */ | ||
| 913 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP timed out"); | ||
| 914 | return -ETIMEDOUT; | ||
| 915 | } | ||
| 916 | if (resp->status || | ||
| 917 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 918 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP failed "); | ||
| 919 | dev_err(&rcfw->pdev->dev, | ||
| 920 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 921 | resp->status, le16_to_cpu(req.cookie), | ||
| 922 | le16_to_cpu(resp->cookie)); | ||
| 923 | return -EINVAL; | ||
| 924 | } | ||
| 925 | /* Extract the context from the side buffer */ | ||
| 926 | qp->state = sb->en_sqd_async_notify_state & | ||
| 927 | CREQ_QUERY_QP_RESP_SB_STATE_MASK; | ||
| 928 | qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & | ||
| 929 | CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? | ||
| 930 | true : false; | ||
| 931 | qp->access = sb->access; | ||
| 932 | qp->pkey_index = le16_to_cpu(sb->pkey); | ||
| 933 | qp->qkey = le32_to_cpu(sb->qkey); | ||
| 934 | |||
| 935 | temp32[0] = le32_to_cpu(sb->dgid[0]); | ||
| 936 | temp32[1] = le32_to_cpu(sb->dgid[1]); | ||
| 937 | temp32[2] = le32_to_cpu(sb->dgid[2]); | ||
| 938 | temp32[3] = le32_to_cpu(sb->dgid[3]); | ||
| 939 | memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); | ||
| 940 | |||
| 941 | qp->ah.flow_label = le32_to_cpu(sb->flow_label); | ||
| 942 | |||
| 943 | qp->ah.sgid_index = 0; | ||
| 944 | for (i = 0; i < res->sgid_tbl.max; i++) { | ||
| 945 | if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { | ||
| 946 | qp->ah.sgid_index = i; | ||
| 947 | break; | ||
| 948 | } | ||
| 949 | } | ||
| 950 | if (i == res->sgid_tbl.max) | ||
| 951 | dev_warn(&res->pdev->dev, "QPLIB: SGID not found??"); | ||
| 952 | |||
| 953 | qp->ah.hop_limit = sb->hop_limit; | ||
| 954 | qp->ah.traffic_class = sb->traffic_class; | ||
| 955 | memcpy(qp->ah.dmac, sb->dest_mac, 6); | ||
| 956 | qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & | ||
| 957 | CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >> | ||
| 958 | CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; | ||
| 959 | qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & | ||
| 960 | CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >> | ||
| 961 | CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT; | ||
| 962 | qp->timeout = sb->timeout; | ||
| 963 | qp->retry_cnt = sb->retry_cnt; | ||
| 964 | qp->rnr_retry = sb->rnr_retry; | ||
| 965 | qp->min_rnr_timer = sb->min_rnr_timer; | ||
| 966 | qp->rq.psn = le32_to_cpu(sb->rq_psn); | ||
| 967 | qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); | ||
| 968 | qp->sq.psn = le32_to_cpu(sb->sq_psn); | ||
| 969 | qp->max_dest_rd_atomic = | ||
| 970 | IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); | ||
| 971 | qp->sq.max_wqe = qp->sq.hwq.max_elements; | ||
| 972 | qp->rq.max_wqe = qp->rq.hwq.max_elements; | ||
| 973 | qp->sq.max_sge = le16_to_cpu(sb->sq_sge); | ||
| 974 | qp->rq.max_sge = le16_to_cpu(sb->rq_sge); | ||
| 975 | qp->max_inline_data = le32_to_cpu(sb->max_inline_data); | ||
| 976 | qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); | ||
| 977 | memcpy(qp->smac, sb->src_mac, 6); | ||
| 978 | qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); | ||
| 979 | return 0; | ||
| 980 | } | ||
| 981 | |||
| 982 | static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) | ||
| 983 | { | ||
| 984 | struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; | ||
| 985 | struct cq_base *hw_cqe, **hw_cqe_ptr; | ||
| 986 | int i; | ||
| 987 | |||
| 988 | for (i = 0; i < cq_hwq->max_elements; i++) { | ||
| 989 | hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr; | ||
| 990 | hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)]; | ||
| 991 | if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements)) | ||
| 992 | continue; | ||
| 993 | switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { | ||
| 994 | case CQ_BASE_CQE_TYPE_REQ: | ||
| 995 | case CQ_BASE_CQE_TYPE_TERMINAL: | ||
| 996 | { | ||
| 997 | struct cq_req *cqe = (struct cq_req *)hw_cqe; | ||
| 998 | |||
| 999 | if (qp == le64_to_cpu(cqe->qp_handle)) | ||
| 1000 | cqe->qp_handle = 0; | ||
| 1001 | break; | ||
| 1002 | } | ||
| 1003 | case CQ_BASE_CQE_TYPE_RES_RC: | ||
| 1004 | case CQ_BASE_CQE_TYPE_RES_UD: | ||
| 1005 | case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: | ||
| 1006 | { | ||
| 1007 | struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; | ||
| 1008 | |||
| 1009 | if (qp == le64_to_cpu(cqe->qp_handle)) | ||
| 1010 | cqe->qp_handle = 0; | ||
| 1011 | break; | ||
| 1012 | } | ||
| 1013 | default: | ||
| 1014 | break; | ||
| 1015 | } | ||
| 1016 | } | ||
| 1017 | } | ||
| 1018 | |||
| 1019 | int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, | ||
| 1020 | struct bnxt_qplib_qp *qp) | ||
| 1021 | { | ||
| 1022 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 1023 | struct cmdq_destroy_qp req; | ||
| 1024 | struct creq_destroy_qp_resp *resp; | ||
| 1025 | unsigned long flags; | ||
| 1026 | u16 cmd_flags = 0; | ||
| 1027 | |||
| 1028 | RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags); | ||
| 1029 | |||
| 1030 | req.qp_cid = cpu_to_le32(qp->id); | ||
| 1031 | resp = (struct creq_destroy_qp_resp *) | ||
| 1032 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 1033 | NULL, 0); | ||
| 1034 | if (!resp) { | ||
| 1035 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP send failed"); | ||
| 1036 | return -EINVAL; | ||
| 1037 | } | ||
| 1038 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 1039 | /* Cmd timed out */ | ||
| 1040 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP timed out"); | ||
| 1041 | return -ETIMEDOUT; | ||
| 1042 | } | ||
| 1043 | if (resp->status || | ||
| 1044 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 1045 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP failed "); | ||
| 1046 | dev_err(&rcfw->pdev->dev, | ||
| 1047 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 1048 | resp->status, le16_to_cpu(req.cookie), | ||
| 1049 | le16_to_cpu(resp->cookie)); | ||
| 1050 | return -EINVAL; | ||
| 1051 | } | ||
| 1052 | |||
| 1053 | /* Must walk the associated CQs to nullified the QP ptr */ | ||
| 1054 | spin_lock_irqsave(&qp->scq->hwq.lock, flags); | ||
| 1055 | |||
| 1056 | __clean_cq(qp->scq, (u64)(unsigned long)qp); | ||
| 1057 | |||
| 1058 | if (qp->rcq && qp->rcq != qp->scq) { | ||
| 1059 | spin_lock(&qp->rcq->hwq.lock); | ||
| 1060 | __clean_cq(qp->rcq, (u64)(unsigned long)qp); | ||
| 1061 | spin_unlock(&qp->rcq->hwq.lock); | ||
| 1062 | } | ||
| 1063 | |||
| 1064 | spin_unlock_irqrestore(&qp->scq->hwq.lock, flags); | ||
| 1065 | |||
| 1066 | bnxt_qplib_free_qp_hdr_buf(res, qp); | ||
| 1067 | bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq); | ||
| 1068 | kfree(qp->sq.swq); | ||
| 1069 | |||
| 1070 | bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq); | ||
| 1071 | kfree(qp->rq.swq); | ||
| 1072 | |||
| 1073 | if (qp->irrq.max_elements) | ||
| 1074 | bnxt_qplib_free_hwq(res->pdev, &qp->irrq); | ||
| 1075 | if (qp->orrq.max_elements) | ||
| 1076 | bnxt_qplib_free_hwq(res->pdev, &qp->orrq); | ||
| 1077 | |||
| 1078 | return 0; | ||
| 1079 | } | ||
| 1080 | |||
| 1081 | void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, | ||
| 1082 | struct bnxt_qplib_sge *sge) | ||
| 1083 | { | ||
| 1084 | struct bnxt_qplib_q *sq = &qp->sq; | ||
| 1085 | u32 sw_prod; | ||
| 1086 | |||
| 1087 | memset(sge, 0, sizeof(*sge)); | ||
| 1088 | |||
| 1089 | if (qp->sq_hdr_buf) { | ||
| 1090 | sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | ||
| 1091 | sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + | ||
| 1092 | sw_prod * qp->sq_hdr_buf_size); | ||
| 1093 | sge->lkey = 0xFFFFFFFF; | ||
| 1094 | sge->size = qp->sq_hdr_buf_size; | ||
| 1095 | return qp->sq_hdr_buf + sw_prod * sge->size; | ||
| 1096 | } | ||
| 1097 | return NULL; | ||
| 1098 | } | ||
| 1099 | |||
| 1100 | u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) | ||
| 1101 | { | ||
| 1102 | struct bnxt_qplib_q *rq = &qp->rq; | ||
| 1103 | |||
| 1104 | return HWQ_CMP(rq->hwq.prod, &rq->hwq); | ||
| 1105 | } | ||
| 1106 | |||
| 1107 | dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) | ||
| 1108 | { | ||
| 1109 | return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); | ||
| 1110 | } | ||
| 1111 | |||
| 1112 | void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, | ||
| 1113 | struct bnxt_qplib_sge *sge) | ||
| 1114 | { | ||
| 1115 | struct bnxt_qplib_q *rq = &qp->rq; | ||
| 1116 | u32 sw_prod; | ||
| 1117 | |||
| 1118 | memset(sge, 0, sizeof(*sge)); | ||
| 1119 | |||
| 1120 | if (qp->rq_hdr_buf) { | ||
| 1121 | sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | ||
| 1122 | sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + | ||
| 1123 | sw_prod * qp->rq_hdr_buf_size); | ||
| 1124 | sge->lkey = 0xFFFFFFFF; | ||
| 1125 | sge->size = qp->rq_hdr_buf_size; | ||
| 1126 | return qp->rq_hdr_buf + sw_prod * sge->size; | ||
| 1127 | } | ||
| 1128 | return NULL; | ||
| 1129 | } | ||
| 1130 | |||
| 1131 | void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) | ||
| 1132 | { | ||
| 1133 | struct bnxt_qplib_q *sq = &qp->sq; | ||
| 1134 | struct dbr_dbr db_msg = { 0 }; | ||
| 1135 | u32 sw_prod; | ||
| 1136 | |||
| 1137 | sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | ||
| 1138 | |||
| 1139 | db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & | ||
| 1140 | DBR_DBR_INDEX_MASK); | ||
| 1141 | db_msg.type_xid = | ||
| 1142 | cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | ||
| 1143 | DBR_DBR_TYPE_SQ); | ||
| 1144 | /* Flush all the WQE writes to HW */ | ||
| 1145 | wmb(); | ||
| 1146 | __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | ||
| 1147 | } | ||
| 1148 | |||
| 1149 | int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, | ||
| 1150 | struct bnxt_qplib_swqe *wqe) | ||
| 1151 | { | ||
| 1152 | struct bnxt_qplib_q *sq = &qp->sq; | ||
| 1153 | struct bnxt_qplib_swq *swq; | ||
| 1154 | struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; | ||
| 1155 | struct sq_sge *hw_sge; | ||
| 1156 | u32 sw_prod; | ||
| 1157 | u8 wqe_size16; | ||
| 1158 | int i, rc = 0, data_len = 0, pkt_num = 0; | ||
| 1159 | __le32 temp32; | ||
| 1160 | |||
| 1161 | if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { | ||
| 1162 | rc = -EINVAL; | ||
| 1163 | goto done; | ||
| 1164 | } | ||
| 1165 | if (HWQ_CMP((sq->hwq.prod + 1), &sq->hwq) == | ||
| 1166 | HWQ_CMP(sq->hwq.cons, &sq->hwq)) { | ||
| 1167 | rc = -ENOMEM; | ||
| 1168 | goto done; | ||
| 1169 | } | ||
| 1170 | sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | ||
| 1171 | swq = &sq->swq[sw_prod]; | ||
| 1172 | swq->wr_id = wqe->wr_id; | ||
| 1173 | swq->type = wqe->type; | ||
| 1174 | swq->flags = wqe->flags; | ||
| 1175 | if (qp->sig_type) | ||
| 1176 | swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; | ||
| 1177 | swq->start_psn = sq->psn & BTH_PSN_MASK; | ||
| 1178 | |||
| 1179 | hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; | ||
| 1180 | hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] | ||
| 1181 | [get_sqe_idx(sw_prod)]; | ||
| 1182 | |||
| 1183 | memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); | ||
| 1184 | |||
| 1185 | if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { | ||
| 1186 | /* Copy the inline data */ | ||
| 1187 | if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { | ||
| 1188 | dev_warn(&sq->hwq.pdev->dev, | ||
| 1189 | "QPLIB: Inline data length > 96 detected"); | ||
| 1190 | data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH; | ||
| 1191 | } else { | ||
| 1192 | data_len = wqe->inline_len; | ||
| 1193 | } | ||
| 1194 | memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); | ||
| 1195 | wqe_size16 = (data_len + 15) >> 4; | ||
| 1196 | } else { | ||
| 1197 | for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data; | ||
| 1198 | i < wqe->num_sge; i++, hw_sge++) { | ||
| 1199 | hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); | ||
| 1200 | hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); | ||
| 1201 | hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); | ||
| 1202 | data_len += wqe->sg_list[i].size; | ||
| 1203 | } | ||
| 1204 | /* Each SGE entry = 1 WQE size16 */ | ||
| 1205 | wqe_size16 = wqe->num_sge; | ||
| 1206 | } | ||
| 1207 | |||
| 1208 | /* Specifics */ | ||
| 1209 | switch (wqe->type) { | ||
| 1210 | case BNXT_QPLIB_SWQE_TYPE_SEND: | ||
| 1211 | if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { | ||
| 1212 | /* Assemble info for Raw Ethertype QPs */ | ||
| 1213 | struct sq_send_raweth_qp1 *sqe = | ||
| 1214 | (struct sq_send_raweth_qp1 *)hw_sq_send_hdr; | ||
| 1215 | |||
| 1216 | sqe->wqe_type = wqe->type; | ||
| 1217 | sqe->flags = wqe->flags; | ||
| 1218 | sqe->wqe_size = wqe_size16 + | ||
| 1219 | ((offsetof(typeof(*sqe), data) + 15) >> 4); | ||
| 1220 | sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); | ||
| 1221 | sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); | ||
| 1222 | sqe->length = cpu_to_le32(data_len); | ||
| 1223 | sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & | ||
| 1224 | SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << | ||
| 1225 | SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); | ||
| 1226 | |||
| 1227 | break; | ||
| 1228 | } | ||
| 1229 | /* else, just fall thru */ | ||
| 1230 | case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: | ||
| 1231 | case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: | ||
| 1232 | { | ||
| 1233 | struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr; | ||
| 1234 | |||
| 1235 | sqe->wqe_type = wqe->type; | ||
| 1236 | sqe->flags = wqe->flags; | ||
| 1237 | sqe->wqe_size = wqe_size16 + | ||
| 1238 | ((offsetof(typeof(*sqe), data) + 15) >> 4); | ||
| 1239 | sqe->inv_key_or_imm_data = cpu_to_le32( | ||
| 1240 | wqe->send.inv_key); | ||
| 1241 | if (qp->type == CMDQ_CREATE_QP_TYPE_UD) { | ||
| 1242 | sqe->q_key = cpu_to_le32(wqe->send.q_key); | ||
| 1243 | sqe->dst_qp = cpu_to_le32( | ||
| 1244 | wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); | ||
| 1245 | sqe->length = cpu_to_le32(data_len); | ||
| 1246 | sqe->avid = cpu_to_le32(wqe->send.avid & | ||
| 1247 | SQ_SEND_AVID_MASK); | ||
| 1248 | sq->psn = (sq->psn + 1) & BTH_PSN_MASK; | ||
| 1249 | } else { | ||
| 1250 | sqe->length = cpu_to_le32(data_len); | ||
| 1251 | sqe->dst_qp = 0; | ||
| 1252 | sqe->avid = 0; | ||
| 1253 | if (qp->mtu) | ||
| 1254 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | ||
| 1255 | if (!pkt_num) | ||
| 1256 | pkt_num = 1; | ||
| 1257 | sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | ||
| 1258 | } | ||
| 1259 | break; | ||
| 1260 | } | ||
| 1261 | case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: | ||
| 1262 | case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: | ||
| 1263 | case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: | ||
| 1264 | { | ||
| 1265 | struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr; | ||
| 1266 | |||
| 1267 | sqe->wqe_type = wqe->type; | ||
| 1268 | sqe->flags = wqe->flags; | ||
| 1269 | sqe->wqe_size = wqe_size16 + | ||
| 1270 | ((offsetof(typeof(*sqe), data) + 15) >> 4); | ||
| 1271 | sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); | ||
| 1272 | sqe->length = cpu_to_le32((u32)data_len); | ||
| 1273 | sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); | ||
| 1274 | sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); | ||
| 1275 | if (qp->mtu) | ||
| 1276 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | ||
| 1277 | if (!pkt_num) | ||
| 1278 | pkt_num = 1; | ||
| 1279 | sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | ||
| 1280 | break; | ||
| 1281 | } | ||
| 1282 | case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: | ||
| 1283 | case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: | ||
| 1284 | { | ||
| 1285 | struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr; | ||
| 1286 | |||
| 1287 | sqe->wqe_type = wqe->type; | ||
| 1288 | sqe->flags = wqe->flags; | ||
| 1289 | sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); | ||
| 1290 | sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); | ||
| 1291 | sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); | ||
| 1292 | sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); | ||
| 1293 | if (qp->mtu) | ||
| 1294 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; | ||
| 1295 | if (!pkt_num) | ||
| 1296 | pkt_num = 1; | ||
| 1297 | sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; | ||
| 1298 | break; | ||
| 1299 | } | ||
| 1300 | case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: | ||
| 1301 | { | ||
| 1302 | struct sq_localinvalidate *sqe = | ||
| 1303 | (struct sq_localinvalidate *)hw_sq_send_hdr; | ||
| 1304 | |||
| 1305 | sqe->wqe_type = wqe->type; | ||
| 1306 | sqe->flags = wqe->flags; | ||
| 1307 | sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); | ||
| 1308 | |||
| 1309 | break; | ||
| 1310 | } | ||
| 1311 | case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: | ||
| 1312 | { | ||
| 1313 | struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr; | ||
| 1314 | |||
| 1315 | sqe->wqe_type = wqe->type; | ||
| 1316 | sqe->flags = wqe->flags; | ||
| 1317 | sqe->access_cntl = wqe->frmr.access_cntl | | ||
| 1318 | SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; | ||
| 1319 | sqe->zero_based_page_size_log = | ||
| 1320 | (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << | ||
| 1321 | SQ_FR_PMR_PAGE_SIZE_LOG_SFT | | ||
| 1322 | (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); | ||
| 1323 | sqe->l_key = cpu_to_le32(wqe->frmr.l_key); | ||
| 1324 | temp32 = cpu_to_le32(wqe->frmr.length); | ||
| 1325 | memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); | ||
| 1326 | sqe->numlevels_pbl_page_size_log = | ||
| 1327 | ((wqe->frmr.pbl_pg_sz_log << | ||
| 1328 | SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & | ||
| 1329 | SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | | ||
| 1330 | ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & | ||
| 1331 | SQ_FR_PMR_NUMLEVELS_MASK); | ||
| 1332 | |||
| 1333 | for (i = 0; i < wqe->frmr.page_list_len; i++) | ||
| 1334 | wqe->frmr.pbl_ptr[i] = cpu_to_le64( | ||
| 1335 | wqe->frmr.page_list[i] | | ||
| 1336 | PTU_PTE_VALID); | ||
| 1337 | sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); | ||
| 1338 | sqe->va = cpu_to_le64(wqe->frmr.va); | ||
| 1339 | |||
| 1340 | break; | ||
| 1341 | } | ||
| 1342 | case BNXT_QPLIB_SWQE_TYPE_BIND_MW: | ||
| 1343 | { | ||
| 1344 | struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr; | ||
| 1345 | |||
| 1346 | sqe->wqe_type = wqe->type; | ||
| 1347 | sqe->flags = wqe->flags; | ||
| 1348 | sqe->access_cntl = wqe->bind.access_cntl; | ||
| 1349 | sqe->mw_type_zero_based = wqe->bind.mw_type | | ||
| 1350 | (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); | ||
| 1351 | sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); | ||
| 1352 | sqe->l_key = cpu_to_le32(wqe->bind.r_key); | ||
| 1353 | sqe->va = cpu_to_le64(wqe->bind.va); | ||
| 1354 | temp32 = cpu_to_le32(wqe->bind.length); | ||
| 1355 | memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); | ||
| 1356 | break; | ||
| 1357 | } | ||
| 1358 | default: | ||
| 1359 | /* Bad wqe, return error */ | ||
| 1360 | rc = -EINVAL; | ||
| 1361 | goto done; | ||
| 1362 | } | ||
| 1363 | swq->next_psn = sq->psn & BTH_PSN_MASK; | ||
| 1364 | if (swq->psn_search) { | ||
| 1365 | swq->psn_search->opcode_start_psn = cpu_to_le32( | ||
| 1366 | ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & | ||
| 1367 | SQ_PSN_SEARCH_START_PSN_MASK) | | ||
| 1368 | ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & | ||
| 1369 | SQ_PSN_SEARCH_OPCODE_MASK)); | ||
| 1370 | swq->psn_search->flags_next_psn = cpu_to_le32( | ||
| 1371 | ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & | ||
| 1372 | SQ_PSN_SEARCH_NEXT_PSN_MASK)); | ||
| 1373 | } | ||
| 1374 | |||
| 1375 | sq->hwq.prod++; | ||
| 1376 | done: | ||
| 1377 | return rc; | ||
| 1378 | } | ||
| 1379 | |||
| 1380 | void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) | ||
| 1381 | { | ||
| 1382 | struct bnxt_qplib_q *rq = &qp->rq; | ||
| 1383 | struct dbr_dbr db_msg = { 0 }; | ||
| 1384 | u32 sw_prod; | ||
| 1385 | |||
| 1386 | sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | ||
| 1387 | db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & | ||
| 1388 | DBR_DBR_INDEX_MASK); | ||
| 1389 | db_msg.type_xid = | ||
| 1390 | cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | ||
| 1391 | DBR_DBR_TYPE_RQ); | ||
| 1392 | |||
| 1393 | /* Flush the writes to HW Rx WQE before the ringing Rx DB */ | ||
| 1394 | wmb(); | ||
| 1395 | __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | ||
| 1396 | } | ||
| 1397 | |||
| 1398 | int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, | ||
| 1399 | struct bnxt_qplib_swqe *wqe) | ||
| 1400 | { | ||
| 1401 | struct bnxt_qplib_q *rq = &qp->rq; | ||
| 1402 | struct rq_wqe *rqe, **rqe_ptr; | ||
| 1403 | struct sq_sge *hw_sge; | ||
| 1404 | u32 sw_prod; | ||
| 1405 | int i, rc = 0; | ||
| 1406 | |||
| 1407 | if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { | ||
| 1408 | dev_err(&rq->hwq.pdev->dev, | ||
| 1409 | "QPLIB: FP: QP (0x%x) is in the 0x%x state", | ||
| 1410 | qp->id, qp->state); | ||
| 1411 | rc = -EINVAL; | ||
| 1412 | goto done; | ||
| 1413 | } | ||
| 1414 | if (HWQ_CMP((rq->hwq.prod + 1), &rq->hwq) == | ||
| 1415 | HWQ_CMP(rq->hwq.cons, &rq->hwq)) { | ||
| 1416 | dev_err(&rq->hwq.pdev->dev, | ||
| 1417 | "QPLIB: FP: QP (0x%x) RQ is full!", qp->id); | ||
| 1418 | rc = -EINVAL; | ||
| 1419 | goto done; | ||
| 1420 | } | ||
| 1421 | sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | ||
| 1422 | rq->swq[sw_prod].wr_id = wqe->wr_id; | ||
| 1423 | |||
| 1424 | rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr; | ||
| 1425 | rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)]; | ||
| 1426 | |||
| 1427 | memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); | ||
| 1428 | |||
| 1429 | /* Calculate wqe_size16 and data_len */ | ||
| 1430 | for (i = 0, hw_sge = (struct sq_sge *)rqe->data; | ||
| 1431 | i < wqe->num_sge; i++, hw_sge++) { | ||
| 1432 | hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); | ||
| 1433 | hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); | ||
| 1434 | hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); | ||
| 1435 | } | ||
| 1436 | rqe->wqe_type = wqe->type; | ||
| 1437 | rqe->flags = wqe->flags; | ||
| 1438 | rqe->wqe_size = wqe->num_sge + | ||
| 1439 | ((offsetof(typeof(*rqe), data) + 15) >> 4); | ||
| 1440 | |||
| 1441 | /* Supply the rqe->wr_id index to the wr_id_tbl for now */ | ||
| 1442 | rqe->wr_id[0] = cpu_to_le32(sw_prod); | ||
| 1443 | |||
| 1444 | rq->hwq.prod++; | ||
| 1445 | done: | ||
| 1446 | return rc; | ||
| 1447 | } | ||
| 1448 | |||
| 1449 | /* CQ */ | ||
| 1450 | |||
| 1451 | /* Spinlock must be held */ | ||
| 1452 | static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq) | ||
| 1453 | { | ||
| 1454 | struct dbr_dbr db_msg = { 0 }; | ||
| 1455 | |||
| 1456 | db_msg.type_xid = | ||
| 1457 | cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | ||
| 1458 | DBR_DBR_TYPE_CQ_ARMENA); | ||
| 1459 | /* Flush memory writes before enabling the CQ */ | ||
| 1460 | wmb(); | ||
| 1461 | __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64)); | ||
| 1462 | } | ||
| 1463 | |||
| 1464 | static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type) | ||
| 1465 | { | ||
| 1466 | struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; | ||
| 1467 | struct dbr_dbr db_msg = { 0 }; | ||
| 1468 | u32 sw_cons; | ||
| 1469 | |||
| 1470 | /* Ring DB */ | ||
| 1471 | sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq); | ||
| 1472 | db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) & | ||
| 1473 | DBR_DBR_INDEX_MASK); | ||
| 1474 | db_msg.type_xid = | ||
| 1475 | cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | | ||
| 1476 | arm_type); | ||
| 1477 | /* flush memory writes before arming the CQ */ | ||
| 1478 | wmb(); | ||
| 1479 | __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); | ||
| 1480 | } | ||
| 1481 | |||
| 1482 | int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) | ||
| 1483 | { | ||
| 1484 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 1485 | struct cmdq_create_cq req; | ||
| 1486 | struct creq_create_cq_resp *resp; | ||
| 1487 | struct bnxt_qplib_pbl *pbl; | ||
| 1488 | u16 cmd_flags = 0; | ||
| 1489 | int rc; | ||
| 1490 | |||
| 1491 | cq->hwq.max_elements = cq->max_wqe; | ||
| 1492 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead, | ||
| 1493 | cq->nmap, &cq->hwq.max_elements, | ||
| 1494 | BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0, | ||
| 1495 | PAGE_SIZE, HWQ_TYPE_QUEUE); | ||
| 1496 | if (rc) | ||
| 1497 | goto exit; | ||
| 1498 | |||
| 1499 | RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags); | ||
| 1500 | |||
| 1501 | if (!cq->dpi) { | ||
| 1502 | dev_err(&rcfw->pdev->dev, | ||
| 1503 | "QPLIB: FP: CREATE_CQ failed due to NULL DPI"); | ||
| 1504 | return -EINVAL; | ||
| 1505 | } | ||
| 1506 | req.dpi = cpu_to_le32(cq->dpi->dpi); | ||
| 1507 | req.cq_handle = cpu_to_le64(cq->cq_handle); | ||
| 1508 | |||
| 1509 | req.cq_size = cpu_to_le32(cq->hwq.max_elements); | ||
| 1510 | pbl = &cq->hwq.pbl[PBL_LVL_0]; | ||
| 1511 | req.pg_size_lvl = cpu_to_le32( | ||
| 1512 | ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << | ||
| 1513 | CMDQ_CREATE_CQ_LVL_SFT) | | ||
| 1514 | (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K : | ||
| 1515 | pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K : | ||
| 1516 | pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K : | ||
| 1517 | pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M : | ||
| 1518 | pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M : | ||
| 1519 | pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G : | ||
| 1520 | CMDQ_CREATE_CQ_PG_SIZE_PG_4K)); | ||
| 1521 | |||
| 1522 | req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); | ||
| 1523 | |||
| 1524 | req.cq_fco_cnq_id = cpu_to_le32( | ||
| 1525 | (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << | ||
| 1526 | CMDQ_CREATE_CQ_CNQ_ID_SFT); | ||
| 1527 | |||
| 1528 | resp = (struct creq_create_cq_resp *) | ||
| 1529 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 1530 | NULL, 0); | ||
| 1531 | if (!resp) { | ||
| 1532 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ send failed"); | ||
| 1533 | return -EINVAL; | ||
| 1534 | } | ||
| 1535 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 1536 | /* Cmd timed out */ | ||
| 1537 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ timed out"); | ||
| 1538 | rc = -ETIMEDOUT; | ||
| 1539 | goto fail; | ||
| 1540 | } | ||
| 1541 | if (resp->status || | ||
| 1542 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 1543 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ failed "); | ||
| 1544 | dev_err(&rcfw->pdev->dev, | ||
| 1545 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 1546 | resp->status, le16_to_cpu(req.cookie), | ||
| 1547 | le16_to_cpu(resp->cookie)); | ||
| 1548 | rc = -EINVAL; | ||
| 1549 | goto fail; | ||
| 1550 | } | ||
| 1551 | cq->id = le32_to_cpu(resp->xid); | ||
| 1552 | cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem; | ||
| 1553 | cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; | ||
| 1554 | init_waitqueue_head(&cq->waitq); | ||
| 1555 | |||
| 1556 | bnxt_qplib_arm_cq_enable(cq); | ||
| 1557 | return 0; | ||
| 1558 | |||
| 1559 | fail: | ||
| 1560 | bnxt_qplib_free_hwq(res->pdev, &cq->hwq); | ||
| 1561 | exit: | ||
| 1562 | return rc; | ||
| 1563 | } | ||
| 1564 | |||
| 1565 | int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) | ||
| 1566 | { | ||
| 1567 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 1568 | struct cmdq_destroy_cq req; | ||
| 1569 | struct creq_destroy_cq_resp *resp; | ||
| 1570 | u16 cmd_flags = 0; | ||
| 1571 | |||
| 1572 | RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags); | ||
| 1573 | |||
| 1574 | req.cq_cid = cpu_to_le32(cq->id); | ||
| 1575 | resp = (struct creq_destroy_cq_resp *) | ||
| 1576 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 1577 | NULL, 0); | ||
| 1578 | if (!resp) { | ||
| 1579 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ send failed"); | ||
| 1580 | return -EINVAL; | ||
| 1581 | } | ||
| 1582 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 1583 | /* Cmd timed out */ | ||
| 1584 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ timed out"); | ||
| 1585 | return -ETIMEDOUT; | ||
| 1586 | } | ||
| 1587 | if (resp->status || | ||
| 1588 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 1589 | dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ failed "); | ||
| 1590 | dev_err(&rcfw->pdev->dev, | ||
| 1591 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 1592 | resp->status, le16_to_cpu(req.cookie), | ||
| 1593 | le16_to_cpu(resp->cookie)); | ||
| 1594 | return -EINVAL; | ||
| 1595 | } | ||
| 1596 | bnxt_qplib_free_hwq(res->pdev, &cq->hwq); | ||
| 1597 | return 0; | ||
| 1598 | } | ||
| 1599 | |||
| 1600 | static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, | ||
| 1601 | struct bnxt_qplib_cqe **pcqe, int *budget) | ||
| 1602 | { | ||
| 1603 | u32 sw_prod, sw_cons; | ||
| 1604 | struct bnxt_qplib_cqe *cqe; | ||
| 1605 | int rc = 0; | ||
| 1606 | |||
| 1607 | /* Now complete all outstanding SQEs with FLUSHED_ERR */ | ||
| 1608 | sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | ||
| 1609 | cqe = *pcqe; | ||
| 1610 | while (*budget) { | ||
| 1611 | sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | ||
| 1612 | if (sw_cons == sw_prod) { | ||
| 1613 | sq->flush_in_progress = false; | ||
| 1614 | break; | ||
| 1615 | } | ||
| 1616 | memset(cqe, 0, sizeof(*cqe)); | ||
| 1617 | cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; | ||
| 1618 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | ||
| 1619 | cqe->qp_handle = (u64)(unsigned long)qp; | ||
| 1620 | cqe->wr_id = sq->swq[sw_cons].wr_id; | ||
| 1621 | cqe->src_qp = qp->id; | ||
| 1622 | cqe->type = sq->swq[sw_cons].type; | ||
| 1623 | cqe++; | ||
| 1624 | (*budget)--; | ||
| 1625 | sq->hwq.cons++; | ||
| 1626 | } | ||
| 1627 | *pcqe = cqe; | ||
| 1628 | if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) | ||
| 1629 | /* Out of budget */ | ||
| 1630 | rc = -EAGAIN; | ||
| 1631 | |||
| 1632 | return rc; | ||
| 1633 | } | ||
| 1634 | |||
| 1635 | static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, | ||
| 1636 | int opcode, struct bnxt_qplib_cqe **pcqe, int *budget) | ||
| 1637 | { | ||
| 1638 | struct bnxt_qplib_cqe *cqe; | ||
| 1639 | u32 sw_prod, sw_cons; | ||
| 1640 | int rc = 0; | ||
| 1641 | |||
| 1642 | /* Flush the rest of the RQ */ | ||
| 1643 | sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | ||
| 1644 | cqe = *pcqe; | ||
| 1645 | while (*budget) { | ||
| 1646 | sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); | ||
| 1647 | if (sw_cons == sw_prod) | ||
| 1648 | break; | ||
| 1649 | memset(cqe, 0, sizeof(*cqe)); | ||
| 1650 | cqe->status = | ||
| 1651 | CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; | ||
| 1652 | cqe->opcode = opcode; | ||
| 1653 | cqe->qp_handle = (unsigned long)qp; | ||
| 1654 | cqe->wr_id = rq->swq[sw_cons].wr_id; | ||
| 1655 | cqe++; | ||
| 1656 | (*budget)--; | ||
| 1657 | rq->hwq.cons++; | ||
| 1658 | } | ||
| 1659 | *pcqe = cqe; | ||
| 1660 | if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) | ||
| 1661 | /* Out of budget */ | ||
| 1662 | rc = -EAGAIN; | ||
| 1663 | |||
| 1664 | return rc; | ||
| 1665 | } | ||
| 1666 | |||
| 1667 | static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, | ||
| 1668 | struct cq_req *hwcqe, | ||
| 1669 | struct bnxt_qplib_cqe **pcqe, int *budget) | ||
| 1670 | { | ||
| 1671 | struct bnxt_qplib_qp *qp; | ||
| 1672 | struct bnxt_qplib_q *sq; | ||
| 1673 | struct bnxt_qplib_cqe *cqe; | ||
| 1674 | u32 sw_cons, cqe_cons; | ||
| 1675 | int rc = 0; | ||
| 1676 | |||
| 1677 | qp = (struct bnxt_qplib_qp *)((unsigned long) | ||
| 1678 | le64_to_cpu(hwcqe->qp_handle)); | ||
| 1679 | if (!qp) { | ||
| 1680 | dev_err(&cq->hwq.pdev->dev, | ||
| 1681 | "QPLIB: FP: Process Req qp is NULL"); | ||
| 1682 | return -EINVAL; | ||
| 1683 | } | ||
| 1684 | sq = &qp->sq; | ||
| 1685 | |||
| 1686 | cqe_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); | ||
| 1687 | if (cqe_cons > sq->hwq.max_elements) { | ||
| 1688 | dev_err(&cq->hwq.pdev->dev, | ||
| 1689 | "QPLIB: FP: CQ Process req reported "); | ||
| 1690 | dev_err(&cq->hwq.pdev->dev, | ||
| 1691 | "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", | ||
| 1692 | cqe_cons, sq->hwq.max_elements); | ||
| 1693 | return -EINVAL; | ||
| 1694 | } | ||
| 1695 | /* If we were in the middle of flushing the SQ, continue */ | ||
| 1696 | if (sq->flush_in_progress) | ||
| 1697 | goto flush; | ||
| 1698 | |||
| 1699 | /* Require to walk the sq's swq to fabricate CQEs for all previously | ||
| 1700 | * signaled SWQEs due to CQE aggregation from the current sq cons | ||
| 1701 | * to the cqe_cons | ||
| 1702 | */ | ||
| 1703 | cqe = *pcqe; | ||
| 1704 | while (*budget) { | ||
| 1705 | sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | ||
| 1706 | if (sw_cons == cqe_cons) | ||
| 1707 | break; | ||
| 1708 | memset(cqe, 0, sizeof(*cqe)); | ||
| 1709 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | ||
| 1710 | cqe->qp_handle = (u64)(unsigned long)qp; | ||
| 1711 | cqe->src_qp = qp->id; | ||
| 1712 | cqe->wr_id = sq->swq[sw_cons].wr_id; | ||
| 1713 | cqe->type = sq->swq[sw_cons].type; | ||
| 1714 | |||
| 1715 | /* For the last CQE, check for status. For errors, regardless | ||
| 1716 | * of the request being signaled or not, it must complete with | ||
| 1717 | * the hwcqe error status | ||
| 1718 | */ | ||
| 1719 | if (HWQ_CMP((sw_cons + 1), &sq->hwq) == cqe_cons && | ||
| 1720 | hwcqe->status != CQ_REQ_STATUS_OK) { | ||
| 1721 | cqe->status = hwcqe->status; | ||
| 1722 | dev_err(&cq->hwq.pdev->dev, | ||
| 1723 | "QPLIB: FP: CQ Processed Req "); | ||
| 1724 | dev_err(&cq->hwq.pdev->dev, | ||
| 1725 | "QPLIB: wr_id[%d] = 0x%llx with status 0x%x", | ||
| 1726 | sw_cons, cqe->wr_id, cqe->status); | ||
| 1727 | cqe++; | ||
| 1728 | (*budget)--; | ||
| 1729 | sq->flush_in_progress = true; | ||
| 1730 | /* Must block new posting of SQ and RQ */ | ||
| 1731 | qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; | ||
| 1732 | } else { | ||
| 1733 | if (sq->swq[sw_cons].flags & | ||
| 1734 | SQ_SEND_FLAGS_SIGNAL_COMP) { | ||
| 1735 | cqe->status = CQ_REQ_STATUS_OK; | ||
| 1736 | cqe++; | ||
| 1737 | (*budget)--; | ||
| 1738 | } | ||
| 1739 | } | ||
| 1740 | sq->hwq.cons++; | ||
| 1741 | } | ||
| 1742 | *pcqe = cqe; | ||
| 1743 | if (!*budget && HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_cons) { | ||
| 1744 | /* Out of budget */ | ||
| 1745 | rc = -EAGAIN; | ||
| 1746 | goto done; | ||
| 1747 | } | ||
| 1748 | if (!sq->flush_in_progress) | ||
| 1749 | goto done; | ||
| 1750 | flush: | ||
| 1751 | /* Require to walk the sq's swq to fabricate CQEs for all | ||
| 1752 | * previously posted SWQEs due to the error CQE received | ||
| 1753 | */ | ||
| 1754 | rc = __flush_sq(sq, qp, pcqe, budget); | ||
| 1755 | if (!rc) | ||
| 1756 | sq->flush_in_progress = false; | ||
| 1757 | done: | ||
| 1758 | return rc; | ||
| 1759 | } | ||
| 1760 | |||
| 1761 | static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, | ||
| 1762 | struct cq_res_rc *hwcqe, | ||
| 1763 | struct bnxt_qplib_cqe **pcqe, | ||
| 1764 | int *budget) | ||
| 1765 | { | ||
| 1766 | struct bnxt_qplib_qp *qp; | ||
| 1767 | struct bnxt_qplib_q *rq; | ||
| 1768 | struct bnxt_qplib_cqe *cqe; | ||
| 1769 | u32 wr_id_idx; | ||
| 1770 | int rc = 0; | ||
| 1771 | |||
| 1772 | qp = (struct bnxt_qplib_qp *)((unsigned long) | ||
| 1773 | le64_to_cpu(hwcqe->qp_handle)); | ||
| 1774 | if (!qp) { | ||
| 1775 | dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL"); | ||
| 1776 | return -EINVAL; | ||
| 1777 | } | ||
| 1778 | cqe = *pcqe; | ||
| 1779 | cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | ||
| 1780 | cqe->length = le32_to_cpu(hwcqe->length); | ||
| 1781 | cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); | ||
| 1782 | cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); | ||
| 1783 | cqe->flags = le16_to_cpu(hwcqe->flags); | ||
| 1784 | cqe->status = hwcqe->status; | ||
| 1785 | cqe->qp_handle = (u64)(unsigned long)qp; | ||
| 1786 | |||
| 1787 | wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & | ||
| 1788 | CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; | ||
| 1789 | rq = &qp->rq; | ||
| 1790 | if (wr_id_idx > rq->hwq.max_elements) { | ||
| 1791 | dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC "); | ||
| 1792 | dev_err(&cq->hwq.pdev->dev, | ||
| 1793 | "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x", | ||
| 1794 | wr_id_idx, rq->hwq.max_elements); | ||
| 1795 | return -EINVAL; | ||
| 1796 | } | ||
| 1797 | if (rq->flush_in_progress) | ||
| 1798 | goto flush_rq; | ||
| 1799 | |||
| 1800 | cqe->wr_id = rq->swq[wr_id_idx].wr_id; | ||
| 1801 | cqe++; | ||
| 1802 | (*budget)--; | ||
| 1803 | rq->hwq.cons++; | ||
| 1804 | *pcqe = cqe; | ||
| 1805 | |||
| 1806 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | ||
| 1807 | rq->flush_in_progress = true; | ||
| 1808 | flush_rq: | ||
| 1809 | rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget); | ||
| 1810 | if (!rc) | ||
| 1811 | rq->flush_in_progress = false; | ||
| 1812 | } | ||
| 1813 | return rc; | ||
| 1814 | } | ||
| 1815 | |||
| 1816 | static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, | ||
| 1817 | struct cq_res_ud *hwcqe, | ||
| 1818 | struct bnxt_qplib_cqe **pcqe, | ||
| 1819 | int *budget) | ||
| 1820 | { | ||
| 1821 | struct bnxt_qplib_qp *qp; | ||
| 1822 | struct bnxt_qplib_q *rq; | ||
| 1823 | struct bnxt_qplib_cqe *cqe; | ||
| 1824 | u32 wr_id_idx; | ||
| 1825 | int rc = 0; | ||
| 1826 | |||
| 1827 | qp = (struct bnxt_qplib_qp *)((unsigned long) | ||
| 1828 | le64_to_cpu(hwcqe->qp_handle)); | ||
| 1829 | if (!qp) { | ||
| 1830 | dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL"); | ||
| 1831 | return -EINVAL; | ||
| 1832 | } | ||
| 1833 | cqe = *pcqe; | ||
| 1834 | cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | ||
| 1835 | cqe->length = le32_to_cpu(hwcqe->length); | ||
| 1836 | cqe->invrkey = le32_to_cpu(hwcqe->imm_data); | ||
| 1837 | cqe->flags = le16_to_cpu(hwcqe->flags); | ||
| 1838 | cqe->status = hwcqe->status; | ||
| 1839 | cqe->qp_handle = (u64)(unsigned long)qp; | ||
| 1840 | memcpy(cqe->smac, hwcqe->src_mac, 6); | ||
| 1841 | wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) | ||
| 1842 | & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK; | ||
| 1843 | cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | | ||
| 1844 | ((le32_to_cpu( | ||
| 1845 | hwcqe->src_qp_high_srq_or_rq_wr_id) & | ||
| 1846 | CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8); | ||
| 1847 | |||
| 1848 | rq = &qp->rq; | ||
| 1849 | if (wr_id_idx > rq->hwq.max_elements) { | ||
| 1850 | dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD "); | ||
| 1851 | dev_err(&cq->hwq.pdev->dev, | ||
| 1852 | "QPLIB: wr_id idx %#x exceeded RQ max %#x", | ||
| 1853 | wr_id_idx, rq->hwq.max_elements); | ||
| 1854 | return -EINVAL; | ||
| 1855 | } | ||
| 1856 | if (rq->flush_in_progress) | ||
| 1857 | goto flush_rq; | ||
| 1858 | |||
| 1859 | cqe->wr_id = rq->swq[wr_id_idx].wr_id; | ||
| 1860 | cqe++; | ||
| 1861 | (*budget)--; | ||
| 1862 | rq->hwq.cons++; | ||
| 1863 | *pcqe = cqe; | ||
| 1864 | |||
| 1865 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | ||
| 1866 | rq->flush_in_progress = true; | ||
| 1867 | flush_rq: | ||
| 1868 | rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget); | ||
| 1869 | if (!rc) | ||
| 1870 | rq->flush_in_progress = false; | ||
| 1871 | } | ||
| 1872 | return rc; | ||
| 1873 | } | ||
| 1874 | |||
| 1875 | static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, | ||
| 1876 | struct cq_res_raweth_qp1 *hwcqe, | ||
| 1877 | struct bnxt_qplib_cqe **pcqe, | ||
| 1878 | int *budget) | ||
| 1879 | { | ||
| 1880 | struct bnxt_qplib_qp *qp; | ||
| 1881 | struct bnxt_qplib_q *rq; | ||
| 1882 | struct bnxt_qplib_cqe *cqe; | ||
| 1883 | u32 wr_id_idx; | ||
| 1884 | int rc = 0; | ||
| 1885 | |||
| 1886 | qp = (struct bnxt_qplib_qp *)((unsigned long) | ||
| 1887 | le64_to_cpu(hwcqe->qp_handle)); | ||
| 1888 | if (!qp) { | ||
| 1889 | dev_err(&cq->hwq.pdev->dev, | ||
| 1890 | "QPLIB: process_cq Raw/QP1 qp is NULL"); | ||
| 1891 | return -EINVAL; | ||
| 1892 | } | ||
| 1893 | cqe = *pcqe; | ||
| 1894 | cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; | ||
| 1895 | cqe->flags = le16_to_cpu(hwcqe->flags); | ||
| 1896 | cqe->qp_handle = (u64)(unsigned long)qp; | ||
| 1897 | |||
| 1898 | wr_id_idx = | ||
| 1899 | le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) | ||
| 1900 | & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; | ||
| 1901 | cqe->src_qp = qp->id; | ||
| 1902 | if (qp->id == 1 && !cqe->length) { | ||
| 1903 | /* Add workaround for the length misdetection */ | ||
| 1904 | cqe->length = 296; | ||
| 1905 | } else { | ||
| 1906 | cqe->length = le16_to_cpu(hwcqe->length); | ||
| 1907 | } | ||
| 1908 | cqe->pkey_index = qp->pkey_index; | ||
| 1909 | memcpy(cqe->smac, qp->smac, 6); | ||
| 1910 | |||
| 1911 | cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); | ||
| 1912 | cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); | ||
| 1913 | |||
| 1914 | rq = &qp->rq; | ||
| 1915 | if (wr_id_idx > rq->hwq.max_elements) { | ||
| 1916 | dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id "); | ||
| 1917 | dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x", | ||
| 1918 | wr_id_idx, rq->hwq.max_elements); | ||
| 1919 | return -EINVAL; | ||
| 1920 | } | ||
| 1921 | if (rq->flush_in_progress) | ||
| 1922 | goto flush_rq; | ||
| 1923 | |||
| 1924 | cqe->wr_id = rq->swq[wr_id_idx].wr_id; | ||
| 1925 | cqe++; | ||
| 1926 | (*budget)--; | ||
| 1927 | rq->hwq.cons++; | ||
| 1928 | *pcqe = cqe; | ||
| 1929 | |||
| 1930 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { | ||
| 1931 | rq->flush_in_progress = true; | ||
| 1932 | flush_rq: | ||
| 1933 | rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe, | ||
| 1934 | budget); | ||
| 1935 | if (!rc) | ||
| 1936 | rq->flush_in_progress = false; | ||
| 1937 | } | ||
| 1938 | return rc; | ||
| 1939 | } | ||
| 1940 | |||
| 1941 | static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, | ||
| 1942 | struct cq_terminal *hwcqe, | ||
| 1943 | struct bnxt_qplib_cqe **pcqe, | ||
| 1944 | int *budget) | ||
| 1945 | { | ||
| 1946 | struct bnxt_qplib_qp *qp; | ||
| 1947 | struct bnxt_qplib_q *sq, *rq; | ||
| 1948 | struct bnxt_qplib_cqe *cqe; | ||
| 1949 | u32 sw_cons = 0, cqe_cons; | ||
| 1950 | int rc = 0; | ||
| 1951 | u8 opcode = 0; | ||
| 1952 | |||
| 1953 | /* Check the Status */ | ||
| 1954 | if (hwcqe->status != CQ_TERMINAL_STATUS_OK) | ||
| 1955 | dev_warn(&cq->hwq.pdev->dev, | ||
| 1956 | "QPLIB: FP: CQ Process Terminal Error status = 0x%x", | ||
| 1957 | hwcqe->status); | ||
| 1958 | |||
| 1959 | qp = (struct bnxt_qplib_qp *)((unsigned long) | ||
| 1960 | le64_to_cpu(hwcqe->qp_handle)); | ||
| 1961 | if (!qp) { | ||
| 1962 | dev_err(&cq->hwq.pdev->dev, | ||
| 1963 | "QPLIB: FP: CQ Process terminal qp is NULL"); | ||
| 1964 | return -EINVAL; | ||
| 1965 | } | ||
| 1966 | /* Must block new posting of SQ and RQ */ | ||
| 1967 | qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; | ||
| 1968 | |||
| 1969 | sq = &qp->sq; | ||
| 1970 | rq = &qp->rq; | ||
| 1971 | |||
| 1972 | cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); | ||
| 1973 | if (cqe_cons == 0xFFFF) | ||
| 1974 | goto do_rq; | ||
| 1975 | |||
| 1976 | if (cqe_cons > sq->hwq.max_elements) { | ||
| 1977 | dev_err(&cq->hwq.pdev->dev, | ||
| 1978 | "QPLIB: FP: CQ Process terminal reported "); | ||
| 1979 | dev_err(&cq->hwq.pdev->dev, | ||
| 1980 | "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", | ||
| 1981 | cqe_cons, sq->hwq.max_elements); | ||
| 1982 | goto do_rq; | ||
| 1983 | } | ||
| 1984 | /* If we were in the middle of flushing, continue */ | ||
| 1985 | if (sq->flush_in_progress) | ||
| 1986 | goto flush_sq; | ||
| 1987 | |||
| 1988 | /* Terminal CQE can also include aggregated successful CQEs prior. | ||
| 1989 | * So we must complete all CQEs from the current sq's cons to the | ||
| 1990 | * cq_cons with status OK | ||
| 1991 | */ | ||
| 1992 | cqe = *pcqe; | ||
| 1993 | while (*budget) { | ||
| 1994 | sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | ||
| 1995 | if (sw_cons == cqe_cons) | ||
| 1996 | break; | ||
| 1997 | if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { | ||
| 1998 | memset(cqe, 0, sizeof(*cqe)); | ||
| 1999 | cqe->status = CQ_REQ_STATUS_OK; | ||
| 2000 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; | ||
| 2001 | cqe->qp_handle = (u64)(unsigned long)qp; | ||
| 2002 | cqe->src_qp = qp->id; | ||
| 2003 | cqe->wr_id = sq->swq[sw_cons].wr_id; | ||
| 2004 | cqe->type = sq->swq[sw_cons].type; | ||
| 2005 | cqe++; | ||
| 2006 | (*budget)--; | ||
| 2007 | } | ||
| 2008 | sq->hwq.cons++; | ||
| 2009 | } | ||
| 2010 | *pcqe = cqe; | ||
| 2011 | if (!(*budget) && sw_cons != cqe_cons) { | ||
| 2012 | /* Out of budget */ | ||
| 2013 | rc = -EAGAIN; | ||
| 2014 | goto sq_done; | ||
| 2015 | } | ||
| 2016 | sq->flush_in_progress = true; | ||
| 2017 | flush_sq: | ||
| 2018 | rc = __flush_sq(sq, qp, pcqe, budget); | ||
| 2019 | if (!rc) | ||
| 2020 | sq->flush_in_progress = false; | ||
| 2021 | sq_done: | ||
| 2022 | if (rc) | ||
| 2023 | return rc; | ||
| 2024 | do_rq: | ||
| 2025 | cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); | ||
| 2026 | if (cqe_cons == 0xFFFF) { | ||
| 2027 | goto done; | ||
| 2028 | } else if (cqe_cons > rq->hwq.max_elements) { | ||
| 2029 | dev_err(&cq->hwq.pdev->dev, | ||
| 2030 | "QPLIB: FP: CQ Processed terminal "); | ||
| 2031 | dev_err(&cq->hwq.pdev->dev, | ||
| 2032 | "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x", | ||
| 2033 | cqe_cons, rq->hwq.max_elements); | ||
| 2034 | goto done; | ||
| 2035 | } | ||
| 2036 | /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR | ||
| 2037 | * from the current rq->cons to the rq->prod regardless what the | ||
| 2038 | * rq->cons the terminal CQE indicates | ||
| 2039 | */ | ||
| 2040 | rq->flush_in_progress = true; | ||
| 2041 | switch (qp->type) { | ||
| 2042 | case CMDQ_CREATE_QP1_TYPE_GSI: | ||
| 2043 | opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; | ||
| 2044 | break; | ||
| 2045 | case CMDQ_CREATE_QP_TYPE_RC: | ||
| 2046 | opcode = CQ_BASE_CQE_TYPE_RES_RC; | ||
| 2047 | break; | ||
| 2048 | case CMDQ_CREATE_QP_TYPE_UD: | ||
| 2049 | opcode = CQ_BASE_CQE_TYPE_RES_UD; | ||
| 2050 | break; | ||
| 2051 | } | ||
| 2052 | |||
| 2053 | rc = __flush_rq(rq, qp, opcode, pcqe, budget); | ||
| 2054 | if (!rc) | ||
| 2055 | rq->flush_in_progress = false; | ||
| 2056 | done: | ||
| 2057 | return rc; | ||
| 2058 | } | ||
| 2059 | |||
| 2060 | static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, | ||
| 2061 | struct cq_cutoff *hwcqe) | ||
| 2062 | { | ||
| 2063 | /* Check the Status */ | ||
| 2064 | if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { | ||
| 2065 | dev_err(&cq->hwq.pdev->dev, | ||
| 2066 | "QPLIB: FP: CQ Process Cutoff Error status = 0x%x", | ||
| 2067 | hwcqe->status); | ||
| 2068 | return -EINVAL; | ||
| 2069 | } | ||
| 2070 | clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); | ||
| 2071 | wake_up_interruptible(&cq->waitq); | ||
| 2072 | |||
| 2073 | return 0; | ||
| 2074 | } | ||
| 2075 | |||
| 2076 | int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, | ||
| 2077 | int num_cqes) | ||
| 2078 | { | ||
| 2079 | struct cq_base *hw_cqe, **hw_cqe_ptr; | ||
| 2080 | unsigned long flags; | ||
| 2081 | u32 sw_cons, raw_cons; | ||
| 2082 | int budget, rc = 0; | ||
| 2083 | |||
| 2084 | spin_lock_irqsave(&cq->hwq.lock, flags); | ||
| 2085 | raw_cons = cq->hwq.cons; | ||
| 2086 | budget = num_cqes; | ||
| 2087 | |||
| 2088 | while (budget) { | ||
| 2089 | sw_cons = HWQ_CMP(raw_cons, &cq->hwq); | ||
| 2090 | hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; | ||
| 2091 | hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)]; | ||
| 2092 | |||
| 2093 | /* Check for Valid bit */ | ||
| 2094 | if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) | ||
| 2095 | break; | ||
| 2096 | |||
| 2097 | /* From the device's respective CQE format to qplib_wc*/ | ||
| 2098 | switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { | ||
| 2099 | case CQ_BASE_CQE_TYPE_REQ: | ||
| 2100 | rc = bnxt_qplib_cq_process_req(cq, | ||
| 2101 | (struct cq_req *)hw_cqe, | ||
| 2102 | &cqe, &budget); | ||
| 2103 | break; | ||
| 2104 | case CQ_BASE_CQE_TYPE_RES_RC: | ||
| 2105 | rc = bnxt_qplib_cq_process_res_rc(cq, | ||
| 2106 | (struct cq_res_rc *) | ||
| 2107 | hw_cqe, &cqe, | ||
| 2108 | &budget); | ||
| 2109 | break; | ||
| 2110 | case CQ_BASE_CQE_TYPE_RES_UD: | ||
| 2111 | rc = bnxt_qplib_cq_process_res_ud | ||
| 2112 | (cq, (struct cq_res_ud *)hw_cqe, &cqe, | ||
| 2113 | &budget); | ||
| 2114 | break; | ||
| 2115 | case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: | ||
| 2116 | rc = bnxt_qplib_cq_process_res_raweth_qp1 | ||
| 2117 | (cq, (struct cq_res_raweth_qp1 *) | ||
| 2118 | hw_cqe, &cqe, &budget); | ||
| 2119 | break; | ||
| 2120 | case CQ_BASE_CQE_TYPE_TERMINAL: | ||
| 2121 | rc = bnxt_qplib_cq_process_terminal | ||
| 2122 | (cq, (struct cq_terminal *)hw_cqe, | ||
| 2123 | &cqe, &budget); | ||
| 2124 | break; | ||
| 2125 | case CQ_BASE_CQE_TYPE_CUT_OFF: | ||
| 2126 | bnxt_qplib_cq_process_cutoff | ||
| 2127 | (cq, (struct cq_cutoff *)hw_cqe); | ||
| 2128 | /* Done processing this CQ */ | ||
| 2129 | goto exit; | ||
| 2130 | default: | ||
| 2131 | dev_err(&cq->hwq.pdev->dev, | ||
| 2132 | "QPLIB: process_cq unknown type 0x%lx", | ||
| 2133 | hw_cqe->cqe_type_toggle & | ||
| 2134 | CQ_BASE_CQE_TYPE_MASK); | ||
| 2135 | rc = -EINVAL; | ||
| 2136 | break; | ||
| 2137 | } | ||
| 2138 | if (rc < 0) { | ||
| 2139 | if (rc == -EAGAIN) | ||
| 2140 | break; | ||
| 2141 | /* Error while processing the CQE, just skip to the | ||
| 2142 | * next one | ||
| 2143 | */ | ||
| 2144 | dev_err(&cq->hwq.pdev->dev, | ||
| 2145 | "QPLIB: process_cqe error rc = 0x%x", rc); | ||
| 2146 | } | ||
| 2147 | raw_cons++; | ||
| 2148 | } | ||
| 2149 | if (cq->hwq.cons != raw_cons) { | ||
| 2150 | cq->hwq.cons = raw_cons; | ||
| 2151 | bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ); | ||
| 2152 | } | ||
| 2153 | exit: | ||
| 2154 | spin_unlock_irqrestore(&cq->hwq.lock, flags); | ||
| 2155 | return num_cqes - budget; | ||
| 2156 | } | ||
| 2157 | |||
| 2158 | void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) | ||
| 2159 | { | ||
| 2160 | unsigned long flags; | ||
| 2161 | |||
| 2162 | spin_lock_irqsave(&cq->hwq.lock, flags); | ||
| 2163 | if (arm_type) | ||
| 2164 | bnxt_qplib_arm_cq(cq, arm_type); | ||
| 2165 | |||
| 2166 | spin_unlock_irqrestore(&cq->hwq.lock, flags); | ||
| 2167 | } | ||
| @@ -0,0 +1,439 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: Fast Path Operators (header) | ||
| 37 | */ | ||
| 38 | |||
| 39 | #ifndef __BNXT_QPLIB_FP_H__ | ||
| 40 | #define __BNXT_QPLIB_FP_H__ | ||
| 41 | |||
| 42 | struct bnxt_qplib_sge { | ||
| 43 | u64 addr; | ||
| 44 | u32 lkey; | ||
| 45 | u32 size; | ||
| 46 | }; | ||
| 47 | |||
| 48 | #define BNXT_QPLIB_MAX_SQE_ENTRY_SIZE sizeof(struct sq_send) | ||
| 49 | |||
| 50 | #define SQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_SQE_ENTRY_SIZE) | ||
| 51 | #define SQE_MAX_IDX_PER_PG (SQE_CNT_PER_PG - 1) | ||
| 52 | |||
| 53 | static inline u32 get_sqe_pg(u32 val) | ||
| 54 | { | ||
| 55 | return ((val & ~SQE_MAX_IDX_PER_PG) / SQE_CNT_PER_PG); | ||
| 56 | } | ||
| 57 | |||
| 58 | static inline u32 get_sqe_idx(u32 val) | ||
| 59 | { | ||
| 60 | return (val & SQE_MAX_IDX_PER_PG); | ||
| 61 | } | ||
| 62 | |||
| 63 | #define BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE sizeof(struct sq_psn_search) | ||
| 64 | |||
| 65 | #define PSNE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE) | ||
| 66 | #define PSNE_MAX_IDX_PER_PG (PSNE_CNT_PER_PG - 1) | ||
| 67 | |||
| 68 | static inline u32 get_psne_pg(u32 val) | ||
| 69 | { | ||
| 70 | return ((val & ~PSNE_MAX_IDX_PER_PG) / PSNE_CNT_PER_PG); | ||
| 71 | } | ||
| 72 | |||
| 73 | static inline u32 get_psne_idx(u32 val) | ||
| 74 | { | ||
| 75 | return (val & PSNE_MAX_IDX_PER_PG); | ||
| 76 | } | ||
| 77 | |||
| 78 | #define BNXT_QPLIB_QP_MAX_SGL 6 | ||
| 79 | |||
| 80 | struct bnxt_qplib_swq { | ||
| 81 | u64 wr_id; | ||
| 82 | u8 type; | ||
| 83 | u8 flags; | ||
| 84 | u32 start_psn; | ||
| 85 | u32 next_psn; | ||
| 86 | struct sq_psn_search *psn_search; | ||
| 87 | }; | ||
| 88 | |||
| 89 | struct bnxt_qplib_swqe { | ||
| 90 | /* General */ | ||
| 91 | u64 wr_id; | ||
| 92 | u8 reqs_type; | ||
| 93 | u8 type; | ||
| 94 | #define BNXT_QPLIB_SWQE_TYPE_SEND 0 | ||
| 95 | #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 | ||
| 96 | #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 | ||
| 97 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 | ||
| 98 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 | ||
| 99 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 | ||
| 100 | #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 | ||
| 101 | #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 | ||
| 102 | #define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 | ||
| 103 | #define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 | ||
| 104 | #define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 | ||
| 105 | #define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 | ||
| 106 | #define BNXT_QPLIB_SWQE_TYPE_RECV 128 | ||
| 107 | #define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 | ||
| 108 | u8 flags; | ||
| 109 | #define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0) | ||
| 110 | #define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1) | ||
| 111 | #define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2) | ||
| 112 | #define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3) | ||
| 113 | #define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4) | ||
| 114 | struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL]; | ||
| 115 | int num_sge; | ||
| 116 | /* Max inline data is 96 bytes */ | ||
| 117 | u32 inline_len; | ||
| 118 | #define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96 | ||
| 119 | u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH]; | ||
| 120 | |||
| 121 | union { | ||
| 122 | /* Send, with imm, inval key */ | ||
| 123 | struct { | ||
| 124 | union { | ||
| 125 | __be32 imm_data; | ||
| 126 | u32 inv_key; | ||
| 127 | }; | ||
| 128 | u32 q_key; | ||
| 129 | u32 dst_qp; | ||
| 130 | u16 avid; | ||
| 131 | } send; | ||
| 132 | |||
| 133 | /* Send Raw Ethernet and QP1 */ | ||
| 134 | struct { | ||
| 135 | u16 lflags; | ||
| 136 | u16 cfa_action; | ||
| 137 | u32 cfa_meta; | ||
| 138 | } rawqp1; | ||
| 139 | |||
| 140 | /* RDMA write, with imm, read */ | ||
| 141 | struct { | ||
| 142 | union { | ||
| 143 | __be32 imm_data; | ||
| 144 | u32 inv_key; | ||
| 145 | }; | ||
| 146 | u64 remote_va; | ||
| 147 | u32 r_key; | ||
| 148 | } rdma; | ||
| 149 | |||
| 150 | /* Atomic cmp/swap, fetch/add */ | ||
| 151 | struct { | ||
| 152 | u64 remote_va; | ||
| 153 | u32 r_key; | ||
| 154 | u64 swap_data; | ||
| 155 | u64 cmp_data; | ||
| 156 | } atomic; | ||
| 157 | |||
| 158 | /* Local Invalidate */ | ||
| 159 | struct { | ||
| 160 | u32 inv_l_key; | ||
| 161 | } local_inv; | ||
| 162 | |||
| 163 | /* FR-PMR */ | ||
| 164 | struct { | ||
| 165 | u8 access_cntl; | ||
| 166 | u8 pg_sz_log; | ||
| 167 | bool zero_based; | ||
| 168 | u32 l_key; | ||
| 169 | u32 length; | ||
| 170 | u8 pbl_pg_sz_log; | ||
| 171 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 | ||
| 172 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 | ||
| 173 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 | ||
| 174 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 | ||
| 175 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 | ||
| 176 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 | ||
| 177 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 | ||
| 178 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 | ||
| 179 | u8 levels; | ||
| 180 | #define PAGE_SHIFT_4K 12 | ||
| 181 | __le64 *pbl_ptr; | ||
| 182 | dma_addr_t pbl_dma_ptr; | ||
| 183 | u64 *page_list; | ||
| 184 | u16 page_list_len; | ||
| 185 | u64 va; | ||
| 186 | } frmr; | ||
| 187 | |||
| 188 | /* Bind */ | ||
| 189 | struct { | ||
| 190 | u8 access_cntl; | ||
| 191 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0) | ||
| 192 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1) | ||
| 193 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2) | ||
| 194 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3) | ||
| 195 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4) | ||
| 196 | bool zero_based; | ||
| 197 | u8 mw_type; | ||
| 198 | u32 parent_l_key; | ||
| 199 | u32 r_key; | ||
| 200 | u64 va; | ||
| 201 | u32 length; | ||
| 202 | } bind; | ||
| 203 | }; | ||
| 204 | }; | ||
| 205 | |||
| 206 | #define BNXT_QPLIB_MAX_RQE_ENTRY_SIZE sizeof(struct rq_wqe) | ||
| 207 | |||
| 208 | #define RQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_RQE_ENTRY_SIZE) | ||
| 209 | #define RQE_MAX_IDX_PER_PG (RQE_CNT_PER_PG - 1) | ||
| 210 | #define RQE_PG(x) (((x) & ~RQE_MAX_IDX_PER_PG) / RQE_CNT_PER_PG) | ||
| 211 | #define RQE_IDX(x) ((x) & RQE_MAX_IDX_PER_PG) | ||
| 212 | |||
| 213 | struct bnxt_qplib_q { | ||
| 214 | struct bnxt_qplib_hwq hwq; | ||
| 215 | struct bnxt_qplib_swq *swq; | ||
| 216 | struct scatterlist *sglist; | ||
| 217 | u32 nmap; | ||
| 218 | u32 max_wqe; | ||
| 219 | u16 max_sge; | ||
| 220 | u32 psn; | ||
| 221 | bool flush_in_progress; | ||
| 222 | }; | ||
| 223 | |||
| 224 | struct bnxt_qplib_qp { | ||
| 225 | struct bnxt_qplib_pd *pd; | ||
| 226 | struct bnxt_qplib_dpi *dpi; | ||
| 227 | u64 qp_handle; | ||
| 228 | u32 id; | ||
| 229 | u8 type; | ||
| 230 | u8 sig_type; | ||
| 231 | u32 modify_flags; | ||
| 232 | u8 state; | ||
| 233 | u8 cur_qp_state; | ||
| 234 | u32 max_inline_data; | ||
| 235 | u32 mtu; | ||
| 236 | u8 path_mtu; | ||
| 237 | bool en_sqd_async_notify; | ||
| 238 | u16 pkey_index; | ||
| 239 | u32 qkey; | ||
| 240 | u32 dest_qp_id; | ||
| 241 | u8 access; | ||
| 242 | u8 timeout; | ||
| 243 | u8 retry_cnt; | ||
| 244 | u8 rnr_retry; | ||
| 245 | u32 min_rnr_timer; | ||
| 246 | u32 max_rd_atomic; | ||
| 247 | u32 max_dest_rd_atomic; | ||
| 248 | u32 dest_qpn; | ||
| 249 | u8 smac[6]; | ||
| 250 | u16 vlan_id; | ||
| 251 | u8 nw_type; | ||
| 252 | struct bnxt_qplib_ah ah; | ||
| 253 | |||
| 254 | #define BTH_PSN_MASK ((1 << 24) - 1) | ||
| 255 | /* SQ */ | ||
| 256 | struct bnxt_qplib_q sq; | ||
| 257 | /* RQ */ | ||
| 258 | struct bnxt_qplib_q rq; | ||
| 259 | /* SRQ */ | ||
| 260 | struct bnxt_qplib_srq *srq; | ||
| 261 | /* CQ */ | ||
| 262 | struct bnxt_qplib_cq *scq; | ||
| 263 | struct bnxt_qplib_cq *rcq; | ||
| 264 | /* IRRQ and ORRQ */ | ||
| 265 | struct bnxt_qplib_hwq irrq; | ||
| 266 | struct bnxt_qplib_hwq orrq; | ||
| 267 | /* Header buffer for QP1 */ | ||
| 268 | int sq_hdr_buf_size; | ||
| 269 | int rq_hdr_buf_size; | ||
| 270 | /* | ||
| 271 | * Buffer space for ETH(14), IP or GRH(40), UDP header(8) | ||
| 272 | * and ib_bth + ib_deth (20). | ||
| 273 | * Max required is 82 when RoCE V2 is enabled | ||
| 274 | */ | ||
| 275 | #define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 | ||
| 276 | /* Ethernet header = 14 */ | ||
| 277 | /* ib_grh = 40 (provided by MAD) */ | ||
| 278 | /* ib_bth + ib_deth = 20 */ | ||
| 279 | /* MAD = 256 (provided by MAD) */ | ||
| 280 | /* iCRC = 4 */ | ||
| 281 | #define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 | ||
| 282 | #define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 | ||
| 283 | #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 | ||
| 284 | #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 | ||
| 285 | #define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 | ||
| 286 | void *sq_hdr_buf; | ||
| 287 | dma_addr_t sq_hdr_buf_map; | ||
| 288 | void *rq_hdr_buf; | ||
| 289 | dma_addr_t rq_hdr_buf_map; | ||
| 290 | }; | ||
| 291 | |||
| 292 | #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) | ||
| 293 | |||
| 294 | #define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE) | ||
| 295 | #define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1) | ||
| 296 | #define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG) | ||
| 297 | #define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG) | ||
| 298 | |||
| 299 | #define ROCE_CQE_CMP_V 0 | ||
| 300 | #define CQE_CMP_VALID(hdr, raw_cons, cp_bit) \ | ||
| 301 | (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ | ||
| 302 | !((raw_cons) & (cp_bit))) | ||
| 303 | |||
| 304 | struct bnxt_qplib_cqe { | ||
| 305 | u8 status; | ||
| 306 | u8 type; | ||
| 307 | u8 opcode; | ||
| 308 | u32 length; | ||
| 309 | u64 wr_id; | ||
| 310 | union { | ||
| 311 | __be32 immdata; | ||
| 312 | u32 invrkey; | ||
| 313 | }; | ||
| 314 | u64 qp_handle; | ||
| 315 | u64 mr_handle; | ||
| 316 | u16 flags; | ||
| 317 | u8 smac[6]; | ||
| 318 | u32 src_qp; | ||
| 319 | u16 raweth_qp1_flags; | ||
| 320 | u16 raweth_qp1_errors; | ||
| 321 | u16 raweth_qp1_cfa_code; | ||
| 322 | u32 raweth_qp1_flags2; | ||
| 323 | u32 raweth_qp1_metadata; | ||
| 324 | u8 raweth_qp1_payload_offset; | ||
| 325 | u16 pkey_index; | ||
| 326 | }; | ||
| 327 | |||
| 328 | #define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 | ||
| 329 | struct bnxt_qplib_cq { | ||
| 330 | struct bnxt_qplib_dpi *dpi; | ||
| 331 | void __iomem *dbr_base; | ||
| 332 | u32 max_wqe; | ||
| 333 | u32 id; | ||
| 334 | u16 count; | ||
| 335 | u16 period; | ||
| 336 | struct bnxt_qplib_hwq hwq; | ||
| 337 | u32 cnq_hw_ring_id; | ||
| 338 | bool resize_in_progress; | ||
| 339 | struct scatterlist *sghead; | ||
| 340 | u32 nmap; | ||
| 341 | u64 cq_handle; | ||
| 342 | |||
| 343 | #define CQ_RESIZE_WAIT_TIME_MS 500 | ||
| 344 | unsigned long flags; | ||
| 345 | #define CQ_FLAGS_RESIZE_IN_PROG 1 | ||
| 346 | wait_queue_head_t waitq; | ||
| 347 | }; | ||
| 348 | |||
| 349 | #define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) | ||
| 350 | #define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) | ||
| 351 | #define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2) | ||
| 352 | #define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1) | ||
| 353 | #define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1) | ||
| 354 | #define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1) | ||
| 355 | |||
| 356 | #define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base) | ||
| 357 | |||
| 358 | #define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE) | ||
| 359 | #define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1) | ||
| 360 | #define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG) | ||
| 361 | #define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG) | ||
| 362 | |||
| 363 | #define NQE_CMP_VALID(hdr, raw_cons, cp_bit) \ | ||
| 364 | (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \ | ||
| 365 | !((raw_cons) & (cp_bit))) | ||
| 366 | |||
| 367 | #define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) | ||
| 368 | |||
| 369 | #define NQ_CONS_PCI_BAR_REGION 2 | ||
| 370 | #define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) | ||
| 371 | #define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID | ||
| 372 | #define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK | ||
| 373 | #define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \ | ||
| 374 | NQ_DB_IDX_VALID) | ||
| 375 | #define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \ | ||
| 376 | NQ_DB_IDX_VALID | \ | ||
| 377 | NQ_DB_IRQ_DIS) | ||
| 378 | #define NQ_DB_REARM(db, raw_cons, cp_bit) \ | ||
| 379 | writel(NQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db) | ||
| 380 | #define NQ_DB(db, raw_cons, cp_bit) \ | ||
| 381 | writel(NQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db) | ||
| 382 | |||
| 383 | struct bnxt_qplib_nq { | ||
| 384 | struct pci_dev *pdev; | ||
| 385 | |||
| 386 | int vector; | ||
| 387 | int budget; | ||
| 388 | bool requested; | ||
| 389 | struct tasklet_struct worker; | ||
| 390 | struct bnxt_qplib_hwq hwq; | ||
| 391 | |||
| 392 | u16 bar_reg; | ||
| 393 | u16 bar_reg_off; | ||
| 394 | u16 ring_id; | ||
| 395 | void __iomem *bar_reg_iomem; | ||
| 396 | |||
| 397 | int (*cqn_handler) | ||
| 398 | (struct bnxt_qplib_nq *nq, | ||
| 399 | struct bnxt_qplib_cq *cq); | ||
| 400 | int (*srqn_handler) | ||
| 401 | (struct bnxt_qplib_nq *nq, | ||
| 402 | void *srq, | ||
| 403 | u8 event); | ||
| 404 | }; | ||
| 405 | |||
| 406 | void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); | ||
| 407 | int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, | ||
| 408 | int msix_vector, int bar_reg_offset, | ||
| 409 | int (*cqn_handler)(struct bnxt_qplib_nq *nq, | ||
| 410 | struct bnxt_qplib_cq *cq), | ||
| 411 | int (*srqn_handler)(struct bnxt_qplib_nq *nq, | ||
| 412 | void *srq, | ||
| 413 | u8 event)); | ||
| 414 | int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | ||
| 415 | int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | ||
| 416 | int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | ||
| 417 | int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | ||
| 418 | int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | ||
| 419 | void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, | ||
| 420 | struct bnxt_qplib_sge *sge); | ||
| 421 | void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, | ||
| 422 | struct bnxt_qplib_sge *sge); | ||
| 423 | u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); | ||
| 424 | dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, | ||
| 425 | u32 index); | ||
| 426 | void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); | ||
| 427 | int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, | ||
| 428 | struct bnxt_qplib_swqe *wqe); | ||
| 429 | void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); | ||
| 430 | int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, | ||
| 431 | struct bnxt_qplib_swqe *wqe); | ||
| 432 | int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); | ||
| 433 | int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); | ||
| 434 | int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, | ||
| 435 | int num); | ||
| 436 | void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); | ||
| 437 | void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq); | ||
| 438 | int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq); | ||
| 439 | #endif /* __BNXT_QPLIB_FP_H__ */ | ||
| @@ -0,0 +1,694 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: RDMA Controller HW interface | ||
| 37 | */ | ||
| 38 | #include <linux/interrupt.h> | ||
| 39 | #include <linux/spinlock.h> | ||
| 40 | #include <linux/pci.h> | ||
| 41 | #include <linux/prefetch.h> | ||
| 42 | #include "roce_hsi.h" | ||
| 43 | #include "qplib_res.h" | ||
| 44 | #include "qplib_rcfw.h" | ||
| 45 | static void bnxt_qplib_service_creq(unsigned long data); | ||
| 46 | |||
| 47 | /* Hardware communication channel */ | ||
| 48 | int bnxt_qplib_rcfw_wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) | ||
| 49 | { | ||
| 50 | u16 cbit; | ||
| 51 | int rc; | ||
| 52 | |||
| 53 | cookie &= RCFW_MAX_COOKIE_VALUE; | ||
| 54 | cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; | ||
| 55 | if (!test_bit(cbit, rcfw->cmdq_bitmap)) | ||
| 56 | dev_warn(&rcfw->pdev->dev, | ||
| 57 | "QPLIB: CMD bit %d for cookie 0x%x is not set?", | ||
| 58 | cbit, cookie); | ||
| 59 | |||
| 60 | rc = wait_event_timeout(rcfw->waitq, | ||
| 61 | !test_bit(cbit, rcfw->cmdq_bitmap), | ||
| 62 | msecs_to_jiffies(RCFW_CMD_WAIT_TIME_MS)); | ||
| 63 | if (!rc) { | ||
| 64 | dev_warn(&rcfw->pdev->dev, | ||
| 65 | "QPLIB: Bono Error: timeout %d msec, msg {0x%x}\n", | ||
| 66 | RCFW_CMD_WAIT_TIME_MS, cookie); | ||
| 67 | } | ||
| 68 | |||
| 69 | return rc; | ||
| 70 | }; | ||
| 71 | |||
| 72 | int bnxt_qplib_rcfw_block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) | ||
| 73 | { | ||
| 74 | u32 count = -1; | ||
| 75 | u16 cbit; | ||
| 76 | |||
| 77 | cookie &= RCFW_MAX_COOKIE_VALUE; | ||
| 78 | cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; | ||
| 79 | if (!test_bit(cbit, rcfw->cmdq_bitmap)) | ||
| 80 | goto done; | ||
| 81 | do { | ||
| 82 | bnxt_qplib_service_creq((unsigned long)rcfw); | ||
| 83 | } while (test_bit(cbit, rcfw->cmdq_bitmap) && --count); | ||
| 84 | done: | ||
| 85 | return count; | ||
| 86 | }; | ||
| 87 | |||
| 88 | void *bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, | ||
| 89 | struct cmdq_base *req, void **crsbe, | ||
| 90 | u8 is_block) | ||
| 91 | { | ||
| 92 | struct bnxt_qplib_crsq *crsq = &rcfw->crsq; | ||
| 93 | struct bnxt_qplib_cmdqe *cmdqe, **cmdq_ptr; | ||
| 94 | struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq; | ||
| 95 | struct bnxt_qplib_hwq *crsb = &rcfw->crsb; | ||
| 96 | struct bnxt_qplib_crsqe *crsqe = NULL; | ||
| 97 | struct bnxt_qplib_crsbe **crsb_ptr; | ||
| 98 | u32 sw_prod, cmdq_prod; | ||
| 99 | u8 retry_cnt = 0xFF; | ||
| 100 | dma_addr_t dma_addr; | ||
| 101 | unsigned long flags; | ||
| 102 | u32 size, opcode; | ||
| 103 | u16 cookie, cbit; | ||
| 104 | int pg, idx; | ||
| 105 | u8 *preq; | ||
| 106 | |||
| 107 | retry: | ||
| 108 | opcode = req->opcode; | ||
| 109 | if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) && | ||
| 110 | (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC && | ||
| 111 | opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW)) { | ||
| 112 | dev_err(&rcfw->pdev->dev, | ||
| 113 | "QPLIB: RCFW not initialized, reject opcode 0x%x", | ||
| 114 | opcode); | ||
| 115 | return NULL; | ||
| 116 | } | ||
| 117 | |||
| 118 | if (test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) && | ||
| 119 | opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) { | ||
| 120 | dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!"); | ||
| 121 | return NULL; | ||
| 122 | } | ||
| 123 | |||
| 124 | /* Cmdq are in 16-byte units, each request can consume 1 or more | ||
| 125 | * cmdqe | ||
| 126 | */ | ||
| 127 | spin_lock_irqsave(&cmdq->lock, flags); | ||
| 128 | if (req->cmd_size > cmdq->max_elements - | ||
| 129 | ((HWQ_CMP(cmdq->prod, cmdq) - HWQ_CMP(cmdq->cons, cmdq)) & | ||
| 130 | (cmdq->max_elements - 1))) { | ||
| 131 | dev_err(&rcfw->pdev->dev, "QPLIB: RCFW: CMDQ is full!"); | ||
| 132 | spin_unlock_irqrestore(&cmdq->lock, flags); | ||
| 133 | |||
| 134 | if (!retry_cnt--) | ||
| 135 | return NULL; | ||
| 136 | goto retry; | ||
| 137 | } | ||
| 138 | |||
| 139 | retry_cnt = 0xFF; | ||
| 140 | |||
| 141 | cookie = atomic_inc_return(&rcfw->seq_num) & RCFW_MAX_COOKIE_VALUE; | ||
| 142 | cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; | ||
| 143 | if (is_block) | ||
| 144 | cookie |= RCFW_CMD_IS_BLOCKING; | ||
| 145 | req->cookie = cpu_to_le16(cookie); | ||
| 146 | if (test_and_set_bit(cbit, rcfw->cmdq_bitmap)) { | ||
| 147 | dev_err(&rcfw->pdev->dev, | ||
| 148 | "QPLIB: RCFW MAX outstanding cmd reached!"); | ||
| 149 | atomic_dec(&rcfw->seq_num); | ||
| 150 | spin_unlock_irqrestore(&cmdq->lock, flags); | ||
| 151 | |||
| 152 | if (!retry_cnt--) | ||
| 153 | return NULL; | ||
| 154 | goto retry; | ||
| 155 | } | ||
| 156 | /* Reserve a resp buffer slot if requested */ | ||
| 157 | if (req->resp_size && crsbe) { | ||
| 158 | spin_lock(&crsb->lock); | ||
| 159 | sw_prod = HWQ_CMP(crsb->prod, crsb); | ||
| 160 | crsb_ptr = (struct bnxt_qplib_crsbe **)crsb->pbl_ptr; | ||
| 161 | *crsbe = (void *)&crsb_ptr[get_crsb_pg(sw_prod)] | ||
| 162 | [get_crsb_idx(sw_prod)]; | ||
| 163 | bnxt_qplib_crsb_dma_next(crsb->pbl_dma_ptr, sw_prod, &dma_addr); | ||
| 164 | req->resp_addr = cpu_to_le64(dma_addr); | ||
| 165 | crsb->prod++; | ||
| 166 | spin_unlock(&crsb->lock); | ||
| 167 | |||
| 168 | req->resp_size = (sizeof(struct bnxt_qplib_crsbe) + | ||
| 169 | BNXT_QPLIB_CMDQE_UNITS - 1) / | ||
| 170 | BNXT_QPLIB_CMDQE_UNITS; | ||
| 171 | } | ||
| 172 | cmdq_ptr = (struct bnxt_qplib_cmdqe **)cmdq->pbl_ptr; | ||
| 173 | preq = (u8 *)req; | ||
| 174 | size = req->cmd_size * BNXT_QPLIB_CMDQE_UNITS; | ||
| 175 | do { | ||
| 176 | pg = 0; | ||
| 177 | idx = 0; | ||
| 178 | |||
| 179 | /* Locate the next cmdq slot */ | ||
| 180 | sw_prod = HWQ_CMP(cmdq->prod, cmdq); | ||
| 181 | cmdqe = &cmdq_ptr[get_cmdq_pg(sw_prod)][get_cmdq_idx(sw_prod)]; | ||
| 182 | if (!cmdqe) { | ||
| 183 | dev_err(&rcfw->pdev->dev, | ||
| 184 | "QPLIB: RCFW request failed with no cmdqe!"); | ||
| 185 | goto done; | ||
| 186 | } | ||
| 187 | /* Copy a segment of the req cmd to the cmdq */ | ||
| 188 | memset(cmdqe, 0, sizeof(*cmdqe)); | ||
| 189 | memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe))); | ||
| 190 | preq += min_t(u32, size, sizeof(*cmdqe)); | ||
| 191 | size -= min_t(u32, size, sizeof(*cmdqe)); | ||
| 192 | cmdq->prod++; | ||
| 193 | } while (size > 0); | ||
| 194 | |||
| 195 | cmdq_prod = cmdq->prod; | ||
| 196 | if (rcfw->flags & FIRMWARE_FIRST_FLAG) { | ||
| 197 | /* The very first doorbell write is required to set this flag | ||
| 198 | * which prompts the FW to reset its internal pointers | ||
| 199 | */ | ||
| 200 | cmdq_prod |= FIRMWARE_FIRST_FLAG; | ||
| 201 | rcfw->flags &= ~FIRMWARE_FIRST_FLAG; | ||
| 202 | } | ||
| 203 | sw_prod = HWQ_CMP(crsq->prod, crsq); | ||
| 204 | crsqe = &crsq->crsq[sw_prod]; | ||
| 205 | memset(crsqe, 0, sizeof(*crsqe)); | ||
| 206 | crsq->prod++; | ||
| 207 | crsqe->req_size = req->cmd_size; | ||
| 208 | |||
| 209 | /* ring CMDQ DB */ | ||
| 210 | writel(cmdq_prod, rcfw->cmdq_bar_reg_iomem + | ||
| 211 | rcfw->cmdq_bar_reg_prod_off); | ||
| 212 | writel(RCFW_CMDQ_TRIG_VAL, rcfw->cmdq_bar_reg_iomem + | ||
| 213 | rcfw->cmdq_bar_reg_trig_off); | ||
| 214 | done: | ||
| 215 | spin_unlock_irqrestore(&cmdq->lock, flags); | ||
| 216 | /* Return the CREQ response pointer */ | ||
| 217 | return crsqe ? &crsqe->qp_event : NULL; | ||
| 218 | } | ||
| 219 | |||
| 220 | /* Completions */ | ||
| 221 | static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw, | ||
| 222 | struct creq_func_event *func_event) | ||
| 223 | { | ||
| 224 | switch (func_event->event) { | ||
| 225 | case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: | ||
| 226 | break; | ||
| 227 | case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: | ||
| 228 | break; | ||
| 229 | case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: | ||
| 230 | break; | ||
| 231 | case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: | ||
| 232 | break; | ||
| 233 | case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: | ||
| 234 | break; | ||
| 235 | case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: | ||
| 236 | break; | ||
| 237 | case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: | ||
| 238 | break; | ||
| 239 | case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: | ||
| 240 | /* SRQ ctx error, call srq_handler?? | ||
| 241 | * But there's no SRQ handle! | ||
| 242 | */ | ||
| 243 | break; | ||
| 244 | case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: | ||
| 245 | break; | ||
| 246 | case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: | ||
| 247 | break; | ||
| 248 | case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: | ||
| 249 | break; | ||
| 250 | case CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST: | ||
| 251 | break; | ||
| 252 | case CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED: | ||
| 253 | break; | ||
| 254 | default: | ||
| 255 | return -EINVAL; | ||
| 256 | } | ||
| 257 | return 0; | ||
| 258 | } | ||
| 259 | |||
| 260 | static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, | ||
| 261 | struct creq_qp_event *qp_event) | ||
| 262 | { | ||
| 263 | struct bnxt_qplib_crsq *crsq = &rcfw->crsq; | ||
| 264 | struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq; | ||
| 265 | struct bnxt_qplib_crsqe *crsqe; | ||
| 266 | u16 cbit, cookie, blocked = 0; | ||
| 267 | unsigned long flags; | ||
| 268 | u32 sw_cons; | ||
| 269 | |||
| 270 | switch (qp_event->event) { | ||
| 271 | case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: | ||
| 272 | dev_dbg(&rcfw->pdev->dev, | ||
| 273 | "QPLIB: Received QP error notification"); | ||
| 274 | break; | ||
| 275 | default: | ||
| 276 | /* Command Response */ | ||
| 277 | spin_lock_irqsave(&cmdq->lock, flags); | ||
| 278 | sw_cons = HWQ_CMP(crsq->cons, crsq); | ||
| 279 | crsqe = &crsq->crsq[sw_cons]; | ||
| 280 | crsq->cons++; | ||
| 281 | memcpy(&crsqe->qp_event, qp_event, sizeof(crsqe->qp_event)); | ||
| 282 | |||
| 283 | cookie = le16_to_cpu(crsqe->qp_event.cookie); | ||
| 284 | blocked = cookie & RCFW_CMD_IS_BLOCKING; | ||
| 285 | cookie &= RCFW_MAX_COOKIE_VALUE; | ||
| 286 | cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; | ||
| 287 | if (!test_and_clear_bit(cbit, rcfw->cmdq_bitmap)) | ||
| 288 | dev_warn(&rcfw->pdev->dev, | ||
| 289 | "QPLIB: CMD bit %d was not requested", cbit); | ||
| 290 | |||
| 291 | cmdq->cons += crsqe->req_size; | ||
| 292 | spin_unlock_irqrestore(&cmdq->lock, flags); | ||
| 293 | if (!blocked) | ||
| 294 | wake_up(&rcfw->waitq); | ||
| 295 | break; | ||
| 296 | } | ||
| 297 | return 0; | ||
| 298 | } | ||
| 299 | |||
| 300 | /* SP - CREQ Completion handlers */ | ||
| 301 | static void bnxt_qplib_service_creq(unsigned long data) | ||
| 302 | { | ||
| 303 | struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data; | ||
| 304 | struct bnxt_qplib_hwq *creq = &rcfw->creq; | ||
| 305 | struct creq_base *creqe, **creq_ptr; | ||
| 306 | u32 sw_cons, raw_cons; | ||
| 307 | unsigned long flags; | ||
| 308 | u32 type; | ||
| 309 | |||
| 310 | /* Service the CREQ until empty */ | ||
| 311 | spin_lock_irqsave(&creq->lock, flags); | ||
| 312 | raw_cons = creq->cons; | ||
| 313 | while (1) { | ||
| 314 | sw_cons = HWQ_CMP(raw_cons, creq); | ||
| 315 | creq_ptr = (struct creq_base **)creq->pbl_ptr; | ||
| 316 | creqe = &creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]; | ||
| 317 | if (!CREQ_CMP_VALID(creqe, raw_cons, creq->max_elements)) | ||
| 318 | break; | ||
| 319 | |||
| 320 | type = creqe->type & CREQ_BASE_TYPE_MASK; | ||
| 321 | switch (type) { | ||
| 322 | case CREQ_BASE_TYPE_QP_EVENT: | ||
| 323 | if (!bnxt_qplib_process_qp_event | ||
| 324 | (rcfw, (struct creq_qp_event *)creqe)) | ||
| 325 | rcfw->creq_qp_event_processed++; | ||
| 326 | else { | ||
| 327 | dev_warn(&rcfw->pdev->dev, "QPLIB: crsqe with"); | ||
| 328 | dev_warn(&rcfw->pdev->dev, | ||
| 329 | "QPLIB: type = 0x%x not handled", | ||
| 330 | type); | ||
| 331 | } | ||
| 332 | break; | ||
| 333 | case CREQ_BASE_TYPE_FUNC_EVENT: | ||
| 334 | if (!bnxt_qplib_process_func_event | ||
| 335 | (rcfw, (struct creq_func_event *)creqe)) | ||
| 336 | rcfw->creq_func_event_processed++; | ||
| 337 | else | ||
| 338 | dev_warn | ||
| 339 | (&rcfw->pdev->dev, "QPLIB:aeqe:%#x Not handled", | ||
| 340 | type); | ||
| 341 | break; | ||
| 342 | default: | ||
| 343 | dev_warn(&rcfw->pdev->dev, "QPLIB: creqe with "); | ||
| 344 | dev_warn(&rcfw->pdev->dev, | ||
| 345 | "QPLIB: op_event = 0x%x not handled", type); | ||
| 346 | break; | ||
| 347 | } | ||
| 348 | raw_cons++; | ||
| 349 | } | ||
| 350 | if (creq->cons != raw_cons) { | ||
| 351 | creq->cons = raw_cons; | ||
| 352 | CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, raw_cons, | ||
| 353 | creq->max_elements); | ||
| 354 | } | ||
| 355 | spin_unlock_irqrestore(&creq->lock, flags); | ||
| 356 | } | ||
| 357 | |||
| 358 | static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance) | ||
| 359 | { | ||
| 360 | struct bnxt_qplib_rcfw *rcfw = dev_instance; | ||
| 361 | struct bnxt_qplib_hwq *creq = &rcfw->creq; | ||
| 362 | struct creq_base **creq_ptr; | ||
| 363 | u32 sw_cons; | ||
| 364 | |||
| 365 | /* Prefetch the CREQ element */ | ||
| 366 | sw_cons = HWQ_CMP(creq->cons, creq); | ||
| 367 | creq_ptr = (struct creq_base **)rcfw->creq.pbl_ptr; | ||
| 368 | prefetch(&creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]); | ||
| 369 | |||
| 370 | tasklet_schedule(&rcfw->worker); | ||
| 371 | |||
| 372 | return IRQ_HANDLED; | ||
| 373 | } | ||
| 374 | |||
| 375 | /* RCFW */ | ||
| 376 | int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw) | ||
| 377 | { | ||
| 378 | struct creq_deinitialize_fw_resp *resp; | ||
| 379 | struct cmdq_deinitialize_fw req; | ||
| 380 | u16 cmd_flags = 0; | ||
| 381 | |||
| 382 | RCFW_CMD_PREP(req, DEINITIALIZE_FW, cmd_flags); | ||
| 383 | resp = (struct creq_deinitialize_fw_resp *) | ||
| 384 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 385 | NULL, 0); | ||
| 386 | if (!resp) | ||
| 387 | return -EINVAL; | ||
| 388 | |||
| 389 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) | ||
| 390 | return -ETIMEDOUT; | ||
| 391 | |||
| 392 | if (resp->status || | ||
| 393 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) | ||
| 394 | return -EFAULT; | ||
| 395 | |||
| 396 | clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags); | ||
| 397 | return 0; | ||
| 398 | } | ||
| 399 | |||
| 400 | static int __get_pbl_pg_idx(struct bnxt_qplib_pbl *pbl) | ||
| 401 | { | ||
| 402 | return (pbl->pg_size == ROCE_PG_SIZE_4K ? | ||
| 403 | CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K : | ||
| 404 | pbl->pg_size == ROCE_PG_SIZE_8K ? | ||
| 405 | CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K : | ||
| 406 | pbl->pg_size == ROCE_PG_SIZE_64K ? | ||
| 407 | CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K : | ||
| 408 | pbl->pg_size == ROCE_PG_SIZE_2M ? | ||
| 409 | CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M : | ||
| 410 | pbl->pg_size == ROCE_PG_SIZE_8M ? | ||
| 411 | CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M : | ||
| 412 | pbl->pg_size == ROCE_PG_SIZE_1G ? | ||
| 413 | CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G : | ||
| 414 | CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K); | ||
| 415 | } | ||
| 416 | |||
| 417 | int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, | ||
| 418 | struct bnxt_qplib_ctx *ctx, int is_virtfn) | ||
| 419 | { | ||
| 420 | struct creq_initialize_fw_resp *resp; | ||
| 421 | struct cmdq_initialize_fw req; | ||
| 422 | u16 cmd_flags = 0, level; | ||
| 423 | |||
| 424 | RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags); | ||
| 425 | |||
| 426 | /* | ||
| 427 | * VFs need not setup the HW context area, PF | ||
| 428 | * shall setup this area for VF. Skipping the | ||
| 429 | * HW programming | ||
| 430 | */ | ||
| 431 | if (is_virtfn) | ||
| 432 | goto skip_ctx_setup; | ||
| 433 | |||
| 434 | level = ctx->qpc_tbl.level; | ||
| 435 | req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) | | ||
| 436 | __get_pbl_pg_idx(&ctx->qpc_tbl.pbl[level]); | ||
| 437 | level = ctx->mrw_tbl.level; | ||
| 438 | req.mrw_pg_size_mrw_lvl = (level << CMDQ_INITIALIZE_FW_MRW_LVL_SFT) | | ||
| 439 | __get_pbl_pg_idx(&ctx->mrw_tbl.pbl[level]); | ||
| 440 | level = ctx->srqc_tbl.level; | ||
| 441 | req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) | | ||
| 442 | __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]); | ||
| 443 | level = ctx->cq_tbl.level; | ||
| 444 | req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) | | ||
| 445 | __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]); | ||
| 446 | level = ctx->srqc_tbl.level; | ||
| 447 | req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) | | ||
| 448 | __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]); | ||
| 449 | level = ctx->cq_tbl.level; | ||
| 450 | req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) | | ||
| 451 | __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]); | ||
| 452 | level = ctx->tim_tbl.level; | ||
| 453 | req.tim_pg_size_tim_lvl = (level << CMDQ_INITIALIZE_FW_TIM_LVL_SFT) | | ||
| 454 | __get_pbl_pg_idx(&ctx->tim_tbl.pbl[level]); | ||
| 455 | level = ctx->tqm_pde_level; | ||
| 456 | req.tqm_pg_size_tqm_lvl = (level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) | | ||
| 457 | __get_pbl_pg_idx(&ctx->tqm_pde.pbl[level]); | ||
| 458 | |||
| 459 | req.qpc_page_dir = | ||
| 460 | cpu_to_le64(ctx->qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | ||
| 461 | req.mrw_page_dir = | ||
| 462 | cpu_to_le64(ctx->mrw_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | ||
| 463 | req.srq_page_dir = | ||
| 464 | cpu_to_le64(ctx->srqc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | ||
| 465 | req.cq_page_dir = | ||
| 466 | cpu_to_le64(ctx->cq_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | ||
| 467 | req.tim_page_dir = | ||
| 468 | cpu_to_le64(ctx->tim_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); | ||
| 469 | req.tqm_page_dir = | ||
| 470 | cpu_to_le64(ctx->tqm_pde.pbl[PBL_LVL_0].pg_map_arr[0]); | ||
| 471 | |||
| 472 | req.number_of_qp = cpu_to_le32(ctx->qpc_tbl.max_elements); | ||
| 473 | req.number_of_mrw = cpu_to_le32(ctx->mrw_tbl.max_elements); | ||
| 474 | req.number_of_srq = cpu_to_le32(ctx->srqc_tbl.max_elements); | ||
| 475 | req.number_of_cq = cpu_to_le32(ctx->cq_tbl.max_elements); | ||
| 476 | |||
| 477 | req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf); | ||
| 478 | req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf); | ||
| 479 | req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf); | ||
| 480 | req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf); | ||
| 481 | req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf); | ||
| 482 | |||
| 483 | skip_ctx_setup: | ||
| 484 | req.stat_ctx_id = cpu_to_le32(ctx->stats.fw_id); | ||
| 485 | resp = (struct creq_initialize_fw_resp *) | ||
| 486 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 487 | NULL, 0); | ||
| 488 | if (!resp) { | ||
| 489 | dev_err(&rcfw->pdev->dev, | ||
| 490 | "QPLIB: RCFW: INITIALIZE_FW send failed"); | ||
| 491 | return -EINVAL; | ||
| 492 | } | ||
| 493 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 494 | /* Cmd timed out */ | ||
| 495 | dev_err(&rcfw->pdev->dev, | ||
| 496 | "QPLIB: RCFW: INITIALIZE_FW timed out"); | ||
| 497 | return -ETIMEDOUT; | ||
| 498 | } | ||
| 499 | if (resp->status || | ||
| 500 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 501 | dev_err(&rcfw->pdev->dev, | ||
| 502 | "QPLIB: RCFW: INITIALIZE_FW failed"); | ||
| 503 | return -EINVAL; | ||
| 504 | } | ||
| 505 | set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags); | ||
| 506 | return 0; | ||
| 507 | } | ||
| 508 | |||
| 509 | void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) | ||
| 510 | { | ||
| 511 | bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->crsb); | ||
| 512 | kfree(rcfw->crsq.crsq); | ||
| 513 | bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->cmdq); | ||
| 514 | bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->creq); | ||
| 515 | |||
| 516 | rcfw->pdev = NULL; | ||
| 517 | } | ||
| 518 | |||
| 519 | int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev, | ||
| 520 | struct bnxt_qplib_rcfw *rcfw) | ||
| 521 | { | ||
| 522 | rcfw->pdev = pdev; | ||
| 523 | rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT; | ||
| 524 | if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL, 0, | ||
| 525 | &rcfw->creq.max_elements, | ||
| 526 | BNXT_QPLIB_CREQE_UNITS, 0, PAGE_SIZE, | ||
| 527 | HWQ_TYPE_L2_CMPL)) { | ||
| 528 | dev_err(&rcfw->pdev->dev, | ||
| 529 | "QPLIB: HW channel CREQ allocation failed"); | ||
| 530 | goto fail; | ||
| 531 | } | ||
| 532 | rcfw->cmdq.max_elements = BNXT_QPLIB_CMDQE_MAX_CNT; | ||
| 533 | if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->cmdq, NULL, 0, | ||
| 534 | &rcfw->cmdq.max_elements, | ||
| 535 | BNXT_QPLIB_CMDQE_UNITS, 0, PAGE_SIZE, | ||
| 536 | HWQ_TYPE_CTX)) { | ||
| 537 | dev_err(&rcfw->pdev->dev, | ||
| 538 | "QPLIB: HW channel CMDQ allocation failed"); | ||
| 539 | goto fail; | ||
| 540 | } | ||
| 541 | |||
| 542 | rcfw->crsq.max_elements = rcfw->cmdq.max_elements; | ||
| 543 | rcfw->crsq.crsq = kcalloc(rcfw->crsq.max_elements, | ||
| 544 | sizeof(*rcfw->crsq.crsq), GFP_KERNEL); | ||
| 545 | if (!rcfw->crsq.crsq) | ||
| 546 | goto fail; | ||
| 547 | |||
| 548 | rcfw->crsb.max_elements = BNXT_QPLIB_CRSBE_MAX_CNT; | ||
| 549 | if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->crsb, NULL, 0, | ||
| 550 | &rcfw->crsb.max_elements, | ||
| 551 | BNXT_QPLIB_CRSBE_UNITS, 0, PAGE_SIZE, | ||
| 552 | HWQ_TYPE_CTX)) { | ||
| 553 | dev_err(&rcfw->pdev->dev, | ||
| 554 | "QPLIB: HW channel CRSB allocation failed"); | ||
| 555 | goto fail; | ||
| 556 | } | ||
| 557 | return 0; | ||
| 558 | |||
| 559 | fail: | ||
| 560 | bnxt_qplib_free_rcfw_channel(rcfw); | ||
| 561 | return -ENOMEM; | ||
| 562 | } | ||
| 563 | |||
| 564 | void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) | ||
| 565 | { | ||
| 566 | unsigned long indx; | ||
| 567 | |||
| 568 | /* Make sure the HW channel is stopped! */ | ||
| 569 | synchronize_irq(rcfw->vector); | ||
| 570 | tasklet_disable(&rcfw->worker); | ||
| 571 | tasklet_kill(&rcfw->worker); | ||
| 572 | |||
| 573 | if (rcfw->requested) { | ||
| 574 | free_irq(rcfw->vector, rcfw); | ||
| 575 | rcfw->requested = false; | ||
| 576 | } | ||
| 577 | if (rcfw->cmdq_bar_reg_iomem) | ||
| 578 | iounmap(rcfw->cmdq_bar_reg_iomem); | ||
| 579 | rcfw->cmdq_bar_reg_iomem = NULL; | ||
| 580 | |||
| 581 | if (rcfw->creq_bar_reg_iomem) | ||
| 582 | iounmap(rcfw->creq_bar_reg_iomem); | ||
| 583 | rcfw->creq_bar_reg_iomem = NULL; | ||
| 584 | |||
| 585 | indx = find_first_bit(rcfw->cmdq_bitmap, rcfw->bmap_size); | ||
| 586 | if (indx != rcfw->bmap_size) | ||
| 587 | dev_err(&rcfw->pdev->dev, | ||
| 588 | "QPLIB: disabling RCFW with pending cmd-bit %lx", indx); | ||
| 589 | kfree(rcfw->cmdq_bitmap); | ||
| 590 | rcfw->bmap_size = 0; | ||
| 591 | |||
| 592 | rcfw->aeq_handler = NULL; | ||
| 593 | rcfw->vector = 0; | ||
| 594 | } | ||
| 595 | |||
| 596 | int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, | ||
| 597 | struct bnxt_qplib_rcfw *rcfw, | ||
| 598 | int msix_vector, | ||
| 599 | int cp_bar_reg_off, int virt_fn, | ||
| 600 | int (*aeq_handler)(struct bnxt_qplib_rcfw *, | ||
| 601 | struct creq_func_event *)) | ||
| 602 | { | ||
| 603 | resource_size_t res_base; | ||
| 604 | struct cmdq_init init; | ||
| 605 | u16 bmap_size; | ||
| 606 | int rc; | ||
| 607 | |||
| 608 | /* General */ | ||
| 609 | atomic_set(&rcfw->seq_num, 0); | ||
| 610 | rcfw->flags = FIRMWARE_FIRST_FLAG; | ||
| 611 | bmap_size = BITS_TO_LONGS(RCFW_MAX_OUTSTANDING_CMD * | ||
| 612 | sizeof(unsigned long)); | ||
| 613 | rcfw->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL); | ||
| 614 | if (!rcfw->cmdq_bitmap) | ||
| 615 | return -ENOMEM; | ||
| 616 | rcfw->bmap_size = bmap_size; | ||
| 617 | |||
| 618 | /* CMDQ */ | ||
| 619 | rcfw->cmdq_bar_reg = RCFW_COMM_PCI_BAR_REGION; | ||
| 620 | res_base = pci_resource_start(pdev, rcfw->cmdq_bar_reg); | ||
| 621 | if (!res_base) | ||
| 622 | return -ENOMEM; | ||
| 623 | |||
| 624 | rcfw->cmdq_bar_reg_iomem = ioremap_nocache(res_base + | ||
| 625 | RCFW_COMM_BASE_OFFSET, | ||
| 626 | RCFW_COMM_SIZE); | ||
| 627 | if (!rcfw->cmdq_bar_reg_iomem) { | ||
| 628 | dev_err(&rcfw->pdev->dev, | ||
| 629 | "QPLIB: CMDQ BAR region %d mapping failed", | ||
| 630 | rcfw->cmdq_bar_reg); | ||
| 631 | return -ENOMEM; | ||
| 632 | } | ||
| 633 | |||
| 634 | rcfw->cmdq_bar_reg_prod_off = virt_fn ? RCFW_VF_COMM_PROD_OFFSET : | ||
| 635 | RCFW_PF_COMM_PROD_OFFSET; | ||
| 636 | |||
| 637 | rcfw->cmdq_bar_reg_trig_off = RCFW_COMM_TRIG_OFFSET; | ||
| 638 | |||
| 639 | /* CRSQ */ | ||
| 640 | rcfw->crsq.prod = 0; | ||
| 641 | rcfw->crsq.cons = 0; | ||
| 642 | |||
| 643 | /* CREQ */ | ||
| 644 | rcfw->creq_bar_reg = RCFW_COMM_CONS_PCI_BAR_REGION; | ||
| 645 | res_base = pci_resource_start(pdev, rcfw->creq_bar_reg); | ||
| 646 | if (!res_base) | ||
| 647 | dev_err(&rcfw->pdev->dev, | ||
| 648 | "QPLIB: CREQ BAR region %d resc start is 0!", | ||
| 649 | rcfw->creq_bar_reg); | ||
| 650 | rcfw->creq_bar_reg_iomem = ioremap_nocache(res_base + cp_bar_reg_off, | ||
| 651 | 4); | ||
| 652 | if (!rcfw->creq_bar_reg_iomem) { | ||
| 653 | dev_err(&rcfw->pdev->dev, | ||
| 654 | "QPLIB: CREQ BAR region %d mapping failed", | ||
| 655 | rcfw->creq_bar_reg); | ||
| 656 | return -ENOMEM; | ||
| 657 | } | ||
| 658 | rcfw->creq_qp_event_processed = 0; | ||
| 659 | rcfw->creq_func_event_processed = 0; | ||
| 660 | |||
| 661 | rcfw->vector = msix_vector; | ||
| 662 | if (aeq_handler) | ||
| 663 | rcfw->aeq_handler = aeq_handler; | ||
| 664 | |||
| 665 | tasklet_init(&rcfw->worker, bnxt_qplib_service_creq, | ||
| 666 | (unsigned long)rcfw); | ||
| 667 | |||
| 668 | rcfw->requested = false; | ||
| 669 | rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0, | ||
| 670 | "bnxt_qplib_creq", rcfw); | ||
| 671 | if (rc) { | ||
| 672 | dev_err(&rcfw->pdev->dev, | ||
| 673 | "QPLIB: Failed to request IRQ for CREQ rc = 0x%x", rc); | ||
| 674 | bnxt_qplib_disable_rcfw_channel(rcfw); | ||
| 675 | return rc; | ||
| 676 | } | ||
| 677 | rcfw->requested = true; | ||
| 678 | |||
| 679 | init_waitqueue_head(&rcfw->waitq); | ||
| 680 | |||
| 681 | CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, 0, rcfw->creq.max_elements); | ||
| 682 | |||
| 683 | init.cmdq_pbl = cpu_to_le64(rcfw->cmdq.pbl[PBL_LVL_0].pg_map_arr[0]); | ||
| 684 | init.cmdq_size_cmdq_lvl = cpu_to_le16( | ||
| 685 | ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) & | ||
| 686 | CMDQ_INIT_CMDQ_SIZE_MASK) | | ||
| 687 | ((rcfw->cmdq.level << CMDQ_INIT_CMDQ_LVL_SFT) & | ||
| 688 | CMDQ_INIT_CMDQ_LVL_MASK)); | ||
| 689 | init.creq_ring_id = cpu_to_le16(rcfw->creq_ring_id); | ||
| 690 | |||
| 691 | /* Write to the Bono mailbox register */ | ||
| 692 | __iowrite32_copy(rcfw->cmdq_bar_reg_iomem, &init, sizeof(init) / 4); | ||
| 693 | return 0; | ||
| 694 | } | ||
| @@ -0,0 +1,231 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: RDMA Controller HW interface (header) | ||
| 37 | */ | ||
| 38 | |||
| 39 | #ifndef __BNXT_QPLIB_RCFW_H__ | ||
| 40 | #define __BNXT_QPLIB_RCFW_H__ | ||
| 41 | |||
| 42 | #define RCFW_CMDQ_TRIG_VAL 1 | ||
| 43 | #define RCFW_COMM_PCI_BAR_REGION 0 | ||
| 44 | #define RCFW_COMM_CONS_PCI_BAR_REGION 2 | ||
| 45 | #define RCFW_COMM_BASE_OFFSET 0x600 | ||
| 46 | #define RCFW_PF_COMM_PROD_OFFSET 0xc | ||
| 47 | #define RCFW_VF_COMM_PROD_OFFSET 0xc | ||
| 48 | #define RCFW_COMM_TRIG_OFFSET 0x100 | ||
| 49 | #define RCFW_COMM_SIZE 0x104 | ||
| 50 | |||
| 51 | #define RCFW_DBR_PCI_BAR_REGION 2 | ||
| 52 | |||
| 53 | #define RCFW_CMD_PREP(req, CMD, cmd_flags) \ | ||
| 54 | do { \ | ||
| 55 | memset(&(req), 0, sizeof((req))); \ | ||
| 56 | (req).opcode = CMDQ_BASE_OPCODE_##CMD; \ | ||
| 57 | (req).cmd_size = (sizeof((req)) + \ | ||
| 58 | BNXT_QPLIB_CMDQE_UNITS - 1) / \ | ||
| 59 | BNXT_QPLIB_CMDQE_UNITS; \ | ||
| 60 | (req).flags = cpu_to_le16(cmd_flags); \ | ||
| 61 | } while (0) | ||
| 62 | |||
| 63 | #define RCFW_CMD_WAIT_TIME_MS 20000 /* 20 Seconds timeout */ | ||
| 64 | |||
| 65 | /* CMDQ elements */ | ||
| 66 | #define BNXT_QPLIB_CMDQE_MAX_CNT 256 | ||
| 67 | #define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe) | ||
| 68 | #define BNXT_QPLIB_CMDQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CMDQE_UNITS) | ||
| 69 | |||
| 70 | #define MAX_CMDQ_IDX (BNXT_QPLIB_CMDQE_MAX_CNT - 1) | ||
| 71 | #define MAX_CMDQ_IDX_PER_PG (BNXT_QPLIB_CMDQE_CNT_PER_PG - 1) | ||
| 72 | |||
| 73 | #define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT | ||
| 74 | #define RCFW_MAX_COOKIE_VALUE 0x7FFF | ||
| 75 | #define RCFW_CMD_IS_BLOCKING 0x8000 | ||
| 76 | |||
| 77 | /* Cmdq contains a fix number of a 16-Byte slots */ | ||
| 78 | struct bnxt_qplib_cmdqe { | ||
| 79 | u8 data[16]; | ||
| 80 | }; | ||
| 81 | |||
| 82 | static inline u32 get_cmdq_pg(u32 val) | ||
| 83 | { | ||
| 84 | return (val & ~MAX_CMDQ_IDX_PER_PG) / BNXT_QPLIB_CMDQE_CNT_PER_PG; | ||
| 85 | } | ||
| 86 | |||
| 87 | static inline u32 get_cmdq_idx(u32 val) | ||
| 88 | { | ||
| 89 | return val & MAX_CMDQ_IDX_PER_PG; | ||
| 90 | } | ||
| 91 | |||
| 92 | /* Crsq buf is 1024-Byte */ | ||
| 93 | struct bnxt_qplib_crsbe { | ||
| 94 | u8 data[1024]; | ||
| 95 | }; | ||
| 96 | |||
| 97 | /* CRSQ SB */ | ||
| 98 | #define BNXT_QPLIB_CRSBE_MAX_CNT 4 | ||
| 99 | #define BNXT_QPLIB_CRSBE_UNITS sizeof(struct bnxt_qplib_crsbe) | ||
| 100 | #define BNXT_QPLIB_CRSBE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CRSBE_UNITS) | ||
| 101 | |||
| 102 | #define MAX_CRSB_IDX (BNXT_QPLIB_CRSBE_MAX_CNT - 1) | ||
| 103 | #define MAX_CRSB_IDX_PER_PG (BNXT_QPLIB_CRSBE_CNT_PER_PG - 1) | ||
| 104 | |||
| 105 | static inline u32 get_crsb_pg(u32 val) | ||
| 106 | { | ||
| 107 | return (val & ~MAX_CRSB_IDX_PER_PG) / BNXT_QPLIB_CRSBE_CNT_PER_PG; | ||
| 108 | } | ||
| 109 | |||
| 110 | static inline u32 get_crsb_idx(u32 val) | ||
| 111 | { | ||
| 112 | return val & MAX_CRSB_IDX_PER_PG; | ||
| 113 | } | ||
| 114 | |||
| 115 | static inline void bnxt_qplib_crsb_dma_next(dma_addr_t *pg_map_arr, | ||
| 116 | u32 prod, dma_addr_t *dma_addr) | ||
| 117 | { | ||
| 118 | *dma_addr = pg_map_arr[(prod) / BNXT_QPLIB_CRSBE_CNT_PER_PG]; | ||
| 119 | *dma_addr += ((prod) % BNXT_QPLIB_CRSBE_CNT_PER_PG) * | ||
| 120 | BNXT_QPLIB_CRSBE_UNITS; | ||
| 121 | } | ||
| 122 | |||
| 123 | /* CREQ */ | ||
| 124 | /* Allocate 1 per QP for async error notification for now */ | ||
| 125 | #define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024) | ||
| 126 | #define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */ | ||
| 127 | #define BNXT_QPLIB_CREQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CREQE_UNITS) | ||
| 128 | |||
| 129 | #define MAX_CREQ_IDX (BNXT_QPLIB_CREQE_MAX_CNT - 1) | ||
| 130 | #define MAX_CREQ_IDX_PER_PG (BNXT_QPLIB_CREQE_CNT_PER_PG - 1) | ||
| 131 | |||
| 132 | static inline u32 get_creq_pg(u32 val) | ||
| 133 | { | ||
| 134 | return (val & ~MAX_CREQ_IDX_PER_PG) / BNXT_QPLIB_CREQE_CNT_PER_PG; | ||
| 135 | } | ||
| 136 | |||
| 137 | static inline u32 get_creq_idx(u32 val) | ||
| 138 | { | ||
| 139 | return val & MAX_CREQ_IDX_PER_PG; | ||
| 140 | } | ||
| 141 | |||
| 142 | #define BNXT_QPLIB_CREQE_PER_PG (PAGE_SIZE / sizeof(struct creq_base)) | ||
| 143 | |||
| 144 | #define CREQ_CMP_VALID(hdr, raw_cons, cp_bit) \ | ||
| 145 | (!!((hdr)->v & CREQ_BASE_V) == \ | ||
| 146 | !((raw_cons) & (cp_bit))) | ||
| 147 | |||
| 148 | #define CREQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) | ||
| 149 | #define CREQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID | ||
| 150 | #define CREQ_DB_IRQ_DIS CMPL_DOORBELL_MASK | ||
| 151 | #define CREQ_DB_CP_FLAGS_REARM (CREQ_DB_KEY_CP | \ | ||
| 152 | CREQ_DB_IDX_VALID) | ||
| 153 | #define CREQ_DB_CP_FLAGS (CREQ_DB_KEY_CP | \ | ||
| 154 | CREQ_DB_IDX_VALID | \ | ||
| 155 | CREQ_DB_IRQ_DIS) | ||
| 156 | #define CREQ_DB_REARM(db, raw_cons, cp_bit) \ | ||
| 157 | writel(CREQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db) | ||
| 158 | #define CREQ_DB(db, raw_cons, cp_bit) \ | ||
| 159 | writel(CREQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db) | ||
| 160 | |||
| 161 | /* HWQ */ | ||
| 162 | struct bnxt_qplib_crsqe { | ||
| 163 | struct creq_qp_event qp_event; | ||
| 164 | u32 req_size; | ||
| 165 | }; | ||
| 166 | |||
| 167 | struct bnxt_qplib_crsq { | ||
| 168 | struct bnxt_qplib_crsqe *crsq; | ||
| 169 | u32 prod; | ||
| 170 | u32 cons; | ||
| 171 | u32 max_elements; | ||
| 172 | }; | ||
| 173 | |||
| 174 | /* RCFW Communication Channels */ | ||
| 175 | struct bnxt_qplib_rcfw { | ||
| 176 | struct pci_dev *pdev; | ||
| 177 | int vector; | ||
| 178 | struct tasklet_struct worker; | ||
| 179 | bool requested; | ||
| 180 | unsigned long *cmdq_bitmap; | ||
| 181 | u32 bmap_size; | ||
| 182 | unsigned long flags; | ||
| 183 | #define FIRMWARE_INITIALIZED_FLAG 1 | ||
| 184 | #define FIRMWARE_FIRST_FLAG BIT(31) | ||
| 185 | wait_queue_head_t waitq; | ||
| 186 | int (*aeq_handler)(struct bnxt_qplib_rcfw *, | ||
| 187 | struct creq_func_event *); | ||
| 188 | atomic_t seq_num; | ||
| 189 | |||
| 190 | /* Bar region info */ | ||
| 191 | void __iomem *cmdq_bar_reg_iomem; | ||
| 192 | u16 cmdq_bar_reg; | ||
| 193 | u16 cmdq_bar_reg_prod_off; | ||
| 194 | u16 cmdq_bar_reg_trig_off; | ||
| 195 | u16 creq_ring_id; | ||
| 196 | u16 creq_bar_reg; | ||
| 197 | void __iomem *creq_bar_reg_iomem; | ||
| 198 | |||
| 199 | /* Cmd-Resp and Async Event notification queue */ | ||
| 200 | struct bnxt_qplib_hwq creq; | ||
| 201 | u64 creq_qp_event_processed; | ||
| 202 | u64 creq_func_event_processed; | ||
| 203 | |||
| 204 | /* Actual Cmd and Resp Queues */ | ||
| 205 | struct bnxt_qplib_hwq cmdq; | ||
| 206 | struct bnxt_qplib_crsq crsq; | ||
| 207 | struct bnxt_qplib_hwq crsb; | ||
| 208 | }; | ||
| 209 | |||
| 210 | void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); | ||
| 211 | int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev, | ||
| 212 | struct bnxt_qplib_rcfw *rcfw); | ||
| 213 | void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); | ||
| 214 | int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, | ||
| 215 | struct bnxt_qplib_rcfw *rcfw, | ||
| 216 | int msix_vector, | ||
| 217 | int cp_bar_reg_off, int virt_fn, | ||
| 218 | int (*aeq_handler) | ||
| 219 | (struct bnxt_qplib_rcfw *, | ||
| 220 | struct creq_func_event *)); | ||
| 221 | |||
| 222 | int bnxt_qplib_rcfw_block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie); | ||
| 223 | int bnxt_qplib_rcfw_wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie); | ||
| 224 | void *bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, | ||
| 225 | struct cmdq_base *req, void **crsbe, | ||
| 226 | u8 is_block); | ||
| 227 | |||
| 228 | int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw); | ||
| 229 | int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, | ||
| 230 | struct bnxt_qplib_ctx *ctx, int is_virtfn); | ||
| 231 | #endif /* __BNXT_QPLIB_RCFW_H__ */ | ||
| @@ -0,0 +1,825 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: QPLib resource manager | ||
| 37 | */ | ||
| 38 | |||
| 39 | #include <linux/spinlock.h> | ||
| 40 | #include <linux/pci.h> | ||
| 41 | #include <linux/interrupt.h> | ||
| 42 | #include <linux/inetdevice.h> | ||
| 43 | #include <linux/dma-mapping.h> | ||
| 44 | #include <linux/if_vlan.h> | ||
| 45 | #include "roce_hsi.h" | ||
| 46 | #include "qplib_res.h" | ||
| 47 | #include "qplib_sp.h" | ||
| 48 | #include "qplib_rcfw.h" | ||
| 49 | |||
| 50 | static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev, | ||
| 51 | struct bnxt_qplib_stats *stats); | ||
| 52 | static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev, | ||
| 53 | struct bnxt_qplib_stats *stats); | ||
| 54 | |||
| 55 | /* PBL */ | ||
| 56 | static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl, | ||
| 57 | bool is_umem) | ||
| 58 | { | ||
| 59 | int i; | ||
| 60 | |||
| 61 | if (!is_umem) { | ||
| 62 | for (i = 0; i < pbl->pg_count; i++) { | ||
| 63 | if (pbl->pg_arr[i]) | ||
| 64 | dma_free_coherent(&pdev->dev, pbl->pg_size, | ||
| 65 | (void *)((unsigned long) | ||
| 66 | pbl->pg_arr[i] & | ||
| 67 | PAGE_MASK), | ||
| 68 | pbl->pg_map_arr[i]); | ||
| 69 | else | ||
| 70 | dev_warn(&pdev->dev, | ||
| 71 | "QPLIB: PBL free pg_arr[%d] empty?!", | ||
| 72 | i); | ||
| 73 | pbl->pg_arr[i] = NULL; | ||
| 74 | } | ||
| 75 | } | ||
| 76 | kfree(pbl->pg_arr); | ||
| 77 | pbl->pg_arr = NULL; | ||
| 78 | kfree(pbl->pg_map_arr); | ||
| 79 | pbl->pg_map_arr = NULL; | ||
| 80 | pbl->pg_count = 0; | ||
| 81 | pbl->pg_size = 0; | ||
| 82 | } | ||
| 83 | |||
| 84 | static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl, | ||
| 85 | struct scatterlist *sghead, u32 pages, u32 pg_size) | ||
| 86 | { | ||
| 87 | struct scatterlist *sg; | ||
| 88 | bool is_umem = false; | ||
| 89 | int i; | ||
| 90 | |||
| 91 | /* page ptr arrays */ | ||
| 92 | pbl->pg_arr = kcalloc(pages, sizeof(void *), GFP_KERNEL); | ||
| 93 | if (!pbl->pg_arr) | ||
| 94 | return -ENOMEM; | ||
| 95 | |||
| 96 | pbl->pg_map_arr = kcalloc(pages, sizeof(dma_addr_t), GFP_KERNEL); | ||
| 97 | if (!pbl->pg_map_arr) { | ||
| 98 | kfree(pbl->pg_arr); | ||
| 99 | pbl->pg_arr = NULL; | ||
| 100 | return -ENOMEM; | ||
| 101 | } | ||
| 102 | pbl->pg_count = 0; | ||
| 103 | pbl->pg_size = pg_size; | ||
| 104 | |||
| 105 | if (!sghead) { | ||
| 106 | for (i = 0; i < pages; i++) { | ||
| 107 | pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev, | ||
| 108 | pbl->pg_size, | ||
| 109 | &pbl->pg_map_arr[i], | ||
| 110 | GFP_KERNEL); | ||
| 111 | if (!pbl->pg_arr[i]) | ||
| 112 | goto fail; | ||
| 113 | memset(pbl->pg_arr[i], 0, pbl->pg_size); | ||
| 114 | pbl->pg_count++; | ||
| 115 | } | ||
| 116 | } else { | ||
| 117 | i = 0; | ||
| 118 | is_umem = true; | ||
| 119 | for_each_sg(sghead, sg, pages, i) { | ||
| 120 | pbl->pg_map_arr[i] = sg_dma_address(sg); | ||
| 121 | pbl->pg_arr[i] = sg_virt(sg); | ||
| 122 | if (!pbl->pg_arr[i]) | ||
| 123 | goto fail; | ||
| 124 | |||
| 125 | pbl->pg_count++; | ||
| 126 | } | ||
| 127 | } | ||
| 128 | |||
| 129 | return 0; | ||
| 130 | |||
| 131 | fail: | ||
| 132 | __free_pbl(pdev, pbl, is_umem); | ||
| 133 | return -ENOMEM; | ||
| 134 | } | ||
| 135 | |||
| 136 | /* HWQ */ | ||
| 137 | void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq) | ||
| 138 | { | ||
| 139 | int i; | ||
| 140 | |||
| 141 | if (!hwq->max_elements) | ||
| 142 | return; | ||
| 143 | if (hwq->level >= PBL_LVL_MAX) | ||
| 144 | return; | ||
| 145 | |||
| 146 | for (i = 0; i < hwq->level + 1; i++) { | ||
| 147 | if (i == hwq->level) | ||
| 148 | __free_pbl(pdev, &hwq->pbl[i], hwq->is_user); | ||
| 149 | else | ||
| 150 | __free_pbl(pdev, &hwq->pbl[i], false); | ||
| 151 | } | ||
| 152 | |||
| 153 | hwq->level = PBL_LVL_MAX; | ||
| 154 | hwq->max_elements = 0; | ||
| 155 | hwq->element_size = 0; | ||
| 156 | hwq->prod = 0; | ||
| 157 | hwq->cons = 0; | ||
| 158 | hwq->cp_bit = 0; | ||
| 159 | } | ||
| 160 | |||
| 161 | /* All HWQs are power of 2 in size */ | ||
| 162 | int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq, | ||
| 163 | struct scatterlist *sghead, int nmap, | ||
| 164 | u32 *elements, u32 element_size, u32 aux, | ||
| 165 | u32 pg_size, enum bnxt_qplib_hwq_type hwq_type) | ||
| 166 | { | ||
| 167 | u32 pages, slots, size, aux_pages = 0, aux_size = 0; | ||
| 168 | dma_addr_t *src_phys_ptr, **dst_virt_ptr; | ||
| 169 | int i, rc; | ||
| 170 | |||
| 171 | hwq->level = PBL_LVL_MAX; | ||
| 172 | |||
| 173 | slots = roundup_pow_of_two(*elements); | ||
| 174 | if (aux) { | ||
| 175 | aux_size = roundup_pow_of_two(aux); | ||
| 176 | aux_pages = (slots * aux_size) / pg_size; | ||
| 177 | if ((slots * aux_size) % pg_size) | ||
| 178 | aux_pages++; | ||
| 179 | } | ||
| 180 | size = roundup_pow_of_two(element_size); | ||
| 181 | |||
| 182 | if (!sghead) { | ||
| 183 | hwq->is_user = false; | ||
| 184 | pages = (slots * size) / pg_size + aux_pages; | ||
| 185 | if ((slots * size) % pg_size) | ||
| 186 | pages++; | ||
| 187 | if (!pages) | ||
| 188 | return -EINVAL; | ||
| 189 | } else { | ||
| 190 | hwq->is_user = true; | ||
| 191 | pages = nmap; | ||
| 192 | } | ||
| 193 | |||
| 194 | /* Alloc the 1st memory block; can be a PDL/PTL/PBL */ | ||
| 195 | if (sghead && (pages == MAX_PBL_LVL_0_PGS)) | ||
| 196 | rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], sghead, | ||
| 197 | pages, pg_size); | ||
| 198 | else | ||
| 199 | rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], NULL, 1, pg_size); | ||
| 200 | if (rc) | ||
| 201 | goto fail; | ||
| 202 | |||
| 203 | hwq->level = PBL_LVL_0; | ||
| 204 | |||
| 205 | if (pages > MAX_PBL_LVL_0_PGS) { | ||
| 206 | if (pages > MAX_PBL_LVL_1_PGS) { | ||
| 207 | /* 2 levels of indirection */ | ||
| 208 | rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], NULL, | ||
| 209 | MAX_PBL_LVL_1_PGS_FOR_LVL_2, pg_size); | ||
| 210 | if (rc) | ||
| 211 | goto fail; | ||
| 212 | /* Fill in lvl0 PBL */ | ||
| 213 | dst_virt_ptr = | ||
| 214 | (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; | ||
| 215 | src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; | ||
| 216 | for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) | ||
| 217 | dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = | ||
| 218 | src_phys_ptr[i] | PTU_PDE_VALID; | ||
| 219 | hwq->level = PBL_LVL_1; | ||
| 220 | |||
| 221 | rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_2], sghead, | ||
| 222 | pages, pg_size); | ||
| 223 | if (rc) | ||
| 224 | goto fail; | ||
| 225 | |||
| 226 | /* Fill in lvl1 PBL */ | ||
| 227 | dst_virt_ptr = | ||
| 228 | (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr; | ||
| 229 | src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr; | ||
| 230 | for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) { | ||
| 231 | dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = | ||
| 232 | src_phys_ptr[i] | PTU_PTE_VALID; | ||
| 233 | } | ||
| 234 | if (hwq_type == HWQ_TYPE_QUEUE) { | ||
| 235 | /* Find the last pg of the size */ | ||
| 236 | i = hwq->pbl[PBL_LVL_2].pg_count; | ||
| 237 | dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= | ||
| 238 | PTU_PTE_LAST; | ||
| 239 | if (i > 1) | ||
| 240 | dst_virt_ptr[PTR_PG(i - 2)] | ||
| 241 | [PTR_IDX(i - 2)] |= | ||
| 242 | PTU_PTE_NEXT_TO_LAST; | ||
| 243 | } | ||
| 244 | hwq->level = PBL_LVL_2; | ||
| 245 | } else { | ||
| 246 | u32 flag = hwq_type == HWQ_TYPE_L2_CMPL ? 0 : | ||
| 247 | PTU_PTE_VALID; | ||
| 248 | |||
| 249 | /* 1 level of indirection */ | ||
| 250 | rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], sghead, | ||
| 251 | pages, pg_size); | ||
| 252 | if (rc) | ||
| 253 | goto fail; | ||
| 254 | /* Fill in lvl0 PBL */ | ||
| 255 | dst_virt_ptr = | ||
| 256 | (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; | ||
| 257 | src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; | ||
| 258 | for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) { | ||
| 259 | dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = | ||
| 260 | src_phys_ptr[i] | flag; | ||
| 261 | } | ||
| 262 | if (hwq_type == HWQ_TYPE_QUEUE) { | ||
| 263 | /* Find the last pg of the size */ | ||
| 264 | i = hwq->pbl[PBL_LVL_1].pg_count; | ||
| 265 | dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= | ||
| 266 | PTU_PTE_LAST; | ||
| 267 | if (i > 1) | ||
| 268 | dst_virt_ptr[PTR_PG(i - 2)] | ||
| 269 | [PTR_IDX(i - 2)] |= | ||
| 270 | PTU_PTE_NEXT_TO_LAST; | ||
| 271 | } | ||
| 272 | hwq->level = PBL_LVL_1; | ||
| 273 | } | ||
| 274 | } | ||
| 275 | hwq->pdev = pdev; | ||
| 276 | spin_lock_init(&hwq->lock); | ||
| 277 | hwq->prod = 0; | ||
| 278 | hwq->cons = 0; | ||
| 279 | *elements = hwq->max_elements = slots; | ||
| 280 | hwq->element_size = size; | ||
| 281 | |||
| 282 | /* For direct access to the elements */ | ||
| 283 | hwq->pbl_ptr = hwq->pbl[hwq->level].pg_arr; | ||
| 284 | hwq->pbl_dma_ptr = hwq->pbl[hwq->level].pg_map_arr; | ||
| 285 | |||
| 286 | return 0; | ||
| 287 | |||
| 288 | fail: | ||
| 289 | bnxt_qplib_free_hwq(pdev, hwq); | ||
| 290 | return -ENOMEM; | ||
| 291 | } | ||
| 292 | |||
| 293 | /* Context Tables */ | ||
| 294 | void bnxt_qplib_free_ctx(struct pci_dev *pdev, | ||
| 295 | struct bnxt_qplib_ctx *ctx) | ||
| 296 | { | ||
| 297 | int i; | ||
| 298 | |||
| 299 | bnxt_qplib_free_hwq(pdev, &ctx->qpc_tbl); | ||
| 300 | bnxt_qplib_free_hwq(pdev, &ctx->mrw_tbl); | ||
| 301 | bnxt_qplib_free_hwq(pdev, &ctx->srqc_tbl); | ||
| 302 | bnxt_qplib_free_hwq(pdev, &ctx->cq_tbl); | ||
| 303 | bnxt_qplib_free_hwq(pdev, &ctx->tim_tbl); | ||
| 304 | for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) | ||
| 305 | bnxt_qplib_free_hwq(pdev, &ctx->tqm_tbl[i]); | ||
| 306 | bnxt_qplib_free_hwq(pdev, &ctx->tqm_pde); | ||
| 307 | bnxt_qplib_free_stats_ctx(pdev, &ctx->stats); | ||
| 308 | } | ||
| 309 | |||
| 310 | /* | ||
| 311 | * Routine: bnxt_qplib_alloc_ctx | ||
| 312 | * Description: | ||
| 313 | * Context tables are memories which are used by the chip fw. | ||
| 314 | * The 6 tables defined are: | ||
| 315 | * QPC ctx - holds QP states | ||
| 316 | * MRW ctx - holds memory region and window | ||
| 317 | * SRQ ctx - holds shared RQ states | ||
| 318 | * CQ ctx - holds completion queue states | ||
| 319 | * TQM ctx - holds Tx Queue Manager context | ||
| 320 | * TIM ctx - holds timer context | ||
| 321 | * Depending on the size of the tbl requested, either a 1 Page Buffer List | ||
| 322 | * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used | ||
| 323 | * instead. | ||
| 324 | * Table might be employed as follows: | ||
| 325 | * For 0 < ctx size <= 1 PAGE, 0 level of ind is used | ||
| 326 | * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used | ||
| 327 | * For 512 < ctx size <= MAX, 2 levels of ind is used | ||
| 328 | * Returns: | ||
| 329 | * 0 if success, else -ERRORS | ||
| 330 | */ | ||
| 331 | int bnxt_qplib_alloc_ctx(struct pci_dev *pdev, | ||
| 332 | struct bnxt_qplib_ctx *ctx, | ||
| 333 | bool virt_fn) | ||
| 334 | { | ||
| 335 | int i, j, k, rc = 0; | ||
| 336 | int fnz_idx = -1; | ||
| 337 | __le64 **pbl_ptr; | ||
| 338 | |||
| 339 | if (virt_fn) | ||
| 340 | goto stats_alloc; | ||
| 341 | |||
| 342 | /* QPC Tables */ | ||
| 343 | ctx->qpc_tbl.max_elements = ctx->qpc_count; | ||
| 344 | rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->qpc_tbl, NULL, 0, | ||
| 345 | &ctx->qpc_tbl.max_elements, | ||
| 346 | BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE, 0, | ||
| 347 | PAGE_SIZE, HWQ_TYPE_CTX); | ||
| 348 | if (rc) | ||
| 349 | goto fail; | ||
| 350 | |||
| 351 | /* MRW Tables */ | ||
| 352 | ctx->mrw_tbl.max_elements = ctx->mrw_count; | ||
| 353 | rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->mrw_tbl, NULL, 0, | ||
| 354 | &ctx->mrw_tbl.max_elements, | ||
| 355 | BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE, 0, | ||
| 356 | PAGE_SIZE, HWQ_TYPE_CTX); | ||
| 357 | if (rc) | ||
| 358 | goto fail; | ||
| 359 | |||
| 360 | /* SRQ Tables */ | ||
| 361 | ctx->srqc_tbl.max_elements = ctx->srqc_count; | ||
| 362 | rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->srqc_tbl, NULL, 0, | ||
| 363 | &ctx->srqc_tbl.max_elements, | ||
| 364 | BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE, 0, | ||
| 365 | PAGE_SIZE, HWQ_TYPE_CTX); | ||
| 366 | if (rc) | ||
| 367 | goto fail; | ||
| 368 | |||
| 369 | /* CQ Tables */ | ||
| 370 | ctx->cq_tbl.max_elements = ctx->cq_count; | ||
| 371 | rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->cq_tbl, NULL, 0, | ||
| 372 | &ctx->cq_tbl.max_elements, | ||
| 373 | BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE, 0, | ||
| 374 | PAGE_SIZE, HWQ_TYPE_CTX); | ||
| 375 | if (rc) | ||
| 376 | goto fail; | ||
| 377 | |||
| 378 | /* TQM Buffer */ | ||
| 379 | ctx->tqm_pde.max_elements = 512; | ||
| 380 | rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_pde, NULL, 0, | ||
| 381 | &ctx->tqm_pde.max_elements, sizeof(u64), | ||
| 382 | 0, PAGE_SIZE, HWQ_TYPE_CTX); | ||
| 383 | if (rc) | ||
| 384 | goto fail; | ||
| 385 | |||
| 386 | for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) { | ||
| 387 | if (!ctx->tqm_count[i]) | ||
| 388 | continue; | ||
| 389 | ctx->tqm_tbl[i].max_elements = ctx->qpc_count * | ||
| 390 | ctx->tqm_count[i]; | ||
| 391 | rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_tbl[i], NULL, 0, | ||
| 392 | &ctx->tqm_tbl[i].max_elements, 1, | ||
| 393 | 0, PAGE_SIZE, HWQ_TYPE_CTX); | ||
| 394 | if (rc) | ||
| 395 | goto fail; | ||
| 396 | } | ||
| 397 | pbl_ptr = (__le64 **)ctx->tqm_pde.pbl_ptr; | ||
| 398 | for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ; | ||
| 399 | i++, j += MAX_TQM_ALLOC_BLK_SIZE) { | ||
| 400 | if (!ctx->tqm_tbl[i].max_elements) | ||
| 401 | continue; | ||
| 402 | if (fnz_idx == -1) | ||
| 403 | fnz_idx = i; | ||
| 404 | switch (ctx->tqm_tbl[i].level) { | ||
| 405 | case PBL_LVL_2: | ||
| 406 | for (k = 0; k < ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_count; | ||
| 407 | k++) | ||
| 408 | pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)] = | ||
| 409 | cpu_to_le64( | ||
| 410 | ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_map_arr[k] | ||
| 411 | | PTU_PTE_VALID); | ||
| 412 | break; | ||
| 413 | case PBL_LVL_1: | ||
| 414 | case PBL_LVL_0: | ||
| 415 | default: | ||
| 416 | pbl_ptr[PTR_PG(j)][PTR_IDX(j)] = cpu_to_le64( | ||
| 417 | ctx->tqm_tbl[i].pbl[PBL_LVL_0].pg_map_arr[0] | | ||
| 418 | PTU_PTE_VALID); | ||
| 419 | break; | ||
| 420 | } | ||
| 421 | } | ||
| 422 | if (fnz_idx == -1) | ||
| 423 | fnz_idx = 0; | ||
| 424 | ctx->tqm_pde_level = ctx->tqm_tbl[fnz_idx].level == PBL_LVL_2 ? | ||
| 425 | PBL_LVL_2 : ctx->tqm_tbl[fnz_idx].level + 1; | ||
| 426 | |||
| 427 | /* TIM Buffer */ | ||
| 428 | ctx->tim_tbl.max_elements = ctx->qpc_count * 16; | ||
| 429 | rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tim_tbl, NULL, 0, | ||
| 430 | &ctx->tim_tbl.max_elements, 1, | ||
| 431 | 0, PAGE_SIZE, HWQ_TYPE_CTX); | ||
| 432 | if (rc) | ||
| 433 | goto fail; | ||
| 434 | |||
| 435 | stats_alloc: | ||
| 436 | /* Stats */ | ||
| 437 | rc = bnxt_qplib_alloc_stats_ctx(pdev, &ctx->stats); | ||
| 438 | if (rc) | ||
| 439 | goto fail; | ||
| 440 | |||
| 441 | return 0; | ||
| 442 | |||
| 443 | fail: | ||
| 444 | bnxt_qplib_free_ctx(pdev, ctx); | ||
| 445 | return rc; | ||
| 446 | } | ||
| 447 | |||
| 448 | /* GUID */ | ||
| 449 | void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid) | ||
| 450 | { | ||
| 451 | u8 mac[ETH_ALEN]; | ||
| 452 | |||
| 453 | /* MAC-48 to EUI-64 mapping */ | ||
| 454 | memcpy(mac, dev_addr, ETH_ALEN); | ||
| 455 | guid[0] = mac[0] ^ 2; | ||
| 456 | guid[1] = mac[1]; | ||
| 457 | guid[2] = mac[2]; | ||
| 458 | guid[3] = 0xff; | ||
| 459 | guid[4] = 0xfe; | ||
| 460 | guid[5] = mac[3]; | ||
| 461 | guid[6] = mac[4]; | ||
| 462 | guid[7] = mac[5]; | ||
| 463 | } | ||
| 464 | |||
| 465 | static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res, | ||
| 466 | struct bnxt_qplib_sgid_tbl *sgid_tbl) | ||
| 467 | { | ||
| 468 | kfree(sgid_tbl->tbl); | ||
| 469 | kfree(sgid_tbl->hw_id); | ||
| 470 | kfree(sgid_tbl->ctx); | ||
| 471 | sgid_tbl->tbl = NULL; | ||
| 472 | sgid_tbl->hw_id = NULL; | ||
| 473 | sgid_tbl->ctx = NULL; | ||
| 474 | sgid_tbl->max = 0; | ||
| 475 | sgid_tbl->active = 0; | ||
| 476 | } | ||
| 477 | |||
| 478 | static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res, | ||
| 479 | struct bnxt_qplib_sgid_tbl *sgid_tbl, | ||
| 480 | u16 max) | ||
| 481 | { | ||
| 482 | sgid_tbl->tbl = kcalloc(max, sizeof(struct bnxt_qplib_gid), GFP_KERNEL); | ||
| 483 | if (!sgid_tbl->tbl) | ||
| 484 | return -ENOMEM; | ||
| 485 | |||
| 486 | sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL); | ||
| 487 | if (!sgid_tbl->hw_id) | ||
| 488 | goto out_free1; | ||
| 489 | |||
| 490 | sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL); | ||
| 491 | if (!sgid_tbl->ctx) | ||
| 492 | goto out_free2; | ||
| 493 | |||
| 494 | sgid_tbl->max = max; | ||
| 495 | return 0; | ||
| 496 | out_free2: | ||
| 497 | kfree(sgid_tbl->hw_id); | ||
| 498 | sgid_tbl->hw_id = NULL; | ||
| 499 | out_free1: | ||
| 500 | kfree(sgid_tbl->tbl); | ||
| 501 | sgid_tbl->tbl = NULL; | ||
| 502 | return -ENOMEM; | ||
| 503 | }; | ||
| 504 | |||
| 505 | static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res, | ||
| 506 | struct bnxt_qplib_sgid_tbl *sgid_tbl) | ||
| 507 | { | ||
| 508 | int i; | ||
| 509 | |||
| 510 | for (i = 0; i < sgid_tbl->max; i++) { | ||
| 511 | if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, | ||
| 512 | sizeof(bnxt_qplib_gid_zero))) | ||
| 513 | bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i], true); | ||
| 514 | } | ||
| 515 | memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max); | ||
| 516 | memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); | ||
| 517 | sgid_tbl->active = 0; | ||
| 518 | } | ||
| 519 | |||
| 520 | static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, | ||
| 521 | struct net_device *netdev) | ||
| 522 | { | ||
| 523 | memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max); | ||
| 524 | memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); | ||
| 525 | } | ||
| 526 | |||
| 527 | static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res, | ||
| 528 | struct bnxt_qplib_pkey_tbl *pkey_tbl) | ||
| 529 | { | ||
| 530 | if (!pkey_tbl->tbl) | ||
| 531 | dev_dbg(&res->pdev->dev, "QPLIB: PKEY tbl not present"); | ||
| 532 | else | ||
| 533 | kfree(pkey_tbl->tbl); | ||
| 534 | |||
| 535 | pkey_tbl->tbl = NULL; | ||
| 536 | pkey_tbl->max = 0; | ||
| 537 | pkey_tbl->active = 0; | ||
| 538 | } | ||
| 539 | |||
| 540 | static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res, | ||
| 541 | struct bnxt_qplib_pkey_tbl *pkey_tbl, | ||
| 542 | u16 max) | ||
| 543 | { | ||
| 544 | pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL); | ||
| 545 | if (!pkey_tbl->tbl) | ||
| 546 | return -ENOMEM; | ||
| 547 | |||
| 548 | pkey_tbl->max = max; | ||
| 549 | return 0; | ||
| 550 | }; | ||
| 551 | |||
| 552 | /* PDs */ | ||
| 553 | int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd) | ||
| 554 | { | ||
| 555 | u32 bit_num; | ||
| 556 | |||
| 557 | bit_num = find_first_bit(pdt->tbl, pdt->max); | ||
| 558 | if (bit_num == pdt->max) | ||
| 559 | return -ENOMEM; | ||
| 560 | |||
| 561 | /* Found unused PD */ | ||
| 562 | clear_bit(bit_num, pdt->tbl); | ||
| 563 | pd->id = bit_num; | ||
| 564 | return 0; | ||
| 565 | } | ||
| 566 | |||
| 567 | int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, | ||
| 568 | struct bnxt_qplib_pd_tbl *pdt, | ||
| 569 | struct bnxt_qplib_pd *pd) | ||
| 570 | { | ||
| 571 | if (test_and_set_bit(pd->id, pdt->tbl)) { | ||
| 572 | dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d", | ||
| 573 | pd->id); | ||
| 574 | return -EINVAL; | ||
| 575 | } | ||
| 576 | pd->id = 0; | ||
| 577 | return 0; | ||
| 578 | } | ||
| 579 | |||
| 580 | static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt) | ||
| 581 | { | ||
| 582 | kfree(pdt->tbl); | ||
| 583 | pdt->tbl = NULL; | ||
| 584 | pdt->max = 0; | ||
| 585 | } | ||
| 586 | |||
| 587 | static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, | ||
| 588 | struct bnxt_qplib_pd_tbl *pdt, | ||
| 589 | u32 max) | ||
| 590 | { | ||
| 591 | u32 bytes; | ||
| 592 | |||
| 593 | bytes = max >> 3; | ||
| 594 | if (!bytes) | ||
| 595 | bytes = 1; | ||
| 596 | pdt->tbl = kmalloc(bytes, GFP_KERNEL); | ||
| 597 | if (!pdt->tbl) | ||
| 598 | return -ENOMEM; | ||
| 599 | |||
| 600 | pdt->max = max; | ||
| 601 | memset((u8 *)pdt->tbl, 0xFF, bytes); | ||
| 602 | |||
| 603 | return 0; | ||
| 604 | } | ||
| 605 | |||
| 606 | /* DPIs */ | ||
| 607 | int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit, | ||
| 608 | struct bnxt_qplib_dpi *dpi, | ||
| 609 | void *app) | ||
| 610 | { | ||
| 611 | u32 bit_num; | ||
| 612 | |||
| 613 | bit_num = find_first_bit(dpit->tbl, dpit->max); | ||
| 614 | if (bit_num == dpit->max) | ||
| 615 | return -ENOMEM; | ||
| 616 | |||
| 617 | /* Found unused DPI */ | ||
| 618 | clear_bit(bit_num, dpit->tbl); | ||
| 619 | dpit->app_tbl[bit_num] = app; | ||
| 620 | |||
| 621 | dpi->dpi = bit_num; | ||
| 622 | dpi->dbr = dpit->dbr_bar_reg_iomem + (bit_num * PAGE_SIZE); | ||
| 623 | dpi->umdbr = dpit->unmapped_dbr + (bit_num * PAGE_SIZE); | ||
| 624 | |||
| 625 | return 0; | ||
| 626 | } | ||
| 627 | |||
| 628 | int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, | ||
| 629 | struct bnxt_qplib_dpi_tbl *dpit, | ||
| 630 | struct bnxt_qplib_dpi *dpi) | ||
| 631 | { | ||
| 632 | if (dpi->dpi >= dpit->max) { | ||
| 633 | dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d", dpi->dpi); | ||
| 634 | return -EINVAL; | ||
| 635 | } | ||
| 636 | if (test_and_set_bit(dpi->dpi, dpit->tbl)) { | ||
| 637 | dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d", | ||
| 638 | dpi->dpi); | ||
| 639 | return -EINVAL; | ||
| 640 | } | ||
| 641 | if (dpit->app_tbl) | ||
| 642 | dpit->app_tbl[dpi->dpi] = NULL; | ||
| 643 | memset(dpi, 0, sizeof(*dpi)); | ||
| 644 | |||
| 645 | return 0; | ||
| 646 | } | ||
| 647 | |||
| 648 | static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res *res, | ||
| 649 | struct bnxt_qplib_dpi_tbl *dpit) | ||
| 650 | { | ||
| 651 | kfree(dpit->tbl); | ||
| 652 | kfree(dpit->app_tbl); | ||
| 653 | if (dpit->dbr_bar_reg_iomem) | ||
| 654 | pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); | ||
| 655 | memset(dpit, 0, sizeof(*dpit)); | ||
| 656 | } | ||
| 657 | |||
| 658 | static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, | ||
| 659 | struct bnxt_qplib_dpi_tbl *dpit, | ||
| 660 | u32 dbr_offset) | ||
| 661 | { | ||
| 662 | u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION; | ||
| 663 | resource_size_t bar_reg_base; | ||
| 664 | u32 dbr_len, bytes; | ||
| 665 | |||
| 666 | if (dpit->dbr_bar_reg_iomem) { | ||
| 667 | dev_err(&res->pdev->dev, | ||
| 668 | "QPLIB: DBR BAR region %d already mapped", dbr_bar_reg); | ||
| 669 | return -EALREADY; | ||
| 670 | } | ||
| 671 | |||
| 672 | bar_reg_base = pci_resource_start(res->pdev, dbr_bar_reg); | ||
| 673 | if (!bar_reg_base) { | ||
| 674 | dev_err(&res->pdev->dev, | ||
| 675 | "QPLIB: BAR region %d resc start failed", dbr_bar_reg); | ||
| 676 | return -ENOMEM; | ||
| 677 | } | ||
| 678 | |||
| 679 | dbr_len = pci_resource_len(res->pdev, dbr_bar_reg) - dbr_offset; | ||
| 680 | if (!dbr_len || ((dbr_len & (PAGE_SIZE - 1)) != 0)) { | ||
| 681 | dev_err(&res->pdev->dev, "QPLIB: Invalid DBR length %d", | ||
| 682 | dbr_len); | ||
| 683 | return -ENOMEM; | ||
| 684 | } | ||
| 685 | |||
| 686 | dpit->dbr_bar_reg_iomem = ioremap_nocache(bar_reg_base + dbr_offset, | ||
| 687 | dbr_len); | ||
| 688 | if (!dpit->dbr_bar_reg_iomem) { | ||
| 689 | dev_err(&res->pdev->dev, | ||
| 690 | "QPLIB: FP: DBR BAR region %d mapping failed", | ||
| 691 | dbr_bar_reg); | ||
| 692 | return -ENOMEM; | ||
| 693 | } | ||
| 694 | |||
| 695 | dpit->unmapped_dbr = bar_reg_base + dbr_offset; | ||
| 696 | dpit->max = dbr_len / PAGE_SIZE; | ||
| 697 | |||
| 698 | dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL); | ||
| 699 | if (!dpit->app_tbl) { | ||
| 700 | pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); | ||
| 701 | dev_err(&res->pdev->dev, | ||
| 702 | "QPLIB: DPI app tbl allocation failed"); | ||
| 703 | return -ENOMEM; | ||
| 704 | } | ||
| 705 | |||
| 706 | bytes = dpit->max >> 3; | ||
| 707 | if (!bytes) | ||
| 708 | bytes = 1; | ||
| 709 | |||
| 710 | dpit->tbl = kmalloc(bytes, GFP_KERNEL); | ||
| 711 | if (!dpit->tbl) { | ||
| 712 | pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); | ||
| 713 | kfree(dpit->app_tbl); | ||
| 714 | dpit->app_tbl = NULL; | ||
| 715 | dev_err(&res->pdev->dev, | ||
| 716 | "QPLIB: DPI tbl allocation failed for size = %d", | ||
| 717 | bytes); | ||
| 718 | return -ENOMEM; | ||
| 719 | } | ||
| 720 | |||
| 721 | memset((u8 *)dpit->tbl, 0xFF, bytes); | ||
| 722 | |||
| 723 | return 0; | ||
| 724 | } | ||
| 725 | |||
| 726 | /* PKEYs */ | ||
| 727 | static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl) | ||
| 728 | { | ||
| 729 | memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max); | ||
| 730 | pkey_tbl->active = 0; | ||
| 731 | } | ||
| 732 | |||
| 733 | static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res, | ||
| 734 | struct bnxt_qplib_pkey_tbl *pkey_tbl) | ||
| 735 | { | ||
| 736 | u16 pkey = 0xFFFF; | ||
| 737 | |||
| 738 | memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max); | ||
| 739 | |||
| 740 | /* pkey default = 0xFFFF */ | ||
| 741 | bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false); | ||
| 742 | } | ||
| 743 | |||
| 744 | /* Stats */ | ||
| 745 | static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev, | ||
| 746 | struct bnxt_qplib_stats *stats) | ||
| 747 | { | ||
| 748 | if (stats->dma) { | ||
| 749 | dma_free_coherent(&pdev->dev, stats->size, | ||
| 750 | stats->dma, stats->dma_map); | ||
| 751 | } | ||
| 752 | memset(stats, 0, sizeof(*stats)); | ||
| 753 | stats->fw_id = -1; | ||
| 754 | } | ||
| 755 | |||
| 756 | static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev, | ||
| 757 | struct bnxt_qplib_stats *stats) | ||
| 758 | { | ||
| 759 | memset(stats, 0, sizeof(*stats)); | ||
| 760 | stats->fw_id = -1; | ||
| 761 | stats->size = sizeof(struct ctx_hw_stats); | ||
| 762 | stats->dma = dma_alloc_coherent(&pdev->dev, stats->size, | ||
| 763 | &stats->dma_map, GFP_KERNEL); | ||
| 764 | if (!stats->dma) { | ||
| 765 | dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed"); | ||
| 766 | return -ENOMEM; | ||
| 767 | } | ||
| 768 | return 0; | ||
| 769 | } | ||
| 770 | |||
| 771 | void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res) | ||
| 772 | { | ||
| 773 | bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl); | ||
| 774 | bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl); | ||
| 775 | } | ||
| 776 | |||
| 777 | int bnxt_qplib_init_res(struct bnxt_qplib_res *res) | ||
| 778 | { | ||
| 779 | bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev); | ||
| 780 | bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl); | ||
| 781 | |||
| 782 | return 0; | ||
| 783 | } | ||
| 784 | |||
| 785 | void bnxt_qplib_free_res(struct bnxt_qplib_res *res) | ||
| 786 | { | ||
| 787 | bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl); | ||
| 788 | bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl); | ||
| 789 | bnxt_qplib_free_pd_tbl(&res->pd_tbl); | ||
| 790 | bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl); | ||
| 791 | |||
| 792 | res->netdev = NULL; | ||
| 793 | res->pdev = NULL; | ||
| 794 | } | ||
| 795 | |||
| 796 | int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, | ||
| 797 | struct net_device *netdev, | ||
| 798 | struct bnxt_qplib_dev_attr *dev_attr) | ||
| 799 | { | ||
| 800 | int rc = 0; | ||
| 801 | |||
| 802 | res->pdev = pdev; | ||
| 803 | res->netdev = netdev; | ||
| 804 | |||
| 805 | rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid); | ||
| 806 | if (rc) | ||
| 807 | goto fail; | ||
| 808 | |||
| 809 | rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey); | ||
| 810 | if (rc) | ||
| 811 | goto fail; | ||
| 812 | |||
| 813 | rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd); | ||
| 814 | if (rc) | ||
| 815 | goto fail; | ||
| 816 | |||
| 817 | rc = bnxt_qplib_alloc_dpi_tbl(res, &res->dpi_tbl, dev_attr->l2_db_size); | ||
| 818 | if (rc) | ||
| 819 | goto fail; | ||
| 820 | |||
| 821 | return 0; | ||
| 822 | fail: | ||
| 823 | bnxt_qplib_free_res(res); | ||
| 824 | return rc; | ||
| 825 | } | ||
| @@ -0,0 +1,223 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: QPLib resource manager (header) | ||
| 37 | */ | ||
| 38 | |||
| 39 | #ifndef __BNXT_QPLIB_RES_H__ | ||
| 40 | #define __BNXT_QPLIB_RES_H__ | ||
| 41 | |||
| 42 | extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; | ||
| 43 | |||
| 44 | #define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *)) | ||
| 45 | #define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1) | ||
| 46 | #define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG) | ||
| 47 | #define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG) | ||
| 48 | |||
| 49 | #define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1)) | ||
| 50 | |||
| 51 | enum bnxt_qplib_hwq_type { | ||
| 52 | HWQ_TYPE_CTX, | ||
| 53 | HWQ_TYPE_QUEUE, | ||
| 54 | HWQ_TYPE_L2_CMPL | ||
| 55 | }; | ||
| 56 | |||
| 57 | #define MAX_PBL_LVL_0_PGS 1 | ||
| 58 | #define MAX_PBL_LVL_1_PGS 512 | ||
| 59 | #define MAX_PBL_LVL_1_PGS_SHIFT 9 | ||
| 60 | #define MAX_PBL_LVL_1_PGS_FOR_LVL_2 256 | ||
| 61 | #define MAX_PBL_LVL_2_PGS (256 * 512) | ||
| 62 | |||
| 63 | enum bnxt_qplib_pbl_lvl { | ||
| 64 | PBL_LVL_0, | ||
| 65 | PBL_LVL_1, | ||
| 66 | PBL_LVL_2, | ||
| 67 | PBL_LVL_MAX | ||
| 68 | }; | ||
| 69 | |||
| 70 | #define ROCE_PG_SIZE_4K (4 * 1024) | ||
| 71 | #define ROCE_PG_SIZE_8K (8 * 1024) | ||
| 72 | #define ROCE_PG_SIZE_64K (64 * 1024) | ||
| 73 | #define ROCE_PG_SIZE_2M (2 * 1024 * 1024) | ||
| 74 | #define ROCE_PG_SIZE_8M (8 * 1024 * 1024) | ||
| 75 | #define ROCE_PG_SIZE_1G (1024 * 1024 * 1024) | ||
| 76 | |||
| 77 | struct bnxt_qplib_pbl { | ||
| 78 | u32 pg_count; | ||
| 79 | u32 pg_size; | ||
| 80 | void **pg_arr; | ||
| 81 | dma_addr_t *pg_map_arr; | ||
| 82 | }; | ||
| 83 | |||
| 84 | struct bnxt_qplib_hwq { | ||
| 85 | struct pci_dev *pdev; | ||
| 86 | /* lock to protect qplib_hwq */ | ||
| 87 | spinlock_t lock; | ||
| 88 | struct bnxt_qplib_pbl pbl[PBL_LVL_MAX]; | ||
| 89 | enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */ | ||
| 90 | /* ptr for easy access to the PBL entries */ | ||
| 91 | void **pbl_ptr; | ||
| 92 | /* ptr for easy access to the dma_addr */ | ||
| 93 | dma_addr_t *pbl_dma_ptr; | ||
| 94 | u32 max_elements; | ||
| 95 | u16 element_size; /* Size of each entry */ | ||
| 96 | |||
| 97 | u32 prod; /* raw */ | ||
| 98 | u32 cons; /* raw */ | ||
| 99 | u8 cp_bit; | ||
| 100 | u8 is_user; | ||
| 101 | }; | ||
| 102 | |||
| 103 | /* Tables */ | ||
| 104 | struct bnxt_qplib_pd_tbl { | ||
| 105 | unsigned long *tbl; | ||
| 106 | u32 max; | ||
| 107 | }; | ||
| 108 | |||
| 109 | struct bnxt_qplib_sgid_tbl { | ||
| 110 | struct bnxt_qplib_gid *tbl; | ||
| 111 | u16 *hw_id; | ||
| 112 | u16 max; | ||
| 113 | u16 active; | ||
| 114 | void *ctx; | ||
| 115 | }; | ||
| 116 | |||
| 117 | struct bnxt_qplib_pkey_tbl { | ||
| 118 | u16 *tbl; | ||
| 119 | u16 max; | ||
| 120 | u16 active; | ||
| 121 | }; | ||
| 122 | |||
| 123 | struct bnxt_qplib_dpi { | ||
| 124 | u32 dpi; | ||
| 125 | void __iomem *dbr; | ||
| 126 | u64 umdbr; | ||
| 127 | }; | ||
| 128 | |||
| 129 | struct bnxt_qplib_dpi_tbl { | ||
| 130 | void **app_tbl; | ||
| 131 | unsigned long *tbl; | ||
| 132 | u16 max; | ||
| 133 | void __iomem *dbr_bar_reg_iomem; | ||
| 134 | u64 unmapped_dbr; | ||
| 135 | }; | ||
| 136 | |||
| 137 | struct bnxt_qplib_stats { | ||
| 138 | dma_addr_t dma_map; | ||
| 139 | void *dma; | ||
| 140 | u32 size; | ||
| 141 | u32 fw_id; | ||
| 142 | }; | ||
| 143 | |||
| 144 | struct bnxt_qplib_vf_res { | ||
| 145 | u32 max_qp_per_vf; | ||
| 146 | u32 max_mrw_per_vf; | ||
| 147 | u32 max_srq_per_vf; | ||
| 148 | u32 max_cq_per_vf; | ||
| 149 | u32 max_gid_per_vf; | ||
| 150 | }; | ||
| 151 | |||
| 152 | #define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448 | ||
| 153 | #define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64 | ||
| 154 | #define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64 | ||
| 155 | #define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128 | ||
| 156 | |||
| 157 | struct bnxt_qplib_ctx { | ||
| 158 | u32 qpc_count; | ||
| 159 | struct bnxt_qplib_hwq qpc_tbl; | ||
| 160 | u32 mrw_count; | ||
| 161 | struct bnxt_qplib_hwq mrw_tbl; | ||
| 162 | u32 srqc_count; | ||
| 163 | struct bnxt_qplib_hwq srqc_tbl; | ||
| 164 | u32 cq_count; | ||
| 165 | struct bnxt_qplib_hwq cq_tbl; | ||
| 166 | struct bnxt_qplib_hwq tim_tbl; | ||
| 167 | #define MAX_TQM_ALLOC_REQ 32 | ||
| 168 | #define MAX_TQM_ALLOC_BLK_SIZE 8 | ||
| 169 | u8 tqm_count[MAX_TQM_ALLOC_REQ]; | ||
| 170 | struct bnxt_qplib_hwq tqm_pde; | ||
| 171 | u32 tqm_pde_level; | ||
| 172 | struct bnxt_qplib_hwq tqm_tbl[MAX_TQM_ALLOC_REQ]; | ||
| 173 | struct bnxt_qplib_stats stats; | ||
| 174 | struct bnxt_qplib_vf_res vf_res; | ||
| 175 | }; | ||
| 176 | |||
| 177 | struct bnxt_qplib_res { | ||
| 178 | struct pci_dev *pdev; | ||
| 179 | struct net_device *netdev; | ||
| 180 | |||
| 181 | struct bnxt_qplib_rcfw *rcfw; | ||
| 182 | |||
| 183 | struct bnxt_qplib_pd_tbl pd_tbl; | ||
| 184 | struct bnxt_qplib_sgid_tbl sgid_tbl; | ||
| 185 | struct bnxt_qplib_pkey_tbl pkey_tbl; | ||
| 186 | struct bnxt_qplib_dpi_tbl dpi_tbl; | ||
| 187 | }; | ||
| 188 | |||
| 189 | #define to_bnxt_qplib(ptr, type, member) \ | ||
| 190 | container_of(ptr, type, member) | ||
| 191 | |||
| 192 | struct bnxt_qplib_pd; | ||
| 193 | struct bnxt_qplib_dev_attr; | ||
| 194 | |||
| 195 | void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq); | ||
| 196 | int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq, | ||
| 197 | struct scatterlist *sl, int nmap, u32 *elements, | ||
| 198 | u32 elements_per_page, u32 aux, u32 pg_size, | ||
| 199 | enum bnxt_qplib_hwq_type hwq_type); | ||
| 200 | void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid); | ||
| 201 | int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pd_tbl, | ||
| 202 | struct bnxt_qplib_pd *pd); | ||
| 203 | int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, | ||
| 204 | struct bnxt_qplib_pd_tbl *pd_tbl, | ||
| 205 | struct bnxt_qplib_pd *pd); | ||
| 206 | int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit, | ||
| 207 | struct bnxt_qplib_dpi *dpi, | ||
| 208 | void *app); | ||
| 209 | int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, | ||
| 210 | struct bnxt_qplib_dpi_tbl *dpi_tbl, | ||
| 211 | struct bnxt_qplib_dpi *dpi); | ||
| 212 | void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res); | ||
| 213 | int bnxt_qplib_init_res(struct bnxt_qplib_res *res); | ||
| 214 | void bnxt_qplib_free_res(struct bnxt_qplib_res *res); | ||
| 215 | int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, | ||
| 216 | struct net_device *netdev, | ||
| 217 | struct bnxt_qplib_dev_attr *dev_attr); | ||
| 218 | void bnxt_qplib_free_ctx(struct pci_dev *pdev, | ||
| 219 | struct bnxt_qplib_ctx *ctx); | ||
| 220 | int bnxt_qplib_alloc_ctx(struct pci_dev *pdev, | ||
| 221 | struct bnxt_qplib_ctx *ctx, | ||
| 222 | bool virt_fn); | ||
| 223 | #endif /* __BNXT_QPLIB_RES_H__ */ | ||
| @@ -0,0 +1,838 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: Slow Path Operators | ||
| 37 | */ | ||
| 38 | |||
| 39 | #include <linux/interrupt.h> | ||
| 40 | #include <linux/spinlock.h> | ||
| 41 | #include <linux/sched.h> | ||
| 42 | #include <linux/pci.h> | ||
| 43 | |||
| 44 | #include "roce_hsi.h" | ||
| 45 | |||
| 46 | #include "qplib_res.h" | ||
| 47 | #include "qplib_rcfw.h" | ||
| 48 | #include "qplib_sp.h" | ||
| 49 | |||
| 50 | const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0, | ||
| 51 | 0, 0, 0, 0, 0, 0, 0, 0 } }; | ||
| 52 | |||
| 53 | /* Device */ | ||
| 54 | int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, | ||
| 55 | struct bnxt_qplib_dev_attr *attr) | ||
| 56 | { | ||
| 57 | struct cmdq_query_func req; | ||
| 58 | struct creq_query_func_resp *resp; | ||
| 59 | struct creq_query_func_resp_sb *sb; | ||
| 60 | u16 cmd_flags = 0; | ||
| 61 | u32 temp; | ||
| 62 | u8 *tqm_alloc; | ||
| 63 | int i; | ||
| 64 | |||
| 65 | RCFW_CMD_PREP(req, QUERY_FUNC, cmd_flags); | ||
| 66 | |||
| 67 | req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; | ||
| 68 | resp = (struct creq_query_func_resp *) | ||
| 69 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void **)&sb, | ||
| 70 | 0); | ||
| 71 | if (!resp) { | ||
| 72 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC send failed"); | ||
| 73 | return -EINVAL; | ||
| 74 | } | ||
| 75 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 76 | /* Cmd timed out */ | ||
| 77 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC timed out"); | ||
| 78 | return -ETIMEDOUT; | ||
| 79 | } | ||
| 80 | if (resp->status || | ||
| 81 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 82 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_FUNC failed "); | ||
| 83 | dev_err(&rcfw->pdev->dev, | ||
| 84 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 85 | resp->status, le16_to_cpu(req.cookie), | ||
| 86 | le16_to_cpu(resp->cookie)); | ||
| 87 | return -EINVAL; | ||
| 88 | } | ||
| 89 | /* Extract the context from the side buffer */ | ||
| 90 | attr->max_qp = le32_to_cpu(sb->max_qp); | ||
| 91 | attr->max_qp_rd_atom = | ||
| 92 | sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? | ||
| 93 | BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom; | ||
| 94 | attr->max_qp_init_rd_atom = | ||
| 95 | sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? | ||
| 96 | BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom; | ||
| 97 | attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr); | ||
| 98 | attr->max_qp_sges = sb->max_sge; | ||
| 99 | attr->max_cq = le32_to_cpu(sb->max_cq); | ||
| 100 | attr->max_cq_wqes = le32_to_cpu(sb->max_cqe); | ||
| 101 | attr->max_cq_sges = attr->max_qp_sges; | ||
| 102 | attr->max_mr = le32_to_cpu(sb->max_mr); | ||
| 103 | attr->max_mw = le32_to_cpu(sb->max_mw); | ||
| 104 | |||
| 105 | attr->max_mr_size = le64_to_cpu(sb->max_mr_size); | ||
| 106 | attr->max_pd = 64 * 1024; | ||
| 107 | attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp); | ||
| 108 | attr->max_ah = le32_to_cpu(sb->max_ah); | ||
| 109 | |||
| 110 | attr->max_fmr = le32_to_cpu(sb->max_fmr); | ||
| 111 | attr->max_map_per_fmr = sb->max_map_per_fmr; | ||
| 112 | |||
| 113 | attr->max_srq = le16_to_cpu(sb->max_srq); | ||
| 114 | attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; | ||
| 115 | attr->max_srq_sges = sb->max_srq_sge; | ||
| 116 | /* Bono only reports 1 PKEY for now, but it can support > 1 */ | ||
| 117 | attr->max_pkey = le32_to_cpu(sb->max_pkeys); | ||
| 118 | |||
| 119 | attr->max_inline_data = le32_to_cpu(sb->max_inline_data); | ||
| 120 | attr->l2_db_size = (sb->l2_db_space_size + 1) * PAGE_SIZE; | ||
| 121 | attr->max_sgid = le32_to_cpu(sb->max_gid); | ||
| 122 | |||
| 123 | strlcpy(attr->fw_ver, "20.6.28.0", sizeof(attr->fw_ver)); | ||
| 124 | |||
| 125 | for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) { | ||
| 126 | temp = le32_to_cpu(sb->tqm_alloc_reqs[i]); | ||
| 127 | tqm_alloc = (u8 *)&temp; | ||
| 128 | attr->tqm_alloc_reqs[i * 4] = *tqm_alloc; | ||
| 129 | attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc); | ||
| 130 | attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc); | ||
| 131 | attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc); | ||
| 132 | } | ||
| 133 | return 0; | ||
| 134 | } | ||
| 135 | |||
| 136 | /* SGID */ | ||
| 137 | int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, | ||
| 138 | struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, | ||
| 139 | struct bnxt_qplib_gid *gid) | ||
| 140 | { | ||
| 141 | if (index > sgid_tbl->max) { | ||
| 142 | dev_err(&res->pdev->dev, | ||
| 143 | "QPLIB: Index %d exceeded SGID table max (%d)", | ||
| 144 | index, sgid_tbl->max); | ||
| 145 | return -EINVAL; | ||
| 146 | } | ||
| 147 | memcpy(gid, &sgid_tbl->tbl[index], sizeof(*gid)); | ||
| 148 | return 0; | ||
| 149 | } | ||
| 150 | |||
| 151 | int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, | ||
| 152 | struct bnxt_qplib_gid *gid, bool update) | ||
| 153 | { | ||
| 154 | struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, | ||
| 155 | struct bnxt_qplib_res, | ||
| 156 | sgid_tbl); | ||
| 157 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 158 | int index; | ||
| 159 | |||
| 160 | if (!sgid_tbl) { | ||
| 161 | dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated"); | ||
| 162 | return -EINVAL; | ||
| 163 | } | ||
| 164 | /* Do we need a sgid_lock here? */ | ||
| 165 | if (!sgid_tbl->active) { | ||
| 166 | dev_err(&res->pdev->dev, | ||
| 167 | "QPLIB: SGID table has no active entries"); | ||
| 168 | return -ENOMEM; | ||
| 169 | } | ||
| 170 | for (index = 0; index < sgid_tbl->max; index++) { | ||
| 171 | if (!memcmp(&sgid_tbl->tbl[index], gid, sizeof(*gid))) | ||
| 172 | break; | ||
| 173 | } | ||
| 174 | if (index == sgid_tbl->max) { | ||
| 175 | dev_warn(&res->pdev->dev, "GID not found in the SGID table"); | ||
| 176 | return 0; | ||
| 177 | } | ||
| 178 | /* Remove GID from the SGID table */ | ||
| 179 | if (update) { | ||
| 180 | struct cmdq_delete_gid req; | ||
| 181 | struct creq_delete_gid_resp *resp; | ||
| 182 | u16 cmd_flags = 0; | ||
| 183 | |||
| 184 | RCFW_CMD_PREP(req, DELETE_GID, cmd_flags); | ||
| 185 | if (sgid_tbl->hw_id[index] == 0xFFFF) { | ||
| 186 | dev_err(&res->pdev->dev, | ||
| 187 | "QPLIB: GID entry contains an invalid HW id"); | ||
| 188 | return -EINVAL; | ||
| 189 | } | ||
| 190 | req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]); | ||
| 191 | resp = (struct creq_delete_gid_resp *) | ||
| 192 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, NULL, | ||
| 193 | 0); | ||
| 194 | if (!resp) { | ||
| 195 | dev_err(&res->pdev->dev, | ||
| 196 | "QPLIB: SP: DELETE_GID send failed"); | ||
| 197 | return -EINVAL; | ||
| 198 | } | ||
| 199 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, | ||
| 200 | le16_to_cpu(req.cookie))) { | ||
| 201 | /* Cmd timed out */ | ||
| 202 | dev_err(&res->pdev->dev, | ||
| 203 | "QPLIB: SP: DELETE_GID timed out"); | ||
| 204 | return -ETIMEDOUT; | ||
| 205 | } | ||
| 206 | if (resp->status || | ||
| 207 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 208 | dev_err(&res->pdev->dev, | ||
| 209 | "QPLIB: SP: DELETE_GID failed "); | ||
| 210 | dev_err(&res->pdev->dev, | ||
| 211 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 212 | resp->status, le16_to_cpu(req.cookie), | ||
| 213 | le16_to_cpu(resp->cookie)); | ||
| 214 | return -EINVAL; | ||
| 215 | } | ||
| 216 | } | ||
| 217 | memcpy(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, | ||
| 218 | sizeof(bnxt_qplib_gid_zero)); | ||
| 219 | sgid_tbl->active--; | ||
| 220 | dev_dbg(&res->pdev->dev, | ||
| 221 | "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x", | ||
| 222 | index, sgid_tbl->hw_id[index], sgid_tbl->active); | ||
| 223 | sgid_tbl->hw_id[index] = (u16)-1; | ||
| 224 | |||
| 225 | /* unlock */ | ||
| 226 | return 0; | ||
| 227 | } | ||
| 228 | |||
| 229 | int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, | ||
| 230 | struct bnxt_qplib_gid *gid, u8 *smac, u16 vlan_id, | ||
| 231 | bool update, u32 *index) | ||
| 232 | { | ||
| 233 | struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, | ||
| 234 | struct bnxt_qplib_res, | ||
| 235 | sgid_tbl); | ||
| 236 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 237 | int i, free_idx, rc = 0; | ||
| 238 | |||
| 239 | if (!sgid_tbl) { | ||
| 240 | dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated"); | ||
| 241 | return -EINVAL; | ||
| 242 | } | ||
| 243 | /* Do we need a sgid_lock here? */ | ||
| 244 | if (sgid_tbl->active == sgid_tbl->max) { | ||
| 245 | dev_err(&res->pdev->dev, "QPLIB: SGID table is full"); | ||
| 246 | return -ENOMEM; | ||
| 247 | } | ||
| 248 | free_idx = sgid_tbl->max; | ||
| 249 | for (i = 0; i < sgid_tbl->max; i++) { | ||
| 250 | if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid))) { | ||
| 251 | dev_dbg(&res->pdev->dev, | ||
| 252 | "QPLIB: SGID entry already exist in entry %d!", | ||
| 253 | i); | ||
| 254 | *index = i; | ||
| 255 | return -EALREADY; | ||
| 256 | } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, | ||
| 257 | sizeof(bnxt_qplib_gid_zero)) && | ||
| 258 | free_idx == sgid_tbl->max) { | ||
| 259 | free_idx = i; | ||
| 260 | } | ||
| 261 | } | ||
| 262 | if (free_idx == sgid_tbl->max) { | ||
| 263 | dev_err(&res->pdev->dev, | ||
| 264 | "QPLIB: SGID table is FULL but count is not MAX??"); | ||
| 265 | return -ENOMEM; | ||
| 266 | } | ||
| 267 | if (update) { | ||
| 268 | struct cmdq_add_gid req; | ||
| 269 | struct creq_add_gid_resp *resp; | ||
| 270 | u16 cmd_flags = 0; | ||
| 271 | u32 temp32[4]; | ||
| 272 | u16 temp16[3]; | ||
| 273 | |||
| 274 | RCFW_CMD_PREP(req, ADD_GID, cmd_flags); | ||
| 275 | |||
| 276 | memcpy(temp32, gid->data, sizeof(struct bnxt_qplib_gid)); | ||
| 277 | req.gid[0] = cpu_to_be32(temp32[3]); | ||
| 278 | req.gid[1] = cpu_to_be32(temp32[2]); | ||
| 279 | req.gid[2] = cpu_to_be32(temp32[1]); | ||
| 280 | req.gid[3] = cpu_to_be32(temp32[0]); | ||
| 281 | if (vlan_id != 0xFFFF) | ||
| 282 | req.vlan = cpu_to_le16((vlan_id & | ||
| 283 | CMDQ_ADD_GID_VLAN_VLAN_ID_MASK) | | ||
| 284 | CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | | ||
| 285 | CMDQ_ADD_GID_VLAN_VLAN_EN); | ||
| 286 | |||
| 287 | /* MAC in network format */ | ||
| 288 | memcpy(temp16, smac, 6); | ||
| 289 | req.src_mac[0] = cpu_to_be16(temp16[0]); | ||
| 290 | req.src_mac[1] = cpu_to_be16(temp16[1]); | ||
| 291 | req.src_mac[2] = cpu_to_be16(temp16[2]); | ||
| 292 | |||
| 293 | resp = (struct creq_add_gid_resp *) | ||
| 294 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 295 | NULL, 0); | ||
| 296 | if (!resp) { | ||
| 297 | dev_err(&res->pdev->dev, | ||
| 298 | "QPLIB: SP: ADD_GID send failed"); | ||
| 299 | return -EINVAL; | ||
| 300 | } | ||
| 301 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, | ||
| 302 | le16_to_cpu(req.cookie))) { | ||
| 303 | /* Cmd timed out */ | ||
| 304 | dev_err(&res->pdev->dev, | ||
| 305 | "QPIB: SP: ADD_GID timed out"); | ||
| 306 | return -ETIMEDOUT; | ||
| 307 | } | ||
| 308 | if (resp->status || | ||
| 309 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 310 | dev_err(&res->pdev->dev, "QPLIB: SP: ADD_GID failed "); | ||
| 311 | dev_err(&res->pdev->dev, | ||
| 312 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 313 | resp->status, le16_to_cpu(req.cookie), | ||
| 314 | le16_to_cpu(resp->cookie)); | ||
| 315 | return -EINVAL; | ||
| 316 | } | ||
| 317 | sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp->xid); | ||
| 318 | } | ||
| 319 | /* Add GID to the sgid_tbl */ | ||
| 320 | memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid)); | ||
| 321 | sgid_tbl->active++; | ||
| 322 | dev_dbg(&res->pdev->dev, | ||
| 323 | "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x", | ||
| 324 | free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active); | ||
| 325 | |||
| 326 | *index = free_idx; | ||
| 327 | /* unlock */ | ||
| 328 | return rc; | ||
| 329 | } | ||
| 330 | |||
| 331 | /* pkeys */ | ||
| 332 | int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res, | ||
| 333 | struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index, | ||
| 334 | u16 *pkey) | ||
| 335 | { | ||
| 336 | if (index == 0xFFFF) { | ||
| 337 | *pkey = 0xFFFF; | ||
| 338 | return 0; | ||
| 339 | } | ||
| 340 | if (index > pkey_tbl->max) { | ||
| 341 | dev_err(&res->pdev->dev, | ||
| 342 | "QPLIB: Index %d exceeded PKEY table max (%d)", | ||
| 343 | index, pkey_tbl->max); | ||
| 344 | return -EINVAL; | ||
| 345 | } | ||
| 346 | memcpy(pkey, &pkey_tbl->tbl[index], sizeof(*pkey)); | ||
| 347 | return 0; | ||
| 348 | } | ||
| 349 | |||
| 350 | int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res, | ||
| 351 | struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, | ||
| 352 | bool update) | ||
| 353 | { | ||
| 354 | int i, rc = 0; | ||
| 355 | |||
| 356 | if (!pkey_tbl) { | ||
| 357 | dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated"); | ||
| 358 | return -EINVAL; | ||
| 359 | } | ||
| 360 | |||
| 361 | /* Do we need a pkey_lock here? */ | ||
| 362 | if (!pkey_tbl->active) { | ||
| 363 | dev_err(&res->pdev->dev, | ||
| 364 | "QPLIB: PKEY table has no active entries"); | ||
| 365 | return -ENOMEM; | ||
| 366 | } | ||
| 367 | for (i = 0; i < pkey_tbl->max; i++) { | ||
| 368 | if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey))) | ||
| 369 | break; | ||
| 370 | } | ||
| 371 | if (i == pkey_tbl->max) { | ||
| 372 | dev_err(&res->pdev->dev, | ||
| 373 | "QPLIB: PKEY 0x%04x not found in the pkey table", | ||
| 374 | *pkey); | ||
| 375 | return -ENOMEM; | ||
| 376 | } | ||
| 377 | memset(&pkey_tbl->tbl[i], 0, sizeof(*pkey)); | ||
| 378 | pkey_tbl->active--; | ||
| 379 | |||
| 380 | /* unlock */ | ||
| 381 | return rc; | ||
| 382 | } | ||
| 383 | |||
| 384 | int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res, | ||
| 385 | struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, | ||
| 386 | bool update) | ||
| 387 | { | ||
| 388 | int i, free_idx, rc = 0; | ||
| 389 | |||
| 390 | if (!pkey_tbl) { | ||
| 391 | dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated"); | ||
| 392 | return -EINVAL; | ||
| 393 | } | ||
| 394 | |||
| 395 | /* Do we need a pkey_lock here? */ | ||
| 396 | if (pkey_tbl->active == pkey_tbl->max) { | ||
| 397 | dev_err(&res->pdev->dev, "QPLIB: PKEY table is full"); | ||
| 398 | return -ENOMEM; | ||
| 399 | } | ||
| 400 | free_idx = pkey_tbl->max; | ||
| 401 | for (i = 0; i < pkey_tbl->max; i++) { | ||
| 402 | if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey))) | ||
| 403 | return -EALREADY; | ||
| 404 | else if (!pkey_tbl->tbl[i] && free_idx == pkey_tbl->max) | ||
| 405 | free_idx = i; | ||
| 406 | } | ||
| 407 | if (free_idx == pkey_tbl->max) { | ||
| 408 | dev_err(&res->pdev->dev, | ||
| 409 | "QPLIB: PKEY table is FULL but count is not MAX??"); | ||
| 410 | return -ENOMEM; | ||
| 411 | } | ||
| 412 | /* Add PKEY to the pkey_tbl */ | ||
| 413 | memcpy(&pkey_tbl->tbl[free_idx], pkey, sizeof(*pkey)); | ||
| 414 | pkey_tbl->active++; | ||
| 415 | |||
| 416 | /* unlock */ | ||
| 417 | return rc; | ||
| 418 | } | ||
| 419 | |||
| 420 | /* AH */ | ||
| 421 | int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah) | ||
| 422 | { | ||
| 423 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 424 | struct cmdq_create_ah req; | ||
| 425 | struct creq_create_ah_resp *resp; | ||
| 426 | u16 cmd_flags = 0; | ||
| 427 | u32 temp32[4]; | ||
| 428 | u16 temp16[3]; | ||
| 429 | |||
| 430 | RCFW_CMD_PREP(req, CREATE_AH, cmd_flags); | ||
| 431 | |||
| 432 | memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid)); | ||
| 433 | req.dgid[0] = cpu_to_le32(temp32[0]); | ||
| 434 | req.dgid[1] = cpu_to_le32(temp32[1]); | ||
| 435 | req.dgid[2] = cpu_to_le32(temp32[2]); | ||
| 436 | req.dgid[3] = cpu_to_le32(temp32[3]); | ||
| 437 | |||
| 438 | req.type = ah->nw_type; | ||
| 439 | req.hop_limit = ah->hop_limit; | ||
| 440 | req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]); | ||
| 441 | req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label & | ||
| 442 | CMDQ_CREATE_AH_FLOW_LABEL_MASK) | | ||
| 443 | CMDQ_CREATE_AH_DEST_VLAN_ID_MASK); | ||
| 444 | req.pd_id = cpu_to_le32(ah->pd->id); | ||
| 445 | req.traffic_class = ah->traffic_class; | ||
| 446 | |||
| 447 | /* MAC in network format */ | ||
| 448 | memcpy(temp16, ah->dmac, 6); | ||
| 449 | req.dest_mac[0] = cpu_to_le16(temp16[0]); | ||
| 450 | req.dest_mac[1] = cpu_to_le16(temp16[1]); | ||
| 451 | req.dest_mac[2] = cpu_to_le16(temp16[2]); | ||
| 452 | |||
| 453 | resp = (struct creq_create_ah_resp *) | ||
| 454 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 455 | NULL, 1); | ||
| 456 | if (!resp) { | ||
| 457 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH send failed"); | ||
| 458 | return -EINVAL; | ||
| 459 | } | ||
| 460 | if (!bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 461 | /* Cmd timed out */ | ||
| 462 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH timed out"); | ||
| 463 | return -ETIMEDOUT; | ||
| 464 | } | ||
| 465 | if (resp->status || | ||
| 466 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 467 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: CREATE_AH failed "); | ||
| 468 | dev_err(&rcfw->pdev->dev, | ||
| 469 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 470 | resp->status, le16_to_cpu(req.cookie), | ||
| 471 | le16_to_cpu(resp->cookie)); | ||
| 472 | return -EINVAL; | ||
| 473 | } | ||
| 474 | ah->id = le32_to_cpu(resp->xid); | ||
| 475 | return 0; | ||
| 476 | } | ||
| 477 | |||
| 478 | int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah) | ||
| 479 | { | ||
| 480 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 481 | struct cmdq_destroy_ah req; | ||
| 482 | struct creq_destroy_ah_resp *resp; | ||
| 483 | u16 cmd_flags = 0; | ||
| 484 | |||
| 485 | /* Clean up the AH table in the device */ | ||
| 486 | RCFW_CMD_PREP(req, DESTROY_AH, cmd_flags); | ||
| 487 | |||
| 488 | req.ah_cid = cpu_to_le32(ah->id); | ||
| 489 | |||
| 490 | resp = (struct creq_destroy_ah_resp *) | ||
| 491 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 492 | NULL, 1); | ||
| 493 | if (!resp) { | ||
| 494 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH send failed"); | ||
| 495 | return -EINVAL; | ||
| 496 | } | ||
| 497 | if (!bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 498 | /* Cmd timed out */ | ||
| 499 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH timed out"); | ||
| 500 | return -ETIMEDOUT; | ||
| 501 | } | ||
| 502 | if (resp->status || | ||
| 503 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 504 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: DESTROY_AH failed "); | ||
| 505 | dev_err(&rcfw->pdev->dev, | ||
| 506 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 507 | resp->status, le16_to_cpu(req.cookie), | ||
| 508 | le16_to_cpu(resp->cookie)); | ||
| 509 | return -EINVAL; | ||
| 510 | } | ||
| 511 | return 0; | ||
| 512 | } | ||
| 513 | |||
| 514 | /* MRW */ | ||
| 515 | int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) | ||
| 516 | { | ||
| 517 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 518 | struct cmdq_deallocate_key req; | ||
| 519 | struct creq_deallocate_key_resp *resp; | ||
| 520 | u16 cmd_flags = 0; | ||
| 521 | |||
| 522 | if (mrw->lkey == 0xFFFFFFFF) { | ||
| 523 | dev_info(&res->pdev->dev, | ||
| 524 | "QPLIB: SP: Free a reserved lkey MRW"); | ||
| 525 | return 0; | ||
| 526 | } | ||
| 527 | |||
| 528 | RCFW_CMD_PREP(req, DEALLOCATE_KEY, cmd_flags); | ||
| 529 | |||
| 530 | req.mrw_flags = mrw->type; | ||
| 531 | |||
| 532 | if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || | ||
| 533 | (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || | ||
| 534 | (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) | ||
| 535 | req.key = cpu_to_le32(mrw->rkey); | ||
| 536 | else | ||
| 537 | req.key = cpu_to_le32(mrw->lkey); | ||
| 538 | |||
| 539 | resp = (struct creq_deallocate_key_resp *) | ||
| 540 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 541 | NULL, 0); | ||
| 542 | if (!resp) { | ||
| 543 | dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR send failed"); | ||
| 544 | return -EINVAL; | ||
| 545 | } | ||
| 546 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 547 | /* Cmd timed out */ | ||
| 548 | dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR timed out"); | ||
| 549 | return -ETIMEDOUT; | ||
| 550 | } | ||
| 551 | if (resp->status || | ||
| 552 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 553 | dev_err(&res->pdev->dev, "QPLIB: SP: FREE_MR failed "); | ||
| 554 | dev_err(&res->pdev->dev, | ||
| 555 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 556 | resp->status, le16_to_cpu(req.cookie), | ||
| 557 | le16_to_cpu(resp->cookie)); | ||
| 558 | return -EINVAL; | ||
| 559 | } | ||
| 560 | /* Free the qplib's MRW memory */ | ||
| 561 | if (mrw->hwq.max_elements) | ||
| 562 | bnxt_qplib_free_hwq(res->pdev, &mrw->hwq); | ||
| 563 | |||
| 564 | return 0; | ||
| 565 | } | ||
| 566 | |||
| 567 | int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) | ||
| 568 | { | ||
| 569 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 570 | struct cmdq_allocate_mrw req; | ||
| 571 | struct creq_allocate_mrw_resp *resp; | ||
| 572 | u16 cmd_flags = 0; | ||
| 573 | unsigned long tmp; | ||
| 574 | |||
| 575 | RCFW_CMD_PREP(req, ALLOCATE_MRW, cmd_flags); | ||
| 576 | |||
| 577 | req.pd_id = cpu_to_le32(mrw->pd->id); | ||
| 578 | req.mrw_flags = mrw->type; | ||
| 579 | if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR && | ||
| 580 | mrw->flags & BNXT_QPLIB_FR_PMR) || | ||
| 581 | mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A || | ||
| 582 | mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B) | ||
| 583 | req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY; | ||
| 584 | tmp = (unsigned long)mrw; | ||
| 585 | req.mrw_handle = cpu_to_le64(tmp); | ||
| 586 | |||
| 587 | resp = (struct creq_allocate_mrw_resp *) | ||
| 588 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 589 | NULL, 0); | ||
| 590 | if (!resp) { | ||
| 591 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW send failed"); | ||
| 592 | return -EINVAL; | ||
| 593 | } | ||
| 594 | if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) { | ||
| 595 | /* Cmd timed out */ | ||
| 596 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW timed out"); | ||
| 597 | return -ETIMEDOUT; | ||
| 598 | } | ||
| 599 | if (resp->status || | ||
| 600 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 601 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: ALLOC_MRW failed "); | ||
| 602 | dev_err(&rcfw->pdev->dev, | ||
| 603 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 604 | resp->status, le16_to_cpu(req.cookie), | ||
| 605 | le16_to_cpu(resp->cookie)); | ||
| 606 | return -EINVAL; | ||
| 607 | } | ||
| 608 | if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || | ||
| 609 | (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || | ||
| 610 | (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) | ||
| 611 | mrw->rkey = le32_to_cpu(resp->xid); | ||
| 612 | else | ||
| 613 | mrw->lkey = le32_to_cpu(resp->xid); | ||
| 614 | return 0; | ||
| 615 | } | ||
| 616 | |||
| 617 | int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, | ||
| 618 | bool block) | ||
| 619 | { | ||
| 620 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 621 | struct cmdq_deregister_mr req; | ||
| 622 | struct creq_deregister_mr_resp *resp; | ||
| 623 | u16 cmd_flags = 0; | ||
| 624 | int rc; | ||
| 625 | |||
| 626 | RCFW_CMD_PREP(req, DEREGISTER_MR, cmd_flags); | ||
| 627 | |||
| 628 | req.lkey = cpu_to_le32(mrw->lkey); | ||
| 629 | resp = (struct creq_deregister_mr_resp *) | ||
| 630 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 631 | NULL, block); | ||
| 632 | if (!resp) { | ||
| 633 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: DEREG_MR send failed"); | ||
| 634 | return -EINVAL; | ||
| 635 | } | ||
| 636 | if (block) | ||
| 637 | rc = bnxt_qplib_rcfw_block_for_resp(rcfw, | ||
| 638 | le16_to_cpu(req.cookie)); | ||
| 639 | else | ||
| 640 | rc = bnxt_qplib_rcfw_wait_for_resp(rcfw, | ||
| 641 | le16_to_cpu(req.cookie)); | ||
| 642 | if (!rc) { | ||
| 643 | /* Cmd timed out */ | ||
| 644 | dev_err(&res->pdev->dev, "QPLIB: SP: DEREG_MR timed out"); | ||
| 645 | return -ETIMEDOUT; | ||
| 646 | } | ||
| 647 | if (resp->status || | ||
| 648 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 649 | dev_err(&rcfw->pdev->dev, "QPLIB: SP: DEREG_MR failed "); | ||
| 650 | dev_err(&rcfw->pdev->dev, | ||
| 651 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 652 | resp->status, le16_to_cpu(req.cookie), | ||
| 653 | le16_to_cpu(resp->cookie)); | ||
| 654 | return -EINVAL; | ||
| 655 | } | ||
| 656 | |||
| 657 | /* Free the qplib's MR memory */ | ||
| 658 | if (mrw->hwq.max_elements) { | ||
| 659 | mrw->va = 0; | ||
| 660 | mrw->total_size = 0; | ||
| 661 | bnxt_qplib_free_hwq(res->pdev, &mrw->hwq); | ||
| 662 | } | ||
| 663 | |||
| 664 | return 0; | ||
| 665 | } | ||
| 666 | |||
| 667 | int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, | ||
| 668 | u64 *pbl_tbl, int num_pbls, bool block) | ||
| 669 | { | ||
| 670 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 671 | struct cmdq_register_mr req; | ||
| 672 | struct creq_register_mr_resp *resp; | ||
| 673 | u16 cmd_flags = 0, level; | ||
| 674 | int pg_ptrs, pages, i, rc; | ||
| 675 | dma_addr_t **pbl_ptr; | ||
| 676 | u32 pg_size; | ||
| 677 | |||
| 678 | if (num_pbls) { | ||
| 679 | pg_ptrs = roundup_pow_of_two(num_pbls); | ||
| 680 | pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT; | ||
| 681 | if (!pages) | ||
| 682 | pages++; | ||
| 683 | |||
| 684 | if (pages > MAX_PBL_LVL_1_PGS) { | ||
| 685 | dev_err(&res->pdev->dev, "QPLIB: SP: Reg MR pages "); | ||
| 686 | dev_err(&res->pdev->dev, | ||
| 687 | "requested (0x%x) exceeded max (0x%x)", | ||
| 688 | pages, MAX_PBL_LVL_1_PGS); | ||
| 689 | return -ENOMEM; | ||
| 690 | } | ||
| 691 | /* Free the hwq if it already exist, must be a rereg */ | ||
| 692 | if (mr->hwq.max_elements) | ||
| 693 | bnxt_qplib_free_hwq(res->pdev, &mr->hwq); | ||
| 694 | |||
| 695 | mr->hwq.max_elements = pages; | ||
| 696 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL, 0, | ||
| 697 | &mr->hwq.max_elements, | ||
| 698 | PAGE_SIZE, 0, PAGE_SIZE, | ||
| 699 | HWQ_TYPE_CTX); | ||
| 700 | if (rc) { | ||
| 701 | dev_err(&res->pdev->dev, | ||
| 702 | "SP: Reg MR memory allocation failed"); | ||
| 703 | return -ENOMEM; | ||
| 704 | } | ||
| 705 | /* Write to the hwq */ | ||
| 706 | pbl_ptr = (dma_addr_t **)mr->hwq.pbl_ptr; | ||
| 707 | for (i = 0; i < num_pbls; i++) | ||
| 708 | pbl_ptr[PTR_PG(i)][PTR_IDX(i)] = | ||
| 709 | (pbl_tbl[i] & PAGE_MASK) | PTU_PTE_VALID; | ||
| 710 | } | ||
| 711 | |||
| 712 | RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags); | ||
| 713 | |||
| 714 | /* Configure the request */ | ||
| 715 | if (mr->hwq.level == PBL_LVL_MAX) { | ||
| 716 | level = 0; | ||
| 717 | req.pbl = 0; | ||
| 718 | pg_size = PAGE_SIZE; | ||
| 719 | } else { | ||
| 720 | level = mr->hwq.level + 1; | ||
| 721 | req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); | ||
| 722 | pg_size = mr->hwq.pbl[PBL_LVL_0].pg_size; | ||
| 723 | } | ||
| 724 | req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) | | ||
| 725 | ((ilog2(pg_size) << | ||
| 726 | CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) & | ||
| 727 | CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK); | ||
| 728 | req.access = (mr->flags & 0xFFFF); | ||
| 729 | req.va = cpu_to_le64(mr->va); | ||
| 730 | req.key = cpu_to_le32(mr->lkey); | ||
| 731 | req.mr_size = cpu_to_le64(mr->total_size); | ||
| 732 | |||
| 733 | resp = (struct creq_register_mr_resp *) | ||
| 734 | bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, | ||
| 735 | NULL, block); | ||
| 736 | if (!resp) { | ||
| 737 | dev_err(&res->pdev->dev, "SP: REG_MR send failed"); | ||
| 738 | rc = -EINVAL; | ||
| 739 | goto fail; | ||
| 740 | } | ||
| 741 | if (block) | ||
| 742 | rc = bnxt_qplib_rcfw_block_for_resp(rcfw, | ||
| 743 | le16_to_cpu(req.cookie)); | ||
| 744 | else | ||
| 745 | rc = bnxt_qplib_rcfw_wait_for_resp(rcfw, | ||
| 746 | le16_to_cpu(req.cookie)); | ||
| 747 | if (!rc) { | ||
| 748 | /* Cmd timed out */ | ||
| 749 | dev_err(&res->pdev->dev, "SP: REG_MR timed out"); | ||
| 750 | rc = -ETIMEDOUT; | ||
| 751 | goto fail; | ||
| 752 | } | ||
| 753 | if (resp->status || | ||
| 754 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 755 | dev_err(&res->pdev->dev, "QPLIB: SP: REG_MR failed "); | ||
| 756 | dev_err(&res->pdev->dev, | ||
| 757 | "QPLIB: SP: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 758 | resp->status, le16_to_cpu(req.cookie), | ||
| 759 | le16_to_cpu(resp->cookie)); | ||
| 760 | rc = -EINVAL; | ||
| 761 | goto fail; | ||
| 762 | } | ||
| 763 | return 0; | ||
| 764 | |||
| 765 | fail: | ||
| 766 | if (mr->hwq.max_elements) | ||
| 767 | bnxt_qplib_free_hwq(res->pdev, &mr->hwq); | ||
| 768 | return rc; | ||
| 769 | } | ||
| 770 | |||
| 771 | int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, | ||
| 772 | struct bnxt_qplib_frpl *frpl, | ||
| 773 | int max_pg_ptrs) | ||
| 774 | { | ||
| 775 | int pg_ptrs, pages, rc; | ||
| 776 | |||
| 777 | /* Re-calculate the max to fit the HWQ allocation model */ | ||
| 778 | pg_ptrs = roundup_pow_of_two(max_pg_ptrs); | ||
| 779 | pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT; | ||
| 780 | if (!pages) | ||
| 781 | pages++; | ||
| 782 | |||
| 783 | if (pages > MAX_PBL_LVL_1_PGS) | ||
| 784 | return -ENOMEM; | ||
| 785 | |||
| 786 | frpl->hwq.max_elements = pages; | ||
| 787 | rc = bnxt_qplib_alloc_init_hwq(res->pdev, &frpl->hwq, NULL, 0, | ||
| 788 | &frpl->hwq.max_elements, PAGE_SIZE, 0, | ||
| 789 | PAGE_SIZE, HWQ_TYPE_CTX); | ||
| 790 | if (!rc) | ||
| 791 | frpl->max_pg_ptrs = pg_ptrs; | ||
| 792 | |||
| 793 | return rc; | ||
| 794 | } | ||
| 795 | |||
| 796 | int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, | ||
| 797 | struct bnxt_qplib_frpl *frpl) | ||
| 798 | { | ||
| 799 | bnxt_qplib_free_hwq(res->pdev, &frpl->hwq); | ||
| 800 | return 0; | ||
| 801 | } | ||
| 802 | |||
| 803 | int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids) | ||
| 804 | { | ||
| 805 | struct bnxt_qplib_rcfw *rcfw = res->rcfw; | ||
| 806 | struct cmdq_map_tc_to_cos req; | ||
| 807 | struct creq_map_tc_to_cos_resp *resp; | ||
| 808 | u16 cmd_flags = 0; | ||
| 809 | int tleft; | ||
| 810 | |||
| 811 | RCFW_CMD_PREP(req, MAP_TC_TO_COS, cmd_flags); | ||
| 812 | req.cos0 = cpu_to_le16(cids[0]); | ||
| 813 | req.cos1 = cpu_to_le16(cids[1]); | ||
| 814 | |||
| 815 | resp = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, NULL, 0); | ||
| 816 | if (!resp) { | ||
| 817 | dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS send failed"); | ||
| 818 | return -EINVAL; | ||
| 819 | } | ||
| 820 | |||
| 821 | tleft = bnxt_qplib_rcfw_block_for_resp(rcfw, le16_to_cpu(req.cookie)); | ||
| 822 | if (!tleft) { | ||
| 823 | dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS timed out"); | ||
| 824 | return -ETIMEDOUT; | ||
| 825 | } | ||
| 826 | |||
| 827 | if (resp->status || | ||
| 828 | le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) { | ||
| 829 | dev_err(&res->pdev->dev, "QPLIB: SP: MAP_TC2COS failed "); | ||
| 830 | dev_err(&res->pdev->dev, | ||
| 831 | "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x", | ||
| 832 | resp->status, le16_to_cpu(req.cookie), | ||
| 833 | le16_to_cpu(resp->cookie)); | ||
| 834 | return -EINVAL; | ||
| 835 | } | ||
| 836 | |||
| 837 | return 0; | ||
| 838 | } | ||
| @@ -0,0 +1,160 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: Slow Path Operators (header) | ||
| 37 | * | ||
| 38 | */ | ||
| 39 | |||
| 40 | #ifndef __BNXT_QPLIB_SP_H__ | ||
| 41 | #define __BNXT_QPLIB_SP_H__ | ||
| 42 | |||
| 43 | struct bnxt_qplib_dev_attr { | ||
| 44 | char fw_ver[32]; | ||
| 45 | u16 max_sgid; | ||
| 46 | u16 max_mrw; | ||
| 47 | u32 max_qp; | ||
| 48 | #define BNXT_QPLIB_MAX_OUT_RD_ATOM 126 | ||
| 49 | u32 max_qp_rd_atom; | ||
| 50 | u32 max_qp_init_rd_atom; | ||
| 51 | u32 max_qp_wqes; | ||
| 52 | u32 max_qp_sges; | ||
| 53 | u32 max_cq; | ||
| 54 | u32 max_cq_wqes; | ||
| 55 | u32 max_cq_sges; | ||
| 56 | u32 max_mr; | ||
| 57 | u64 max_mr_size; | ||
| 58 | u32 max_pd; | ||
| 59 | u32 max_mw; | ||
| 60 | u32 max_raw_ethy_qp; | ||
| 61 | u32 max_ah; | ||
| 62 | u32 max_fmr; | ||
| 63 | u32 max_map_per_fmr; | ||
| 64 | u32 max_srq; | ||
| 65 | u32 max_srq_wqes; | ||
| 66 | u32 max_srq_sges; | ||
| 67 | u32 max_pkey; | ||
| 68 | u32 max_inline_data; | ||
| 69 | u32 l2_db_size; | ||
| 70 | u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; | ||
| 71 | }; | ||
| 72 | |||
| 73 | struct bnxt_qplib_pd { | ||
| 74 | u32 id; | ||
| 75 | }; | ||
| 76 | |||
| 77 | struct bnxt_qplib_gid { | ||
| 78 | u8 data[16]; | ||
| 79 | }; | ||
| 80 | |||
| 81 | struct bnxt_qplib_ah { | ||
| 82 | struct bnxt_qplib_gid dgid; | ||
| 83 | struct bnxt_qplib_pd *pd; | ||
| 84 | u32 id; | ||
| 85 | u8 sgid_index; | ||
| 86 | /* For Query AH if the hw table and SW table are differnt */ | ||
| 87 | u8 host_sgid_index; | ||
| 88 | u8 traffic_class; | ||
| 89 | u32 flow_label; | ||
| 90 | u8 hop_limit; | ||
| 91 | u8 sl; | ||
| 92 | u8 dmac[6]; | ||
| 93 | u16 vlan_id; | ||
| 94 | u8 nw_type; | ||
| 95 | }; | ||
| 96 | |||
| 97 | struct bnxt_qplib_mrw { | ||
| 98 | struct bnxt_qplib_pd *pd; | ||
| 99 | int type; | ||
| 100 | u32 flags; | ||
| 101 | #define BNXT_QPLIB_FR_PMR 0x80000000 | ||
| 102 | u32 lkey; | ||
| 103 | u32 rkey; | ||
| 104 | #define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF | ||
| 105 | u64 va; | ||
| 106 | u64 total_size; | ||
| 107 | u32 npages; | ||
| 108 | u64 mr_handle; | ||
| 109 | struct bnxt_qplib_hwq hwq; | ||
| 110 | }; | ||
| 111 | |||
| 112 | struct bnxt_qplib_frpl { | ||
| 113 | int max_pg_ptrs; | ||
| 114 | struct bnxt_qplib_hwq hwq; | ||
| 115 | }; | ||
| 116 | |||
| 117 | #define BNXT_QPLIB_ACCESS_LOCAL_WRITE BIT(0) | ||
| 118 | #define BNXT_QPLIB_ACCESS_REMOTE_READ BIT(1) | ||
| 119 | #define BNXT_QPLIB_ACCESS_REMOTE_WRITE BIT(2) | ||
| 120 | #define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC BIT(3) | ||
| 121 | #define BNXT_QPLIB_ACCESS_MW_BIND BIT(4) | ||
| 122 | #define BNXT_QPLIB_ACCESS_ZERO_BASED BIT(5) | ||
| 123 | #define BNXT_QPLIB_ACCESS_ON_DEMAND BIT(6) | ||
| 124 | |||
| 125 | int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, | ||
| 126 | struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, | ||
| 127 | struct bnxt_qplib_gid *gid); | ||
| 128 | int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, | ||
| 129 | struct bnxt_qplib_gid *gid, bool update); | ||
| 130 | int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, | ||
| 131 | struct bnxt_qplib_gid *gid, u8 *mac, u16 vlan_id, | ||
| 132 | bool update, u32 *index); | ||
| 133 | int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res, | ||
| 134 | struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index, | ||
| 135 | u16 *pkey); | ||
| 136 | int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res, | ||
| 137 | struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, | ||
| 138 | bool update); | ||
| 139 | int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res, | ||
| 140 | struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, | ||
| 141 | bool update); | ||
| 142 | int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, | ||
| 143 | struct bnxt_qplib_dev_attr *attr); | ||
| 144 | int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah); | ||
| 145 | int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah); | ||
| 146 | int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, | ||
| 147 | struct bnxt_qplib_mrw *mrw); | ||
| 148 | int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, | ||
| 149 | bool block); | ||
| 150 | int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, | ||
| 151 | u64 *pbl_tbl, int num_pbls, bool block); | ||
| 152 | int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr); | ||
| 153 | int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res, | ||
| 154 | struct bnxt_qplib_mrw *mr, int max); | ||
| 155 | int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, | ||
| 156 | struct bnxt_qplib_frpl *frpl, int max); | ||
| 157 | int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, | ||
| 158 | struct bnxt_qplib_frpl *frpl); | ||
| 159 | int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids); | ||
| 160 | #endif /* __BNXT_QPLIB_SP_H__*/ | ||
| @@ -0,0 +1,2821 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: RoCE HSI File - Autogenerated | ||
| 37 | */ | ||
| 38 | |||
| 39 | #ifndef __BNXT_RE_HSI_H__ | ||
| 40 | #define __BNXT_RE_HSI_H__ | ||
| 41 | |||
| 42 | /* include bnxt_hsi.h from bnxt_en driver */ | ||
| 43 | #include "bnxt_hsi.h" | ||
| 44 | |||
| 45 | /* CMP Door Bell Format (4 bytes) */ | ||
| 46 | struct cmpl_doorbell { | ||
| 47 | __le32 key_mask_valid_idx; | ||
| 48 | #define CMPL_DOORBELL_IDX_MASK 0xffffffUL | ||
| 49 | #define CMPL_DOORBELL_IDX_SFT 0 | ||
| 50 | #define CMPL_DOORBELL_RESERVED_MASK 0x3000000UL | ||
| 51 | #define CMPL_DOORBELL_RESERVED_SFT 24 | ||
| 52 | #define CMPL_DOORBELL_IDX_VALID 0x4000000UL | ||
| 53 | #define CMPL_DOORBELL_MASK 0x8000000UL | ||
| 54 | #define CMPL_DOORBELL_KEY_MASK 0xf0000000UL | ||
| 55 | #define CMPL_DOORBELL_KEY_SFT 28 | ||
| 56 | #define CMPL_DOORBELL_KEY_CMPL (0x2UL << 28) | ||
| 57 | }; | ||
| 58 | |||
| 59 | /* Status Door Bell Format (4 bytes) */ | ||
| 60 | struct status_doorbell { | ||
| 61 | __le32 key_idx; | ||
| 62 | #define STATUS_DOORBELL_IDX_MASK 0xffffffUL | ||
| 63 | #define STATUS_DOORBELL_IDX_SFT 0 | ||
| 64 | #define STATUS_DOORBELL_RESERVED_MASK 0xf000000UL | ||
| 65 | #define STATUS_DOORBELL_RESERVED_SFT 24 | ||
| 66 | #define STATUS_DOORBELL_KEY_MASK 0xf0000000UL | ||
| 67 | #define STATUS_DOORBELL_KEY_SFT 28 | ||
| 68 | #define STATUS_DOORBELL_KEY_STAT (0x3UL << 28) | ||
| 69 | }; | ||
| 70 | |||
| 71 | /* RoCE Host Structures */ | ||
| 72 | |||
| 73 | /* Doorbell Structures */ | ||
| 74 | /* 64b Doorbell Format (8 bytes) */ | ||
| 75 | struct dbr_dbr { | ||
| 76 | __le32 index; | ||
| 77 | #define DBR_DBR_INDEX_MASK 0xfffffUL | ||
| 78 | #define DBR_DBR_INDEX_SFT 0 | ||
| 79 | #define DBR_DBR_RESERVED12_MASK 0xfff00000UL | ||
| 80 | #define DBR_DBR_RESERVED12_SFT 20 | ||
| 81 | __le32 type_xid; | ||
| 82 | #define DBR_DBR_XID_MASK 0xfffffUL | ||
| 83 | #define DBR_DBR_XID_SFT 0 | ||
| 84 | #define DBR_DBR_RESERVED8_MASK 0xff00000UL | ||
| 85 | #define DBR_DBR_RESERVED8_SFT 20 | ||
| 86 | #define DBR_DBR_TYPE_MASK 0xf0000000UL | ||
| 87 | #define DBR_DBR_TYPE_SFT 28 | ||
| 88 | #define DBR_DBR_TYPE_SQ (0x0UL << 28) | ||
| 89 | #define DBR_DBR_TYPE_RQ (0x1UL << 28) | ||
| 90 | #define DBR_DBR_TYPE_SRQ (0x2UL << 28) | ||
| 91 | #define DBR_DBR_TYPE_SRQ_ARM (0x3UL << 28) | ||
| 92 | #define DBR_DBR_TYPE_CQ (0x4UL << 28) | ||
| 93 | #define DBR_DBR_TYPE_CQ_ARMSE (0x5UL << 28) | ||
| 94 | #define DBR_DBR_TYPE_CQ_ARMALL (0x6UL << 28) | ||
| 95 | #define DBR_DBR_TYPE_CQ_ARMENA (0x7UL << 28) | ||
| 96 | #define DBR_DBR_TYPE_SRQ_ARMENA (0x8UL << 28) | ||
| 97 | #define DBR_DBR_TYPE_CQ_CUTOFF_ACK (0x9UL << 28) | ||
| 98 | #define DBR_DBR_TYPE_NULL (0xfUL << 28) | ||
| 99 | }; | ||
| 100 | |||
| 101 | /* 32b Doorbell Format (4 bytes) */ | ||
| 102 | struct dbr_dbr32 { | ||
| 103 | __le32 type_abs_incr_xid; | ||
| 104 | #define DBR_DBR32_XID_MASK 0xfffffUL | ||
| 105 | #define DBR_DBR32_XID_SFT 0 | ||
| 106 | #define DBR_DBR32_RESERVED4_MASK 0xf00000UL | ||
| 107 | #define DBR_DBR32_RESERVED4_SFT 20 | ||
| 108 | #define DBR_DBR32_INCR_MASK 0xf000000UL | ||
| 109 | #define DBR_DBR32_INCR_SFT 24 | ||
| 110 | #define DBR_DBR32_ABS 0x10000000UL | ||
| 111 | #define DBR_DBR32_TYPE_MASK 0xe0000000UL | ||
| 112 | #define DBR_DBR32_TYPE_SFT 29 | ||
| 113 | #define DBR_DBR32_TYPE_SQ (0x0UL << 29) | ||
| 114 | }; | ||
| 115 | |||
| 116 | /* SQ WQE Structures */ | ||
| 117 | /* Base SQ WQE (8 bytes) */ | ||
| 118 | struct sq_base { | ||
| 119 | u8 wqe_type; | ||
| 120 | #define SQ_BASE_WQE_TYPE_SEND 0x0UL | ||
| 121 | #define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD 0x1UL | ||
| 122 | #define SQ_BASE_WQE_TYPE_SEND_W_INVALID 0x2UL | ||
| 123 | #define SQ_BASE_WQE_TYPE_WRITE_WQE 0x4UL | ||
| 124 | #define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD 0x5UL | ||
| 125 | #define SQ_BASE_WQE_TYPE_READ_WQE 0x6UL | ||
| 126 | #define SQ_BASE_WQE_TYPE_ATOMIC_CS 0x8UL | ||
| 127 | #define SQ_BASE_WQE_TYPE_ATOMIC_FA 0xbUL | ||
| 128 | #define SQ_BASE_WQE_TYPE_LOCAL_INVALID 0xcUL | ||
| 129 | #define SQ_BASE_WQE_TYPE_FR_PMR 0xdUL | ||
| 130 | #define SQ_BASE_WQE_TYPE_BIND 0xeUL | ||
| 131 | u8 unused_0[7]; | ||
| 132 | }; | ||
| 133 | |||
| 134 | /* WQE SGE (16 bytes) */ | ||
| 135 | struct sq_sge { | ||
| 136 | __le64 va_or_pa; | ||
| 137 | __le32 l_key; | ||
| 138 | __le32 size; | ||
| 139 | }; | ||
| 140 | |||
| 141 | /* PSN Search Structure (8 bytes) */ | ||
| 142 | struct sq_psn_search { | ||
| 143 | __le32 opcode_start_psn; | ||
| 144 | #define SQ_PSN_SEARCH_START_PSN_MASK 0xffffffUL | ||
| 145 | #define SQ_PSN_SEARCH_START_PSN_SFT 0 | ||
| 146 | #define SQ_PSN_SEARCH_OPCODE_MASK 0xff000000UL | ||
| 147 | #define SQ_PSN_SEARCH_OPCODE_SFT 24 | ||
| 148 | __le32 flags_next_psn; | ||
| 149 | #define SQ_PSN_SEARCH_NEXT_PSN_MASK 0xffffffUL | ||
| 150 | #define SQ_PSN_SEARCH_NEXT_PSN_SFT 0 | ||
| 151 | #define SQ_PSN_SEARCH_FLAGS_MASK 0xff000000UL | ||
| 152 | #define SQ_PSN_SEARCH_FLAGS_SFT 24 | ||
| 153 | }; | ||
| 154 | |||
| 155 | /* Send SQ WQE (40 bytes) */ | ||
| 156 | struct sq_send { | ||
| 157 | u8 wqe_type; | ||
| 158 | #define SQ_SEND_WQE_TYPE_SEND 0x0UL | ||
| 159 | #define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD 0x1UL | ||
| 160 | #define SQ_SEND_WQE_TYPE_SEND_W_INVALID 0x2UL | ||
| 161 | u8 flags; | ||
| 162 | #define SQ_SEND_FLAGS_SIGNAL_COMP 0x1UL | ||
| 163 | #define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | ||
| 164 | #define SQ_SEND_FLAGS_UC_FENCE 0x4UL | ||
| 165 | #define SQ_SEND_FLAGS_SE 0x8UL | ||
| 166 | #define SQ_SEND_FLAGS_INLINE 0x10UL | ||
| 167 | u8 wqe_size; | ||
| 168 | u8 reserved8_1; | ||
| 169 | __le32 inv_key_or_imm_data; | ||
| 170 | __le32 length; | ||
| 171 | __le32 q_key; | ||
| 172 | __le32 dst_qp; | ||
| 173 | #define SQ_SEND_DST_QP_MASK 0xffffffUL | ||
| 174 | #define SQ_SEND_DST_QP_SFT 0 | ||
| 175 | #define SQ_SEND_RESERVED8_2_MASK 0xff000000UL | ||
| 176 | #define SQ_SEND_RESERVED8_2_SFT 24 | ||
| 177 | __le32 avid; | ||
| 178 | #define SQ_SEND_AVID_MASK 0xfffffUL | ||
| 179 | #define SQ_SEND_AVID_SFT 0 | ||
| 180 | #define SQ_SEND_RESERVED_AVID_MASK 0xfff00000UL | ||
| 181 | #define SQ_SEND_RESERVED_AVID_SFT 20 | ||
| 182 | __le64 reserved64; | ||
| 183 | __le32 data[24]; | ||
| 184 | }; | ||
| 185 | |||
| 186 | /* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */ | ||
| 187 | struct sq_send_raweth_qp1 { | ||
| 188 | u8 wqe_type; | ||
| 189 | #define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND 0x0UL | ||
| 190 | u8 flags; | ||
| 191 | #define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP 0x1UL | ||
| 192 | #define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | ||
| 193 | #define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE 0x4UL | ||
| 194 | #define SQ_SEND_RAWETH_QP1_FLAGS_SE 0x8UL | ||
| 195 | #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE 0x10UL | ||
| 196 | u8 wqe_size; | ||
| 197 | u8 reserved8; | ||
| 198 | __le16 lflags; | ||
| 199 | #define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM 0x1UL | ||
| 200 | #define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM 0x2UL | ||
| 201 | #define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC 0x4UL | ||
| 202 | #define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP 0x8UL | ||
| 203 | #define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM 0x10UL | ||
| 204 | #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_1 0x20UL | ||
| 205 | #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_2 0x40UL | ||
| 206 | #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_3 0x80UL | ||
| 207 | #define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC 0x100UL | ||
| 208 | #define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC 0x200UL | ||
| 209 | __le16 cfa_action; | ||
| 210 | __le32 length; | ||
| 211 | __le32 reserved32_1; | ||
| 212 | __le32 cfa_meta; | ||
| 213 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK 0xfffUL | ||
| 214 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT 0 | ||
| 215 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE 0x1000UL | ||
| 216 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK 0xe000UL | ||
| 217 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT 13 | ||
| 218 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK 0x70000UL | ||
| 219 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT 16 | ||
| 220 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 (0x0UL << 16) | ||
| 221 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 (0x1UL << 16) | ||
| 222 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 (0x2UL << 16) | ||
| 223 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 (0x3UL << 16) | ||
| 224 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 (0x4UL << 16) | ||
| 225 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG (0x5UL << 16) | ||
| 226 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST \ | ||
| 227 | SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG | ||
| 228 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK 0xff80000UL | ||
| 229 | #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19 | ||
| 230 | #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK 0xf0000000UL | ||
| 231 | #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT 28 | ||
| 232 | #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE (0x0UL << 28) | ||
| 233 | #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG (0x1UL << 28) | ||
| 234 | #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST \ | ||
| 235 | SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG | ||
| 236 | __le32 reserved32_2; | ||
| 237 | __le64 reserved64; | ||
| 238 | __le32 data[24]; | ||
| 239 | }; | ||
| 240 | |||
| 241 | /* RDMA SQ WQE (40 bytes) */ | ||
| 242 | struct sq_rdma { | ||
| 243 | u8 wqe_type; | ||
| 244 | #define SQ_RDMA_WQE_TYPE_WRITE_WQE 0x4UL | ||
| 245 | #define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD 0x5UL | ||
| 246 | #define SQ_RDMA_WQE_TYPE_READ_WQE 0x6UL | ||
| 247 | u8 flags; | ||
| 248 | #define SQ_RDMA_FLAGS_SIGNAL_COMP 0x1UL | ||
| 249 | #define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | ||
| 250 | #define SQ_RDMA_FLAGS_UC_FENCE 0x4UL | ||
| 251 | #define SQ_RDMA_FLAGS_SE 0x8UL | ||
| 252 | #define SQ_RDMA_FLAGS_INLINE 0x10UL | ||
| 253 | u8 wqe_size; | ||
| 254 | u8 reserved8; | ||
| 255 | __le32 imm_data; | ||
| 256 | __le32 length; | ||
| 257 | __le32 reserved32_1; | ||
| 258 | __le64 remote_va; | ||
| 259 | __le32 remote_key; | ||
| 260 | __le32 reserved32_2; | ||
| 261 | __le32 data[24]; | ||
| 262 | }; | ||
| 263 | |||
| 264 | /* Atomic SQ WQE (40 bytes) */ | ||
| 265 | struct sq_atomic { | ||
| 266 | u8 wqe_type; | ||
| 267 | #define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS 0x8UL | ||
| 268 | #define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA 0xbUL | ||
| 269 | u8 flags; | ||
| 270 | #define SQ_ATOMIC_FLAGS_SIGNAL_COMP 0x1UL | ||
| 271 | #define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | ||
| 272 | #define SQ_ATOMIC_FLAGS_UC_FENCE 0x4UL | ||
| 273 | #define SQ_ATOMIC_FLAGS_SE 0x8UL | ||
| 274 | #define SQ_ATOMIC_FLAGS_INLINE 0x10UL | ||
| 275 | __le16 reserved16; | ||
| 276 | __le32 remote_key; | ||
| 277 | __le64 remote_va; | ||
| 278 | __le64 swap_data; | ||
| 279 | __le64 cmp_data; | ||
| 280 | __le32 data[24]; | ||
| 281 | }; | ||
| 282 | |||
| 283 | /* Local Invalidate SQ WQE (40 bytes) */ | ||
| 284 | struct sq_localinvalidate { | ||
| 285 | u8 wqe_type; | ||
| 286 | #define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID 0xcUL | ||
| 287 | u8 flags; | ||
| 288 | #define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP 0x1UL | ||
| 289 | #define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | ||
| 290 | #define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE 0x4UL | ||
| 291 | #define SQ_LOCALINVALIDATE_FLAGS_SE 0x8UL | ||
| 292 | #define SQ_LOCALINVALIDATE_FLAGS_INLINE 0x10UL | ||
| 293 | __le16 reserved16; | ||
| 294 | __le32 inv_l_key; | ||
| 295 | __le64 reserved64; | ||
| 296 | __le32 reserved128[4]; | ||
| 297 | __le32 data[24]; | ||
| 298 | }; | ||
| 299 | |||
| 300 | /* FR-PMR SQ WQE (40 bytes) */ | ||
| 301 | struct sq_fr_pmr { | ||
| 302 | u8 wqe_type; | ||
| 303 | #define SQ_FR_PMR_WQE_TYPE_FR_PMR 0xdUL | ||
| 304 | u8 flags; | ||
| 305 | #define SQ_FR_PMR_FLAGS_SIGNAL_COMP 0x1UL | ||
| 306 | #define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | ||
| 307 | #define SQ_FR_PMR_FLAGS_UC_FENCE 0x4UL | ||
| 308 | #define SQ_FR_PMR_FLAGS_SE 0x8UL | ||
| 309 | #define SQ_FR_PMR_FLAGS_INLINE 0x10UL | ||
| 310 | u8 access_cntl; | ||
| 311 | #define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE 0x1UL | ||
| 312 | #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ 0x2UL | ||
| 313 | #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE 0x4UL | ||
| 314 | #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL | ||
| 315 | #define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND 0x10UL | ||
| 316 | u8 zero_based_page_size_log; | ||
| 317 | #define SQ_FR_PMR_PAGE_SIZE_LOG_MASK 0x1fUL | ||
| 318 | #define SQ_FR_PMR_PAGE_SIZE_LOG_SFT 0 | ||
| 319 | #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K 0x0UL | ||
| 320 | #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K 0x1UL | ||
| 321 | #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K 0x4UL | ||
| 322 | #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K 0x6UL | ||
| 323 | #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M 0x8UL | ||
| 324 | #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M 0x9UL | ||
| 325 | #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M 0xaUL | ||
| 326 | #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G 0x12UL | ||
| 327 | #define SQ_FR_PMR_ZERO_BASED 0x20UL | ||
| 328 | #define SQ_FR_PMR_RESERVED2_MASK 0xc0UL | ||
| 329 | #define SQ_FR_PMR_RESERVED2_SFT 6 | ||
| 330 | __le32 l_key; | ||
| 331 | u8 length[5]; | ||
| 332 | u8 reserved8_1; | ||
| 333 | u8 reserved8_2; | ||
| 334 | u8 numlevels_pbl_page_size_log; | ||
| 335 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK 0x1fUL | ||
| 336 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT 0 | ||
| 337 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K 0x0UL | ||
| 338 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K 0x1UL | ||
| 339 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K 0x4UL | ||
| 340 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K 0x6UL | ||
| 341 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M 0x8UL | ||
| 342 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M 0x9UL | ||
| 343 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M 0xaUL | ||
| 344 | #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G 0x12UL | ||
| 345 | #define SQ_FR_PMR_RESERVED1 0x20UL | ||
| 346 | #define SQ_FR_PMR_NUMLEVELS_MASK 0xc0UL | ||
| 347 | #define SQ_FR_PMR_NUMLEVELS_SFT 6 | ||
| 348 | #define SQ_FR_PMR_NUMLEVELS_PHYSICAL (0x0UL << 6) | ||
| 349 | #define SQ_FR_PMR_NUMLEVELS_LAYER1 (0x1UL << 6) | ||
| 350 | #define SQ_FR_PMR_NUMLEVELS_LAYER2 (0x2UL << 6) | ||
| 351 | __le64 pblptr; | ||
| 352 | __le64 va; | ||
| 353 | __le32 data[24]; | ||
| 354 | }; | ||
| 355 | |||
| 356 | /* Bind SQ WQE (40 bytes) */ | ||
| 357 | struct sq_bind { | ||
| 358 | u8 wqe_type; | ||
| 359 | #define SQ_BIND_WQE_TYPE_BIND 0xeUL | ||
| 360 | u8 flags; | ||
| 361 | #define SQ_BIND_FLAGS_SIGNAL_COMP 0x1UL | ||
| 362 | #define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL | ||
| 363 | #define SQ_BIND_FLAGS_UC_FENCE 0x4UL | ||
| 364 | #define SQ_BIND_FLAGS_SE 0x8UL | ||
| 365 | #define SQ_BIND_FLAGS_INLINE 0x10UL | ||
| 366 | u8 access_cntl; | ||
| 367 | #define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE 0x1UL | ||
| 368 | #define SQ_BIND_ACCESS_CNTL_REMOTE_READ 0x2UL | ||
| 369 | #define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE 0x4UL | ||
| 370 | #define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL | ||
| 371 | #define SQ_BIND_ACCESS_CNTL_WINDOW_BIND 0x10UL | ||
| 372 | u8 reserved8_1; | ||
| 373 | u8 mw_type_zero_based; | ||
| 374 | #define SQ_BIND_ZERO_BASED 0x1UL | ||
| 375 | #define SQ_BIND_MW_TYPE 0x2UL | ||
| 376 | #define SQ_BIND_MW_TYPE_TYPE1 (0x0UL << 1) | ||
| 377 | #define SQ_BIND_MW_TYPE_TYPE2 (0x1UL << 1) | ||
| 378 | #define SQ_BIND_RESERVED6_MASK 0xfcUL | ||
| 379 | #define SQ_BIND_RESERVED6_SFT 2 | ||
| 380 | u8 reserved8_2; | ||
| 381 | __le16 reserved16; | ||
| 382 | __le32 parent_l_key; | ||
| 383 | __le32 l_key; | ||
| 384 | __le64 va; | ||
| 385 | u8 length[5]; | ||
| 386 | u8 data_reserved24[99]; | ||
| 387 | #define SQ_BIND_RESERVED24_MASK 0xffffff00UL | ||
| 388 | #define SQ_BIND_RESERVED24_SFT 8 | ||
| 389 | #define SQ_BIND_DATA_MASK 0xffffffffUL | ||
| 390 | #define SQ_BIND_DATA_SFT 0 | ||
| 391 | }; | ||
| 392 | |||
| 393 | /* RQ/SRQ WQE Structures */ | ||
| 394 | /* RQ/SRQ WQE (40 bytes) */ | ||
| 395 | struct rq_wqe { | ||
| 396 | u8 wqe_type; | ||
| 397 | #define RQ_WQE_WQE_TYPE_RCV 0x80UL | ||
| 398 | u8 flags; | ||
| 399 | u8 wqe_size; | ||
| 400 | u8 reserved8; | ||
| 401 | __le32 reserved32; | ||
| 402 | __le32 wr_id[2]; | ||
| 403 | #define RQ_WQE_WR_ID_MASK 0xfffffUL | ||
| 404 | #define RQ_WQE_WR_ID_SFT 0 | ||
| 405 | #define RQ_WQE_RESERVED44_MASK 0xfff00000UL | ||
| 406 | #define RQ_WQE_RESERVED44_SFT 20 | ||
| 407 | __le32 reserved128[4]; | ||
| 408 | __le32 data[24]; | ||
| 409 | }; | ||
| 410 | |||
| 411 | /* CQ CQE Structures */ | ||
| 412 | /* Base CQE (32 bytes) */ | ||
| 413 | struct cq_base { | ||
| 414 | __le64 reserved64_1; | ||
| 415 | __le64 reserved64_2; | ||
| 416 | __le64 reserved64_3; | ||
| 417 | u8 cqe_type_toggle; | ||
| 418 | #define CQ_BASE_TOGGLE 0x1UL | ||
| 419 | #define CQ_BASE_CQE_TYPE_MASK 0x1eUL | ||
| 420 | #define CQ_BASE_CQE_TYPE_SFT 1 | ||
| 421 | #define CQ_BASE_CQE_TYPE_REQ (0x0UL << 1) | ||
| 422 | #define CQ_BASE_CQE_TYPE_RES_RC (0x1UL << 1) | ||
| 423 | #define CQ_BASE_CQE_TYPE_RES_UD (0x2UL << 1) | ||
| 424 | #define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1) | ||
| 425 | #define CQ_BASE_CQE_TYPE_TERMINAL (0xeUL << 1) | ||
| 426 | #define CQ_BASE_CQE_TYPE_CUT_OFF (0xfUL << 1) | ||
| 427 | #define CQ_BASE_RESERVED3_MASK 0xe0UL | ||
| 428 | #define CQ_BASE_RESERVED3_SFT 5 | ||
| 429 | u8 status; | ||
| 430 | __le16 reserved16; | ||
| 431 | __le32 reserved32; | ||
| 432 | }; | ||
| 433 | |||
| 434 | /* Requester CQ CQE (32 bytes) */ | ||
| 435 | struct cq_req { | ||
| 436 | __le64 qp_handle; | ||
| 437 | __le16 sq_cons_idx; | ||
| 438 | __le16 reserved16_1; | ||
| 439 | __le32 reserved32_2; | ||
| 440 | __le64 reserved64; | ||
| 441 | u8 cqe_type_toggle; | ||
| 442 | #define CQ_REQ_TOGGLE 0x1UL | ||
| 443 | #define CQ_REQ_CQE_TYPE_MASK 0x1eUL | ||
| 444 | #define CQ_REQ_CQE_TYPE_SFT 1 | ||
| 445 | #define CQ_REQ_CQE_TYPE_REQ (0x0UL << 1) | ||
| 446 | #define CQ_REQ_RESERVED3_MASK 0xe0UL | ||
| 447 | #define CQ_REQ_RESERVED3_SFT 5 | ||
| 448 | u8 status; | ||
| 449 | #define CQ_REQ_STATUS_OK 0x0UL | ||
| 450 | #define CQ_REQ_STATUS_BAD_RESPONSE_ERR 0x1UL | ||
| 451 | #define CQ_REQ_STATUS_LOCAL_LENGTH_ERR 0x2UL | ||
| 452 | #define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR 0x3UL | ||
| 453 | #define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR 0x4UL | ||
| 454 | #define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL | ||
| 455 | #define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL | ||
| 456 | #define CQ_REQ_STATUS_REMOTE_ACCESS_ERR 0x7UL | ||
| 457 | #define CQ_REQ_STATUS_REMOTE_OPERATION_ERR 0x8UL | ||
| 458 | #define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR 0x9UL | ||
| 459 | #define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR 0xaUL | ||
| 460 | #define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR 0xbUL | ||
| 461 | __le16 reserved16_2; | ||
| 462 | __le32 reserved32_1; | ||
| 463 | }; | ||
| 464 | |||
| 465 | /* Responder RC CQE (32 bytes) */ | ||
| 466 | struct cq_res_rc { | ||
| 467 | __le32 length; | ||
| 468 | __le32 imm_data_or_inv_r_key; | ||
| 469 | __le64 qp_handle; | ||
| 470 | __le64 mr_handle; | ||
| 471 | u8 cqe_type_toggle; | ||
| 472 | #define CQ_RES_RC_TOGGLE 0x1UL | ||
| 473 | #define CQ_RES_RC_CQE_TYPE_MASK 0x1eUL | ||
| 474 | #define CQ_RES_RC_CQE_TYPE_SFT 1 | ||
| 475 | #define CQ_RES_RC_CQE_TYPE_RES_RC (0x1UL << 1) | ||
| 476 | #define CQ_RES_RC_RESERVED3_MASK 0xe0UL | ||
| 477 | #define CQ_RES_RC_RESERVED3_SFT 5 | ||
| 478 | u8 status; | ||
| 479 | #define CQ_RES_RC_STATUS_OK 0x0UL | ||
| 480 | #define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR 0x1UL | ||
| 481 | #define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR 0x2UL | ||
| 482 | #define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR 0x3UL | ||
| 483 | #define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL | ||
| 484 | #define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL | ||
| 485 | #define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL | ||
| 486 | #define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL | ||
| 487 | #define CQ_RES_RC_STATUS_HW_FLUSH_ERR 0x8UL | ||
| 488 | __le16 flags; | ||
| 489 | #define CQ_RES_RC_FLAGS_SRQ 0x1UL | ||
| 490 | #define CQ_RES_RC_FLAGS_SRQ_RQ (0x0UL << 0) | ||
| 491 | #define CQ_RES_RC_FLAGS_SRQ_SRQ (0x1UL << 0) | ||
| 492 | #define CQ_RES_RC_FLAGS_SRQ_LAST CQ_RES_RC_FLAGS_SRQ_SRQ | ||
| 493 | #define CQ_RES_RC_FLAGS_IMM 0x2UL | ||
| 494 | #define CQ_RES_RC_FLAGS_INV 0x4UL | ||
| 495 | #define CQ_RES_RC_FLAGS_RDMA 0x8UL | ||
| 496 | #define CQ_RES_RC_FLAGS_RDMA_SEND (0x0UL << 3) | ||
| 497 | #define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE (0x1UL << 3) | ||
| 498 | #define CQ_RES_RC_FLAGS_RDMA_LAST CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE | ||
| 499 | __le32 srq_or_rq_wr_id; | ||
| 500 | #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL | ||
| 501 | #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0 | ||
| 502 | #define CQ_RES_RC_RESERVED12_MASK 0xfff00000UL | ||
| 503 | #define CQ_RES_RC_RESERVED12_SFT 20 | ||
| 504 | }; | ||
| 505 | |||
| 506 | /* Responder UD CQE (32 bytes) */ | ||
| 507 | struct cq_res_ud { | ||
| 508 | __le32 length; | ||
| 509 | #define CQ_RES_UD_LENGTH_MASK 0x3fffUL | ||
| 510 | #define CQ_RES_UD_LENGTH_SFT 0 | ||
| 511 | #define CQ_RES_UD_RESERVED18_MASK 0xffffc000UL | ||
| 512 | #define CQ_RES_UD_RESERVED18_SFT 14 | ||
| 513 | __le32 imm_data; | ||
| 514 | __le64 qp_handle; | ||
| 515 | __le16 src_mac[3]; | ||
| 516 | __le16 src_qp_low; | ||
| 517 | u8 cqe_type_toggle; | ||
| 518 | #define CQ_RES_UD_TOGGLE 0x1UL | ||
| 519 | #define CQ_RES_UD_CQE_TYPE_MASK 0x1eUL | ||
| 520 | #define CQ_RES_UD_CQE_TYPE_SFT 1 | ||
| 521 | #define CQ_RES_UD_CQE_TYPE_RES_UD (0x2UL << 1) | ||
| 522 | #define CQ_RES_UD_RESERVED3_MASK 0xe0UL | ||
| 523 | #define CQ_RES_UD_RESERVED3_SFT 5 | ||
| 524 | u8 status; | ||
| 525 | #define CQ_RES_UD_STATUS_OK 0x0UL | ||
| 526 | #define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR 0x1UL | ||
| 527 | #define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL | ||
| 528 | #define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR 0x3UL | ||
| 529 | #define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL | ||
| 530 | #define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL | ||
| 531 | #define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL | ||
| 532 | #define CQ_RES_UD_STATUS_HW_FLUSH_ERR 0x8UL | ||
| 533 | __le16 flags; | ||
| 534 | #define CQ_RES_UD_FLAGS_SRQ 0x1UL | ||
| 535 | #define CQ_RES_UD_FLAGS_SRQ_RQ (0x0UL << 0) | ||
| 536 | #define CQ_RES_UD_FLAGS_SRQ_SRQ (0x1UL << 0) | ||
| 537 | #define CQ_RES_UD_FLAGS_SRQ_LAST CQ_RES_UD_FLAGS_SRQ_SRQ | ||
| 538 | #define CQ_RES_UD_FLAGS_IMM 0x2UL | ||
| 539 | #define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK 0xcUL | ||
| 540 | #define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT 2 | ||
| 541 | #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 (0x0UL << 2) | ||
| 542 | #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 2) | ||
| 543 | #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 2) | ||
| 544 | #define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST \ | ||
| 545 | CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 | ||
| 546 | __le32 src_qp_high_srq_or_rq_wr_id; | ||
| 547 | #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL | ||
| 548 | #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0 | ||
| 549 | #define CQ_RES_UD_RESERVED4_MASK 0xf00000UL | ||
| 550 | #define CQ_RES_UD_RESERVED4_SFT 20 | ||
| 551 | #define CQ_RES_UD_SRC_QP_HIGH_MASK 0xff000000UL | ||
| 552 | #define CQ_RES_UD_SRC_QP_HIGH_SFT 24 | ||
| 553 | }; | ||
| 554 | |||
| 555 | /* Responder RawEth and QP1 CQE (32 bytes) */ | ||
| 556 | struct cq_res_raweth_qp1 { | ||
| 557 | __le16 length; | ||
| 558 | #define CQ_RES_RAWETH_QP1_LENGTH_MASK 0x3fffUL | ||
| 559 | #define CQ_RES_RAWETH_QP1_LENGTH_SFT 0 | ||
| 560 | #define CQ_RES_RAWETH_QP1_RESERVED2_MASK 0xc000UL | ||
| 561 | #define CQ_RES_RAWETH_QP1_RESERVED2_SFT 14 | ||
| 562 | __le16 raweth_qp1_flags; | ||
| 563 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR 0x1UL | ||
| 564 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_MASK 0x3eUL | ||
| 565 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_SFT 1 | ||
| 566 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK 0x3c0UL | ||
| 567 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT 6 | ||
| 568 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL << 6) | ||
| 569 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP (0x1UL << 6) | ||
| 570 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP (0x2UL << 6) | ||
| 571 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP (0x3UL << 6) | ||
| 572 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE (0x4UL << 6) | ||
| 573 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE (0x5UL << 6) | ||
| 574 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP (0x7UL << 6) | ||
| 575 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP \ | ||
| 576 | (0x8UL << 6) | ||
| 577 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP \ | ||
| 578 | (0x9UL << 6) | ||
| 579 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST \ | ||
| 580 | CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP | ||
| 581 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK 0x3ffUL | ||
| 582 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT 0 | ||
| 583 | #define CQ_RES_RAWETH_QP1_RESERVED6_MASK 0xfc00UL | ||
| 584 | #define CQ_RES_RAWETH_QP1_RESERVED6_SFT 10 | ||
| 585 | __le16 raweth_qp1_errors; | ||
| 586 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_MASK 0xfUL | ||
| 587 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_SFT 0 | ||
| 588 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR 0x10UL | ||
| 589 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR 0x20UL | ||
| 590 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL | ||
| 591 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL | ||
| 592 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR 0x100UL | ||
| 593 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL | ||
| 594 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9 | ||
| 595 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR \ | ||
| 596 | (0x0UL << 9) | ||
| 597 | #define \ | ||
| 598 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \ | ||
| 599 | (0x1UL << 9) | ||
| 600 | #define \ | ||
| 601 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \ | ||
| 602 | (0x2UL << 9) | ||
| 603 | #define \ | ||
| 604 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \ | ||
| 605 | (0x3UL << 9) | ||
| 606 | #define \ | ||
| 607 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \ | ||
| 608 | (0x4UL << 9) | ||
| 609 | #define \ | ||
| 610 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \ | ||
| 611 | (0x5UL << 9) | ||
| 612 | #define \ | ||
| 613 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \ | ||
| 614 | (0x6UL << 9) | ||
| 615 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \ | ||
| 616 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL | ||
| 617 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL | ||
| 618 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12 | ||
| 619 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR \ | ||
| 620 | (0x0UL << 12) | ||
| 621 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION \ | ||
| 622 | (0x1UL << 12) | ||
| 623 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \ | ||
| 624 | (0x2UL << 12) | ||
| 625 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL \ | ||
| 626 | (0x3UL << 12) | ||
| 627 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \ | ||
| 628 | (0x4UL << 12) | ||
| 629 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \ | ||
| 630 | (0x5UL << 12) | ||
| 631 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \ | ||
| 632 | (0x6UL << 12) | ||
| 633 | #define \ | ||
| 634 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL\ | ||
| 635 | (0x7UL << 12) | ||
| 636 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \ | ||
| 637 | (0x8UL << 12) | ||
| 638 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \ | ||
| 639 | CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN | ||
| 640 | __le16 raweth_qp1_cfa_code; | ||
| 641 | __le64 qp_handle; | ||
| 642 | __le32 raweth_qp1_flags2; | ||
| 643 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC 0x1UL | ||
| 644 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC 0x2UL | ||
| 645 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC 0x4UL | ||
| 646 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC 0x8UL | ||
| 647 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL | ||
| 648 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4 | ||
| 649 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE \ | ||
| 650 | (0x0UL << 4) | ||
| 651 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN \ | ||
| 652 | (0x1UL << 4) | ||
| 653 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST\ | ||
| 654 | CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN | ||
| 655 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE 0x100UL | ||
| 656 | __le32 raweth_qp1_metadata; | ||
| 657 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK 0xfffUL | ||
| 658 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT 0 | ||
| 659 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE 0x1000UL | ||
| 660 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK 0xe000UL | ||
| 661 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT 13 | ||
| 662 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK 0xffff0000UL | ||
| 663 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT 16 | ||
| 664 | u8 cqe_type_toggle; | ||
| 665 | #define CQ_RES_RAWETH_QP1_TOGGLE 0x1UL | ||
| 666 | #define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK 0x1eUL | ||
| 667 | #define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT 1 | ||
| 668 | #define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1) | ||
| 669 | #define CQ_RES_RAWETH_QP1_RESERVED3_MASK 0xe0UL | ||
| 670 | #define CQ_RES_RAWETH_QP1_RESERVED3_SFT 5 | ||
| 671 | u8 status; | ||
| 672 | #define CQ_RES_RAWETH_QP1_STATUS_OK 0x0UL | ||
| 673 | #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR 0x1UL | ||
| 674 | #define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL | ||
| 675 | #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR 0x3UL | ||
| 676 | #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL | ||
| 677 | #define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL | ||
| 678 | #define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL | ||
| 679 | #define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR 0x8UL | ||
| 680 | __le16 flags; | ||
| 681 | #define CQ_RES_RAWETH_QP1_FLAGS_SRQ 0x1UL | ||
| 682 | #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ 0x0UL | ||
| 683 | #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ 0x1UL | ||
| 684 | #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST \ | ||
| 685 | CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ | ||
| 686 | __le32 raweth_qp1_payload_offset_srq_or_rq_wr_id; | ||
| 687 | #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL | ||
| 688 | #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT 0 | ||
| 689 | #define CQ_RES_RAWETH_QP1_RESERVED4_MASK 0xf00000UL | ||
| 690 | #define CQ_RES_RAWETH_QP1_RESERVED4_SFT 20 | ||
| 691 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL | ||
| 692 | #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24 | ||
| 693 | }; | ||
| 694 | |||
| 695 | /* Terminal CQE (32 bytes) */ | ||
| 696 | struct cq_terminal { | ||
| 697 | __le64 qp_handle; | ||
| 698 | __le16 sq_cons_idx; | ||
| 699 | __le16 rq_cons_idx; | ||
| 700 | __le32 reserved32_1; | ||
| 701 | __le64 reserved64_3; | ||
| 702 | u8 cqe_type_toggle; | ||
| 703 | #define CQ_TERMINAL_TOGGLE 0x1UL | ||
| 704 | #define CQ_TERMINAL_CQE_TYPE_MASK 0x1eUL | ||
| 705 | #define CQ_TERMINAL_CQE_TYPE_SFT 1 | ||
| 706 | #define CQ_TERMINAL_CQE_TYPE_TERMINAL (0xeUL << 1) | ||
| 707 | #define CQ_TERMINAL_RESERVED3_MASK 0xe0UL | ||
| 708 | #define CQ_TERMINAL_RESERVED3_SFT 5 | ||
| 709 | u8 status; | ||
| 710 | #define CQ_TERMINAL_STATUS_OK 0x0UL | ||
| 711 | __le16 reserved16; | ||
| 712 | __le32 reserved32_2; | ||
| 713 | }; | ||
| 714 | |||
| 715 | /* Cutoff CQE (32 bytes) */ | ||
| 716 | struct cq_cutoff { | ||
| 717 | __le64 reserved64_1; | ||
| 718 | __le64 reserved64_2; | ||
| 719 | __le64 reserved64_3; | ||
| 720 | u8 cqe_type_toggle; | ||
| 721 | #define CQ_CUTOFF_TOGGLE 0x1UL | ||
| 722 | #define CQ_CUTOFF_CQE_TYPE_MASK 0x1eUL | ||
| 723 | #define CQ_CUTOFF_CQE_TYPE_SFT 1 | ||
| 724 | #define CQ_CUTOFF_CQE_TYPE_CUT_OFF (0xfUL << 1) | ||
| 725 | #define CQ_CUTOFF_RESERVED3_MASK 0xe0UL | ||
| 726 | #define CQ_CUTOFF_RESERVED3_SFT 5 | ||
| 727 | u8 status; | ||
| 728 | #define CQ_CUTOFF_STATUS_OK 0x0UL | ||
| 729 | __le16 reserved16; | ||
| 730 | __le32 reserved32; | ||
| 731 | }; | ||
| 732 | |||
| 733 | /* Notification Queue (NQ) Structures */ | ||
| 734 | /* Base NQ Record (16 bytes) */ | ||
| 735 | struct nq_base { | ||
| 736 | __le16 info10_type; | ||
| 737 | #define NQ_BASE_TYPE_MASK 0x3fUL | ||
| 738 | #define NQ_BASE_TYPE_SFT 0 | ||
| 739 | #define NQ_BASE_TYPE_CQ_NOTIFICATION 0x30UL | ||
| 740 | #define NQ_BASE_TYPE_SRQ_EVENT 0x32UL | ||
| 741 | #define NQ_BASE_TYPE_DBQ_EVENT 0x34UL | ||
| 742 | #define NQ_BASE_TYPE_QP_EVENT 0x38UL | ||
| 743 | #define NQ_BASE_TYPE_FUNC_EVENT 0x3aUL | ||
| 744 | #define NQ_BASE_INFO10_MASK 0xffc0UL | ||
| 745 | #define NQ_BASE_INFO10_SFT 6 | ||
| 746 | __le16 info16; | ||
| 747 | __le32 info32; | ||
| 748 | __le32 info63_v[2]; | ||
| 749 | #define NQ_BASE_V 0x1UL | ||
| 750 | #define NQ_BASE_INFO63_MASK 0xfffffffeUL | ||
| 751 | #define NQ_BASE_INFO63_SFT 1 | ||
| 752 | }; | ||
| 753 | |||
| 754 | /* Completion Queue Notification (16 bytes) */ | ||
| 755 | struct nq_cn { | ||
| 756 | __le16 type; | ||
| 757 | #define NQ_CN_TYPE_MASK 0x3fUL | ||
| 758 | #define NQ_CN_TYPE_SFT 0 | ||
| 759 | #define NQ_CN_TYPE_CQ_NOTIFICATION 0x30UL | ||
| 760 | #define NQ_CN_RESERVED9_MASK 0xffc0UL | ||
| 761 | #define NQ_CN_RESERVED9_SFT 6 | ||
| 762 | __le16 reserved16; | ||
| 763 | __le32 cq_handle_low; | ||
| 764 | __le32 v; | ||
| 765 | #define NQ_CN_V 0x1UL | ||
| 766 | #define NQ_CN_RESERVED31_MASK 0xfffffffeUL | ||
| 767 | #define NQ_CN_RESERVED31_SFT 1 | ||
| 768 | __le32 cq_handle_high; | ||
| 769 | }; | ||
| 770 | |||
| 771 | /* SRQ Event Notification (16 bytes) */ | ||
| 772 | struct nq_srq_event { | ||
| 773 | u8 type; | ||
| 774 | #define NQ_SRQ_EVENT_TYPE_MASK 0x3fUL | ||
| 775 | #define NQ_SRQ_EVENT_TYPE_SFT 0 | ||
| 776 | #define NQ_SRQ_EVENT_TYPE_SRQ_EVENT 0x32UL | ||
| 777 | #define NQ_SRQ_EVENT_RESERVED1_MASK 0xc0UL | ||
| 778 | #define NQ_SRQ_EVENT_RESERVED1_SFT 6 | ||
| 779 | u8 event; | ||
| 780 | #define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT 0x1UL | ||
| 781 | __le16 reserved16; | ||
| 782 | __le32 srq_handle_low; | ||
| 783 | __le32 v; | ||
| 784 | #define NQ_SRQ_EVENT_V 0x1UL | ||
| 785 | #define NQ_SRQ_EVENT_RESERVED31_MASK 0xfffffffeUL | ||
| 786 | #define NQ_SRQ_EVENT_RESERVED31_SFT 1 | ||
| 787 | __le32 srq_handle_high; | ||
| 788 | }; | ||
| 789 | |||
| 790 | /* DBQ Async Event Notification (16 bytes) */ | ||
| 791 | struct nq_dbq_event { | ||
| 792 | u8 type; | ||
| 793 | #define NQ_DBQ_EVENT_TYPE_MASK 0x3fUL | ||
| 794 | #define NQ_DBQ_EVENT_TYPE_SFT 0 | ||
| 795 | #define NQ_DBQ_EVENT_TYPE_DBQ_EVENT 0x34UL | ||
| 796 | #define NQ_DBQ_EVENT_RESERVED1_MASK 0xc0UL | ||
| 797 | #define NQ_DBQ_EVENT_RESERVED1_SFT 6 | ||
| 798 | u8 event; | ||
| 799 | #define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT 0x1UL | ||
| 800 | __le16 db_pfid; | ||
| 801 | #define NQ_DBQ_EVENT_DB_PFID_MASK 0xfUL | ||
| 802 | #define NQ_DBQ_EVENT_DB_PFID_SFT 0 | ||
| 803 | #define NQ_DBQ_EVENT_RESERVED12_MASK 0xfff0UL | ||
| 804 | #define NQ_DBQ_EVENT_RESERVED12_SFT 4 | ||
| 805 | __le32 db_dpi; | ||
| 806 | #define NQ_DBQ_EVENT_DB_DPI_MASK 0xfffffUL | ||
| 807 | #define NQ_DBQ_EVENT_DB_DPI_SFT 0 | ||
| 808 | #define NQ_DBQ_EVENT_RESERVED12_2_MASK 0xfff00000UL | ||
| 809 | #define NQ_DBQ_EVENT_RESERVED12_2_SFT 20 | ||
| 810 | __le32 v; | ||
| 811 | #define NQ_DBQ_EVENT_V 0x1UL | ||
| 812 | #define NQ_DBQ_EVENT_RESERVED32_MASK 0xfffffffeUL | ||
| 813 | #define NQ_DBQ_EVENT_RESERVED32_SFT 1 | ||
| 814 | __le32 db_type_db_xid; | ||
| 815 | #define NQ_DBQ_EVENT_DB_XID_MASK 0xfffffUL | ||
| 816 | #define NQ_DBQ_EVENT_DB_XID_SFT 0 | ||
| 817 | #define NQ_DBQ_EVENT_RESERVED8_MASK 0xff00000UL | ||
| 818 | #define NQ_DBQ_EVENT_RESERVED8_SFT 20 | ||
| 819 | #define NQ_DBQ_EVENT_DB_TYPE_MASK 0xf0000000UL | ||
| 820 | #define NQ_DBQ_EVENT_DB_TYPE_SFT 28 | ||
| 821 | }; | ||
| 822 | |||
| 823 | /* Read Request/Response Queue Structures */ | ||
| 824 | /* Input Read Request Queue (IRRQ) Message (32 bytes) */ | ||
| 825 | struct xrrq_irrq { | ||
| 826 | __le16 credits_type; | ||
| 827 | #define XRRQ_IRRQ_TYPE 0x1UL | ||
| 828 | #define XRRQ_IRRQ_TYPE_READ_REQ 0x0UL | ||
| 829 | #define XRRQ_IRRQ_TYPE_ATOMIC_REQ 0x1UL | ||
| 830 | #define XRRQ_IRRQ_RESERVED10_MASK 0x7feUL | ||
| 831 | #define XRRQ_IRRQ_RESERVED10_SFT 1 | ||
| 832 | #define XRRQ_IRRQ_CREDITS_MASK 0xf800UL | ||
| 833 | #define XRRQ_IRRQ_CREDITS_SFT 11 | ||
| 834 | __le16 reserved16; | ||
| 835 | __le32 reserved32; | ||
| 836 | __le32 psn; | ||
| 837 | #define XRRQ_IRRQ_PSN_MASK 0xffffffUL | ||
| 838 | #define XRRQ_IRRQ_PSN_SFT 0 | ||
| 839 | #define XRRQ_IRRQ_RESERVED8_1_MASK 0xff000000UL | ||
| 840 | #define XRRQ_IRRQ_RESERVED8_1_SFT 24 | ||
| 841 | __le32 msn; | ||
| 842 | #define XRRQ_IRRQ_MSN_MASK 0xffffffUL | ||
| 843 | #define XRRQ_IRRQ_MSN_SFT 0 | ||
| 844 | #define XRRQ_IRRQ_RESERVED8_2_MASK 0xff000000UL | ||
| 845 | #define XRRQ_IRRQ_RESERVED8_2_SFT 24 | ||
| 846 | __le64 va_or_atomic_result; | ||
| 847 | __le32 rdma_r_key; | ||
| 848 | __le32 length; | ||
| 849 | }; | ||
| 850 | |||
| 851 | /* Output Read Request Queue (ORRQ) Message (32 bytes) */ | ||
| 852 | struct xrrq_orrq { | ||
| 853 | __le16 num_sges_type; | ||
| 854 | #define XRRQ_ORRQ_TYPE 0x1UL | ||
| 855 | #define XRRQ_ORRQ_TYPE_READ_REQ 0x0UL | ||
| 856 | #define XRRQ_ORRQ_TYPE_ATOMIC_REQ 0x1UL | ||
| 857 | #define XRRQ_ORRQ_RESERVED10_MASK 0x7feUL | ||
| 858 | #define XRRQ_ORRQ_RESERVED10_SFT 1 | ||
| 859 | #define XRRQ_ORRQ_NUM_SGES_MASK 0xf800UL | ||
| 860 | #define XRRQ_ORRQ_NUM_SGES_SFT 11 | ||
| 861 | __le16 reserved16; | ||
| 862 | __le32 length; | ||
| 863 | __le32 psn; | ||
| 864 | #define XRRQ_ORRQ_PSN_MASK 0xffffffUL | ||
| 865 | #define XRRQ_ORRQ_PSN_SFT 0 | ||
| 866 | #define XRRQ_ORRQ_RESERVED8_1_MASK 0xff000000UL | ||
| 867 | #define XRRQ_ORRQ_RESERVED8_1_SFT 24 | ||
| 868 | __le32 end_psn; | ||
| 869 | #define XRRQ_ORRQ_END_PSN_MASK 0xffffffUL | ||
| 870 | #define XRRQ_ORRQ_END_PSN_SFT 0 | ||
| 871 | #define XRRQ_ORRQ_RESERVED8_2_MASK 0xff000000UL | ||
| 872 | #define XRRQ_ORRQ_RESERVED8_2_SFT 24 | ||
| 873 | __le64 first_sge_phy_or_sing_sge_va; | ||
| 874 | __le32 single_sge_l_key; | ||
| 875 | __le32 single_sge_size; | ||
| 876 | }; | ||
| 877 | |||
| 878 | /* Page Buffer List Memory Structures (PBL) */ | ||
| 879 | /* Page Table Entry (PTE) (8 bytes) */ | ||
| 880 | struct ptu_pte { | ||
| 881 | __le32 page_next_to_last_last_valid[2]; | ||
| 882 | #define PTU_PTE_VALID 0x1UL | ||
| 883 | #define PTU_PTE_LAST 0x2UL | ||
| 884 | #define PTU_PTE_NEXT_TO_LAST 0x4UL | ||
| 885 | #define PTU_PTE_PAGE_MASK 0xfffff000UL | ||
| 886 | #define PTU_PTE_PAGE_SFT 12 | ||
| 887 | }; | ||
| 888 | |||
| 889 | /* Page Directory Entry (PDE) (8 bytes) */ | ||
| 890 | struct ptu_pde { | ||
| 891 | __le32 page_valid[2]; | ||
| 892 | #define PTU_PDE_VALID 0x1UL | ||
| 893 | #define PTU_PDE_PAGE_MASK 0xfffff000UL | ||
| 894 | #define PTU_PDE_PAGE_SFT 12 | ||
| 895 | }; | ||
| 896 | |||
| 897 | /* RoCE Fastpath Host Structures */ | ||
| 898 | /* Command Queue (CMDQ) Interface */ | ||
| 899 | /* Init CMDQ (16 bytes) */ | ||
| 900 | struct cmdq_init { | ||
| 901 | __le64 cmdq_pbl; | ||
| 902 | __le16 cmdq_size_cmdq_lvl; | ||
| 903 | #define CMDQ_INIT_CMDQ_LVL_MASK 0x3UL | ||
| 904 | #define CMDQ_INIT_CMDQ_LVL_SFT 0 | ||
| 905 | #define CMDQ_INIT_CMDQ_SIZE_MASK 0xfffcUL | ||
| 906 | #define CMDQ_INIT_CMDQ_SIZE_SFT 2 | ||
| 907 | __le16 creq_ring_id; | ||
| 908 | __le32 prod_idx; | ||
| 909 | }; | ||
| 910 | |||
| 911 | /* Update CMDQ producer index (16 bytes) */ | ||
| 912 | struct cmdq_update { | ||
| 913 | __le64 reserved64; | ||
| 914 | __le32 reserved32; | ||
| 915 | __le32 prod_idx; | ||
| 916 | }; | ||
| 917 | |||
| 918 | /* CMDQ common header structure (16 bytes) */ | ||
| 919 | struct cmdq_base { | ||
| 920 | u8 opcode; | ||
| 921 | #define CMDQ_BASE_OPCODE_CREATE_QP 0x1UL | ||
| 922 | #define CMDQ_BASE_OPCODE_DESTROY_QP 0x2UL | ||
| 923 | #define CMDQ_BASE_OPCODE_MODIFY_QP 0x3UL | ||
| 924 | #define CMDQ_BASE_OPCODE_QUERY_QP 0x4UL | ||
| 925 | #define CMDQ_BASE_OPCODE_CREATE_SRQ 0x5UL | ||
| 926 | #define CMDQ_BASE_OPCODE_DESTROY_SRQ 0x6UL | ||
| 927 | #define CMDQ_BASE_OPCODE_QUERY_SRQ 0x8UL | ||
| 928 | #define CMDQ_BASE_OPCODE_CREATE_CQ 0x9UL | ||
| 929 | #define CMDQ_BASE_OPCODE_DESTROY_CQ 0xaUL | ||
| 930 | #define CMDQ_BASE_OPCODE_RESIZE_CQ 0xcUL | ||
| 931 | #define CMDQ_BASE_OPCODE_ALLOCATE_MRW 0xdUL | ||
| 932 | #define CMDQ_BASE_OPCODE_DEALLOCATE_KEY 0xeUL | ||
| 933 | #define CMDQ_BASE_OPCODE_REGISTER_MR 0xfUL | ||
| 934 | #define CMDQ_BASE_OPCODE_DEREGISTER_MR 0x10UL | ||
| 935 | #define CMDQ_BASE_OPCODE_ADD_GID 0x11UL | ||
| 936 | #define CMDQ_BASE_OPCODE_DELETE_GID 0x12UL | ||
| 937 | #define CMDQ_BASE_OPCODE_MODIFY_GID 0x17UL | ||
| 938 | #define CMDQ_BASE_OPCODE_QUERY_GID 0x18UL | ||
| 939 | #define CMDQ_BASE_OPCODE_CREATE_QP1 0x13UL | ||
| 940 | #define CMDQ_BASE_OPCODE_DESTROY_QP1 0x14UL | ||
| 941 | #define CMDQ_BASE_OPCODE_CREATE_AH 0x15UL | ||
| 942 | #define CMDQ_BASE_OPCODE_DESTROY_AH 0x16UL | ||
| 943 | #define CMDQ_BASE_OPCODE_INITIALIZE_FW 0x80UL | ||
| 944 | #define CMDQ_BASE_OPCODE_DEINITIALIZE_FW 0x81UL | ||
| 945 | #define CMDQ_BASE_OPCODE_STOP_FUNC 0x82UL | ||
| 946 | #define CMDQ_BASE_OPCODE_QUERY_FUNC 0x83UL | ||
| 947 | #define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES 0x84UL | ||
| 948 | #define CMDQ_BASE_OPCODE_READ_CONTEXT 0x85UL | ||
| 949 | #define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST 0x86UL | ||
| 950 | #define CMDQ_BASE_OPCODE_READ_VF_MEMORY 0x87UL | ||
| 951 | #define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST 0x88UL | ||
| 952 | #define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY 0x89UL | ||
| 953 | #define CMDQ_BASE_OPCODE_MAP_TC_TO_COS 0x8aUL | ||
| 954 | #define CMDQ_BASE_OPCODE_QUERY_VERSION 0x8bUL | ||
| 955 | #define CMDQ_BASE_OPCODE_MODIFY_CC 0x8cUL | ||
| 956 | #define CMDQ_BASE_OPCODE_QUERY_CC 0x8dUL | ||
| 957 | u8 cmd_size; | ||
| 958 | __le16 flags; | ||
| 959 | __le16 cookie; | ||
| 960 | u8 resp_size; | ||
| 961 | u8 reserved8; | ||
| 962 | __le64 resp_addr; | ||
| 963 | }; | ||
| 964 | |||
| 965 | /* Create QP command (96 bytes) */ | ||
| 966 | struct cmdq_create_qp { | ||
| 967 | u8 opcode; | ||
| 968 | #define CMDQ_CREATE_QP_OPCODE_CREATE_QP 0x1UL | ||
| 969 | u8 cmd_size; | ||
| 970 | __le16 flags; | ||
| 971 | __le16 cookie; | ||
| 972 | u8 resp_size; | ||
| 973 | u8 reserved8; | ||
| 974 | __le64 resp_addr; | ||
| 975 | __le64 qp_handle; | ||
| 976 | __le32 qp_flags; | ||
| 977 | #define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED 0x1UL | ||
| 978 | #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL | ||
| 979 | #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL | ||
| 980 | #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL | ||
| 981 | u8 type; | ||
| 982 | #define CMDQ_CREATE_QP_TYPE_RC 0x2UL | ||
| 983 | #define CMDQ_CREATE_QP_TYPE_UD 0x4UL | ||
| 984 | #define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE 0x6UL | ||
| 985 | u8 sq_pg_size_sq_lvl; | ||
| 986 | #define CMDQ_CREATE_QP_SQ_LVL_MASK 0xfUL | ||
| 987 | #define CMDQ_CREATE_QP_SQ_LVL_SFT 0 | ||
| 988 | #define CMDQ_CREATE_QP_SQ_LVL_LVL_0 0x0UL | ||
| 989 | #define CMDQ_CREATE_QP_SQ_LVL_LVL_1 0x1UL | ||
| 990 | #define CMDQ_CREATE_QP_SQ_LVL_LVL_2 0x2UL | ||
| 991 | #define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK 0xf0UL | ||
| 992 | #define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT 4 | ||
| 993 | #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 994 | #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 995 | #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 996 | #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 997 | #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 998 | #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 999 | u8 rq_pg_size_rq_lvl; | ||
| 1000 | #define CMDQ_CREATE_QP_RQ_LVL_MASK 0xfUL | ||
| 1001 | #define CMDQ_CREATE_QP_RQ_LVL_SFT 0 | ||
| 1002 | #define CMDQ_CREATE_QP_RQ_LVL_LVL_0 0x0UL | ||
| 1003 | #define CMDQ_CREATE_QP_RQ_LVL_LVL_1 0x1UL | ||
| 1004 | #define CMDQ_CREATE_QP_RQ_LVL_LVL_2 0x2UL | ||
| 1005 | #define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK 0xf0UL | ||
| 1006 | #define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT 4 | ||
| 1007 | #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1008 | #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1009 | #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1010 | #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1011 | #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1012 | #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1013 | u8 unused_0; | ||
| 1014 | __le32 dpi; | ||
| 1015 | __le32 sq_size; | ||
| 1016 | __le32 rq_size; | ||
| 1017 | __le16 sq_fwo_sq_sge; | ||
| 1018 | #define CMDQ_CREATE_QP_SQ_SGE_MASK 0xfUL | ||
| 1019 | #define CMDQ_CREATE_QP_SQ_SGE_SFT 0 | ||
| 1020 | #define CMDQ_CREATE_QP_SQ_FWO_MASK 0xfff0UL | ||
| 1021 | #define CMDQ_CREATE_QP_SQ_FWO_SFT 4 | ||
| 1022 | __le16 rq_fwo_rq_sge; | ||
| 1023 | #define CMDQ_CREATE_QP_RQ_SGE_MASK 0xfUL | ||
| 1024 | #define CMDQ_CREATE_QP_RQ_SGE_SFT 0 | ||
| 1025 | #define CMDQ_CREATE_QP_RQ_FWO_MASK 0xfff0UL | ||
| 1026 | #define CMDQ_CREATE_QP_RQ_FWO_SFT 4 | ||
| 1027 | __le32 scq_cid; | ||
| 1028 | __le32 rcq_cid; | ||
| 1029 | __le32 srq_cid; | ||
| 1030 | __le32 pd_id; | ||
| 1031 | __le64 sq_pbl; | ||
| 1032 | __le64 rq_pbl; | ||
| 1033 | __le64 irrq_addr; | ||
| 1034 | __le64 orrq_addr; | ||
| 1035 | }; | ||
| 1036 | |||
| 1037 | /* Destroy QP command (24 bytes) */ | ||
| 1038 | struct cmdq_destroy_qp { | ||
| 1039 | u8 opcode; | ||
| 1040 | #define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP 0x2UL | ||
| 1041 | u8 cmd_size; | ||
| 1042 | __le16 flags; | ||
| 1043 | __le16 cookie; | ||
| 1044 | u8 resp_size; | ||
| 1045 | u8 reserved8; | ||
| 1046 | __le64 resp_addr; | ||
| 1047 | __le32 qp_cid; | ||
| 1048 | __le32 unused_0; | ||
| 1049 | }; | ||
| 1050 | |||
| 1051 | /* Modify QP command (112 bytes) */ | ||
| 1052 | struct cmdq_modify_qp { | ||
| 1053 | u8 opcode; | ||
| 1054 | #define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP 0x3UL | ||
| 1055 | u8 cmd_size; | ||
| 1056 | __le16 flags; | ||
| 1057 | __le16 cookie; | ||
| 1058 | u8 resp_size; | ||
| 1059 | u8 reserved8; | ||
| 1060 | __le64 resp_addr; | ||
| 1061 | __le32 modify_mask; | ||
| 1062 | #define CMDQ_MODIFY_QP_MODIFY_MASK_STATE 0x1UL | ||
| 1063 | #define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY 0x2UL | ||
| 1064 | #define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS 0x4UL | ||
| 1065 | #define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY 0x8UL | ||
| 1066 | #define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY 0x10UL | ||
| 1067 | #define CMDQ_MODIFY_QP_MODIFY_MASK_DGID 0x20UL | ||
| 1068 | #define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL 0x40UL | ||
| 1069 | #define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX 0x80UL | ||
| 1070 | #define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT 0x100UL | ||
| 1071 | #define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS 0x200UL | ||
| 1072 | #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC 0x400UL | ||
| 1073 | #define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU 0x1000UL | ||
| 1074 | #define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT 0x2000UL | ||
| 1075 | #define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT 0x4000UL | ||
| 1076 | #define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY 0x8000UL | ||
| 1077 | #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN 0x10000UL | ||
| 1078 | #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC 0x20000UL | ||
| 1079 | #define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER 0x40000UL | ||
| 1080 | #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN 0x80000UL | ||
| 1081 | #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC 0x100000UL | ||
| 1082 | #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE 0x200000UL | ||
| 1083 | #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE 0x400000UL | ||
| 1084 | #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE 0x800000UL | ||
| 1085 | #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE 0x1000000UL | ||
| 1086 | #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA 0x2000000UL | ||
| 1087 | #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID 0x4000000UL | ||
| 1088 | #define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC 0x8000000UL | ||
| 1089 | #define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID 0x10000000UL | ||
| 1090 | #define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC 0x20000000UL | ||
| 1091 | #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN 0x40000000UL | ||
| 1092 | #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP 0x80000000UL | ||
| 1093 | __le32 qp_cid; | ||
| 1094 | u8 network_type_en_sqd_async_notify_new_state; | ||
| 1095 | #define CMDQ_MODIFY_QP_NEW_STATE_MASK 0xfUL | ||
| 1096 | #define CMDQ_MODIFY_QP_NEW_STATE_SFT 0 | ||
| 1097 | #define CMDQ_MODIFY_QP_NEW_STATE_RESET 0x0UL | ||
| 1098 | #define CMDQ_MODIFY_QP_NEW_STATE_INIT 0x1UL | ||
| 1099 | #define CMDQ_MODIFY_QP_NEW_STATE_RTR 0x2UL | ||
| 1100 | #define CMDQ_MODIFY_QP_NEW_STATE_RTS 0x3UL | ||
| 1101 | #define CMDQ_MODIFY_QP_NEW_STATE_SQD 0x4UL | ||
| 1102 | #define CMDQ_MODIFY_QP_NEW_STATE_SQE 0x5UL | ||
| 1103 | #define CMDQ_MODIFY_QP_NEW_STATE_ERR 0x6UL | ||
| 1104 | #define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY 0x10UL | ||
| 1105 | #define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK 0xc0UL | ||
| 1106 | #define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT 6 | ||
| 1107 | #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 (0x0UL << 6) | ||
| 1108 | #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 (0x2UL << 6) | ||
| 1109 | #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 (0x3UL << 6) | ||
| 1110 | u8 access; | ||
| 1111 | #define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE 0x1UL | ||
| 1112 | #define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE 0x2UL | ||
| 1113 | #define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ 0x4UL | ||
| 1114 | #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC 0x8UL | ||
| 1115 | __le16 pkey; | ||
| 1116 | __le32 qkey; | ||
| 1117 | __le32 dgid[4]; | ||
| 1118 | __le32 flow_label; | ||
| 1119 | __le16 sgid_index; | ||
| 1120 | u8 hop_limit; | ||
| 1121 | u8 traffic_class; | ||
| 1122 | __le16 dest_mac[3]; | ||
| 1123 | u8 tos_dscp_tos_ecn; | ||
| 1124 | #define CMDQ_MODIFY_QP_TOS_ECN_MASK 0x3UL | ||
| 1125 | #define CMDQ_MODIFY_QP_TOS_ECN_SFT 0 | ||
| 1126 | #define CMDQ_MODIFY_QP_TOS_DSCP_MASK 0xfcUL | ||
| 1127 | #define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2 | ||
| 1128 | u8 path_mtu; | ||
| 1129 | #define CMDQ_MODIFY_QP_PATH_MTU_MASK 0xf0UL | ||
| 1130 | #define CMDQ_MODIFY_QP_PATH_MTU_SFT 4 | ||
| 1131 | #define CMDQ_MODIFY_QP_PATH_MTU_MTU_256 (0x0UL << 4) | ||
| 1132 | #define CMDQ_MODIFY_QP_PATH_MTU_MTU_512 (0x1UL << 4) | ||
| 1133 | #define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 (0x2UL << 4) | ||
| 1134 | #define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 (0x3UL << 4) | ||
| 1135 | #define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 (0x4UL << 4) | ||
| 1136 | #define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 (0x5UL << 4) | ||
| 1137 | u8 timeout; | ||
| 1138 | u8 retry_cnt; | ||
| 1139 | u8 rnr_retry; | ||
| 1140 | u8 min_rnr_timer; | ||
| 1141 | __le32 rq_psn; | ||
| 1142 | __le32 sq_psn; | ||
| 1143 | u8 max_rd_atomic; | ||
| 1144 | u8 max_dest_rd_atomic; | ||
| 1145 | __le16 enable_cc; | ||
| 1146 | #define CMDQ_MODIFY_QP_ENABLE_CC 0x1UL | ||
| 1147 | __le32 sq_size; | ||
| 1148 | __le32 rq_size; | ||
| 1149 | __le16 sq_sge; | ||
| 1150 | __le16 rq_sge; | ||
| 1151 | __le32 max_inline_data; | ||
| 1152 | __le32 dest_qp_id; | ||
| 1153 | __le32 unused_3; | ||
| 1154 | __le16 src_mac[3]; | ||
| 1155 | __le16 vlan_pcp_vlan_dei_vlan_id; | ||
| 1156 | #define CMDQ_MODIFY_QP_VLAN_ID_MASK 0xfffUL | ||
| 1157 | #define CMDQ_MODIFY_QP_VLAN_ID_SFT 0 | ||
| 1158 | #define CMDQ_MODIFY_QP_VLAN_DEI 0x1000UL | ||
| 1159 | #define CMDQ_MODIFY_QP_VLAN_PCP_MASK 0xe000UL | ||
| 1160 | #define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13 | ||
| 1161 | }; | ||
| 1162 | |||
| 1163 | /* Query QP command (24 bytes) */ | ||
| 1164 | struct cmdq_query_qp { | ||
| 1165 | u8 opcode; | ||
| 1166 | #define CMDQ_QUERY_QP_OPCODE_QUERY_QP 0x4UL | ||
| 1167 | u8 cmd_size; | ||
| 1168 | __le16 flags; | ||
| 1169 | __le16 cookie; | ||
| 1170 | u8 resp_size; | ||
| 1171 | u8 reserved8; | ||
| 1172 | __le64 resp_addr; | ||
| 1173 | __le32 qp_cid; | ||
| 1174 | __le32 unused_0; | ||
| 1175 | }; | ||
| 1176 | |||
| 1177 | /* Create SRQ command (48 bytes) */ | ||
| 1178 | struct cmdq_create_srq { | ||
| 1179 | u8 opcode; | ||
| 1180 | #define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ 0x5UL | ||
| 1181 | u8 cmd_size; | ||
| 1182 | __le16 flags; | ||
| 1183 | __le16 cookie; | ||
| 1184 | u8 resp_size; | ||
| 1185 | u8 reserved8; | ||
| 1186 | __le64 resp_addr; | ||
| 1187 | __le64 srq_handle; | ||
| 1188 | __le16 pg_size_lvl; | ||
| 1189 | #define CMDQ_CREATE_SRQ_LVL_MASK 0x3UL | ||
| 1190 | #define CMDQ_CREATE_SRQ_LVL_SFT 0 | ||
| 1191 | #define CMDQ_CREATE_SRQ_LVL_LVL_0 0x0UL | ||
| 1192 | #define CMDQ_CREATE_SRQ_LVL_LVL_1 0x1UL | ||
| 1193 | #define CMDQ_CREATE_SRQ_LVL_LVL_2 0x2UL | ||
| 1194 | #define CMDQ_CREATE_SRQ_PG_SIZE_MASK 0x1cUL | ||
| 1195 | #define CMDQ_CREATE_SRQ_PG_SIZE_SFT 2 | ||
| 1196 | #define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K (0x0UL << 2) | ||
| 1197 | #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K (0x1UL << 2) | ||
| 1198 | #define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K (0x2UL << 2) | ||
| 1199 | #define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M (0x3UL << 2) | ||
| 1200 | #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M (0x4UL << 2) | ||
| 1201 | #define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G (0x5UL << 2) | ||
| 1202 | __le16 eventq_id; | ||
| 1203 | #define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK 0xfffUL | ||
| 1204 | #define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0 | ||
| 1205 | __le16 srq_size; | ||
| 1206 | __le16 srq_fwo; | ||
| 1207 | __le32 dpi; | ||
| 1208 | __le32 pd_id; | ||
| 1209 | __le64 pbl; | ||
| 1210 | }; | ||
| 1211 | |||
| 1212 | /* Destroy SRQ command (24 bytes) */ | ||
| 1213 | struct cmdq_destroy_srq { | ||
| 1214 | u8 opcode; | ||
| 1215 | #define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ 0x6UL | ||
| 1216 | u8 cmd_size; | ||
| 1217 | __le16 flags; | ||
| 1218 | __le16 cookie; | ||
| 1219 | u8 resp_size; | ||
| 1220 | u8 reserved8; | ||
| 1221 | __le64 resp_addr; | ||
| 1222 | __le32 srq_cid; | ||
| 1223 | __le32 unused_0; | ||
| 1224 | }; | ||
| 1225 | |||
| 1226 | /* Query SRQ command (24 bytes) */ | ||
| 1227 | struct cmdq_query_srq { | ||
| 1228 | u8 opcode; | ||
| 1229 | #define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ 0x8UL | ||
| 1230 | u8 cmd_size; | ||
| 1231 | __le16 flags; | ||
| 1232 | __le16 cookie; | ||
| 1233 | u8 resp_size; | ||
| 1234 | u8 reserved8; | ||
| 1235 | __le64 resp_addr; | ||
| 1236 | __le32 srq_cid; | ||
| 1237 | __le32 unused_0; | ||
| 1238 | }; | ||
| 1239 | |||
| 1240 | /* Create CQ command (48 bytes) */ | ||
| 1241 | struct cmdq_create_cq { | ||
| 1242 | u8 opcode; | ||
| 1243 | #define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ 0x9UL | ||
| 1244 | u8 cmd_size; | ||
| 1245 | __le16 flags; | ||
| 1246 | __le16 cookie; | ||
| 1247 | u8 resp_size; | ||
| 1248 | u8 reserved8; | ||
| 1249 | __le64 resp_addr; | ||
| 1250 | __le64 cq_handle; | ||
| 1251 | __le32 pg_size_lvl; | ||
| 1252 | #define CMDQ_CREATE_CQ_LVL_MASK 0x3UL | ||
| 1253 | #define CMDQ_CREATE_CQ_LVL_SFT 0 | ||
| 1254 | #define CMDQ_CREATE_CQ_LVL_LVL_0 0x0UL | ||
| 1255 | #define CMDQ_CREATE_CQ_LVL_LVL_1 0x1UL | ||
| 1256 | #define CMDQ_CREATE_CQ_LVL_LVL_2 0x2UL | ||
| 1257 | #define CMDQ_CREATE_CQ_PG_SIZE_MASK 0x1cUL | ||
| 1258 | #define CMDQ_CREATE_CQ_PG_SIZE_SFT 2 | ||
| 1259 | #define CMDQ_CREATE_CQ_PG_SIZE_PG_4K (0x0UL << 2) | ||
| 1260 | #define CMDQ_CREATE_CQ_PG_SIZE_PG_8K (0x1UL << 2) | ||
| 1261 | #define CMDQ_CREATE_CQ_PG_SIZE_PG_64K (0x2UL << 2) | ||
| 1262 | #define CMDQ_CREATE_CQ_PG_SIZE_PG_2M (0x3UL << 2) | ||
| 1263 | #define CMDQ_CREATE_CQ_PG_SIZE_PG_8M (0x4UL << 2) | ||
| 1264 | #define CMDQ_CREATE_CQ_PG_SIZE_PG_1G (0x5UL << 2) | ||
| 1265 | __le32 cq_fco_cnq_id; | ||
| 1266 | #define CMDQ_CREATE_CQ_CNQ_ID_MASK 0xfffUL | ||
| 1267 | #define CMDQ_CREATE_CQ_CNQ_ID_SFT 0 | ||
| 1268 | #define CMDQ_CREATE_CQ_CQ_FCO_MASK 0xfffff000UL | ||
| 1269 | #define CMDQ_CREATE_CQ_CQ_FCO_SFT 12 | ||
| 1270 | __le32 dpi; | ||
| 1271 | __le32 cq_size; | ||
| 1272 | __le64 pbl; | ||
| 1273 | }; | ||
| 1274 | |||
| 1275 | /* Destroy CQ command (24 bytes) */ | ||
| 1276 | struct cmdq_destroy_cq { | ||
| 1277 | u8 opcode; | ||
| 1278 | #define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ 0xaUL | ||
| 1279 | u8 cmd_size; | ||
| 1280 | __le16 flags; | ||
| 1281 | __le16 cookie; | ||
| 1282 | u8 resp_size; | ||
| 1283 | u8 reserved8; | ||
| 1284 | __le64 resp_addr; | ||
| 1285 | __le32 cq_cid; | ||
| 1286 | __le32 unused_0; | ||
| 1287 | }; | ||
| 1288 | |||
| 1289 | /* Resize CQ command (40 bytes) */ | ||
| 1290 | struct cmdq_resize_cq { | ||
| 1291 | u8 opcode; | ||
| 1292 | #define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ 0xcUL | ||
| 1293 | u8 cmd_size; | ||
| 1294 | __le16 flags; | ||
| 1295 | __le16 cookie; | ||
| 1296 | u8 resp_size; | ||
| 1297 | u8 reserved8; | ||
| 1298 | __le64 resp_addr; | ||
| 1299 | __le32 cq_cid; | ||
| 1300 | __le32 new_cq_size_pg_size_lvl; | ||
| 1301 | #define CMDQ_RESIZE_CQ_LVL_MASK 0x3UL | ||
| 1302 | #define CMDQ_RESIZE_CQ_LVL_SFT 0 | ||
| 1303 | #define CMDQ_RESIZE_CQ_LVL_LVL_0 0x0UL | ||
| 1304 | #define CMDQ_RESIZE_CQ_LVL_LVL_1 0x1UL | ||
| 1305 | #define CMDQ_RESIZE_CQ_LVL_LVL_2 0x2UL | ||
| 1306 | #define CMDQ_RESIZE_CQ_PG_SIZE_MASK 0x1cUL | ||
| 1307 | #define CMDQ_RESIZE_CQ_PG_SIZE_SFT 2 | ||
| 1308 | #define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K (0x0UL << 2) | ||
| 1309 | #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K (0x1UL << 2) | ||
| 1310 | #define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K (0x2UL << 2) | ||
| 1311 | #define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M (0x3UL << 2) | ||
| 1312 | #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M (0x4UL << 2) | ||
| 1313 | #define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G (0x5UL << 2) | ||
| 1314 | #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK 0x1fffe0UL | ||
| 1315 | #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5 | ||
| 1316 | __le64 new_pbl; | ||
| 1317 | __le32 new_cq_fco; | ||
| 1318 | __le32 unused_2; | ||
| 1319 | }; | ||
| 1320 | |||
| 1321 | /* Allocate MRW command (32 bytes) */ | ||
| 1322 | struct cmdq_allocate_mrw { | ||
| 1323 | u8 opcode; | ||
| 1324 | #define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW 0xdUL | ||
| 1325 | u8 cmd_size; | ||
| 1326 | __le16 flags; | ||
| 1327 | __le16 cookie; | ||
| 1328 | u8 resp_size; | ||
| 1329 | u8 reserved8; | ||
| 1330 | __le64 resp_addr; | ||
| 1331 | __le64 mrw_handle; | ||
| 1332 | u8 mrw_flags; | ||
| 1333 | #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK 0xfUL | ||
| 1334 | #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT 0 | ||
| 1335 | #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR 0x0UL | ||
| 1336 | #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR 0x1UL | ||
| 1337 | #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 0x2UL | ||
| 1338 | #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A 0x3UL | ||
| 1339 | #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B 0x4UL | ||
| 1340 | u8 access; | ||
| 1341 | #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_MASK 0x1fUL | ||
| 1342 | #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_SFT 0 | ||
| 1343 | #define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY 0x20UL | ||
| 1344 | __le16 unused_1; | ||
| 1345 | __le32 pd_id; | ||
| 1346 | }; | ||
| 1347 | |||
| 1348 | /* De-allocate key command (24 bytes) */ | ||
| 1349 | struct cmdq_deallocate_key { | ||
| 1350 | u8 opcode; | ||
| 1351 | #define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY 0xeUL | ||
| 1352 | u8 cmd_size; | ||
| 1353 | __le16 flags; | ||
| 1354 | __le16 cookie; | ||
| 1355 | u8 resp_size; | ||
| 1356 | u8 reserved8; | ||
| 1357 | __le64 resp_addr; | ||
| 1358 | u8 mrw_flags; | ||
| 1359 | #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK 0xfUL | ||
| 1360 | #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT 0 | ||
| 1361 | #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR 0x0UL | ||
| 1362 | #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR 0x1UL | ||
| 1363 | #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 0x2UL | ||
| 1364 | #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A 0x3UL | ||
| 1365 | #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B 0x4UL | ||
| 1366 | u8 unused_1[3]; | ||
| 1367 | __le32 key; | ||
| 1368 | }; | ||
| 1369 | |||
| 1370 | /* Register MR command (48 bytes) */ | ||
| 1371 | struct cmdq_register_mr { | ||
| 1372 | u8 opcode; | ||
| 1373 | #define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR 0xfUL | ||
| 1374 | u8 cmd_size; | ||
| 1375 | __le16 flags; | ||
| 1376 | __le16 cookie; | ||
| 1377 | u8 resp_size; | ||
| 1378 | u8 reserved8; | ||
| 1379 | __le64 resp_addr; | ||
| 1380 | u8 log2_pg_size_lvl; | ||
| 1381 | #define CMDQ_REGISTER_MR_LVL_MASK 0x3UL | ||
| 1382 | #define CMDQ_REGISTER_MR_LVL_SFT 0 | ||
| 1383 | #define CMDQ_REGISTER_MR_LVL_LVL_0 0x0UL | ||
| 1384 | #define CMDQ_REGISTER_MR_LVL_LVL_1 0x1UL | ||
| 1385 | #define CMDQ_REGISTER_MR_LVL_LVL_2 0x2UL | ||
| 1386 | #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK 0x7cUL | ||
| 1387 | #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT 2 | ||
| 1388 | u8 access; | ||
| 1389 | #define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE 0x1UL | ||
| 1390 | #define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ 0x2UL | ||
| 1391 | #define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE 0x4UL | ||
| 1392 | #define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC 0x8UL | ||
| 1393 | #define CMDQ_REGISTER_MR_ACCESS_MW_BIND 0x10UL | ||
| 1394 | #define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED 0x20UL | ||
| 1395 | __le16 unused_1; | ||
| 1396 | __le32 key; | ||
| 1397 | __le64 pbl; | ||
| 1398 | __le64 va; | ||
| 1399 | __le64 mr_size; | ||
| 1400 | }; | ||
| 1401 | |||
| 1402 | /* Deregister MR command (24 bytes) */ | ||
| 1403 | struct cmdq_deregister_mr { | ||
| 1404 | u8 opcode; | ||
| 1405 | #define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR 0x10UL | ||
| 1406 | u8 cmd_size; | ||
| 1407 | __le16 flags; | ||
| 1408 | __le16 cookie; | ||
| 1409 | u8 resp_size; | ||
| 1410 | u8 reserved8; | ||
| 1411 | __le64 resp_addr; | ||
| 1412 | __le32 lkey; | ||
| 1413 | __le32 unused_0; | ||
| 1414 | }; | ||
| 1415 | |||
| 1416 | /* Add GID command (48 bytes) */ | ||
| 1417 | struct cmdq_add_gid { | ||
| 1418 | u8 opcode; | ||
| 1419 | #define CMDQ_ADD_GID_OPCODE_ADD_GID 0x11UL | ||
| 1420 | u8 cmd_size; | ||
| 1421 | __le16 flags; | ||
| 1422 | __le16 cookie; | ||
| 1423 | u8 resp_size; | ||
| 1424 | u8 reserved8; | ||
| 1425 | __le64 resp_addr; | ||
| 1426 | __be32 gid[4]; | ||
| 1427 | __be16 src_mac[3]; | ||
| 1428 | __le16 vlan; | ||
| 1429 | #define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK 0xfffUL | ||
| 1430 | #define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT 0 | ||
| 1431 | #define CMDQ_ADD_GID_VLAN_TPID_MASK 0x7000UL | ||
| 1432 | #define CMDQ_ADD_GID_VLAN_TPID_SFT 12 | ||
| 1433 | #define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12) | ||
| 1434 | #define CMDQ_ADD_GID_VLAN_TPID_TPID_8100 (0x1UL << 12) | ||
| 1435 | #define CMDQ_ADD_GID_VLAN_TPID_TPID_9100 (0x2UL << 12) | ||
| 1436 | #define CMDQ_ADD_GID_VLAN_TPID_TPID_9200 (0x3UL << 12) | ||
| 1437 | #define CMDQ_ADD_GID_VLAN_TPID_TPID_9300 (0x4UL << 12) | ||
| 1438 | #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12) | ||
| 1439 | #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12) | ||
| 1440 | #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12) | ||
| 1441 | #define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 | ||
| 1442 | #define CMDQ_ADD_GID_VLAN_VLAN_EN 0x8000UL | ||
| 1443 | __le16 ipid; | ||
| 1444 | __le16 stats_ctx; | ||
| 1445 | #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL | ||
| 1446 | #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT 0 | ||
| 1447 | #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL | ||
| 1448 | __le32 unused_0; | ||
| 1449 | }; | ||
| 1450 | |||
| 1451 | /* Delete GID command (24 bytes) */ | ||
| 1452 | struct cmdq_delete_gid { | ||
| 1453 | u8 opcode; | ||
| 1454 | #define CMDQ_DELETE_GID_OPCODE_DELETE_GID 0x12UL | ||
| 1455 | u8 cmd_size; | ||
| 1456 | __le16 flags; | ||
| 1457 | __le16 cookie; | ||
| 1458 | u8 resp_size; | ||
| 1459 | u8 reserved8; | ||
| 1460 | __le64 resp_addr; | ||
| 1461 | __le16 gid_index; | ||
| 1462 | __le16 unused_0; | ||
| 1463 | __le32 unused_1; | ||
| 1464 | }; | ||
| 1465 | |||
| 1466 | /* Modify GID command (48 bytes) */ | ||
| 1467 | struct cmdq_modify_gid { | ||
| 1468 | u8 opcode; | ||
| 1469 | #define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID 0x17UL | ||
| 1470 | u8 cmd_size; | ||
| 1471 | __le16 flags; | ||
| 1472 | __le16 cookie; | ||
| 1473 | u8 resp_size; | ||
| 1474 | u8 reserved8; | ||
| 1475 | __le64 resp_addr; | ||
| 1476 | __le32 gid[4]; | ||
| 1477 | __le16 src_mac[3]; | ||
| 1478 | __le16 vlan; | ||
| 1479 | #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK 0xfffUL | ||
| 1480 | #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT 0 | ||
| 1481 | #define CMDQ_MODIFY_GID_VLAN_TPID_MASK 0x7000UL | ||
| 1482 | #define CMDQ_MODIFY_GID_VLAN_TPID_SFT 12 | ||
| 1483 | #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12) | ||
| 1484 | #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 (0x1UL << 12) | ||
| 1485 | #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 (0x2UL << 12) | ||
| 1486 | #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 (0x3UL << 12) | ||
| 1487 | #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 (0x4UL << 12) | ||
| 1488 | #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12) | ||
| 1489 | #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12) | ||
| 1490 | #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12) | ||
| 1491 | #define CMDQ_MODIFY_GID_VLAN_TPID_LAST \ | ||
| 1492 | CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 | ||
| 1493 | #define CMDQ_MODIFY_GID_VLAN_VLAN_EN 0x8000UL | ||
| 1494 | __le16 ipid; | ||
| 1495 | __le16 gid_index; | ||
| 1496 | __le16 stats_ctx; | ||
| 1497 | #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL | ||
| 1498 | #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT 0 | ||
| 1499 | #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL | ||
| 1500 | __le16 unused_0; | ||
| 1501 | }; | ||
| 1502 | |||
| 1503 | /* Query GID command (24 bytes) */ | ||
| 1504 | struct cmdq_query_gid { | ||
| 1505 | u8 opcode; | ||
| 1506 | #define CMDQ_QUERY_GID_OPCODE_QUERY_GID 0x18UL | ||
| 1507 | u8 cmd_size; | ||
| 1508 | __le16 flags; | ||
| 1509 | __le16 cookie; | ||
| 1510 | u8 resp_size; | ||
| 1511 | u8 reserved8; | ||
| 1512 | __le64 resp_addr; | ||
| 1513 | __le16 gid_index; | ||
| 1514 | __le16 unused_0; | ||
| 1515 | __le32 unused_1; | ||
| 1516 | }; | ||
| 1517 | |||
| 1518 | /* Create QP1 command (80 bytes) */ | ||
| 1519 | struct cmdq_create_qp1 { | ||
| 1520 | u8 opcode; | ||
| 1521 | #define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 0x13UL | ||
| 1522 | u8 cmd_size; | ||
| 1523 | __le16 flags; | ||
| 1524 | __le16 cookie; | ||
| 1525 | u8 resp_size; | ||
| 1526 | u8 reserved8; | ||
| 1527 | __le64 resp_addr; | ||
| 1528 | __le64 qp_handle; | ||
| 1529 | __le32 qp_flags; | ||
| 1530 | #define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED 0x1UL | ||
| 1531 | #define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION 0x2UL | ||
| 1532 | #define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL | ||
| 1533 | u8 type; | ||
| 1534 | #define CMDQ_CREATE_QP1_TYPE_GSI 0x1UL | ||
| 1535 | u8 sq_pg_size_sq_lvl; | ||
| 1536 | #define CMDQ_CREATE_QP1_SQ_LVL_MASK 0xfUL | ||
| 1537 | #define CMDQ_CREATE_QP1_SQ_LVL_SFT 0 | ||
| 1538 | #define CMDQ_CREATE_QP1_SQ_LVL_LVL_0 0x0UL | ||
| 1539 | #define CMDQ_CREATE_QP1_SQ_LVL_LVL_1 0x1UL | ||
| 1540 | #define CMDQ_CREATE_QP1_SQ_LVL_LVL_2 0x2UL | ||
| 1541 | #define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK 0xf0UL | ||
| 1542 | #define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT 4 | ||
| 1543 | #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1544 | #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1545 | #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1546 | #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1547 | #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1548 | #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1549 | u8 rq_pg_size_rq_lvl; | ||
| 1550 | #define CMDQ_CREATE_QP1_RQ_LVL_MASK 0xfUL | ||
| 1551 | #define CMDQ_CREATE_QP1_RQ_LVL_SFT 0 | ||
| 1552 | #define CMDQ_CREATE_QP1_RQ_LVL_LVL_0 0x0UL | ||
| 1553 | #define CMDQ_CREATE_QP1_RQ_LVL_LVL_1 0x1UL | ||
| 1554 | #define CMDQ_CREATE_QP1_RQ_LVL_LVL_2 0x2UL | ||
| 1555 | #define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK 0xf0UL | ||
| 1556 | #define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT 4 | ||
| 1557 | #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1558 | #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1559 | #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1560 | #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1561 | #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1562 | #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1563 | u8 unused_0; | ||
| 1564 | __le32 dpi; | ||
| 1565 | __le32 sq_size; | ||
| 1566 | __le32 rq_size; | ||
| 1567 | __le16 sq_fwo_sq_sge; | ||
| 1568 | #define CMDQ_CREATE_QP1_SQ_SGE_MASK 0xfUL | ||
| 1569 | #define CMDQ_CREATE_QP1_SQ_SGE_SFT 0 | ||
| 1570 | #define CMDQ_CREATE_QP1_SQ_FWO_MASK 0xfff0UL | ||
| 1571 | #define CMDQ_CREATE_QP1_SQ_FWO_SFT 4 | ||
| 1572 | __le16 rq_fwo_rq_sge; | ||
| 1573 | #define CMDQ_CREATE_QP1_RQ_SGE_MASK 0xfUL | ||
| 1574 | #define CMDQ_CREATE_QP1_RQ_SGE_SFT 0 | ||
| 1575 | #define CMDQ_CREATE_QP1_RQ_FWO_MASK 0xfff0UL | ||
| 1576 | #define CMDQ_CREATE_QP1_RQ_FWO_SFT 4 | ||
| 1577 | __le32 scq_cid; | ||
| 1578 | __le32 rcq_cid; | ||
| 1579 | __le32 srq_cid; | ||
| 1580 | __le32 pd_id; | ||
| 1581 | __le64 sq_pbl; | ||
| 1582 | __le64 rq_pbl; | ||
| 1583 | }; | ||
| 1584 | |||
| 1585 | /* Destroy QP1 command (24 bytes) */ | ||
| 1586 | struct cmdq_destroy_qp1 { | ||
| 1587 | u8 opcode; | ||
| 1588 | #define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 0x14UL | ||
| 1589 | u8 cmd_size; | ||
| 1590 | __le16 flags; | ||
| 1591 | __le16 cookie; | ||
| 1592 | u8 resp_size; | ||
| 1593 | u8 reserved8; | ||
| 1594 | __le64 resp_addr; | ||
| 1595 | __le32 qp1_cid; | ||
| 1596 | __le32 unused_0; | ||
| 1597 | }; | ||
| 1598 | |||
| 1599 | /* Create AH command (64 bytes) */ | ||
| 1600 | struct cmdq_create_ah { | ||
| 1601 | u8 opcode; | ||
| 1602 | #define CMDQ_CREATE_AH_OPCODE_CREATE_AH 0x15UL | ||
| 1603 | u8 cmd_size; | ||
| 1604 | __le16 flags; | ||
| 1605 | __le16 cookie; | ||
| 1606 | u8 resp_size; | ||
| 1607 | u8 reserved8; | ||
| 1608 | __le64 resp_addr; | ||
| 1609 | __le64 ah_handle; | ||
| 1610 | __le32 dgid[4]; | ||
| 1611 | u8 type; | ||
| 1612 | #define CMDQ_CREATE_AH_TYPE_V1 0x0UL | ||
| 1613 | #define CMDQ_CREATE_AH_TYPE_V2IPV4 0x2UL | ||
| 1614 | #define CMDQ_CREATE_AH_TYPE_V2IPV6 0x3UL | ||
| 1615 | u8 hop_limit; | ||
| 1616 | __le16 sgid_index; | ||
| 1617 | __le32 dest_vlan_id_flow_label; | ||
| 1618 | #define CMDQ_CREATE_AH_FLOW_LABEL_MASK 0xfffffUL | ||
| 1619 | #define CMDQ_CREATE_AH_FLOW_LABEL_SFT 0 | ||
| 1620 | #define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK 0xfff00000UL | ||
| 1621 | #define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20 | ||
| 1622 | __le32 pd_id; | ||
| 1623 | __le32 unused_0; | ||
| 1624 | __le16 dest_mac[3]; | ||
| 1625 | u8 traffic_class; | ||
| 1626 | u8 unused_1; | ||
| 1627 | }; | ||
| 1628 | |||
| 1629 | /* Destroy AH command (24 bytes) */ | ||
| 1630 | struct cmdq_destroy_ah { | ||
| 1631 | u8 opcode; | ||
| 1632 | #define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH 0x16UL | ||
| 1633 | u8 cmd_size; | ||
| 1634 | __le16 flags; | ||
| 1635 | __le16 cookie; | ||
| 1636 | u8 resp_size; | ||
| 1637 | u8 reserved8; | ||
| 1638 | __le64 resp_addr; | ||
| 1639 | __le32 ah_cid; | ||
| 1640 | __le32 unused_0; | ||
| 1641 | }; | ||
| 1642 | |||
| 1643 | /* Initialize Firmware command (112 bytes) */ | ||
| 1644 | struct cmdq_initialize_fw { | ||
| 1645 | u8 opcode; | ||
| 1646 | #define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW 0x80UL | ||
| 1647 | u8 cmd_size; | ||
| 1648 | __le16 flags; | ||
| 1649 | __le16 cookie; | ||
| 1650 | u8 resp_size; | ||
| 1651 | u8 reserved8; | ||
| 1652 | __le64 resp_addr; | ||
| 1653 | u8 qpc_pg_size_qpc_lvl; | ||
| 1654 | #define CMDQ_INITIALIZE_FW_QPC_LVL_MASK 0xfUL | ||
| 1655 | #define CMDQ_INITIALIZE_FW_QPC_LVL_SFT 0 | ||
| 1656 | #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 0x0UL | ||
| 1657 | #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 0x1UL | ||
| 1658 | #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 0x2UL | ||
| 1659 | #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK 0xf0UL | ||
| 1660 | #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT 4 | ||
| 1661 | #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1662 | #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1663 | #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1664 | #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1665 | #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1666 | #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1667 | u8 mrw_pg_size_mrw_lvl; | ||
| 1668 | #define CMDQ_INITIALIZE_FW_MRW_LVL_MASK 0xfUL | ||
| 1669 | #define CMDQ_INITIALIZE_FW_MRW_LVL_SFT 0 | ||
| 1670 | #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 0x0UL | ||
| 1671 | #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 0x1UL | ||
| 1672 | #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 0x2UL | ||
| 1673 | #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK 0xf0UL | ||
| 1674 | #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT 4 | ||
| 1675 | #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1676 | #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1677 | #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1678 | #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1679 | #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1680 | #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1681 | u8 srq_pg_size_srq_lvl; | ||
| 1682 | #define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK 0xfUL | ||
| 1683 | #define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT 0 | ||
| 1684 | #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 0x0UL | ||
| 1685 | #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 0x1UL | ||
| 1686 | #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 0x2UL | ||
| 1687 | #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK 0xf0UL | ||
| 1688 | #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT 4 | ||
| 1689 | #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1690 | #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1691 | #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1692 | #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1693 | #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1694 | #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1695 | u8 cq_pg_size_cq_lvl; | ||
| 1696 | #define CMDQ_INITIALIZE_FW_CQ_LVL_MASK 0xfUL | ||
| 1697 | #define CMDQ_INITIALIZE_FW_CQ_LVL_SFT 0 | ||
| 1698 | #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 0x0UL | ||
| 1699 | #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 0x1UL | ||
| 1700 | #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 0x2UL | ||
| 1701 | #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK 0xf0UL | ||
| 1702 | #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT 4 | ||
| 1703 | #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1704 | #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1705 | #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1706 | #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1707 | #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1708 | #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1709 | u8 tqm_pg_size_tqm_lvl; | ||
| 1710 | #define CMDQ_INITIALIZE_FW_TQM_LVL_MASK 0xfUL | ||
| 1711 | #define CMDQ_INITIALIZE_FW_TQM_LVL_SFT 0 | ||
| 1712 | #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 0x0UL | ||
| 1713 | #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 0x1UL | ||
| 1714 | #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 0x2UL | ||
| 1715 | #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK 0xf0UL | ||
| 1716 | #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT 4 | ||
| 1717 | #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1718 | #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1719 | #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1720 | #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1721 | #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1722 | #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1723 | u8 tim_pg_size_tim_lvl; | ||
| 1724 | #define CMDQ_INITIALIZE_FW_TIM_LVL_MASK 0xfUL | ||
| 1725 | #define CMDQ_INITIALIZE_FW_TIM_LVL_SFT 0 | ||
| 1726 | #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 0x0UL | ||
| 1727 | #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 0x1UL | ||
| 1728 | #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 0x2UL | ||
| 1729 | #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK 0xf0UL | ||
| 1730 | #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT 4 | ||
| 1731 | #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K (0x0UL << 4) | ||
| 1732 | #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K (0x1UL << 4) | ||
| 1733 | #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K (0x2UL << 4) | ||
| 1734 | #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M (0x3UL << 4) | ||
| 1735 | #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M (0x4UL << 4) | ||
| 1736 | #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G (0x5UL << 4) | ||
| 1737 | __le16 reserved16; | ||
| 1738 | __le64 qpc_page_dir; | ||
| 1739 | __le64 mrw_page_dir; | ||
| 1740 | __le64 srq_page_dir; | ||
| 1741 | __le64 cq_page_dir; | ||
| 1742 | __le64 tqm_page_dir; | ||
| 1743 | __le64 tim_page_dir; | ||
| 1744 | __le32 number_of_qp; | ||
| 1745 | __le32 number_of_mrw; | ||
| 1746 | __le32 number_of_srq; | ||
| 1747 | __le32 number_of_cq; | ||
| 1748 | __le32 max_qp_per_vf; | ||
| 1749 | __le32 max_mrw_per_vf; | ||
| 1750 | __le32 max_srq_per_vf; | ||
| 1751 | __le32 max_cq_per_vf; | ||
| 1752 | __le32 max_gid_per_vf; | ||
| 1753 | __le32 stat_ctx_id; | ||
| 1754 | }; | ||
| 1755 | |||
| 1756 | /* De-initialize Firmware command (16 bytes) */ | ||
| 1757 | struct cmdq_deinitialize_fw { | ||
| 1758 | u8 opcode; | ||
| 1759 | #define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW 0x81UL | ||
| 1760 | u8 cmd_size; | ||
| 1761 | __le16 flags; | ||
| 1762 | __le16 cookie; | ||
| 1763 | u8 resp_size; | ||
| 1764 | u8 reserved8; | ||
| 1765 | __le64 resp_addr; | ||
| 1766 | }; | ||
| 1767 | |||
| 1768 | /* Stop function command (16 bytes) */ | ||
| 1769 | struct cmdq_stop_func { | ||
| 1770 | u8 opcode; | ||
| 1771 | #define CMDQ_STOP_FUNC_OPCODE_STOP_FUNC 0x82UL | ||
| 1772 | u8 cmd_size; | ||
| 1773 | __le16 flags; | ||
| 1774 | __le16 cookie; | ||
| 1775 | u8 resp_size; | ||
| 1776 | u8 reserved8; | ||
| 1777 | __le64 resp_addr; | ||
| 1778 | }; | ||
| 1779 | |||
| 1780 | /* Query function command (16 bytes) */ | ||
| 1781 | struct cmdq_query_func { | ||
| 1782 | u8 opcode; | ||
| 1783 | #define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC 0x83UL | ||
| 1784 | u8 cmd_size; | ||
| 1785 | __le16 flags; | ||
| 1786 | __le16 cookie; | ||
| 1787 | u8 resp_size; | ||
| 1788 | u8 reserved8; | ||
| 1789 | __le64 resp_addr; | ||
| 1790 | }; | ||
| 1791 | |||
| 1792 | /* Set function resources command (16 bytes) */ | ||
| 1793 | struct cmdq_set_func_resources { | ||
| 1794 | u8 opcode; | ||
| 1795 | #define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL | ||
| 1796 | u8 cmd_size; | ||
| 1797 | __le16 flags; | ||
| 1798 | __le16 cookie; | ||
| 1799 | u8 resp_size; | ||
| 1800 | u8 reserved8; | ||
| 1801 | __le64 resp_addr; | ||
| 1802 | }; | ||
| 1803 | |||
| 1804 | /* Read hardware resource context command (24 bytes) */ | ||
| 1805 | struct cmdq_read_context { | ||
| 1806 | u8 opcode; | ||
| 1807 | #define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT 0x85UL | ||
| 1808 | u8 cmd_size; | ||
| 1809 | __le16 flags; | ||
| 1810 | __le16 cookie; | ||
| 1811 | u8 resp_size; | ||
| 1812 | u8 reserved8; | ||
| 1813 | __le64 resp_addr; | ||
| 1814 | __le32 type_xid; | ||
| 1815 | #define CMDQ_READ_CONTEXT_XID_MASK 0xffffffUL | ||
| 1816 | #define CMDQ_READ_CONTEXT_XID_SFT 0 | ||
| 1817 | #define CMDQ_READ_CONTEXT_TYPE_MASK 0xff000000UL | ||
| 1818 | #define CMDQ_READ_CONTEXT_TYPE_SFT 24 | ||
| 1819 | #define CMDQ_READ_CONTEXT_TYPE_QPC (0x0UL << 24) | ||
| 1820 | #define CMDQ_READ_CONTEXT_TYPE_CQ (0x1UL << 24) | ||
| 1821 | #define CMDQ_READ_CONTEXT_TYPE_MRW (0x2UL << 24) | ||
| 1822 | #define CMDQ_READ_CONTEXT_TYPE_SRQ (0x3UL << 24) | ||
| 1823 | __le32 unused_0; | ||
| 1824 | }; | ||
| 1825 | |||
| 1826 | /* Map TC to COS. Can only be issued from a PF (24 bytes) */ | ||
| 1827 | struct cmdq_map_tc_to_cos { | ||
| 1828 | u8 opcode; | ||
| 1829 | #define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS 0x8aUL | ||
| 1830 | u8 cmd_size; | ||
| 1831 | __le16 flags; | ||
| 1832 | __le16 cookie; | ||
| 1833 | u8 resp_size; | ||
| 1834 | u8 reserved8; | ||
| 1835 | __le64 resp_addr; | ||
| 1836 | __le16 cos0; | ||
| 1837 | #define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE 0xffffUL | ||
| 1838 | __le16 cos1; | ||
| 1839 | #define CMDQ_MAP_TC_TO_COS_COS1_DISABLE 0x8000UL | ||
| 1840 | #define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE 0xffffUL | ||
| 1841 | __le32 unused_0; | ||
| 1842 | }; | ||
| 1843 | |||
| 1844 | /* Query version command (16 bytes) */ | ||
| 1845 | struct cmdq_query_version { | ||
| 1846 | u8 opcode; | ||
| 1847 | #define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION 0x8bUL | ||
| 1848 | u8 cmd_size; | ||
| 1849 | __le16 flags; | ||
| 1850 | __le16 cookie; | ||
| 1851 | u8 resp_size; | ||
| 1852 | u8 reserved8; | ||
| 1853 | __le64 resp_addr; | ||
| 1854 | }; | ||
| 1855 | |||
| 1856 | /* Command-Response Event Queue (CREQ) Structures */ | ||
| 1857 | /* Base CREQ Record (16 bytes) */ | ||
| 1858 | struct creq_base { | ||
| 1859 | u8 type; | ||
| 1860 | #define CREQ_BASE_TYPE_MASK 0x3fUL | ||
| 1861 | #define CREQ_BASE_TYPE_SFT 0 | ||
| 1862 | #define CREQ_BASE_TYPE_QP_EVENT 0x38UL | ||
| 1863 | #define CREQ_BASE_TYPE_FUNC_EVENT 0x3aUL | ||
| 1864 | #define CREQ_BASE_RESERVED2_MASK 0xc0UL | ||
| 1865 | #define CREQ_BASE_RESERVED2_SFT 6 | ||
| 1866 | u8 reserved56[7]; | ||
| 1867 | u8 v; | ||
| 1868 | #define CREQ_BASE_V 0x1UL | ||
| 1869 | #define CREQ_BASE_RESERVED7_MASK 0xfeUL | ||
| 1870 | #define CREQ_BASE_RESERVED7_SFT 1 | ||
| 1871 | u8 event; | ||
| 1872 | __le16 reserved48[3]; | ||
| 1873 | }; | ||
| 1874 | |||
| 1875 | /* RoCE Function Async Event Notification (16 bytes) */ | ||
| 1876 | struct creq_func_event { | ||
| 1877 | u8 type; | ||
| 1878 | #define CREQ_FUNC_EVENT_TYPE_MASK 0x3fUL | ||
| 1879 | #define CREQ_FUNC_EVENT_TYPE_SFT 0 | ||
| 1880 | #define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT 0x3aUL | ||
| 1881 | #define CREQ_FUNC_EVENT_RESERVED2_MASK 0xc0UL | ||
| 1882 | #define CREQ_FUNC_EVENT_RESERVED2_SFT 6 | ||
| 1883 | u8 reserved56[7]; | ||
| 1884 | u8 v; | ||
| 1885 | #define CREQ_FUNC_EVENT_V 0x1UL | ||
| 1886 | #define CREQ_FUNC_EVENT_RESERVED7_MASK 0xfeUL | ||
| 1887 | #define CREQ_FUNC_EVENT_RESERVED7_SFT 1 | ||
| 1888 | u8 event; | ||
| 1889 | #define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR 0x1UL | ||
| 1890 | #define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR 0x2UL | ||
| 1891 | #define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR 0x3UL | ||
| 1892 | #define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR 0x4UL | ||
| 1893 | #define CREQ_FUNC_EVENT_EVENT_CQ_ERROR 0x5UL | ||
| 1894 | #define CREQ_FUNC_EVENT_EVENT_TQM_ERROR 0x6UL | ||
| 1895 | #define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR 0x7UL | ||
| 1896 | #define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR 0x8UL | ||
| 1897 | #define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR 0x9UL | ||
| 1898 | #define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR 0xaUL | ||
| 1899 | #define CREQ_FUNC_EVENT_EVENT_TIM_ERROR 0xbUL | ||
| 1900 | #define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST 0x80UL | ||
| 1901 | #define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED 0x81UL | ||
| 1902 | __le16 reserved48[3]; | ||
| 1903 | }; | ||
| 1904 | |||
| 1905 | /* RoCE Slowpath Command Completion (16 bytes) */ | ||
| 1906 | struct creq_qp_event { | ||
| 1907 | u8 type; | ||
| 1908 | #define CREQ_QP_EVENT_TYPE_MASK 0x3fUL | ||
| 1909 | #define CREQ_QP_EVENT_TYPE_SFT 0 | ||
| 1910 | #define CREQ_QP_EVENT_TYPE_QP_EVENT 0x38UL | ||
| 1911 | #define CREQ_QP_EVENT_RESERVED2_MASK 0xc0UL | ||
| 1912 | #define CREQ_QP_EVENT_RESERVED2_SFT 6 | ||
| 1913 | u8 status; | ||
| 1914 | __le16 cookie; | ||
| 1915 | __le32 reserved32; | ||
| 1916 | u8 v; | ||
| 1917 | #define CREQ_QP_EVENT_V 0x1UL | ||
| 1918 | #define CREQ_QP_EVENT_RESERVED7_MASK 0xfeUL | ||
| 1919 | #define CREQ_QP_EVENT_RESERVED7_SFT 1 | ||
| 1920 | u8 event; | ||
| 1921 | #define CREQ_QP_EVENT_EVENT_CREATE_QP 0x1UL | ||
| 1922 | #define CREQ_QP_EVENT_EVENT_DESTROY_QP 0x2UL | ||
| 1923 | #define CREQ_QP_EVENT_EVENT_MODIFY_QP 0x3UL | ||
| 1924 | #define CREQ_QP_EVENT_EVENT_QUERY_QP 0x4UL | ||
| 1925 | #define CREQ_QP_EVENT_EVENT_CREATE_SRQ 0x5UL | ||
| 1926 | #define CREQ_QP_EVENT_EVENT_DESTROY_SRQ 0x6UL | ||
| 1927 | #define CREQ_QP_EVENT_EVENT_QUERY_SRQ 0x8UL | ||
| 1928 | #define CREQ_QP_EVENT_EVENT_CREATE_CQ 0x9UL | ||
| 1929 | #define CREQ_QP_EVENT_EVENT_DESTROY_CQ 0xaUL | ||
| 1930 | #define CREQ_QP_EVENT_EVENT_RESIZE_CQ 0xcUL | ||
| 1931 | #define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW 0xdUL | ||
| 1932 | #define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY 0xeUL | ||
| 1933 | #define CREQ_QP_EVENT_EVENT_REGISTER_MR 0xfUL | ||
| 1934 | #define CREQ_QP_EVENT_EVENT_DEREGISTER_MR 0x10UL | ||
| 1935 | #define CREQ_QP_EVENT_EVENT_ADD_GID 0x11UL | ||
| 1936 | #define CREQ_QP_EVENT_EVENT_DELETE_GID 0x12UL | ||
| 1937 | #define CREQ_QP_EVENT_EVENT_MODIFY_GID 0x17UL | ||
| 1938 | #define CREQ_QP_EVENT_EVENT_QUERY_GID 0x18UL | ||
| 1939 | #define CREQ_QP_EVENT_EVENT_CREATE_QP1 0x13UL | ||
| 1940 | #define CREQ_QP_EVENT_EVENT_DESTROY_QP1 0x14UL | ||
| 1941 | #define CREQ_QP_EVENT_EVENT_CREATE_AH 0x15UL | ||
| 1942 | #define CREQ_QP_EVENT_EVENT_DESTROY_AH 0x16UL | ||
| 1943 | #define CREQ_QP_EVENT_EVENT_INITIALIZE_FW 0x80UL | ||
| 1944 | #define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW 0x81UL | ||
| 1945 | #define CREQ_QP_EVENT_EVENT_STOP_FUNC 0x82UL | ||
| 1946 | #define CREQ_QP_EVENT_EVENT_QUERY_FUNC 0x83UL | ||
| 1947 | #define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES 0x84UL | ||
| 1948 | #define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS 0x8aUL | ||
| 1949 | #define CREQ_QP_EVENT_EVENT_QUERY_VERSION 0x8bUL | ||
| 1950 | #define CREQ_QP_EVENT_EVENT_MODIFY_CC 0x8cUL | ||
| 1951 | #define CREQ_QP_EVENT_EVENT_QUERY_CC 0x8dUL | ||
| 1952 | #define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION 0xc0UL | ||
| 1953 | __le16 reserved48[3]; | ||
| 1954 | }; | ||
| 1955 | |||
| 1956 | /* Create QP command response (16 bytes) */ | ||
| 1957 | struct creq_create_qp_resp { | ||
| 1958 | u8 type; | ||
| 1959 | #define CREQ_CREATE_QP_RESP_TYPE_MASK 0x3fUL | ||
| 1960 | #define CREQ_CREATE_QP_RESP_TYPE_SFT 0 | ||
| 1961 | #define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT 0x38UL | ||
| 1962 | #define CREQ_CREATE_QP_RESP_RESERVED2_MASK 0xc0UL | ||
| 1963 | #define CREQ_CREATE_QP_RESP_RESERVED2_SFT 6 | ||
| 1964 | u8 status; | ||
| 1965 | __le16 cookie; | ||
| 1966 | __le32 xid; | ||
| 1967 | u8 v; | ||
| 1968 | #define CREQ_CREATE_QP_RESP_V 0x1UL | ||
| 1969 | #define CREQ_CREATE_QP_RESP_RESERVED7_MASK 0xfeUL | ||
| 1970 | #define CREQ_CREATE_QP_RESP_RESERVED7_SFT 1 | ||
| 1971 | u8 event; | ||
| 1972 | #define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP 0x1UL | ||
| 1973 | __le16 reserved48[3]; | ||
| 1974 | }; | ||
| 1975 | |||
| 1976 | /* Destroy QP command response (16 bytes) */ | ||
| 1977 | struct creq_destroy_qp_resp { | ||
| 1978 | u8 type; | ||
| 1979 | #define CREQ_DESTROY_QP_RESP_TYPE_MASK 0x3fUL | ||
| 1980 | #define CREQ_DESTROY_QP_RESP_TYPE_SFT 0 | ||
| 1981 | #define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT 0x38UL | ||
| 1982 | #define CREQ_DESTROY_QP_RESP_RESERVED2_MASK 0xc0UL | ||
| 1983 | #define CREQ_DESTROY_QP_RESP_RESERVED2_SFT 6 | ||
| 1984 | u8 status; | ||
| 1985 | __le16 cookie; | ||
| 1986 | __le32 xid; | ||
| 1987 | u8 v; | ||
| 1988 | #define CREQ_DESTROY_QP_RESP_V 0x1UL | ||
| 1989 | #define CREQ_DESTROY_QP_RESP_RESERVED7_MASK 0xfeUL | ||
| 1990 | #define CREQ_DESTROY_QP_RESP_RESERVED7_SFT 1 | ||
| 1991 | u8 event; | ||
| 1992 | #define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP 0x2UL | ||
| 1993 | __le16 reserved48[3]; | ||
| 1994 | }; | ||
| 1995 | |||
| 1996 | /* Modify QP command response (16 bytes) */ | ||
| 1997 | struct creq_modify_qp_resp { | ||
| 1998 | u8 type; | ||
| 1999 | #define CREQ_MODIFY_QP_RESP_TYPE_MASK 0x3fUL | ||
| 2000 | #define CREQ_MODIFY_QP_RESP_TYPE_SFT 0 | ||
| 2001 | #define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2002 | #define CREQ_MODIFY_QP_RESP_RESERVED2_MASK 0xc0UL | ||
| 2003 | #define CREQ_MODIFY_QP_RESP_RESERVED2_SFT 6 | ||
| 2004 | u8 status; | ||
| 2005 | __le16 cookie; | ||
| 2006 | __le32 xid; | ||
| 2007 | u8 v; | ||
| 2008 | #define CREQ_MODIFY_QP_RESP_V 0x1UL | ||
| 2009 | #define CREQ_MODIFY_QP_RESP_RESERVED7_MASK 0xfeUL | ||
| 2010 | #define CREQ_MODIFY_QP_RESP_RESERVED7_SFT 1 | ||
| 2011 | u8 event; | ||
| 2012 | #define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP 0x3UL | ||
| 2013 | __le16 reserved48[3]; | ||
| 2014 | }; | ||
| 2015 | |||
| 2016 | /* Query QP command response (16 bytes) */ | ||
| 2017 | struct creq_query_qp_resp { | ||
| 2018 | u8 type; | ||
| 2019 | #define CREQ_QUERY_QP_RESP_TYPE_MASK 0x3fUL | ||
| 2020 | #define CREQ_QUERY_QP_RESP_TYPE_SFT 0 | ||
| 2021 | #define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2022 | #define CREQ_QUERY_QP_RESP_RESERVED2_MASK 0xc0UL | ||
| 2023 | #define CREQ_QUERY_QP_RESP_RESERVED2_SFT 6 | ||
| 2024 | u8 status; | ||
| 2025 | __le16 cookie; | ||
| 2026 | __le32 size; | ||
| 2027 | u8 v; | ||
| 2028 | #define CREQ_QUERY_QP_RESP_V 0x1UL | ||
| 2029 | #define CREQ_QUERY_QP_RESP_RESERVED7_MASK 0xfeUL | ||
| 2030 | #define CREQ_QUERY_QP_RESP_RESERVED7_SFT 1 | ||
| 2031 | u8 event; | ||
| 2032 | #define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP 0x4UL | ||
| 2033 | __le16 reserved48[3]; | ||
| 2034 | }; | ||
| 2035 | |||
| 2036 | /* Query QP command response side buffer structure (104 bytes) */ | ||
| 2037 | struct creq_query_qp_resp_sb { | ||
| 2038 | u8 opcode; | ||
| 2039 | #define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP 0x4UL | ||
| 2040 | u8 status; | ||
| 2041 | __le16 cookie; | ||
| 2042 | __le16 flags; | ||
| 2043 | u8 resp_size; | ||
| 2044 | u8 reserved8; | ||
| 2045 | __le32 xid; | ||
| 2046 | u8 en_sqd_async_notify_state; | ||
| 2047 | #define CREQ_QUERY_QP_RESP_SB_STATE_MASK 0xfUL | ||
| 2048 | #define CREQ_QUERY_QP_RESP_SB_STATE_SFT 0 | ||
| 2049 | #define CREQ_QUERY_QP_RESP_SB_STATE_RESET 0x0UL | ||
| 2050 | #define CREQ_QUERY_QP_RESP_SB_STATE_INIT 0x1UL | ||
| 2051 | #define CREQ_QUERY_QP_RESP_SB_STATE_RTR 0x2UL | ||
| 2052 | #define CREQ_QUERY_QP_RESP_SB_STATE_RTS 0x3UL | ||
| 2053 | #define CREQ_QUERY_QP_RESP_SB_STATE_SQD 0x4UL | ||
| 2054 | #define CREQ_QUERY_QP_RESP_SB_STATE_SQE 0x5UL | ||
| 2055 | #define CREQ_QUERY_QP_RESP_SB_STATE_ERR 0x6UL | ||
| 2056 | #define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY 0x10UL | ||
| 2057 | u8 access; | ||
| 2058 | #define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE 0x1UL | ||
| 2059 | #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE 0x2UL | ||
| 2060 | #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ 0x4UL | ||
| 2061 | #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC 0x8UL | ||
| 2062 | __le16 pkey; | ||
| 2063 | __le32 qkey; | ||
| 2064 | __le32 reserved32; | ||
| 2065 | __le32 dgid[4]; | ||
| 2066 | __le32 flow_label; | ||
| 2067 | __le16 sgid_index; | ||
| 2068 | u8 hop_limit; | ||
| 2069 | u8 traffic_class; | ||
| 2070 | __le16 dest_mac[3]; | ||
| 2071 | __le16 path_mtu_dest_vlan_id; | ||
| 2072 | #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK 0xfffUL | ||
| 2073 | #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0 | ||
| 2074 | #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK 0xf000UL | ||
| 2075 | #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT 12 | ||
| 2076 | #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 (0x0UL << 12) | ||
| 2077 | #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 (0x1UL << 12) | ||
| 2078 | #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 (0x2UL << 12) | ||
| 2079 | #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 (0x3UL << 12) | ||
| 2080 | #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 (0x4UL << 12) | ||
| 2081 | #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 (0x5UL << 12) | ||
| 2082 | u8 timeout; | ||
| 2083 | u8 retry_cnt; | ||
| 2084 | u8 rnr_retry; | ||
| 2085 | u8 min_rnr_timer; | ||
| 2086 | __le32 rq_psn; | ||
| 2087 | __le32 sq_psn; | ||
| 2088 | u8 max_rd_atomic; | ||
| 2089 | u8 max_dest_rd_atomic; | ||
| 2090 | u8 tos_dscp_tos_ecn; | ||
| 2091 | #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK 0x3UL | ||
| 2092 | #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT 0 | ||
| 2093 | #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK 0xfcUL | ||
| 2094 | #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2 | ||
| 2095 | u8 enable_cc; | ||
| 2096 | #define CREQ_QUERY_QP_RESP_SB_ENABLE_CC 0x1UL | ||
| 2097 | #define CREQ_QUERY_QP_RESP_SB_RESERVED7_MASK 0xfeUL | ||
| 2098 | #define CREQ_QUERY_QP_RESP_SB_RESERVED7_SFT 1 | ||
| 2099 | __le32 sq_size; | ||
| 2100 | __le32 rq_size; | ||
| 2101 | __le16 sq_sge; | ||
| 2102 | __le16 rq_sge; | ||
| 2103 | __le32 max_inline_data; | ||
| 2104 | __le32 dest_qp_id; | ||
| 2105 | __le32 unused_1; | ||
| 2106 | __le16 src_mac[3]; | ||
| 2107 | __le16 vlan_pcp_vlan_dei_vlan_id; | ||
| 2108 | #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK 0xfffUL | ||
| 2109 | #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT 0 | ||
| 2110 | #define CREQ_QUERY_QP_RESP_SB_VLAN_DEI 0x1000UL | ||
| 2111 | #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK 0xe000UL | ||
| 2112 | #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13 | ||
| 2113 | }; | ||
| 2114 | |||
| 2115 | /* Create SRQ command response (16 bytes) */ | ||
| 2116 | struct creq_create_srq_resp { | ||
| 2117 | u8 type; | ||
| 2118 | #define CREQ_CREATE_SRQ_RESP_TYPE_MASK 0x3fUL | ||
| 2119 | #define CREQ_CREATE_SRQ_RESP_TYPE_SFT 0 | ||
| 2120 | #define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2121 | #define CREQ_CREATE_SRQ_RESP_RESERVED2_MASK 0xc0UL | ||
| 2122 | #define CREQ_CREATE_SRQ_RESP_RESERVED2_SFT 6 | ||
| 2123 | u8 status; | ||
| 2124 | __le16 cookie; | ||
| 2125 | __le32 xid; | ||
| 2126 | u8 v; | ||
| 2127 | #define CREQ_CREATE_SRQ_RESP_V 0x1UL | ||
| 2128 | #define CREQ_CREATE_SRQ_RESP_RESERVED7_MASK 0xfeUL | ||
| 2129 | #define CREQ_CREATE_SRQ_RESP_RESERVED7_SFT 1 | ||
| 2130 | u8 event; | ||
| 2131 | #define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ 0x5UL | ||
| 2132 | __le16 reserved48[3]; | ||
| 2133 | }; | ||
| 2134 | |||
| 2135 | /* Destroy SRQ command response (16 bytes) */ | ||
| 2136 | struct creq_destroy_srq_resp { | ||
| 2137 | u8 type; | ||
| 2138 | #define CREQ_DESTROY_SRQ_RESP_TYPE_MASK 0x3fUL | ||
| 2139 | #define CREQ_DESTROY_SRQ_RESP_TYPE_SFT 0 | ||
| 2140 | #define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2141 | #define CREQ_DESTROY_SRQ_RESP_RESERVED2_MASK 0xc0UL | ||
| 2142 | #define CREQ_DESTROY_SRQ_RESP_RESERVED2_SFT 6 | ||
| 2143 | u8 status; | ||
| 2144 | __le16 cookie; | ||
| 2145 | __le32 xid; | ||
| 2146 | u8 v; | ||
| 2147 | #define CREQ_DESTROY_SRQ_RESP_V 0x1UL | ||
| 2148 | #define CREQ_DESTROY_SRQ_RESP_RESERVED7_MASK 0xfeUL | ||
| 2149 | #define CREQ_DESTROY_SRQ_RESP_RESERVED7_SFT 1 | ||
| 2150 | u8 event; | ||
| 2151 | #define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ 0x6UL | ||
| 2152 | __le16 enable_for_arm[3]; | ||
| 2153 | #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK 0x30000UL | ||
| 2154 | #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16 | ||
| 2155 | #define CREQ_DESTROY_SRQ_RESP_RESERVED46_MASK 0xfffc0000UL | ||
| 2156 | #define CREQ_DESTROY_SRQ_RESP_RESERVED46_SFT 18 | ||
| 2157 | }; | ||
| 2158 | |||
| 2159 | /* Query SRQ command response (16 bytes) */ | ||
| 2160 | struct creq_query_srq_resp { | ||
| 2161 | u8 type; | ||
| 2162 | #define CREQ_QUERY_SRQ_RESP_TYPE_MASK 0x3fUL | ||
| 2163 | #define CREQ_QUERY_SRQ_RESP_TYPE_SFT 0 | ||
| 2164 | #define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2165 | #define CREQ_QUERY_SRQ_RESP_RESERVED2_MASK 0xc0UL | ||
| 2166 | #define CREQ_QUERY_SRQ_RESP_RESERVED2_SFT 6 | ||
| 2167 | u8 status; | ||
| 2168 | __le16 cookie; | ||
| 2169 | __le32 size; | ||
| 2170 | u8 v; | ||
| 2171 | #define CREQ_QUERY_SRQ_RESP_V 0x1UL | ||
| 2172 | #define CREQ_QUERY_SRQ_RESP_RESERVED7_MASK 0xfeUL | ||
| 2173 | #define CREQ_QUERY_SRQ_RESP_RESERVED7_SFT 1 | ||
| 2174 | u8 event; | ||
| 2175 | #define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ 0x8UL | ||
| 2176 | __le16 reserved48[3]; | ||
| 2177 | }; | ||
| 2178 | |||
| 2179 | /* Query SRQ command response side buffer structure (24 bytes) */ | ||
| 2180 | struct creq_query_srq_resp_sb { | ||
| 2181 | u8 opcode; | ||
| 2182 | #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ 0x8UL | ||
| 2183 | u8 status; | ||
| 2184 | __le16 cookie; | ||
| 2185 | __le16 flags; | ||
| 2186 | u8 resp_size; | ||
| 2187 | u8 reserved8; | ||
| 2188 | __le32 xid; | ||
| 2189 | __le16 srq_limit; | ||
| 2190 | __le16 reserved16; | ||
| 2191 | __le32 data[4]; | ||
| 2192 | }; | ||
| 2193 | |||
| 2194 | /* Create CQ command Response (16 bytes) */ | ||
| 2195 | struct creq_create_cq_resp { | ||
| 2196 | u8 type; | ||
| 2197 | #define CREQ_CREATE_CQ_RESP_TYPE_MASK 0x3fUL | ||
| 2198 | #define CREQ_CREATE_CQ_RESP_TYPE_SFT 0 | ||
| 2199 | #define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2200 | #define CREQ_CREATE_CQ_RESP_RESERVED2_MASK 0xc0UL | ||
| 2201 | #define CREQ_CREATE_CQ_RESP_RESERVED2_SFT 6 | ||
| 2202 | u8 status; | ||
| 2203 | __le16 cookie; | ||
| 2204 | __le32 xid; | ||
| 2205 | u8 v; | ||
| 2206 | #define CREQ_CREATE_CQ_RESP_V 0x1UL | ||
| 2207 | #define CREQ_CREATE_CQ_RESP_RESERVED7_MASK 0xfeUL | ||
| 2208 | #define CREQ_CREATE_CQ_RESP_RESERVED7_SFT 1 | ||
| 2209 | u8 event; | ||
| 2210 | #define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ 0x9UL | ||
| 2211 | __le16 reserved48[3]; | ||
| 2212 | }; | ||
| 2213 | |||
| 2214 | /* Destroy CQ command response (16 bytes) */ | ||
| 2215 | struct creq_destroy_cq_resp { | ||
| 2216 | u8 type; | ||
| 2217 | #define CREQ_DESTROY_CQ_RESP_TYPE_MASK 0x3fUL | ||
| 2218 | #define CREQ_DESTROY_CQ_RESP_TYPE_SFT 0 | ||
| 2219 | #define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2220 | #define CREQ_DESTROY_CQ_RESP_RESERVED2_MASK 0xc0UL | ||
| 2221 | #define CREQ_DESTROY_CQ_RESP_RESERVED2_SFT 6 | ||
| 2222 | u8 status; | ||
| 2223 | __le16 cookie; | ||
| 2224 | __le32 xid; | ||
| 2225 | u8 v; | ||
| 2226 | #define CREQ_DESTROY_CQ_RESP_V 0x1UL | ||
| 2227 | #define CREQ_DESTROY_CQ_RESP_RESERVED7_MASK 0xfeUL | ||
| 2228 | #define CREQ_DESTROY_CQ_RESP_RESERVED7_SFT 1 | ||
| 2229 | u8 event; | ||
| 2230 | #define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ 0xaUL | ||
| 2231 | __le16 cq_arm_lvl; | ||
| 2232 | #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK 0x3UL | ||
| 2233 | #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0 | ||
| 2234 | #define CREQ_DESTROY_CQ_RESP_RESERVED14_MASK 0xfffcUL | ||
| 2235 | #define CREQ_DESTROY_CQ_RESP_RESERVED14_SFT 2 | ||
| 2236 | __le16 total_cnq_events; | ||
| 2237 | __le16 reserved16; | ||
| 2238 | }; | ||
| 2239 | |||
| 2240 | /* Resize CQ command response (16 bytes) */ | ||
| 2241 | struct creq_resize_cq_resp { | ||
| 2242 | u8 type; | ||
| 2243 | #define CREQ_RESIZE_CQ_RESP_TYPE_MASK 0x3fUL | ||
| 2244 | #define CREQ_RESIZE_CQ_RESP_TYPE_SFT 0 | ||
| 2245 | #define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2246 | #define CREQ_RESIZE_CQ_RESP_RESERVED2_MASK 0xc0UL | ||
| 2247 | #define CREQ_RESIZE_CQ_RESP_RESERVED2_SFT 6 | ||
| 2248 | u8 status; | ||
| 2249 | __le16 cookie; | ||
| 2250 | __le32 xid; | ||
| 2251 | u8 v; | ||
| 2252 | #define CREQ_RESIZE_CQ_RESP_V 0x1UL | ||
| 2253 | #define CREQ_RESIZE_CQ_RESP_RESERVED7_MASK 0xfeUL | ||
| 2254 | #define CREQ_RESIZE_CQ_RESP_RESERVED7_SFT 1 | ||
| 2255 | u8 event; | ||
| 2256 | #define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ 0xcUL | ||
| 2257 | __le16 reserved48[3]; | ||
| 2258 | }; | ||
| 2259 | |||
| 2260 | /* Allocate MRW command response (16 bytes) */ | ||
| 2261 | struct creq_allocate_mrw_resp { | ||
| 2262 | u8 type; | ||
| 2263 | #define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK 0x3fUL | ||
| 2264 | #define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT 0 | ||
| 2265 | #define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2266 | #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_MASK 0xc0UL | ||
| 2267 | #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_SFT 6 | ||
| 2268 | u8 status; | ||
| 2269 | __le16 cookie; | ||
| 2270 | __le32 xid; | ||
| 2271 | u8 v; | ||
| 2272 | #define CREQ_ALLOCATE_MRW_RESP_V 0x1UL | ||
| 2273 | #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_MASK 0xfeUL | ||
| 2274 | #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_SFT 1 | ||
| 2275 | u8 event; | ||
| 2276 | #define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW 0xdUL | ||
| 2277 | __le16 reserved48[3]; | ||
| 2278 | }; | ||
| 2279 | |||
| 2280 | /* De-allocate key command response (16 bytes) */ | ||
| 2281 | struct creq_deallocate_key_resp { | ||
| 2282 | u8 type; | ||
| 2283 | #define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK 0x3fUL | ||
| 2284 | #define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT 0 | ||
| 2285 | #define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2286 | #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_MASK 0xc0UL | ||
| 2287 | #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_SFT 6 | ||
| 2288 | u8 status; | ||
| 2289 | __le16 cookie; | ||
| 2290 | __le32 xid; | ||
| 2291 | u8 v; | ||
| 2292 | #define CREQ_DEALLOCATE_KEY_RESP_V 0x1UL | ||
| 2293 | #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_MASK 0xfeUL | ||
| 2294 | #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_SFT 1 | ||
| 2295 | u8 event; | ||
| 2296 | #define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY 0xeUL | ||
| 2297 | __le16 reserved16; | ||
| 2298 | __le32 bound_window_info; | ||
| 2299 | }; | ||
| 2300 | |||
| 2301 | /* Register MR command response (16 bytes) */ | ||
| 2302 | struct creq_register_mr_resp { | ||
| 2303 | u8 type; | ||
| 2304 | #define CREQ_REGISTER_MR_RESP_TYPE_MASK 0x3fUL | ||
| 2305 | #define CREQ_REGISTER_MR_RESP_TYPE_SFT 0 | ||
| 2306 | #define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2307 | #define CREQ_REGISTER_MR_RESP_RESERVED2_MASK 0xc0UL | ||
| 2308 | #define CREQ_REGISTER_MR_RESP_RESERVED2_SFT 6 | ||
| 2309 | u8 status; | ||
| 2310 | __le16 cookie; | ||
| 2311 | __le32 xid; | ||
| 2312 | u8 v; | ||
| 2313 | #define CREQ_REGISTER_MR_RESP_V 0x1UL | ||
| 2314 | #define CREQ_REGISTER_MR_RESP_RESERVED7_MASK 0xfeUL | ||
| 2315 | #define CREQ_REGISTER_MR_RESP_RESERVED7_SFT 1 | ||
| 2316 | u8 event; | ||
| 2317 | #define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR 0xfUL | ||
| 2318 | __le16 reserved48[3]; | ||
| 2319 | }; | ||
| 2320 | |||
| 2321 | /* Deregister MR command response (16 bytes) */ | ||
| 2322 | struct creq_deregister_mr_resp { | ||
| 2323 | u8 type; | ||
| 2324 | #define CREQ_DEREGISTER_MR_RESP_TYPE_MASK 0x3fUL | ||
| 2325 | #define CREQ_DEREGISTER_MR_RESP_TYPE_SFT 0 | ||
| 2326 | #define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2327 | #define CREQ_DEREGISTER_MR_RESP_RESERVED2_MASK 0xc0UL | ||
| 2328 | #define CREQ_DEREGISTER_MR_RESP_RESERVED2_SFT 6 | ||
| 2329 | u8 status; | ||
| 2330 | __le16 cookie; | ||
| 2331 | __le32 xid; | ||
| 2332 | u8 v; | ||
| 2333 | #define CREQ_DEREGISTER_MR_RESP_V 0x1UL | ||
| 2334 | #define CREQ_DEREGISTER_MR_RESP_RESERVED7_MASK 0xfeUL | ||
| 2335 | #define CREQ_DEREGISTER_MR_RESP_RESERVED7_SFT 1 | ||
| 2336 | u8 event; | ||
| 2337 | #define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR 0x10UL | ||
| 2338 | __le16 reserved16; | ||
| 2339 | __le32 bound_windows; | ||
| 2340 | }; | ||
| 2341 | |||
| 2342 | /* Add GID command response (16 bytes) */ | ||
| 2343 | struct creq_add_gid_resp { | ||
| 2344 | u8 type; | ||
| 2345 | #define CREQ_ADD_GID_RESP_TYPE_MASK 0x3fUL | ||
| 2346 | #define CREQ_ADD_GID_RESP_TYPE_SFT 0 | ||
| 2347 | #define CREQ_ADD_GID_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2348 | #define CREQ_ADD_GID_RESP_RESERVED2_MASK 0xc0UL | ||
| 2349 | #define CREQ_ADD_GID_RESP_RESERVED2_SFT 6 | ||
| 2350 | u8 status; | ||
| 2351 | __le16 cookie; | ||
| 2352 | __le32 xid; | ||
| 2353 | u8 v; | ||
| 2354 | #define CREQ_ADD_GID_RESP_V 0x1UL | ||
| 2355 | #define CREQ_ADD_GID_RESP_RESERVED7_MASK 0xfeUL | ||
| 2356 | #define CREQ_ADD_GID_RESP_RESERVED7_SFT 1 | ||
| 2357 | u8 event; | ||
| 2358 | #define CREQ_ADD_GID_RESP_EVENT_ADD_GID 0x11UL | ||
| 2359 | __le16 reserved48[3]; | ||
| 2360 | }; | ||
| 2361 | |||
| 2362 | /* Delete GID command response (16 bytes) */ | ||
| 2363 | struct creq_delete_gid_resp { | ||
| 2364 | u8 type; | ||
| 2365 | #define CREQ_DELETE_GID_RESP_TYPE_MASK 0x3fUL | ||
| 2366 | #define CREQ_DELETE_GID_RESP_TYPE_SFT 0 | ||
| 2367 | #define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2368 | #define CREQ_DELETE_GID_RESP_RESERVED2_MASK 0xc0UL | ||
| 2369 | #define CREQ_DELETE_GID_RESP_RESERVED2_SFT 6 | ||
| 2370 | u8 status; | ||
| 2371 | __le16 cookie; | ||
| 2372 | __le32 xid; | ||
| 2373 | u8 v; | ||
| 2374 | #define CREQ_DELETE_GID_RESP_V 0x1UL | ||
| 2375 | #define CREQ_DELETE_GID_RESP_RESERVED7_MASK 0xfeUL | ||
| 2376 | #define CREQ_DELETE_GID_RESP_RESERVED7_SFT 1 | ||
| 2377 | u8 event; | ||
| 2378 | #define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID 0x12UL | ||
| 2379 | __le16 reserved48[3]; | ||
| 2380 | }; | ||
| 2381 | |||
| 2382 | /* Modify GID command response (16 bytes) */ | ||
| 2383 | struct creq_modify_gid_resp { | ||
| 2384 | u8 type; | ||
| 2385 | #define CREQ_MODIFY_GID_RESP_TYPE_MASK 0x3fUL | ||
| 2386 | #define CREQ_MODIFY_GID_RESP_TYPE_SFT 0 | ||
| 2387 | #define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2388 | #define CREQ_MODIFY_GID_RESP_RESERVED2_MASK 0xc0UL | ||
| 2389 | #define CREQ_MODIFY_GID_RESP_RESERVED2_SFT 6 | ||
| 2390 | u8 status; | ||
| 2391 | __le16 cookie; | ||
| 2392 | __le32 xid; | ||
| 2393 | u8 v; | ||
| 2394 | #define CREQ_MODIFY_GID_RESP_V 0x1UL | ||
| 2395 | #define CREQ_MODIFY_GID_RESP_RESERVED7_MASK 0xfeUL | ||
| 2396 | #define CREQ_MODIFY_GID_RESP_RESERVED7_SFT 1 | ||
| 2397 | u8 event; | ||
| 2398 | #define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID 0x11UL | ||
| 2399 | __le16 reserved48[3]; | ||
| 2400 | }; | ||
| 2401 | |||
| 2402 | /* Query GID command response (16 bytes) */ | ||
| 2403 | struct creq_query_gid_resp { | ||
| 2404 | u8 type; | ||
| 2405 | #define CREQ_QUERY_GID_RESP_TYPE_MASK 0x3fUL | ||
| 2406 | #define CREQ_QUERY_GID_RESP_TYPE_SFT 0 | ||
| 2407 | #define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2408 | #define CREQ_QUERY_GID_RESP_RESERVED2_MASK 0xc0UL | ||
| 2409 | #define CREQ_QUERY_GID_RESP_RESERVED2_SFT 6 | ||
| 2410 | u8 status; | ||
| 2411 | __le16 cookie; | ||
| 2412 | __le32 size; | ||
| 2413 | u8 v; | ||
| 2414 | #define CREQ_QUERY_GID_RESP_V 0x1UL | ||
| 2415 | #define CREQ_QUERY_GID_RESP_RESERVED7_MASK 0xfeUL | ||
| 2416 | #define CREQ_QUERY_GID_RESP_RESERVED7_SFT 1 | ||
| 2417 | u8 event; | ||
| 2418 | #define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID 0x18UL | ||
| 2419 | __le16 reserved48[3]; | ||
| 2420 | }; | ||
| 2421 | |||
| 2422 | /* Query GID command response side buffer structure (40 bytes) */ | ||
| 2423 | struct creq_query_gid_resp_sb { | ||
| 2424 | u8 opcode; | ||
| 2425 | #define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID 0x18UL | ||
| 2426 | u8 status; | ||
| 2427 | __le16 cookie; | ||
| 2428 | __le16 flags; | ||
| 2429 | u8 resp_size; | ||
| 2430 | u8 reserved8; | ||
| 2431 | __le32 gid[4]; | ||
| 2432 | __le16 src_mac[3]; | ||
| 2433 | __le16 vlan; | ||
| 2434 | #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK 0xfffUL | ||
| 2435 | #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT 0 | ||
| 2436 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK 0x7000UL | ||
| 2437 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT 12 | ||
| 2438 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 (0x0UL << 12) | ||
| 2439 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 (0x1UL << 12) | ||
| 2440 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 (0x2UL << 12) | ||
| 2441 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 (0x3UL << 12) | ||
| 2442 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 (0x4UL << 12) | ||
| 2443 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 (0x5UL << 12) | ||
| 2444 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 (0x6UL << 12) | ||
| 2445 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 (0x7UL << 12) | ||
| 2446 | #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST \ | ||
| 2447 | CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 | ||
| 2448 | #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN 0x8000UL | ||
| 2449 | __le16 ipid; | ||
| 2450 | __le16 gid_index; | ||
| 2451 | __le32 unused_0; | ||
| 2452 | }; | ||
| 2453 | |||
| 2454 | /* Create QP1 command response (16 bytes) */ | ||
| 2455 | struct creq_create_qp1_resp { | ||
| 2456 | u8 type; | ||
| 2457 | #define CREQ_CREATE_QP1_RESP_TYPE_MASK 0x3fUL | ||
| 2458 | #define CREQ_CREATE_QP1_RESP_TYPE_SFT 0 | ||
| 2459 | #define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2460 | #define CREQ_CREATE_QP1_RESP_RESERVED2_MASK 0xc0UL | ||
| 2461 | #define CREQ_CREATE_QP1_RESP_RESERVED2_SFT 6 | ||
| 2462 | u8 status; | ||
| 2463 | __le16 cookie; | ||
| 2464 | __le32 xid; | ||
| 2465 | u8 v; | ||
| 2466 | #define CREQ_CREATE_QP1_RESP_V 0x1UL | ||
| 2467 | #define CREQ_CREATE_QP1_RESP_RESERVED7_MASK 0xfeUL | ||
| 2468 | #define CREQ_CREATE_QP1_RESP_RESERVED7_SFT 1 | ||
| 2469 | u8 event; | ||
| 2470 | #define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 0x13UL | ||
| 2471 | __le16 reserved48[3]; | ||
| 2472 | }; | ||
| 2473 | |||
| 2474 | /* Destroy QP1 command response (16 bytes) */ | ||
| 2475 | struct creq_destroy_qp1_resp { | ||
| 2476 | u8 type; | ||
| 2477 | #define CREQ_DESTROY_QP1_RESP_TYPE_MASK 0x3fUL | ||
| 2478 | #define CREQ_DESTROY_QP1_RESP_TYPE_SFT 0 | ||
| 2479 | #define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2480 | #define CREQ_DESTROY_QP1_RESP_RESERVED2_MASK 0xc0UL | ||
| 2481 | #define CREQ_DESTROY_QP1_RESP_RESERVED2_SFT 6 | ||
| 2482 | u8 status; | ||
| 2483 | __le16 cookie; | ||
| 2484 | __le32 xid; | ||
| 2485 | u8 v; | ||
| 2486 | #define CREQ_DESTROY_QP1_RESP_V 0x1UL | ||
| 2487 | #define CREQ_DESTROY_QP1_RESP_RESERVED7_MASK 0xfeUL | ||
| 2488 | #define CREQ_DESTROY_QP1_RESP_RESERVED7_SFT 1 | ||
| 2489 | u8 event; | ||
| 2490 | #define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 0x14UL | ||
| 2491 | __le16 reserved48[3]; | ||
| 2492 | }; | ||
| 2493 | |||
| 2494 | /* Create AH command response (16 bytes) */ | ||
| 2495 | struct creq_create_ah_resp { | ||
| 2496 | u8 type; | ||
| 2497 | #define CREQ_CREATE_AH_RESP_TYPE_MASK 0x3fUL | ||
| 2498 | #define CREQ_CREATE_AH_RESP_TYPE_SFT 0 | ||
| 2499 | #define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2500 | #define CREQ_CREATE_AH_RESP_RESERVED2_MASK 0xc0UL | ||
| 2501 | #define CREQ_CREATE_AH_RESP_RESERVED2_SFT 6 | ||
| 2502 | u8 status; | ||
| 2503 | __le16 cookie; | ||
| 2504 | __le32 xid; | ||
| 2505 | u8 v; | ||
| 2506 | #define CREQ_CREATE_AH_RESP_V 0x1UL | ||
| 2507 | #define CREQ_CREATE_AH_RESP_RESERVED7_MASK 0xfeUL | ||
| 2508 | #define CREQ_CREATE_AH_RESP_RESERVED7_SFT 1 | ||
| 2509 | u8 event; | ||
| 2510 | #define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH 0x15UL | ||
| 2511 | __le16 reserved48[3]; | ||
| 2512 | }; | ||
| 2513 | |||
| 2514 | /* Destroy AH command response (16 bytes) */ | ||
| 2515 | struct creq_destroy_ah_resp { | ||
| 2516 | u8 type; | ||
| 2517 | #define CREQ_DESTROY_AH_RESP_TYPE_MASK 0x3fUL | ||
| 2518 | #define CREQ_DESTROY_AH_RESP_TYPE_SFT 0 | ||
| 2519 | #define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2520 | #define CREQ_DESTROY_AH_RESP_RESERVED2_MASK 0xc0UL | ||
| 2521 | #define CREQ_DESTROY_AH_RESP_RESERVED2_SFT 6 | ||
| 2522 | u8 status; | ||
| 2523 | __le16 cookie; | ||
| 2524 | __le32 xid; | ||
| 2525 | u8 v; | ||
| 2526 | #define CREQ_DESTROY_AH_RESP_V 0x1UL | ||
| 2527 | #define CREQ_DESTROY_AH_RESP_RESERVED7_MASK 0xfeUL | ||
| 2528 | #define CREQ_DESTROY_AH_RESP_RESERVED7_SFT 1 | ||
| 2529 | u8 event; | ||
| 2530 | #define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH 0x16UL | ||
| 2531 | __le16 reserved48[3]; | ||
| 2532 | }; | ||
| 2533 | |||
| 2534 | /* Initialize Firmware command response (16 bytes) */ | ||
| 2535 | struct creq_initialize_fw_resp { | ||
| 2536 | u8 type; | ||
| 2537 | #define CREQ_INITIALIZE_FW_RESP_TYPE_MASK 0x3fUL | ||
| 2538 | #define CREQ_INITIALIZE_FW_RESP_TYPE_SFT 0 | ||
| 2539 | #define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2540 | #define CREQ_INITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL | ||
| 2541 | #define CREQ_INITIALIZE_FW_RESP_RESERVED2_SFT 6 | ||
| 2542 | u8 status; | ||
| 2543 | __le16 cookie; | ||
| 2544 | __le32 reserved32; | ||
| 2545 | u8 v; | ||
| 2546 | #define CREQ_INITIALIZE_FW_RESP_V 0x1UL | ||
| 2547 | #define CREQ_INITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL | ||
| 2548 | #define CREQ_INITIALIZE_FW_RESP_RESERVED7_SFT 1 | ||
| 2549 | u8 event; | ||
| 2550 | #define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW 0x80UL | ||
| 2551 | __le16 reserved48[3]; | ||
| 2552 | }; | ||
| 2553 | |||
| 2554 | /* De-initialize Firmware command response (16 bytes) */ | ||
| 2555 | struct creq_deinitialize_fw_resp { | ||
| 2556 | u8 type; | ||
| 2557 | #define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK 0x3fUL | ||
| 2558 | #define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT 0 | ||
| 2559 | #define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2560 | #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL | ||
| 2561 | #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_SFT 6 | ||
| 2562 | u8 status; | ||
| 2563 | __le16 cookie; | ||
| 2564 | __le32 reserved32; | ||
| 2565 | u8 v; | ||
| 2566 | #define CREQ_DEINITIALIZE_FW_RESP_V 0x1UL | ||
| 2567 | #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL | ||
| 2568 | #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_SFT 1 | ||
| 2569 | u8 event; | ||
| 2570 | #define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW 0x81UL | ||
| 2571 | __le16 reserved48[3]; | ||
| 2572 | }; | ||
| 2573 | |||
| 2574 | /* Stop function command response (16 bytes) */ | ||
| 2575 | struct creq_stop_func_resp { | ||
| 2576 | u8 type; | ||
| 2577 | #define CREQ_STOP_FUNC_RESP_TYPE_MASK 0x3fUL | ||
| 2578 | #define CREQ_STOP_FUNC_RESP_TYPE_SFT 0 | ||
| 2579 | #define CREQ_STOP_FUNC_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2580 | #define CREQ_STOP_FUNC_RESP_RESERVED2_MASK 0xc0UL | ||
| 2581 | #define CREQ_STOP_FUNC_RESP_RESERVED2_SFT 6 | ||
| 2582 | u8 status; | ||
| 2583 | __le16 cookie; | ||
| 2584 | __le32 reserved32; | ||
| 2585 | u8 v; | ||
| 2586 | #define CREQ_STOP_FUNC_RESP_V 0x1UL | ||
| 2587 | #define CREQ_STOP_FUNC_RESP_RESERVED7_MASK 0xfeUL | ||
| 2588 | #define CREQ_STOP_FUNC_RESP_RESERVED7_SFT 1 | ||
| 2589 | u8 event; | ||
| 2590 | #define CREQ_STOP_FUNC_RESP_EVENT_STOP_FUNC 0x82UL | ||
| 2591 | __le16 reserved48[3]; | ||
| 2592 | }; | ||
| 2593 | |||
| 2594 | /* Query function command response (16 bytes) */ | ||
| 2595 | struct creq_query_func_resp { | ||
| 2596 | u8 type; | ||
| 2597 | #define CREQ_QUERY_FUNC_RESP_TYPE_MASK 0x3fUL | ||
| 2598 | #define CREQ_QUERY_FUNC_RESP_TYPE_SFT 0 | ||
| 2599 | #define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2600 | #define CREQ_QUERY_FUNC_RESP_RESERVED2_MASK 0xc0UL | ||
| 2601 | #define CREQ_QUERY_FUNC_RESP_RESERVED2_SFT 6 | ||
| 2602 | u8 status; | ||
| 2603 | __le16 cookie; | ||
| 2604 | __le32 size; | ||
| 2605 | u8 v; | ||
| 2606 | #define CREQ_QUERY_FUNC_RESP_V 0x1UL | ||
| 2607 | #define CREQ_QUERY_FUNC_RESP_RESERVED7_MASK 0xfeUL | ||
| 2608 | #define CREQ_QUERY_FUNC_RESP_RESERVED7_SFT 1 | ||
| 2609 | u8 event; | ||
| 2610 | #define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC 0x83UL | ||
| 2611 | __le16 reserved48[3]; | ||
| 2612 | }; | ||
| 2613 | |||
| 2614 | /* Query function command response side buffer structure (88 bytes) */ | ||
| 2615 | struct creq_query_func_resp_sb { | ||
| 2616 | u8 opcode; | ||
| 2617 | #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC 0x83UL | ||
| 2618 | u8 status; | ||
| 2619 | __le16 cookie; | ||
| 2620 | __le16 flags; | ||
| 2621 | u8 resp_size; | ||
| 2622 | u8 reserved8; | ||
| 2623 | __le64 max_mr_size; | ||
| 2624 | __le32 max_qp; | ||
| 2625 | __le16 max_qp_wr; | ||
| 2626 | __le16 dev_cap_flags; | ||
| 2627 | #define CREQ_QUERY_FUNC_RESP_SB_DEV_CAP_FLAGS_RESIZE_QP 0x1UL | ||
| 2628 | __le32 max_cq; | ||
| 2629 | __le32 max_cqe; | ||
| 2630 | __le32 max_pd; | ||
| 2631 | u8 max_sge; | ||
| 2632 | u8 max_srq_sge; | ||
| 2633 | u8 max_qp_rd_atom; | ||
| 2634 | u8 max_qp_init_rd_atom; | ||
| 2635 | __le32 max_mr; | ||
| 2636 | __le32 max_mw; | ||
| 2637 | __le32 max_raw_eth_qp; | ||
| 2638 | __le32 max_ah; | ||
| 2639 | __le32 max_fmr; | ||
| 2640 | __le32 max_srq_wr; | ||
| 2641 | __le32 max_pkeys; | ||
| 2642 | __le32 max_inline_data; | ||
| 2643 | u8 max_map_per_fmr; | ||
| 2644 | u8 l2_db_space_size; | ||
| 2645 | __le16 max_srq; | ||
| 2646 | __le32 max_gid; | ||
| 2647 | __le32 tqm_alloc_reqs[8]; | ||
| 2648 | }; | ||
| 2649 | |||
| 2650 | /* Set resources command response (16 bytes) */ | ||
| 2651 | struct creq_set_func_resources_resp { | ||
| 2652 | u8 type; | ||
| 2653 | #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK 0x3fUL | ||
| 2654 | #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT 0 | ||
| 2655 | #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2656 | #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_MASK 0xc0UL | ||
| 2657 | #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_SFT 6 | ||
| 2658 | u8 status; | ||
| 2659 | __le16 cookie; | ||
| 2660 | __le32 reserved32; | ||
| 2661 | u8 v; | ||
| 2662 | #define CREQ_SET_FUNC_RESOURCES_RESP_V 0x1UL | ||
| 2663 | #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_MASK 0xfeUL | ||
| 2664 | #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_SFT 1 | ||
| 2665 | u8 event; | ||
| 2666 | #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL | ||
| 2667 | __le16 reserved48[3]; | ||
| 2668 | }; | ||
| 2669 | |||
| 2670 | /* Map TC to COS response (16 bytes) */ | ||
| 2671 | struct creq_map_tc_to_cos_resp { | ||
| 2672 | u8 type; | ||
| 2673 | #define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK 0x3fUL | ||
| 2674 | #define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT 0 | ||
| 2675 | #define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2676 | #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_MASK 0xc0UL | ||
| 2677 | #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_SFT 6 | ||
| 2678 | u8 status; | ||
| 2679 | __le16 cookie; | ||
| 2680 | __le32 reserved32; | ||
| 2681 | u8 v; | ||
| 2682 | #define CREQ_MAP_TC_TO_COS_RESP_V 0x1UL | ||
| 2683 | #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_MASK 0xfeUL | ||
| 2684 | #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_SFT 1 | ||
| 2685 | u8 event; | ||
| 2686 | #define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS 0x8aUL | ||
| 2687 | __le16 reserved48[3]; | ||
| 2688 | }; | ||
| 2689 | |||
| 2690 | /* Query version response (16 bytes) */ | ||
| 2691 | struct creq_query_version_resp { | ||
| 2692 | u8 type; | ||
| 2693 | #define CREQ_QUERY_VERSION_RESP_TYPE_MASK 0x3fUL | ||
| 2694 | #define CREQ_QUERY_VERSION_RESP_TYPE_SFT 0 | ||
| 2695 | #define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2696 | #define CREQ_QUERY_VERSION_RESP_RESERVED2_MASK 0xc0UL | ||
| 2697 | #define CREQ_QUERY_VERSION_RESP_RESERVED2_SFT 6 | ||
| 2698 | u8 status; | ||
| 2699 | __le16 cookie; | ||
| 2700 | u8 fw_maj; | ||
| 2701 | u8 fw_minor; | ||
| 2702 | u8 fw_bld; | ||
| 2703 | u8 fw_rsvd; | ||
| 2704 | u8 v; | ||
| 2705 | #define CREQ_QUERY_VERSION_RESP_V 0x1UL | ||
| 2706 | #define CREQ_QUERY_VERSION_RESP_RESERVED7_MASK 0xfeUL | ||
| 2707 | #define CREQ_QUERY_VERSION_RESP_RESERVED7_SFT 1 | ||
| 2708 | u8 event; | ||
| 2709 | #define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION 0x8bUL | ||
| 2710 | __le16 reserved16; | ||
| 2711 | u8 intf_maj; | ||
| 2712 | u8 intf_minor; | ||
| 2713 | u8 intf_bld; | ||
| 2714 | u8 intf_rsvd; | ||
| 2715 | }; | ||
| 2716 | |||
| 2717 | /* Modify congestion control command response (16 bytes) */ | ||
| 2718 | struct creq_modify_cc_resp { | ||
| 2719 | u8 type; | ||
| 2720 | #define CREQ_MODIFY_CC_RESP_TYPE_MASK 0x3fUL | ||
| 2721 | #define CREQ_MODIFY_CC_RESP_TYPE_SFT 0 | ||
| 2722 | #define CREQ_MODIFY_CC_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2723 | #define CREQ_MODIFY_CC_RESP_RESERVED2_MASK 0xc0UL | ||
| 2724 | #define CREQ_MODIFY_CC_RESP_RESERVED2_SFT 6 | ||
| 2725 | u8 status; | ||
| 2726 | __le16 cookie; | ||
| 2727 | __le32 reserved32; | ||
| 2728 | u8 v; | ||
| 2729 | #define CREQ_MODIFY_CC_RESP_V 0x1UL | ||
| 2730 | #define CREQ_MODIFY_CC_RESP_RESERVED7_MASK 0xfeUL | ||
| 2731 | #define CREQ_MODIFY_CC_RESP_RESERVED7_SFT 1 | ||
| 2732 | u8 event; | ||
| 2733 | #define CREQ_MODIFY_CC_RESP_EVENT_MODIFY_CC 0x8cUL | ||
| 2734 | __le16 reserved48[3]; | ||
| 2735 | }; | ||
| 2736 | |||
| 2737 | /* Query congestion control command response (16 bytes) */ | ||
| 2738 | struct creq_query_cc_resp { | ||
| 2739 | u8 type; | ||
| 2740 | #define CREQ_QUERY_CC_RESP_TYPE_MASK 0x3fUL | ||
| 2741 | #define CREQ_QUERY_CC_RESP_TYPE_SFT 0 | ||
| 2742 | #define CREQ_QUERY_CC_RESP_TYPE_QP_EVENT 0x38UL | ||
| 2743 | #define CREQ_QUERY_CC_RESP_RESERVED2_MASK 0xc0UL | ||
| 2744 | #define CREQ_QUERY_CC_RESP_RESERVED2_SFT 6 | ||
| 2745 | u8 status; | ||
| 2746 | __le16 cookie; | ||
| 2747 | __le32 size; | ||
| 2748 | u8 v; | ||
| 2749 | #define CREQ_QUERY_CC_RESP_V 0x1UL | ||
| 2750 | #define CREQ_QUERY_CC_RESP_RESERVED7_MASK 0xfeUL | ||
| 2751 | #define CREQ_QUERY_CC_RESP_RESERVED7_SFT 1 | ||
| 2752 | u8 event; | ||
| 2753 | #define CREQ_QUERY_CC_RESP_EVENT_QUERY_CC 0x8dUL | ||
| 2754 | __le16 reserved48[3]; | ||
| 2755 | }; | ||
| 2756 | |||
| 2757 | /* Query congestion control command response side buffer structure (32 bytes) */ | ||
| 2758 | struct creq_query_cc_resp_sb { | ||
| 2759 | u8 opcode; | ||
| 2760 | #define CREQ_QUERY_CC_RESP_SB_OPCODE_QUERY_CC 0x8dUL | ||
| 2761 | u8 status; | ||
| 2762 | __le16 cookie; | ||
| 2763 | __le16 flags; | ||
| 2764 | u8 resp_size; | ||
| 2765 | u8 reserved8; | ||
| 2766 | u8 enable_cc; | ||
| 2767 | #define CREQ_QUERY_CC_RESP_SB_ENABLE_CC 0x1UL | ||
| 2768 | u8 g; | ||
| 2769 | #define CREQ_QUERY_CC_RESP_SB_G_MASK 0x7UL | ||
| 2770 | #define CREQ_QUERY_CC_RESP_SB_G_SFT 0 | ||
| 2771 | u8 num_phases_per_state; | ||
| 2772 | __le16 init_cr; | ||
| 2773 | u8 unused_2; | ||
| 2774 | __le16 unused_3; | ||
| 2775 | u8 unused_4; | ||
| 2776 | __le16 init_tr; | ||
| 2777 | u8 tos_dscp_tos_ecn; | ||
| 2778 | #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_MASK 0x3UL | ||
| 2779 | #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_SFT 0 | ||
| 2780 | #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_MASK 0xfcUL | ||
| 2781 | #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_SFT 2 | ||
| 2782 | __le64 reserved64; | ||
| 2783 | __le64 reserved64_1; | ||
| 2784 | }; | ||
| 2785 | |||
| 2786 | /* QP error notification event (16 bytes) */ | ||
| 2787 | struct creq_qp_error_notification { | ||
| 2788 | u8 type; | ||
| 2789 | #define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK 0x3fUL | ||
| 2790 | #define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT 0 | ||
| 2791 | #define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT 0x38UL | ||
| 2792 | #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_MASK 0xc0UL | ||
| 2793 | #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_SFT 6 | ||
| 2794 | u8 status; | ||
| 2795 | u8 req_slow_path_state; | ||
| 2796 | u8 req_err_state_reason; | ||
| 2797 | __le32 xid; | ||
| 2798 | u8 v; | ||
| 2799 | #define CREQ_QP_ERROR_NOTIFICATION_V 0x1UL | ||
| 2800 | #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_MASK 0xfeUL | ||
| 2801 | #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_SFT 1 | ||
| 2802 | u8 event; | ||
| 2803 | #define CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION 0xc0UL | ||
| 2804 | u8 res_slow_path_state; | ||
| 2805 | u8 res_err_state_reason; | ||
| 2806 | __le16 sq_cons_idx; | ||
| 2807 | __le16 rq_cons_idx; | ||
| 2808 | }; | ||
| 2809 | |||
| 2810 | /* RoCE Slowpath HSI Specification 1.6.0 */ | ||
| 2811 | #define ROCE_SP_HSI_VERSION_MAJOR 1 | ||
| 2812 | #define ROCE_SP_HSI_VERSION_MINOR 6 | ||
| 2813 | #define ROCE_SP_HSI_VERSION_UPDATE 0 | ||
| 2814 | |||
| 2815 | #define ROCE_SP_HSI_VERSION_STR "1.6.0" | ||
| 2816 | /* | ||
| 2817 | * Following is the signature for ROCE_SP_HSI message field that indicates not | ||
| 2818 | * applicable (All F's). Need to cast it the size of the field if needed. | ||
| 2819 | */ | ||
| 2820 | #define ROCE_SP_HSI_NA_SIGNATURE ((__le32)(-1)) | ||
| 2821 | #endif /* __BNXT_RE_HSI_H__ */ | ||
| @@ -0,0 +1,89 @@ | |||
| 1 | /* | ||
| 2 | * Broadcom NetXtreme-E RoCE driver. | ||
| 3 | * | ||
| 4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | ||
| 5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | ||
| 6 | * | ||
| 7 | * This software is available to you under a choice of one of two | ||
| 8 | * licenses. You may choose to be licensed under the terms of the GNU | ||
| 9 | * General Public License (GPL) Version 2, available from the file | ||
| 10 | * COPYING in the main directory of this source tree, or the | ||
| 11 | * BSD license below: | ||
| 12 | * | ||
| 13 | * Redistribution and use in source and binary forms, with or without | ||
| 14 | * modification, are permitted provided that the following conditions | ||
| 15 | * are met: | ||
| 16 | * | ||
| 17 | * 1. Redistributions of source code must retain the above copyright | ||
| 18 | * notice, this list of conditions and the following disclaimer. | ||
| 19 | * 2. Redistributions in binary form must reproduce the above copyright | ||
| 20 | * notice, this list of conditions and the following disclaimer in | ||
| 21 | * the documentation and/or other materials provided with the | ||
| 22 | * distribution. | ||
| 23 | * | ||
| 24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | ||
| 25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| 26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | ||
| 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | ||
| 31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| 32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | ||
| 33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | ||
| 34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 35 | * | ||
| 36 | * Description: Uverbs ABI header file | ||
| 37 | */ | ||
| 38 | |||
| 39 | #ifndef __BNXT_RE_UVERBS_ABI_H__ | ||
| 40 | #define __BNXT_RE_UVERBS_ABI_H__ | ||
| 41 | |||
| 42 | #define BNXT_RE_ABI_VERSION 1 | ||
| 43 | |||
| 44 | struct bnxt_re_uctx_resp { | ||
| 45 | __u32 dev_id; | ||
| 46 | __u32 max_qp; | ||
| 47 | __u32 pg_size; | ||
| 48 | __u32 cqe_sz; | ||
| 49 | __u32 max_cqd; | ||
| 50 | __u32 rsvd; | ||
| 51 | }; | ||
| 52 | |||
| 53 | struct bnxt_re_pd_resp { | ||
| 54 | __u32 pdid; | ||
| 55 | __u32 dpi; | ||
| 56 | __u64 dbr; | ||
| 57 | }; | ||
| 58 | |||
| 59 | struct bnxt_re_cq_req { | ||
| 60 | __u64 cq_va; | ||
| 61 | __u64 cq_handle; | ||
| 62 | }; | ||
| 63 | |||
| 64 | struct bnxt_re_cq_resp { | ||
| 65 | __u32 cqid; | ||
| 66 | __u32 tail; | ||
| 67 | __u32 phase; | ||
| 68 | __u32 rsvd; | ||
| 69 | }; | ||
| 70 | |||
| 71 | struct bnxt_re_qp_req { | ||
| 72 | __u64 qpsva; | ||
| 73 | __u64 qprva; | ||
| 74 | __u64 qp_handle; | ||
| 75 | }; | ||
| 76 | |||
| 77 | struct bnxt_re_qp_resp { | ||
| 78 | __u32 qpid; | ||
| 79 | __u32 rsvd; | ||
| 80 | }; | ||
| 81 | |||
| 82 | enum bnxt_re_shpg_offt { | ||
| 83 | BNXT_RE_BEG_RESV_OFFT = 0x00, | ||
| 84 | BNXT_RE_AVID_OFFT = 0x10, | ||
| 85 | BNXT_RE_AVID_SIZE = 0x04, | ||
| 86 | BNXT_RE_END_RESV_OFFT = 0xFF0 | ||
| 87 | }; | ||
| 88 | |||
| 89 | #endif /* __BNXT_RE_UVERBS_ABI_H__*/ | ||