Pull rdma updates from Jason Gunthorpe: "A quiet cycle after the larger 5.8 effort. Substantially cleanup and driver work with a few smaller features this time. - Driver updates for hfi1, rxe, mlx5, hns, qedr, usnic, bnxt_re - Removal of dead or redundant code across the drivers - RAW resource tracker dumps to include a device specific data blob for device objects to aide device debugging - Further advance the IOCTL interface, remove the ability to turn it off. Add QUERY_CONTEXT, QUERY_MR, and QUERY_PD commands - Remove stubs related to devices with no pkey table - A shared CQ scheme to allow multiple ULPs to share the CQ rings of a device to give higher performance - Several more static checker, syzkaller and rare crashers fixed" * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (121 commits) RDMA/mlx5: Fix flow destination setting for RDMA TX flow table RDMA/rxe: Remove pkey table RDMA/umem: Add a schedule point in ib_umem_get() RDMA/hns: Fix the unneeded process when getting a general type of CQE error RDMA/hns: Fix error during modify qp RTS2RTS RDMA/hns: Delete unnecessary memset when allocating VF resource RDMA/hns: Remove redundant parameters in set_rc_wqe() RDMA/hns: Remove support for HIP08_A RDMA/hns: Refactor hns_roce_v2_set_hem() RDMA/hns: Remove redundant hardware opcode definitions RDMA/netlink: Remove CAP_NET_RAW check when dump a raw QP RDMA/include: Replace license text with SPDX tags RDMA/rtrs: remove WQ_MEM_RECLAIM for rtrs_wq RDMA/rtrs-clt: add an additional random 8 seconds before reconnecting RDMA/cma: Execute rdma_cm destruction from a handler properly RDMA/cma: Remove unneeded locking for req paths RDMA/cma: Using the standard locking pattern when delivering the removal event RDMA/cma: Simplify DEVICE_REMOVAL for internal_id RDMA/efa: Add EFA 0xefa1 PCI ID RDMA/efa: User/kernel compatibility handshake mechanism ...
Unified · Split
| @@ -3621,6 +3621,7 @@ M: Selvin Xavier <selvin.xavier@broadcom.com> · 3620 unmodified lines | ||
| 3621 | 3621 | M: Devesh Sharma <devesh.sharma@broadcom.com> |
| 3622 | 3622 | M: Somnath Kotur <somnath.kotur@broadcom.com> |
| 3623 | 3623 | M: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com> |
| 3624 | +M: Naresh Kumar PBS <nareshkumar.pbs@broadcom.com> | |
| 3624 | 3625 | L: linux-rdma@vger.kernel.org |
| 3625 | 3626 | S: Supported |
| 3626 | 3627 | W: http://www.broadcom.com |
| @@ -37,14 +37,6 @@ config INFINIBAND_USER_ACCESS · 36 unmodified lines | ||
| 37 | 37 | libibverbs, libibcm and a hardware driver library from |
| 38 | 38 | rdma-core <https://github.com/linux-rdma/rdma-core>. |
| 39 | 39 | |
| 40 | −config INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI | |
| 41 | − bool "Allow experimental legacy verbs in new ioctl uAPI (EXPERIMENTAL)" | |
| 42 | − depends on INFINIBAND_USER_ACCESS | |
| 43 | − help | |
| 44 | − IOCTL based uAPI support for Infiniband is enabled by default for | |
| 45 | − new verbs only. This allows userspace to invoke the IOCTL based uAPI | |
| 46 | − for current legacy verbs too. | |
| 47 | − | |
| 48 | 40 | config INFINIBAND_USER_MEM |
| 49 | 41 | bool |
| 50 | 42 | depends on INFINIBAND_USER_ACCESS != n |
| @@ -1054,7 +1054,7 @@ int ib_get_cached_pkey(struct ib_device *device, · 1053 unmodified lines | ||
| 1054 | 1054 | |
| 1055 | 1055 | cache = device->port_data[port_num].cache.pkey; |
| 1056 | 1056 | |
| 1057 | − if (index < 0 || index >= cache->table_len) | |
| 1057 | + if (!cache || index < 0 || index >= cache->table_len) | |
| 1058 | 1058 | ret = -EINVAL; |
| 1059 | 1059 | else |
| 1060 | 1060 | *pkey = cache->table[index]; |
| @@ -1099,6 +1099,10 @@ int ib_find_cached_pkey(struct ib_device *device, · 38 unmodified lines | ||
| 1099 | 1099 | read_lock_irqsave(&device->cache_lock, flags); |
| 1100 | 1100 | |
| 1101 | 1101 | cache = device->port_data[port_num].cache.pkey; |
| 1102 | + if (!cache) { | |
| 1103 | + ret = -EINVAL; | |
| 1104 | + goto err; | |
| 1105 | + } | |
| 1102 | 1106 | |
| 1103 | 1107 | *index = -1; |
| 1104 | 1108 | |
| @@ -1117,6 +1121,7 @@ int ib_find_cached_pkey(struct ib_device *device, · 12 unmodified lines | ||
| 1117 | 1121 | ret = 0; |
| 1118 | 1122 | } |
| 1119 | 1123 | |
| 1124 | +err: | |
| 1120 | 1125 | read_unlock_irqrestore(&device->cache_lock, flags); |
| 1121 | 1126 | |
| 1122 | 1127 | return ret; |
| @@ -1139,6 +1144,10 @@ int ib_find_exact_cached_pkey(struct ib_device *device, · 16 unmodified lines | ||
| 1139 | 1144 | read_lock_irqsave(&device->cache_lock, flags); |
| 1140 | 1145 | |
| 1141 | 1146 | cache = device->port_data[port_num].cache.pkey; |
| 1147 | + if (!cache) { | |
| 1148 | + ret = -EINVAL; | |
| 1149 | + goto err; | |
| 1150 | + } | |
| 1142 | 1151 | |
| 1143 | 1152 | *index = -1; |
| 1144 | 1153 | |
| @@ -1149,6 +1158,7 @@ int ib_find_exact_cached_pkey(struct ib_device *device, · 4 unmodified lines | ||
| 1149 | 1158 | break; |
| 1150 | 1159 | } |
| 1151 | 1160 | |
| 1161 | +err: | |
| 1152 | 1162 | read_unlock_irqrestore(&device->cache_lock, flags); |
| 1153 | 1163 | |
| 1154 | 1164 | return ret; |
| @@ -1425,23 +1435,26 @@ ib_cache_update(struct ib_device *device, u8 port, bool enforce_security) · 270 unmodified lines | ||
| 1425 | 1435 | goto err; |
| 1426 | 1436 | } |
| 1427 | 1437 | |
| 1428 | − pkey_cache = kmalloc(struct_size(pkey_cache, table, | |
| 1429 | − tprops->pkey_tbl_len), | |
| 1430 | − GFP_KERNEL); | |
| 1431 | − if (!pkey_cache) { | |
| 1432 | − ret = -ENOMEM; | |
| 1433 | − goto err; | |
| 1434 | − } | |
| 1438 | + if (tprops->pkey_tbl_len) { | |
| 1439 | + pkey_cache = kmalloc(struct_size(pkey_cache, table, | |
| 1440 | + tprops->pkey_tbl_len), | |
| 1441 | + GFP_KERNEL); | |
| 1442 | + if (!pkey_cache) { | |
| 1443 | + ret = -ENOMEM; | |
| 1444 | + goto err; | |
| 1445 | + } | |
| 1435 | 1446 | |
| 1436 | − pkey_cache->table_len = tprops->pkey_tbl_len; | |
| 1447 | + pkey_cache->table_len = tprops->pkey_tbl_len; | |
| 1437 | 1448 | |
| 1438 | − for (i = 0; i < pkey_cache->table_len; ++i) { | |
| 1439 | − ret = ib_query_pkey(device, port, i, pkey_cache->table + i); | |
| 1440 | − if (ret) { | |
| 1441 | − dev_warn(&device->dev, | |
| 1442 | − "ib_query_pkey failed (%d) for index %d\n", | |
| 1443 | − ret, i); | |
| 1444 | − goto err; | |
| 1449 | + for (i = 0; i < pkey_cache->table_len; ++i) { | |
| 1450 | + ret = ib_query_pkey(device, port, i, | |
| 1451 | + pkey_cache->table + i); | |
| 1452 | + if (ret) { | |
| 1453 | + dev_warn(&device->dev, | |
| 1454 | + "ib_query_pkey failed (%d) for index %d\n", | |
| 1455 | + ret, i); | |
| 1456 | + goto err; | |
| 1457 | + } | |
| 1445 | 1458 | } |
| 1446 | 1459 | } |
| 1447 | 1460 | |
| @@ -428,19 +428,6 @@ static int cma_comp_exch(struct rdma_id_private *id_priv, · 427 unmodified lines | ||
| 428 | 428 | return ret; |
| 429 | 429 | } |
| 430 | 430 | |
| 431 | −static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv, | |
| 432 | − enum rdma_cm_state exch) | |
| 433 | −{ | |
| 434 | − unsigned long flags; | |
| 435 | − enum rdma_cm_state old; | |
| 436 | − | |
| 437 | − spin_lock_irqsave(&id_priv->lock, flags); | |
| 438 | − old = id_priv->state; | |
| 439 | − id_priv->state = exch; | |
| 440 | − spin_unlock_irqrestore(&id_priv->lock, flags); | |
| 441 | − return old; | |
| 442 | −} | |
| 443 | − | |
| 444 | 431 | static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr) |
| 445 | 432 | { |
| 446 | 433 | return hdr->ip_version >> 4; |
| @@ -1829,23 +1816,11 @@ static void cma_leave_mc_groups(struct rdma_id_private *id_priv) · 1382 unmodified lines | ||
| 1829 | 1816 | } |
| 1830 | 1817 | } |
| 1831 | 1818 | |
| 1832 | −void rdma_destroy_id(struct rdma_cm_id *id) | |
| 1819 | +static void _destroy_id(struct rdma_id_private *id_priv, | |
| 1820 | + enum rdma_cm_state state) | |
| 1833 | 1821 | { |
| 1834 | − struct rdma_id_private *id_priv; | |
| 1835 | − enum rdma_cm_state state; | |
| 1836 | − | |
| 1837 | − id_priv = container_of(id, struct rdma_id_private, id); | |
| 1838 | − trace_cm_id_destroy(id_priv); | |
| 1839 | − state = cma_exch(id_priv, RDMA_CM_DESTROYING); | |
| 1840 | 1822 | cma_cancel_operation(id_priv, state); |
| 1841 | 1823 | |
| 1842 | − /* | |
| 1843 | − * Wait for any active callback to finish. New callbacks will find | |
| 1844 | − * the id_priv state set to destroying and abort. | |
| 1845 | − */ | |
| 1846 | − mutex_lock(&id_priv->handler_mutex); | |
| 1847 | − mutex_unlock(&id_priv->handler_mutex); | |
| 1848 | − | |
| 1849 | 1824 | rdma_restrack_del(&id_priv->res); |
| 1850 | 1825 | if (id_priv->cma_dev) { |
| 1851 | 1826 | if (rdma_cap_ib_cm(id_priv->id.device, 1)) { |
| @@ -1874,6 +1849,42 @@ void rdma_destroy_id(struct rdma_cm_id *id) · 22 unmodified lines | ||
| 1874 | 1849 | put_net(id_priv->id.route.addr.dev_addr.net); |
| 1875 | 1850 | kfree(id_priv); |
| 1876 | 1851 | } |
| 1852 | + | |
| 1853 | +/* | |
| 1854 | + * destroy an ID from within the handler_mutex. This ensures that no other | |
| 1855 | + * handlers can start running concurrently. | |
| 1856 | + */ | |
| 1857 | +static void destroy_id_handler_unlock(struct rdma_id_private *id_priv) | |
| 1858 | + __releases(&idprv->handler_mutex) | |
| 1859 | +{ | |
| 1860 | + enum rdma_cm_state state; | |
| 1861 | + unsigned long flags; | |
| 1862 | + | |
| 1863 | + trace_cm_id_destroy(id_priv); | |
| 1864 | + | |
| 1865 | + /* | |
| 1866 | + * Setting the state to destroyed under the handler mutex provides a | |
| 1867 | + * fence against calling handler callbacks. If this is invoked due to | |
| 1868 | + * the failure of a handler callback then it guarentees that no future | |
| 1869 | + * handlers will be called. | |
| 1870 | + */ | |
| 1871 | + lockdep_assert_held(&id_priv->handler_mutex); | |
| 1872 | + spin_lock_irqsave(&id_priv->lock, flags); | |
| 1873 | + state = id_priv->state; | |
| 1874 | + id_priv->state = RDMA_CM_DESTROYING; | |
| 1875 | + spin_unlock_irqrestore(&id_priv->lock, flags); | |
| 1876 | + mutex_unlock(&id_priv->handler_mutex); | |
| 1877 | + _destroy_id(id_priv, state); | |
| 1878 | +} | |
| 1879 | + | |
| 1880 | +void rdma_destroy_id(struct rdma_cm_id *id) | |
| 1881 | +{ | |
| 1882 | + struct rdma_id_private *id_priv = | |
| 1883 | + container_of(id, struct rdma_id_private, id); | |
| 1884 | + | |
| 1885 | + mutex_lock(&id_priv->handler_mutex); | |
| 1886 | + destroy_id_handler_unlock(id_priv); | |
| 1887 | +} | |
| 1877 | 1888 | EXPORT_SYMBOL(rdma_destroy_id); |
| 1878 | 1889 | |
| 1879 | 1890 | static int cma_rep_recv(struct rdma_id_private *id_priv) |
| @@ -1925,6 +1936,8 @@ static int cma_cm_event_handler(struct rdma_id_private *id_priv, · 45 unmodified lines | ||
| 1925 | 1936 | { |
| 1926 | 1937 | int ret; |
| 1927 | 1938 | |
| 1939 | + lockdep_assert_held(&id_priv->handler_mutex); | |
| 1940 | + | |
| 1928 | 1941 | trace_cm_event_handler(id_priv, event); |
| 1929 | 1942 | ret = id_priv->id.event_handler(&id_priv->id, event); |
| 1930 | 1943 | trace_cm_event_done(id_priv, event, ret); |
| @@ -1936,7 +1949,7 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, · 5 unmodified lines | ||
| 1936 | 1949 | { |
| 1937 | 1950 | struct rdma_id_private *id_priv = cm_id->context; |
| 1938 | 1951 | struct rdma_cm_event event = {}; |
| 1939 | − int ret = 0; | |
| 1952 | + int ret; | |
| 1940 | 1953 | |
| 1941 | 1954 | mutex_lock(&id_priv->handler_mutex); |
| 1942 | 1955 | if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && |
| @@ -2005,14 +2018,12 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, · 62 unmodified lines | ||
| 2005 | 2018 | if (ret) { |
| 2006 | 2019 | /* Destroy the CM ID by returning a non-zero value. */ |
| 2007 | 2020 | id_priv->cm_id.ib = NULL; |
| 2008 | − cma_exch(id_priv, RDMA_CM_DESTROYING); | |
| 2009 | − mutex_unlock(&id_priv->handler_mutex); | |
| 2010 | − rdma_destroy_id(&id_priv->id); | |
| 2021 | + destroy_id_handler_unlock(id_priv); | |
| 2011 | 2022 | return ret; |
| 2012 | 2023 | } |
| 2013 | 2024 | out: |
| 2014 | 2025 | mutex_unlock(&id_priv->handler_mutex); |
| 2015 | − return ret; | |
| 2026 | + return 0; | |
| 2016 | 2027 | } |
| 2017 | 2028 | |
| 2018 | 2029 | static struct rdma_id_private * |
| @@ -2174,7 +2185,7 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, · 155 unmodified lines | ||
| 2174 | 2185 | mutex_lock(&listen_id->handler_mutex); |
| 2175 | 2186 | if (listen_id->state != RDMA_CM_LISTEN) { |
| 2176 | 2187 | ret = -ECONNABORTED; |
| 2177 | − goto err1; | |
| 2188 | + goto err_unlock; | |
| 2178 | 2189 | } |
| 2179 | 2190 | |
| 2180 | 2191 | offset = cma_user_data_offset(listen_id); |
| @@ -2191,55 +2202,38 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, · 10 unmodified lines | ||
| 2191 | 2202 | } |
| 2192 | 2203 | if (!conn_id) { |
| 2193 | 2204 | ret = -ENOMEM; |
| 2194 | − goto err1; | |
| 2205 | + goto err_unlock; | |
| 2195 | 2206 | } |
| 2196 | 2207 | |
| 2197 | 2208 | mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); |
| 2198 | 2209 | ret = cma_ib_acquire_dev(conn_id, listen_id, &req); |
| 2199 | − if (ret) | |
| 2200 | − goto err2; | |
| 2210 | + if (ret) { | |
| 2211 | + destroy_id_handler_unlock(conn_id); | |
| 2212 | + goto err_unlock; | |
| 2213 | + } | |
| 2201 | 2214 | |
| 2202 | 2215 | conn_id->cm_id.ib = cm_id; |
| 2203 | 2216 | cm_id->context = conn_id; |
| 2204 | 2217 | cm_id->cm_handler = cma_ib_handler; |
| 2205 | 2218 | |
| 2206 | − /* | |
| 2207 | − * Protect against the user destroying conn_id from another thread | |
| 2208 | − * until we're done accessing it. | |
| 2209 | − */ | |
| 2210 | − cma_id_get(conn_id); | |
| 2211 | 2219 | ret = cma_cm_event_handler(conn_id, &event); |
| 2212 | − if (ret) | |
| 2213 | − goto err3; | |
| 2214 | − /* | |
| 2215 | − * Acquire mutex to prevent user executing rdma_destroy_id() | |
| 2216 | − * while we're accessing the cm_id. | |
| 2217 | − */ | |
| 2218 | − mutex_lock(&lock); | |
| 2220 | + if (ret) { | |
| 2221 | + /* Destroy the CM ID by returning a non-zero value. */ | |
| 2222 | + conn_id->cm_id.ib = NULL; | |
| 2223 | + mutex_unlock(&listen_id->handler_mutex); | |
| 2224 | + destroy_id_handler_unlock(conn_id); | |
| 2225 | + goto net_dev_put; | |
| 2226 | + } | |
| 2227 | + | |
| 2219 | 2228 | if (cma_comp(conn_id, RDMA_CM_CONNECT) && |
| 2220 | 2229 | (conn_id->id.qp_type != IB_QPT_UD)) { |
| 2221 | 2230 | trace_cm_send_mra(cm_id->context); |
| 2222 | 2231 | ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); |
| 2223 | 2232 | } |
| 2224 | − mutex_unlock(&lock); | |
| 2225 | 2233 | mutex_unlock(&conn_id->handler_mutex); |
| 2226 | − mutex_unlock(&listen_id->handler_mutex); | |
| 2227 | − cma_id_put(conn_id); | |
| 2228 | − if (net_dev) | |
| 2229 | − dev_put(net_dev); | |
| 2230 | − return 0; | |
| 2231 | 2234 | |
| 2232 | −err3: | |
| 2233 | − cma_id_put(conn_id); | |
| 2234 | − /* Destroy the CM ID by returning a non-zero value. */ | |
| 2235 | − conn_id->cm_id.ib = NULL; | |
| 2236 | −err2: | |
| 2237 | − cma_exch(conn_id, RDMA_CM_DESTROYING); | |
| 2238 | − mutex_unlock(&conn_id->handler_mutex); | |
| 2239 | −err1: | |
| 2235 | +err_unlock: | |
| 2240 | 2236 | mutex_unlock(&listen_id->handler_mutex); |
| 2241 | − if (conn_id) | |
| 2242 | − rdma_destroy_id(&conn_id->id); | |
| 2243 | 2237 | |
| 2244 | 2238 | net_dev_put: |
| 2245 | 2239 | if (net_dev) |
| @@ -2339,9 +2333,7 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) · 93 unmodified lines | ||
| 2339 | 2333 | if (ret) { |
| 2340 | 2334 | /* Destroy the CM ID by returning a non-zero value. */ |
| 2341 | 2335 | id_priv->cm_id.iw = NULL; |
| 2342 | − cma_exch(id_priv, RDMA_CM_DESTROYING); | |
| 2343 | − mutex_unlock(&id_priv->handler_mutex); | |
| 2344 | − rdma_destroy_id(&id_priv->id); | |
| 2336 | + destroy_id_handler_unlock(id_priv); | |
| 2345 | 2337 | return ret; |
| 2346 | 2338 | } |
| 2347 | 2339 | |
| @@ -2388,16 +2380,16 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, · 40 unmodified lines | ||
| 2388 | 2380 | |
| 2389 | 2381 | ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr); |
| 2390 | 2382 | if (ret) { |
| 2391 | − mutex_unlock(&conn_id->handler_mutex); | |
| 2392 | − rdma_destroy_id(new_cm_id); | |
| 2393 | − goto out; | |
| 2383 | + mutex_unlock(&listen_id->handler_mutex); | |
| 2384 | + destroy_id_handler_unlock(conn_id); | |
| 2385 | + return ret; | |
| 2394 | 2386 | } |
| 2395 | 2387 | |
| 2396 | 2388 | ret = cma_iw_acquire_dev(conn_id, listen_id); |
| 2397 | 2389 | if (ret) { |
| 2398 | − mutex_unlock(&conn_id->handler_mutex); | |
| 2399 | − rdma_destroy_id(new_cm_id); | |
| 2400 | − goto out; | |
| 2390 | + mutex_unlock(&listen_id->handler_mutex); | |
| 2391 | + destroy_id_handler_unlock(conn_id); | |
| 2392 | + return ret; | |
| 2401 | 2393 | } |
| 2402 | 2394 | |
| 2403 | 2395 | conn_id->cm_id.iw = cm_id; |
| @@ -2407,25 +2399,16 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, · 3 unmodified lines | ||
| 2407 | 2399 | memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr)); |
| 2408 | 2400 | memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr)); |
| 2409 | 2401 | |
| 2410 | − /* | |
| 2411 | − * Protect against the user destroying conn_id from another thread | |
| 2412 | − * until we're done accessing it. | |
| 2413 | − */ | |
| 2414 | − cma_id_get(conn_id); | |
| 2415 | 2402 | ret = cma_cm_event_handler(conn_id, &event); |
| 2416 | 2403 | if (ret) { |
| 2417 | 2404 | /* User wants to destroy the CM ID */ |
| 2418 | 2405 | conn_id->cm_id.iw = NULL; |
| 2419 | − cma_exch(conn_id, RDMA_CM_DESTROYING); | |
| 2420 | − mutex_unlock(&conn_id->handler_mutex); | |
| 2421 | 2406 | mutex_unlock(&listen_id->handler_mutex); |
| 2422 | − cma_id_put(conn_id); | |
| 2423 | − rdma_destroy_id(&conn_id->id); | |
| 2407 | + destroy_id_handler_unlock(conn_id); | |
| 2424 | 2408 | return ret; |
| 2425 | 2409 | } |
| 2426 | 2410 | |
| 2427 | 2411 | mutex_unlock(&conn_id->handler_mutex); |
| 2428 | − cma_id_put(conn_id); | |
| 2429 | 2412 | |
| 2430 | 2413 | out: |
| 2431 | 2414 | mutex_unlock(&listen_id->handler_mutex); |
| @@ -2482,6 +2465,10 @@ static int cma_listen_handler(struct rdma_cm_id *id, · 50 unmodified lines | ||
| 2482 | 2465 | { |
| 2483 | 2466 | struct rdma_id_private *id_priv = id->context; |
| 2484 | 2467 | |
| 2468 | + /* Listening IDs are always destroyed on removal */ | |
| 2469 | + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) | |
| 2470 | + return -1; | |
| 2471 | + | |
| 2485 | 2472 | id->context = id_priv->id.context; |
| 2486 | 2473 | id->event_handler = id_priv->id.event_handler; |
| 2487 | 2474 | trace_cm_event_handler(id_priv, event); |
| @@ -2657,21 +2644,21 @@ static void cma_work_handler(struct work_struct *_work) · 169 unmodified lines | ||
| 2657 | 2644 | { |
| 2658 | 2645 | struct cma_work *work = container_of(_work, struct cma_work, work); |
| 2659 | 2646 | struct rdma_id_private *id_priv = work->id; |
| 2660 | − int destroy = 0; | |
| 2661 | 2647 | |
| 2662 | 2648 | mutex_lock(&id_priv->handler_mutex); |
| 2663 | 2649 | if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) |
| 2664 | − goto out; | |
| 2650 | + goto out_unlock; | |
| 2665 | 2651 | |
| 2666 | 2652 | if (cma_cm_event_handler(id_priv, &work->event)) { |
| 2667 | − cma_exch(id_priv, RDMA_CM_DESTROYING); | |
| 2668 | − destroy = 1; | |
| 2653 | + cma_id_put(id_priv); | |
| 2654 | + destroy_id_handler_unlock(id_priv); | |
| 2655 | + goto out_free; | |
| 2669 | 2656 | } |
| 2670 | −out: | |
| 2657 | + | |
| 2658 | +out_unlock: | |
| 2671 | 2659 | mutex_unlock(&id_priv->handler_mutex); |
| 2672 | 2660 | cma_id_put(id_priv); |
| 2673 | − if (destroy) | |
| 2674 | − rdma_destroy_id(&id_priv->id); | |
| 2661 | +out_free: | |
| 2675 | 2662 | kfree(work); |
| 2676 | 2663 | } |
| 2677 | 2664 | |
| @@ -2679,23 +2666,22 @@ static void cma_ndev_work_handler(struct work_struct *_work) · 1 unmodified line | ||
| 2679 | 2666 | { |
| 2680 | 2667 | struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work); |
| 2681 | 2668 | struct rdma_id_private *id_priv = work->id; |
| 2682 | − int destroy = 0; | |
| 2683 | 2669 | |
| 2684 | 2670 | mutex_lock(&id_priv->handler_mutex); |
| 2685 | 2671 | if (id_priv->state == RDMA_CM_DESTROYING || |
| 2686 | 2672 | id_priv->state == RDMA_CM_DEVICE_REMOVAL) |
| 2687 | − goto out; | |
| 2673 | + goto out_unlock; | |
| 2688 | 2674 | |
| 2689 | 2675 | if (cma_cm_event_handler(id_priv, &work->event)) { |
| 2690 | − cma_exch(id_priv, RDMA_CM_DESTROYING); | |
| 2691 | − destroy = 1; | |
| 2676 | + cma_id_put(id_priv); | |
| 2677 | + destroy_id_handler_unlock(id_priv); | |
| 2678 | + goto out_free; | |
| 2692 | 2679 | } |
| 2693 | 2680 | |
| 2694 | −out: | |
| 2681 | +out_unlock: | |
| 2695 | 2682 | mutex_unlock(&id_priv->handler_mutex); |
| 2696 | 2683 | cma_id_put(id_priv); |
| 2697 | − if (destroy) | |
| 2698 | − rdma_destroy_id(&id_priv->id); | |
| 2684 | +out_free: | |
| 2699 | 2685 | kfree(work); |
| 2700 | 2686 | } |
| 2701 | 2687 | |
| @@ -3171,9 +3157,7 @@ static void addr_handler(int status, struct sockaddr *src_addr, · 469 unmodified lines | ||
| 3171 | 3157 | event.event = RDMA_CM_EVENT_ADDR_RESOLVED; |
| 3172 | 3158 | |
| 3173 | 3159 | if (cma_cm_event_handler(id_priv, &event)) { |
| 3174 | − cma_exch(id_priv, RDMA_CM_DESTROYING); | |
| 3175 | − mutex_unlock(&id_priv->handler_mutex); | |
| 3176 | − rdma_destroy_id(&id_priv->id); | |
| 3160 | + destroy_id_handler_unlock(id_priv); | |
| 3177 | 3161 | return; |
| 3178 | 3162 | } |
| 3179 | 3163 | out: |
| @@ -3790,7 +3774,7 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, · 610 unmodified lines | ||
| 3790 | 3774 | struct rdma_cm_event event = {}; |
| 3791 | 3775 | const struct ib_cm_sidr_rep_event_param *rep = |
| 3792 | 3776 | &ib_event->param.sidr_rep_rcvd; |
| 3793 | − int ret = 0; | |
| 3777 | + int ret; | |
| 3794 | 3778 | |
| 3795 | 3779 | mutex_lock(&id_priv->handler_mutex); |
| 3796 | 3780 | if (id_priv->state != RDMA_CM_CONNECT) |
| @@ -3840,14 +3824,12 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, · 43 unmodified lines | ||
| 3840 | 3824 | if (ret) { |
| 3841 | 3825 | /* Destroy the CM ID by returning a non-zero value. */ |
| 3842 | 3826 | id_priv->cm_id.ib = NULL; |
| 3843 | − cma_exch(id_priv, RDMA_CM_DESTROYING); | |
| 3844 | − mutex_unlock(&id_priv->handler_mutex); | |
| 3845 | − rdma_destroy_id(&id_priv->id); | |
| 3827 | + destroy_id_handler_unlock(id_priv); | |
| 3846 | 3828 | return ret; |
| 3847 | 3829 | } |
| 3848 | 3830 | out: |
| 3849 | 3831 | mutex_unlock(&id_priv->handler_mutex); |
| 3850 | − return ret; | |
| 3832 | + return 0; | |
| 3851 | 3833 | } |
| 3852 | 3834 | |
| 3853 | 3835 | static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, |
| @@ -4372,9 +4354,7 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) · 518 unmodified lines | ||
| 4372 | 4354 | |
| 4373 | 4355 | rdma_destroy_ah_attr(&event.param.ud.ah_attr); |
| 4374 | 4356 | if (ret) { |
| 4375 | − cma_exch(id_priv, RDMA_CM_DESTROYING); | |
| 4376 | − mutex_unlock(&id_priv->handler_mutex); | |
| 4377 | − rdma_destroy_id(&id_priv->id); | |
| 4357 | + destroy_id_handler_unlock(id_priv); | |
| 4378 | 4358 | return 0; |
| 4379 | 4359 | } |
| 4380 | 4360 | |
| @@ -4789,50 +4769,59 @@ free_cma_dev: · 408 unmodified lines | ||
| 4789 | 4769 | return ret; |
| 4790 | 4770 | } |
| 4791 | 4771 | |
| 4792 | −static int cma_remove_id_dev(struct rdma_id_private *id_priv) | |
| 4772 | +static void cma_send_device_removal_put(struct rdma_id_private *id_priv) | |
| 4793 | 4773 | { |
| 4794 | − struct rdma_cm_event event = {}; | |
| 4774 | + struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL }; | |
| 4795 | 4775 | enum rdma_cm_state state; |
| 4796 | − int ret = 0; | |
| 4797 | − | |
| 4798 | − /* Record that we want to remove the device */ | |
| 4799 | − state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL); | |
| 4800 | − if (state == RDMA_CM_DESTROYING) | |
| 4801 | − return 0; | |
| 4776 | + unsigned long flags; | |
| 4802 | 4777 | |
| 4803 | − cma_cancel_operation(id_priv, state); | |
| 4804 | 4778 | mutex_lock(&id_priv->handler_mutex); |
| 4779 | + /* Record that we want to remove the device */ | |
| 4780 | + spin_lock_irqsave(&id_priv->lock, flags); | |
| 4781 | + state = id_priv->state; | |
| 4782 | + if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) { | |
| 4783 | + spin_unlock_irqrestore(&id_priv->lock, flags); | |
| 4784 | + mutex_unlock(&id_priv->handler_mutex); | |
| 4785 | + cma_id_put(id_priv); | |
| 4786 | + return; | |
| 4787 | + } | |
| 4788 | + id_priv->state = RDMA_CM_DEVICE_REMOVAL; | |
| 4789 | + spin_unlock_irqrestore(&id_priv->lock, flags); | |
| 4805 | 4790 | |
| 4806 | − /* Check for destruction from another callback. */ | |
| 4807 | − if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL)) | |
| 4808 | − goto out; | |
| 4809 | − | |
| 4810 | − event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; | |
| 4811 | − ret = cma_cm_event_handler(id_priv, &event); | |
| 4812 | −out: | |
| 4791 | + if (cma_cm_event_handler(id_priv, &event)) { | |
| 4792 | + /* | |
| 4793 | + * At this point the ULP promises it won't call | |
| 4794 | + * rdma_destroy_id() concurrently | |
| 4795 | + */ | |
| 4796 | + cma_id_put(id_priv); | |
| 4797 | + mutex_unlock(&id_priv->handler_mutex); | |
| 4798 | + trace_cm_id_destroy(id_priv); | |
| 4799 | + _destroy_id(id_priv, state); | |
| 4800 | + return; | |
| 4801 | + } | |
| 4813 | 4802 | mutex_unlock(&id_priv->handler_mutex); |
| 4814 | − return ret; | |
| 4803 | + | |
| 4804 | + /* | |
| 4805 | + * If this races with destroy then the thread that first assigns state | |
| 4806 | + * to a destroying does the cancel. | |
| 4807 | + */ | |
| 4808 | + cma_cancel_operation(id_priv, state); | |
| 4809 | + cma_id_put(id_priv); | |
| 4815 | 4810 | } |
| 4816 | 4811 | |
| 4817 | 4812 | static void cma_process_remove(struct cma_device *cma_dev) |
| 4818 | 4813 | { |
| 4819 | − struct rdma_id_private *id_priv; | |
| 4820 | − int ret; | |
| 4821 | − | |
| 4822 | 4814 | mutex_lock(&lock); |
| 4823 | 4815 | while (!list_empty(&cma_dev->id_list)) { |
| 4824 | − id_priv = list_entry(cma_dev->id_list.next, | |
| 4825 | − struct rdma_id_private, list); | |
| 4816 | + struct rdma_id_private *id_priv = list_first_entry( | |
| 4817 | + &cma_dev->id_list, struct rdma_id_private, list); | |
| 4826 | 4818 | |
| 4827 | 4819 | list_del(&id_priv->listen_list); |
| 4828 | 4820 | list_del_init(&id_priv->list); |
| 4829 | 4821 | cma_id_get(id_priv); |
| 4830 | 4822 | mutex_unlock(&lock); |
| 4831 | 4823 | |
| 4832 | − ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv); | |
| 4833 | − cma_id_put(id_priv); | |
| 4834 | − if (ret) | |
| 4835 | − rdma_destroy_id(&id_priv->id); | |
| 4824 | + cma_send_device_removal_put(id_priv); | |
| 4836 | 4825 | |
| 4837 | 4826 | mutex_lock(&lock); |
| 4838 | 4827 | } |
| @@ -8,7 +8,7 @@ · 7 unmodified lines | ||
| 8 | 8 | #include "core_priv.h" |
| 9 | 9 | #include "restrack.h" |
| 10 | 10 | |
| 11 | −#define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE) | |
| 11 | +#define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID) | |
| 12 | 12 | |
| 13 | 13 | static int __counter_set_mode(struct rdma_counter_mode *curr, |
| 14 | 14 | enum rdma_nl_counter_mode new_mode, |
| @@ -149,23 +149,13 @@ static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter, · 134 unmodified lines | ||
| 149 | 149 | struct auto_mode_param *param = &counter->mode.param; |
| 150 | 150 | bool match = true; |
| 151 | 151 | |
| 152 | − /* | |
| 153 | − * Ensure that counter belongs to the right PID. This operation can | |
| 154 | − * race with user space which kills the process and leaves QP and | |
| 155 | − * counters orphans. | |
| 156 | − * | |
| 157 | − * It is not a big deal because exitted task will leave both QP and | |
| 158 | − * counter in the same bucket of zombie process. Just ensure that | |
| 159 | − * process is still alive before procedding. | |
| 160 | − * | |
| 161 | − */ | |
| 162 | − if (task_pid_nr(counter->res.task) != task_pid_nr(qp->res.task) || | |
| 163 | − !task_pid_nr(qp->res.task)) | |
| 164 | − return false; | |
| 165 | − | |
| 166 | 152 | if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE) |
| 167 | 153 | match &= (param->qp_type == qp->qp_type); |
| 168 | 154 | |
| 155 | + if (auto_mask & RDMA_COUNTER_MASK_PID) | |
| 156 | + match &= (task_pid_nr(counter->res.task) == | |
| 157 | + task_pid_nr(qp->res.task)); | |
| 158 | + | |
| 169 | 159 | return match; |
| 170 | 160 | } |
| 171 | 161 | |
| @@ -288,7 +278,7 @@ int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port) · 116 unmodified lines | ||
| 288 | 278 | struct rdma_counter *counter; |
| 289 | 279 | int ret; |
| 290 | 280 | |
| 291 | − if (!qp->res.valid) | |
| 281 | + if (!qp->res.valid || rdma_is_kernel_res(&qp->res)) | |
| 292 | 282 | return 0; |
| 293 | 283 | |
| 294 | 284 | if (!rdma_is_port_valid(dev, port)) |
| @@ -483,7 +473,7 @@ int rdma_counter_bind_qpn(struct ib_device *dev, u8 port, · 188 unmodified lines | ||
| 483 | 473 | goto err; |
| 484 | 474 | } |
| 485 | 475 | |
| 486 | − if (counter->res.task != qp->res.task) { | |
| 476 | + if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) { | |
| 487 | 477 | ret = -EINVAL; |
| 488 | 478 | goto err_task; |
| 489 | 479 | } |
| @@ -272,7 +272,6 @@ static void ib_device_check_mandatory(struct ib_device *device) · 271 unmodified lines | ||
| 272 | 272 | } mandatory_table[] = { |
| 273 | 273 | IB_MANDATORY_FUNC(query_device), |
| 274 | 274 | IB_MANDATORY_FUNC(query_port), |
| 275 | − IB_MANDATORY_FUNC(query_pkey), | |
| 276 | 275 | IB_MANDATORY_FUNC(alloc_pd), |
| 277 | 276 | IB_MANDATORY_FUNC(dealloc_pd), |
| 278 | 277 | IB_MANDATORY_FUNC(create_qp), |
| @@ -1343,6 +1342,10 @@ out: · 1064 unmodified lines | ||
| 1343 | 1342 | return ret; |
| 1344 | 1343 | } |
| 1345 | 1344 | |
| 1345 | +static void prevent_dealloc_device(struct ib_device *ib_dev) | |
| 1346 | +{ | |
| 1347 | +} | |
| 1348 | + | |
| 1346 | 1349 | /** |
| 1347 | 1350 | * ib_register_device - Register an IB device with IB core |
| 1348 | 1351 | * @device: Device to register |
| @@ -1413,11 +1416,11 @@ int ib_register_device(struct ib_device *device, const char *name) · 64 unmodified lines | ||
| 1413 | 1416 | * possibility for a parallel unregistration along with this |
| 1414 | 1417 | * error flow. Since we have a refcount here we know any |
| 1415 | 1418 | * parallel flow is stopped in disable_device and will see the |
| 1416 | − * NULL pointers, causing the responsibility to | |
| 1419 | + * special dealloc_driver pointer, causing the responsibility to | |
| 1417 | 1420 | * ib_dealloc_device() to revert back to this thread. |
| 1418 | 1421 | */ |
| 1419 | 1422 | dealloc_fn = device->ops.dealloc_driver; |
| 1420 | − device->ops.dealloc_driver = NULL; | |
| 1423 | + device->ops.dealloc_driver = prevent_dealloc_device; | |
| 1421 | 1424 | ib_device_put(device); |
| 1422 | 1425 | __ib_unregister_device(device); |
| 1423 | 1426 | device->ops.dealloc_driver = dealloc_fn; |
| @@ -1466,7 +1469,8 @@ static void __ib_unregister_device(struct ib_device *ib_dev) · 42 unmodified lines | ||
| 1466 | 1469 | * Drivers using the new flow may not call ib_dealloc_device except |
| 1467 | 1470 | * in error unwind prior to registration success. |
| 1468 | 1471 | */ |
| 1469 | − if (ib_dev->ops.dealloc_driver) { | |
| 1472 | + if (ib_dev->ops.dealloc_driver && | |
| 1473 | + ib_dev->ops.dealloc_driver != prevent_dealloc_device) { | |
| 1470 | 1474 | WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1); |
| 1471 | 1475 | ib_dealloc_device(ib_dev); |
| 1472 | 1476 | } |
| @@ -2361,6 +2365,9 @@ int ib_query_pkey(struct ib_device *device, · 888 unmodified lines | ||
| 2361 | 2365 | if (!rdma_is_port_valid(device, port_num)) |
| 2362 | 2366 | return -EINVAL; |
| 2363 | 2367 | |
| 2368 | + if (!device->ops.query_pkey) | |
| 2369 | + return -EOPNOTSUPP; | |
| 2370 | + | |
| 2364 | 2371 | return device->ops.query_pkey(device, port_num, index, pkey); |
| 2365 | 2372 | } |
| 2366 | 2373 | EXPORT_SYMBOL(ib_query_pkey); |
| @@ -2621,8 +2628,14 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) · 254 unmodified lines | ||
| 2621 | 2628 | SET_DEVICE_OP(dev_ops, drain_rq); |
| 2622 | 2629 | SET_DEVICE_OP(dev_ops, drain_sq); |
| 2623 | 2630 | SET_DEVICE_OP(dev_ops, enable_driver); |
| 2624 | − SET_DEVICE_OP(dev_ops, fill_res_entry); | |
| 2625 | − SET_DEVICE_OP(dev_ops, fill_stat_entry); | |
| 2631 | + SET_DEVICE_OP(dev_ops, fill_res_cm_id_entry); | |
| 2632 | + SET_DEVICE_OP(dev_ops, fill_res_cq_entry); | |
| 2633 | + SET_DEVICE_OP(dev_ops, fill_res_cq_entry_raw); | |
| 2634 | + SET_DEVICE_OP(dev_ops, fill_res_mr_entry); | |
| 2635 | + SET_DEVICE_OP(dev_ops, fill_res_mr_entry_raw); | |
| 2636 | + SET_DEVICE_OP(dev_ops, fill_res_qp_entry); | |
| 2637 | + SET_DEVICE_OP(dev_ops, fill_res_qp_entry_raw); | |
| 2638 | + SET_DEVICE_OP(dev_ops, fill_stat_mr_entry); | |
| 2626 | 2639 | SET_DEVICE_OP(dev_ops, get_dev_fw_str); |
| 2627 | 2640 | SET_DEVICE_OP(dev_ops, get_dma_mr); |
| 2628 | 2641 | SET_DEVICE_OP(dev_ops, get_hw_stats); |
| @@ -2667,6 +2680,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) · 38 unmodified lines | ||
| 2667 | 2680 | SET_DEVICE_OP(dev_ops, query_port); |
| 2668 | 2681 | SET_DEVICE_OP(dev_ops, query_qp); |
| 2669 | 2682 | SET_DEVICE_OP(dev_ops, query_srq); |
| 2683 | + SET_DEVICE_OP(dev_ops, query_ucontext); | |
| 2670 | 2684 | SET_DEVICE_OP(dev_ops, rdma_netdev_get_params); |
| 2671 | 2685 | SET_DEVICE_OP(dev_ops, read_counters); |
| 2672 | 2686 | SET_DEVICE_OP(dev_ops, reg_dm_mr); |
| @@ -2679,10 +2693,12 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) · 6 unmodified lines | ||
| 2679 | 2693 | SET_DEVICE_OP(dev_ops, set_vf_link_state); |
| 2680 | 2694 | |
| 2681 | 2695 | SET_OBJ_SIZE(dev_ops, ib_ah); |
| 2696 | + SET_OBJ_SIZE(dev_ops, ib_counters); | |
| 2682 | 2697 | SET_OBJ_SIZE(dev_ops, ib_cq); |
| 2683 | 2698 | SET_OBJ_SIZE(dev_ops, ib_pd); |
| 2684 | 2699 | SET_OBJ_SIZE(dev_ops, ib_srq); |
| 2685 | 2700 | SET_OBJ_SIZE(dev_ops, ib_ucontext); |
| 2701 | + SET_OBJ_SIZE(dev_ops, ib_xrcd); | |
| 2686 | 2702 | } |
| 2687 | 2703 | EXPORT_SYMBOL(ib_set_device_ops); |
| 2688 | 2704 | |
| @@ -402,7 +402,7 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, · 401 unmodified lines | ||
| 402 | 402 | INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends); |
| 403 | 403 | INIT_LIST_HEAD(&mad_agent_priv->local_list); |
| 404 | 404 | INIT_WORK(&mad_agent_priv->local_work, local_completions); |
| 405 | − atomic_set(&mad_agent_priv->refcount, 1); | |
| 405 | + refcount_set(&mad_agent_priv->refcount, 1); | |
| 406 | 406 | init_completion(&mad_agent_priv->comp); |
| 407 | 407 | |
| 408 | 408 | ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type); |
| @@ -484,7 +484,7 @@ EXPORT_SYMBOL(ib_register_mad_agent); · 75 unmodified lines | ||
| 484 | 484 | |
| 485 | 485 | static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv) |
| 486 | 486 | { |
| 487 | − if (atomic_dec_and_test(&mad_agent_priv->refcount)) | |
| 487 | + if (refcount_dec_and_test(&mad_agent_priv->refcount)) | |
| 488 | 488 | complete(&mad_agent_priv->comp); |
| 489 | 489 | } |
| 490 | 490 | |
| @@ -718,7 +718,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv, · 227 unmodified lines | ||
| 718 | 718 | * Reference MAD agent until receive |
| 719 | 719 | * side of local completion handled |
| 720 | 720 | */ |
| 721 | − atomic_inc(&mad_agent_priv->refcount); | |
| 721 | + refcount_inc(&mad_agent_priv->refcount); | |
| 722 | 722 | } else |
| 723 | 723 | kfree(mad_priv); |
| 724 | 724 | break; |
| @@ -758,7 +758,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv, · 33 unmodified lines | ||
| 758 | 758 | local->return_wc_byte_len = mad_size; |
| 759 | 759 | } |
| 760 | 760 | /* Reference MAD agent until send side of local completion handled */ |
| 761 | − atomic_inc(&mad_agent_priv->refcount); | |
| 761 | + refcount_inc(&mad_agent_priv->refcount); | |
| 762 | 762 | /* Queue local completion to local list */ |
| 763 | 763 | spin_lock_irqsave(&mad_agent_priv->lock, flags); |
| 764 | 764 | list_add_tail(&local->completion_list, &mad_agent_priv->local_list); |
| @@ -916,7 +916,7 @@ struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent, · 151 unmodified lines | ||
| 916 | 916 | } |
| 917 | 917 | |
| 918 | 918 | mad_send_wr->send_buf.mad_agent = mad_agent; |
| 919 | − atomic_inc(&mad_agent_priv->refcount); | |
| 919 | + refcount_inc(&mad_agent_priv->refcount); | |
| 920 | 920 | return &mad_send_wr->send_buf; |
| 921 | 921 | } |
| 922 | 922 | EXPORT_SYMBOL(ib_create_send_mad); |
| @@ -1131,7 +1131,7 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf, · 208 unmodified lines | ||
| 1131 | 1131 | mad_send_wr->status = IB_WC_SUCCESS; |
| 1132 | 1132 | |
| 1133 | 1133 | /* Reference MAD agent until send completes */ |
| 1134 | − atomic_inc(&mad_agent_priv->refcount); | |
| 1134 | + refcount_inc(&mad_agent_priv->refcount); | |
| 1135 | 1135 | spin_lock_irqsave(&mad_agent_priv->lock, flags); |
| 1136 | 1136 | list_add_tail(&mad_send_wr->agent_list, |
| 1137 | 1137 | &mad_agent_priv->send_list); |
| @@ -1148,7 +1148,7 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf, · 10 unmodified lines | ||
| 1148 | 1148 | spin_lock_irqsave(&mad_agent_priv->lock, flags); |
| 1149 | 1149 | list_del(&mad_send_wr->agent_list); |
| 1150 | 1150 | spin_unlock_irqrestore(&mad_agent_priv->lock, flags); |
| 1151 | − atomic_dec(&mad_agent_priv->refcount); | |
| 1151 | + deref_mad_agent(mad_agent_priv); | |
| 1152 | 1152 | goto error; |
| 1153 | 1153 | } |
| 1154 | 1154 | } |
| @@ -1554,7 +1554,7 @@ find_mad_agent(struct ib_mad_port_private *port_priv, · 399 unmodified lines | ||
| 1554 | 1554 | hi_tid = be64_to_cpu(mad_hdr->tid) >> 32; |
| 1555 | 1555 | rcu_read_lock(); |
| 1556 | 1556 | mad_agent = xa_load(&ib_mad_clients, hi_tid); |
| 1557 | − if (mad_agent && !atomic_inc_not_zero(&mad_agent->refcount)) | |
| 1557 | + if (mad_agent && !refcount_inc_not_zero(&mad_agent->refcount)) | |
| 1558 | 1558 | mad_agent = NULL; |
| 1559 | 1559 | rcu_read_unlock(); |
| 1560 | 1560 | } else { |
| @@ -1606,7 +1606,7 @@ find_mad_agent(struct ib_mad_port_private *port_priv, · 45 unmodified lines | ||
| 1606 | 1606 | } |
| 1607 | 1607 | } |
| 1608 | 1608 | if (mad_agent) |
| 1609 | − atomic_inc(&mad_agent->refcount); | |
| 1609 | + refcount_inc(&mad_agent->refcount); | |
| 1610 | 1610 | out: |
| 1611 | 1611 | spin_unlock_irqrestore(&port_priv->reg_lock, flags); |
| 1612 | 1612 | } |
| @@ -1831,7 +1831,7 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, · 218 unmodified lines | ||
| 1831 | 1831 | mad_agent_priv->agent.recv_handler( |
| 1832 | 1832 | &mad_agent_priv->agent, NULL, |
| 1833 | 1833 | mad_recv_wc); |
| 1834 | − atomic_dec(&mad_agent_priv->refcount); | |
| 1834 | + deref_mad_agent(mad_agent_priv); | |
| 1835 | 1835 | } else { |
| 1836 | 1836 | /* not user rmpp, revert to normal behavior and |
| 1837 | 1837 | * drop the mad */ |
| @@ -1848,7 +1848,7 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, · 10 unmodified lines | ||
| 1848 | 1848 | &mad_agent_priv->agent, |
| 1849 | 1849 | &mad_send_wr->send_buf, |
| 1850 | 1850 | mad_recv_wc); |
| 1851 | − atomic_dec(&mad_agent_priv->refcount); | |
| 1851 | + deref_mad_agent(mad_agent_priv); | |
| 1852 | 1852 | |
| 1853 | 1853 | mad_send_wc.status = IB_WC_SUCCESS; |
| 1854 | 1854 | mad_send_wc.vendor_err = 0; |
| @@ -2438,7 +2438,7 @@ static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv) · 583 unmodified lines | ||
| 2438 | 2438 | list_del(&mad_send_wr->agent_list); |
| 2439 | 2439 | mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, |
| 2440 | 2440 | &mad_send_wc); |
| 2441 | − atomic_dec(&mad_agent_priv->refcount); | |
| 2441 | + deref_mad_agent(mad_agent_priv); | |
| 2442 | 2442 | } |
| 2443 | 2443 | } |
| 2444 | 2444 | |
| @@ -2572,7 +2572,7 @@ static void local_completions(struct work_struct *work) · 127 unmodified lines | ||
| 2572 | 2572 | &local->mad_send_wr->send_buf, |
| 2573 | 2573 | &local->mad_priv->header.recv_wc); |
| 2574 | 2574 | spin_lock_irqsave(&recv_mad_agent->lock, flags); |
| 2575 | − atomic_dec(&recv_mad_agent->refcount); | |
| 2575 | + deref_mad_agent(recv_mad_agent); | |
| 2576 | 2576 | spin_unlock_irqrestore(&recv_mad_agent->lock, flags); |
| 2577 | 2577 | } |
| 2578 | 2578 | |
| @@ -2585,7 +2585,7 @@ local_send_completion: · 6 unmodified lines | ||
| 2585 | 2585 | &mad_send_wc); |
| 2586 | 2586 | |
| 2587 | 2587 | spin_lock_irqsave(&mad_agent_priv->lock, flags); |
| 2588 | − atomic_dec(&mad_agent_priv->refcount); | |
| 2588 | + deref_mad_agent(mad_agent_priv); | |
| 2589 | 2589 | if (free_mad) |
| 2590 | 2590 | kfree(local->mad_priv); |
| 2591 | 2591 | kfree(local); |
| @@ -2671,7 +2671,7 @@ static void timeout_sends(struct work_struct *work) · 79 unmodified lines | ||
| 2671 | 2671 | mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, |
| 2672 | 2672 | &mad_send_wc); |
| 2673 | 2673 | |
| 2674 | − atomic_dec(&mad_agent_priv->refcount); | |
| 2674 | + deref_mad_agent(mad_agent_priv); | |
| 2675 | 2675 | spin_lock_irqsave(&mad_agent_priv->lock, flags); |
| 2676 | 2676 | } |
| 2677 | 2677 | spin_unlock_irqrestore(&mad_agent_priv->lock, flags); |
| @@ -103,7 +103,7 @@ struct ib_mad_agent_private { · 102 unmodified lines | ||
| 103 | 103 | struct work_struct local_work; |
| 104 | 104 | struct list_head rmpp_list; |
| 105 | 105 | |
| 106 | − atomic_t refcount; | |
| 106 | + refcount_t refcount; | |
| 107 | 107 | union { |
| 108 | 108 | struct completion comp; |
| 109 | 109 | struct rcu_head rcu; |
| @@ -40,8 +40,7 @@ · 39 unmodified lines | ||
| 40 | 40 | enum rmpp_state { |
| 41 | 41 | RMPP_STATE_ACTIVE, |
| 42 | 42 | RMPP_STATE_TIMEOUT, |
| 43 | − RMPP_STATE_COMPLETE, | |
| 44 | − RMPP_STATE_CANCELING | |
| 43 | + RMPP_STATE_COMPLETE | |
| 45 | 44 | }; |
| 46 | 45 | |
| 47 | 46 | struct mad_rmpp_recv { |
| @@ -52,7 +51,7 @@ struct mad_rmpp_recv { · 4 unmodified lines | ||
| 52 | 51 | struct completion comp; |
| 53 | 52 | enum rmpp_state state; |
| 54 | 53 | spinlock_t lock; |
| 55 | − atomic_t refcount; | |
| 54 | + refcount_t refcount; | |
| 56 | 55 | |
| 57 | 56 | struct ib_ah *ah; |
| 58 | 57 | struct ib_mad_recv_wc *rmpp_wc; |
| @@ -73,7 +72,7 @@ struct mad_rmpp_recv { · 14 unmodified lines | ||
| 73 | 72 | |
| 74 | 73 | static inline void deref_rmpp_recv(struct mad_rmpp_recv *rmpp_recv) |
| 75 | 74 | { |
| 76 | − if (atomic_dec_and_test(&rmpp_recv->refcount)) | |
| 75 | + if (refcount_dec_and_test(&rmpp_recv->refcount)) | |
| 77 | 76 | complete(&rmpp_recv->comp); |
| 78 | 77 | } |
| 79 | 78 | |
| @@ -91,23 +90,19 @@ void ib_cancel_rmpp_recvs(struct ib_mad_agent_private *agent) · 11 unmodified lines | ||
| 91 | 90 | unsigned long flags; |
| 92 | 91 | |
| 93 | 92 | spin_lock_irqsave(&agent->lock, flags); |
| 94 | − list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) { | |
| 95 | − if (rmpp_recv->state != RMPP_STATE_COMPLETE) | |
| 96 | − ib_free_recv_mad(rmpp_recv->rmpp_wc); | |
| 97 | − rmpp_recv->state = RMPP_STATE_CANCELING; | |
| 98 | − } | |
| 99 | − spin_unlock_irqrestore(&agent->lock, flags); | |
| 100 | − | |
| 101 | 93 | list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) { |
| 102 | 94 | cancel_delayed_work(&rmpp_recv->timeout_work); |
| 103 | 95 | cancel_delayed_work(&rmpp_recv->cleanup_work); |
| 104 | 96 | } |
| 97 | + spin_unlock_irqrestore(&agent->lock, flags); | |
| 105 | 98 | |
| 106 | 99 | flush_workqueue(agent->qp_info->port_priv->wq); |
| 107 | 100 | |
| 108 | 101 | list_for_each_entry_safe(rmpp_recv, temp_rmpp_recv, |
| 109 | 102 | &agent->rmpp_list, list) { |
| 110 | 103 | list_del(&rmpp_recv->list); |
| 104 | + if (rmpp_recv->state != RMPP_STATE_COMPLETE) | |
| 105 | + ib_free_recv_mad(rmpp_recv->rmpp_wc); | |
| 111 | 106 | destroy_rmpp_recv(rmpp_recv); |
| 112 | 107 | } |
| 113 | 108 | } |
| @@ -272,10 +267,6 @@ static void recv_cleanup_handler(struct work_struct *work) · 158 unmodified lines | ||
| 272 | 267 | unsigned long flags; |
| 273 | 268 | |
| 274 | 269 | spin_lock_irqsave(&rmpp_recv->agent->lock, flags); |
| 275 | − if (rmpp_recv->state == RMPP_STATE_CANCELING) { | |
| 276 | − spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags); | |
| 277 | − return; | |
| 278 | − } | |
| 279 | 270 | list_del(&rmpp_recv->list); |
| 280 | 271 | spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags); |
| 281 | 272 | destroy_rmpp_recv(rmpp_recv); |
| @@ -305,7 +296,7 @@ create_rmpp_recv(struct ib_mad_agent_private *agent, · 23 unmodified lines | ||
| 305 | 296 | INIT_DELAYED_WORK(&rmpp_recv->cleanup_work, recv_cleanup_handler); |
| 306 | 297 | spin_lock_init(&rmpp_recv->lock); |
| 307 | 298 | rmpp_recv->state = RMPP_STATE_ACTIVE; |
| 308 | − atomic_set(&rmpp_recv->refcount, 1); | |
| 299 | + refcount_set(&rmpp_recv->refcount, 1); | |
| 309 | 300 | |
| 310 | 301 | rmpp_recv->rmpp_wc = mad_recv_wc; |
| 311 | 302 | rmpp_recv->cur_seg_buf = &mad_recv_wc->recv_buf; |
| @@ -357,7 +348,7 @@ acquire_rmpp_recv(struct ib_mad_agent_private *agent, · 45 unmodified lines | ||
| 357 | 348 | spin_lock_irqsave(&agent->lock, flags); |
| 358 | 349 | rmpp_recv = find_rmpp_recv(agent, mad_recv_wc); |
| 359 | 350 | if (rmpp_recv) |
| 360 | − atomic_inc(&rmpp_recv->refcount); | |
| 351 | + refcount_inc(&rmpp_recv->refcount); | |
| 361 | 352 | spin_unlock_irqrestore(&agent->lock, flags); |
| 362 | 353 | return rmpp_recv; |
| 363 | 354 | } |
| @@ -553,7 +544,7 @@ start_rmpp(struct ib_mad_agent_private *agent, · 189 unmodified lines | ||
| 553 | 544 | destroy_rmpp_recv(rmpp_recv); |
| 554 | 545 | return continue_rmpp(agent, mad_recv_wc); |
| 555 | 546 | } |
| 556 | − atomic_inc(&rmpp_recv->refcount); | |
| 547 | + refcount_inc(&rmpp_recv->refcount); | |
| 557 | 548 | |
| 558 | 549 | if (get_last_flag(&mad_recv_wc->recv_buf)) { |
| 559 | 550 | rmpp_recv->state = RMPP_STATE_COMPLETE; |
| @@ -114,6 +114,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { · 113 unmodified lines | ||
| 114 | 114 | [RDMA_NLDEV_ATTR_RES_PS] = { .type = NLA_U32 }, |
| 115 | 115 | [RDMA_NLDEV_ATTR_RES_QP] = { .type = NLA_NESTED }, |
| 116 | 116 | [RDMA_NLDEV_ATTR_RES_QP_ENTRY] = { .type = NLA_NESTED }, |
| 117 | + [RDMA_NLDEV_ATTR_RES_RAW] = { .type = NLA_BINARY }, | |
| 117 | 118 | [RDMA_NLDEV_ATTR_RES_RKEY] = { .type = NLA_U32 }, |
| 118 | 119 | [RDMA_NLDEV_ATTR_RES_RQPN] = { .type = NLA_U32 }, |
| 119 | 120 | [RDMA_NLDEV_ATTR_RES_RQ_PSN] = { .type = NLA_U32 }, |
| @@ -446,27 +447,11 @@ static int fill_res_name_pid(struct sk_buff *msg, · 326 unmodified lines | ||
| 446 | 447 | return err ? -EMSGSIZE : 0; |
| 447 | 448 | } |
| 448 | 449 | |
| 449 | −static bool fill_res_entry(struct ib_device *dev, struct sk_buff *msg, | |
| 450 | − struct rdma_restrack_entry *res) | |
| 450 | +static int fill_res_qp_entry_query(struct sk_buff *msg, | |
| 451 | + struct rdma_restrack_entry *res, | |
| 452 | + struct ib_device *dev, | |
| 453 | + struct ib_qp *qp) | |
| 451 | 454 | { |
| 452 | − if (!dev->ops.fill_res_entry) | |
| 453 | − return false; | |
| 454 | − return dev->ops.fill_res_entry(msg, res); | |
| 455 | −} | |
| 456 | − | |
| 457 | −static bool fill_stat_entry(struct ib_device *dev, struct sk_buff *msg, | |
| 458 | − struct rdma_restrack_entry *res) | |
| 459 | −{ | |
| 460 | − if (!dev->ops.fill_stat_entry) | |
| 461 | − return false; | |
| 462 | − return dev->ops.fill_stat_entry(msg, res); | |
| 463 | −} | |
| 464 | − | |
| 465 | −static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin, | |
| 466 | − struct rdma_restrack_entry *res, uint32_t port) | |
| 467 | −{ | |
| 468 | − struct ib_qp *qp = container_of(res, struct ib_qp, res); | |
| 469 | − struct ib_device *dev = qp->device; | |
| 470 | 455 | struct ib_qp_init_attr qp_init_attr; |
| 471 | 456 | struct ib_qp_attr qp_attr; |
| 472 | 457 | int ret; |
| @@ -475,16 +460,6 @@ static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin, · 2 unmodified lines | ||
| 475 | 460 | if (ret) |
| 476 | 461 | return ret; |
| 477 | 462 | |
| 478 | − if (port && port != qp_attr.port_num) | |
| 479 | − return -EAGAIN; | |
| 480 | − | |
| 481 | − /* In create_qp() port is not set yet */ | |
| 482 | − if (qp_attr.port_num && | |
| 483 | − nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp_attr.port_num)) | |
| 484 | − goto err; | |
| 485 | − | |
| 486 | − if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num)) | |
| 487 | − goto err; | |
| 488 | 463 | if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) { |
| 489 | 464 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN, |
| 490 | 465 | qp_attr.dest_qp_num)) |
| @@ -508,19 +483,53 @@ static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin, · 17 unmodified lines | ||
| 508 | 483 | if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state)) |
| 509 | 484 | goto err; |
| 510 | 485 | |
| 486 | + if (dev->ops.fill_res_qp_entry) | |
| 487 | + return dev->ops.fill_res_qp_entry(msg, qp); | |
| 488 | + return 0; | |
| 489 | + | |
| 490 | +err: return -EMSGSIZE; | |
| 491 | +} | |
| 492 | + | |
| 493 | +static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin, | |
| 494 | + struct rdma_restrack_entry *res, uint32_t port) | |
| 495 | +{ | |
| 496 | + struct ib_qp *qp = container_of(res, struct ib_qp, res); | |
| 497 | + struct ib_device *dev = qp->device; | |
| 498 | + int ret; | |
| 499 | + | |
| 500 | + if (port && port != qp->port) | |
| 501 | + return -EAGAIN; | |
| 502 | + | |
| 503 | + /* In create_qp() port is not set yet */ | |
| 504 | + if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port)) | |
| 505 | + return -EINVAL; | |
| 506 | + | |
| 507 | + ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num); | |
| 508 | + if (ret) | |
| 509 | + return -EMSGSIZE; | |
| 510 | + | |
| 511 | 511 | if (!rdma_is_kernel_res(res) && |
| 512 | 512 | nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id)) |
| 513 | − goto err; | |
| 513 | + return -EMSGSIZE; | |
| 514 | 514 | |
| 515 | − if (fill_res_name_pid(msg, res)) | |
| 516 | − goto err; | |
| 515 | + ret = fill_res_name_pid(msg, res); | |
| 516 | + if (ret) | |
| 517 | + return -EMSGSIZE; | |
| 517 | 518 | |
| 518 | − if (fill_res_entry(dev, msg, res)) | |
| 519 | − goto err; | |
| 519 | + return fill_res_qp_entry_query(msg, res, dev, qp); | |
| 520 | +} | |
| 520 | 521 | |
| 521 | − return 0; | |
| 522 | +static int fill_res_qp_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, | |
| 523 | + struct rdma_restrack_entry *res, uint32_t port) | |
| 524 | +{ | |
| 525 | + struct ib_qp *qp = container_of(res, struct ib_qp, res); | |
| 526 | + struct ib_device *dev = qp->device; | |
| 522 | 527 | |
| 523 | −err: return -EMSGSIZE; | |
| 528 | + if (port && port != qp->port) | |
| 529 | + return -EAGAIN; | |
| 530 | + if (!dev->ops.fill_res_qp_entry_raw) | |
| 531 | + return -EINVAL; | |
| 532 | + return dev->ops.fill_res_qp_entry_raw(msg, qp); | |
| 524 | 533 | } |
| 525 | 534 | |
| 526 | 535 | static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin, |
| @@ -568,9 +577,8 @@ static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin, · 41 unmodified lines | ||
| 568 | 577 | if (fill_res_name_pid(msg, res)) |
| 569 | 578 | goto err; |
| 570 | 579 | |
| 571 | − if (fill_res_entry(dev, msg, res)) | |
| 572 | − goto err; | |
| 573 | − | |
| 580 | + if (dev->ops.fill_res_cm_id_entry) | |
| 581 | + return dev->ops.fill_res_cm_id_entry(msg, cm_id); | |
| 574 | 582 | return 0; |
| 575 | 583 | |
| 576 | 584 | err: return -EMSGSIZE; |
| @@ -583,35 +591,42 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin, · 6 unmodified lines | ||
| 583 | 591 | struct ib_device *dev = cq->device; |
| 584 | 592 | |
| 585 | 593 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq->cqe)) |
| 586 | − goto err; | |
| 594 | + return -EMSGSIZE; | |
| 587 | 595 | if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT, |
| 588 | 596 | atomic_read(&cq->usecnt), RDMA_NLDEV_ATTR_PAD)) |
| 589 | − goto err; | |
| 597 | + return -EMSGSIZE; | |
| 590 | 598 | |
| 591 | 599 | /* Poll context is only valid for kernel CQs */ |
| 592 | 600 | if (rdma_is_kernel_res(res) && |
| 593 | 601 | nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_POLL_CTX, cq->poll_ctx)) |
| 594 | − goto err; | |
| 602 | + return -EMSGSIZE; | |
| 595 | 603 | |
| 596 | 604 | if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, (cq->dim != NULL))) |
| 597 | − goto err; | |
| 605 | + return -EMSGSIZE; | |
| 598 | 606 | |
| 599 | 607 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id)) |
| 600 | − goto err; | |
| 608 | + return -EMSGSIZE; | |
| 601 | 609 | if (!rdma_is_kernel_res(res) && |
| 602 | 610 | nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, |
| 603 | 611 | cq->uobject->uevent.uobject.context->res.id)) |
| 604 | − goto err; | |
| 612 | + return -EMSGSIZE; | |
| 605 | 613 | |
| 606 | 614 | if (fill_res_name_pid(msg, res)) |
| 607 | − goto err; | |
| 615 | + return -EMSGSIZE; | |
| 608 | 616 | |
| 609 | − if (fill_res_entry(dev, msg, res)) | |
| 610 | − goto err; | |
| 617 | + return (dev->ops.fill_res_cq_entry) ? | |
| 618 | + dev->ops.fill_res_cq_entry(msg, cq) : 0; | |
| 619 | +} | |
| 611 | 620 | |
| 612 | − return 0; | |
| 621 | +static int fill_res_cq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, | |
| 622 | + struct rdma_restrack_entry *res, uint32_t port) | |
| 623 | +{ | |
| 624 | + struct ib_cq *cq = container_of(res, struct ib_cq, res); | |
| 625 | + struct ib_device *dev = cq->device; | |
| 613 | 626 | |
| 614 | −err: return -EMSGSIZE; | |
| 627 | + if (!dev->ops.fill_res_cq_entry_raw) | |
| 628 | + return -EINVAL; | |
| 629 | + return dev->ops.fill_res_cq_entry_raw(msg, cq); | |
| 615 | 630 | } |
| 616 | 631 | |
| 617 | 632 | static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, |
| @@ -622,38 +637,45 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, · 4 unmodified lines | ||
| 622 | 637 | |
| 623 | 638 | if (has_cap_net_admin) { |
| 624 | 639 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey)) |
| 625 | − goto err; | |
| 640 | + return -EMSGSIZE; | |
| 626 | 641 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey)) |
| 627 | − goto err; | |
| 642 | + return -EMSGSIZE; | |
| 628 | 643 | } |
| 629 | 644 | |
| 630 | 645 | if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length, |
| 631 | 646 | RDMA_NLDEV_ATTR_PAD)) |
| 632 | − goto err; | |
| 647 | + return -EMSGSIZE; | |
| 633 | 648 | |
| 634 | 649 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) |
| 635 | − goto err; | |
| 650 | + return -EMSGSIZE; | |
| 636 | 651 | |
| 637 | 652 | if (!rdma_is_kernel_res(res) && |
| 638 | 653 | nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id)) |
| 639 | − goto err; | |
| 654 | + return -EMSGSIZE; | |
| 640 | 655 | |
| 641 | 656 | if (fill_res_name_pid(msg, res)) |
| 642 | − goto err; | |
| 657 | + return -EMSGSIZE; | |
| 643 | 658 | |
| 644 | − if (fill_res_entry(dev, msg, res)) | |
| 645 | − goto err; | |
| 659 | + return (dev->ops.fill_res_mr_entry) ? | |
| 660 | + dev->ops.fill_res_mr_entry(msg, mr) : | |
| 661 | + 0; | |
| 662 | +} | |
| 646 | 663 | |
| 647 | − return 0; | |
| 664 | +static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, | |
| 665 | + struct rdma_restrack_entry *res, uint32_t port) | |
| 666 | +{ | |
| 667 | + struct ib_mr *mr = container_of(res, struct ib_mr, res); | |
| 668 | + struct ib_device *dev = mr->pd->device; | |
| 648 | 669 | |
| 649 | −err: return -EMSGSIZE; | |
| 670 | + if (!dev->ops.fill_res_mr_entry_raw) | |
| 671 | + return -EINVAL; | |
| 672 | + return dev->ops.fill_res_mr_entry_raw(msg, mr); | |
| 650 | 673 | } |
| 651 | 674 | |
| 652 | 675 | static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin, |
| 653 | 676 | struct rdma_restrack_entry *res, uint32_t port) |
| 654 | 677 | { |
| 655 | 678 | struct ib_pd *pd = container_of(res, struct ib_pd, res); |
| 656 | − struct ib_device *dev = pd->device; | |
| 657 | 679 | |
| 658 | 680 | if (has_cap_net_admin) { |
| 659 | 681 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY, |
| @@ -676,13 +698,7 @@ static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin, · 16 unmodified lines | ||
| 676 | 698 | pd->uobject->context->res.id)) |
| 677 | 699 | goto err; |
| 678 | 700 | |
| 679 | − if (fill_res_name_pid(msg, res)) | |
| 680 | − goto err; | |
| 681 | − | |
| 682 | − if (fill_res_entry(dev, msg, res)) | |
| 683 | − goto err; | |
| 684 | − | |
| 685 | − return 0; | |
| 701 | + return fill_res_name_pid(msg, res); | |
| 686 | 702 | |
| 687 | 703 | err: return -EMSGSIZE; |
| 688 | 704 | } |
| @@ -695,11 +711,16 @@ static int fill_stat_counter_mode(struct sk_buff *msg, · 6 unmodified lines | ||
| 695 | 711 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, m->mode)) |
| 696 | 712 | return -EMSGSIZE; |
| 697 | 713 | |
| 698 | − if (m->mode == RDMA_COUNTER_MODE_AUTO) | |
| 714 | + if (m->mode == RDMA_COUNTER_MODE_AUTO) { | |
| 699 | 715 | if ((m->mask & RDMA_COUNTER_MASK_QP_TYPE) && |
| 700 | 716 | nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, m->param.qp_type)) |
| 701 | 717 | return -EMSGSIZE; |
| 702 | 718 | |
| 719 | + if ((m->mask & RDMA_COUNTER_MASK_PID) && | |
| 720 | + fill_res_name_pid(msg, &counter->res)) | |
| 721 | + return -EMSGSIZE; | |
| 722 | + } | |
| 723 | + | |
| 703 | 724 | return 0; |
| 704 | 725 | } |
| 705 | 726 | |
| @@ -738,9 +759,6 @@ static int fill_stat_counter_qps(struct sk_buff *msg, · 32 unmodified lines | ||
| 738 | 759 | xa_lock(&rt->xa); |
| 739 | 760 | xa_for_each(&rt->xa, id, res) { |
| 740 | 761 | qp = container_of(res, struct ib_qp, res); |
| 741 | − if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) | |
| 742 | − continue; | |
| 743 | − | |
| 744 | 762 | if (!qp->counter || (qp->counter->id != counter->id)) |
| 745 | 763 | continue; |
| 746 | 764 | |
| @@ -793,9 +811,8 @@ static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, · 46 unmodified lines | ||
| 793 | 811 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) |
| 794 | 812 | goto err; |
| 795 | 813 | |
| 796 | − if (fill_stat_entry(dev, msg, res)) | |
| 797 | − goto err; | |
| 798 | − | |
| 814 | + if (dev->ops.fill_stat_mr_entry) | |
| 815 | + return dev->ops.fill_stat_mr_entry(msg, mr); | |
| 799 | 816 | return 0; |
| 800 | 817 | |
| 801 | 818 | err: |
| @@ -840,7 +857,6 @@ static int fill_res_counter_entry(struct sk_buff *msg, bool has_cap_net_admin, · 38 unmodified lines | ||
| 840 | 857 | |
| 841 | 858 | if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, counter->port) || |
| 842 | 859 | nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, counter->id) || |
| 843 | − fill_res_name_pid(msg, &counter->res) || | |
| 844 | 860 | fill_stat_counter_mode(msg, counter) || |
| 845 | 861 | fill_stat_counter_qps(msg, counter) || |
| 846 | 862 | fill_stat_counter_hwcounters(msg, counter)) |
| @@ -1177,7 +1193,6 @@ static int nldev_res_get_dumpit(struct sk_buff *skb, · 330 unmodified lines | ||
| 1177 | 1193 | |
| 1178 | 1194 | struct nldev_fill_res_entry { |
| 1179 | 1195 | enum rdma_nldev_attr nldev_attr; |
| 1180 | − enum rdma_nldev_command nldev_cmd; | |
| 1181 | 1196 | u8 flags; |
| 1182 | 1197 | u32 entry; |
| 1183 | 1198 | u32 id; |
| @@ -1189,40 +1204,34 @@ enum nldev_res_flags { · 5 unmodified lines | ||
| 1189 | 1204 | |
| 1190 | 1205 | static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = { |
| 1191 | 1206 | [RDMA_RESTRACK_QP] = { |
| 1192 | − .nldev_cmd = RDMA_NLDEV_CMD_RES_QP_GET, | |
| 1193 | 1207 | .nldev_attr = RDMA_NLDEV_ATTR_RES_QP, |
| 1194 | 1208 | .entry = RDMA_NLDEV_ATTR_RES_QP_ENTRY, |
| 1195 | 1209 | .id = RDMA_NLDEV_ATTR_RES_LQPN, |
| 1196 | 1210 | }, |
| 1197 | 1211 | [RDMA_RESTRACK_CM_ID] = { |
| 1198 | − .nldev_cmd = RDMA_NLDEV_CMD_RES_CM_ID_GET, | |
| 1199 | 1212 | .nldev_attr = RDMA_NLDEV_ATTR_RES_CM_ID, |
| 1200 | 1213 | .entry = RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY, |
| 1201 | 1214 | .id = RDMA_NLDEV_ATTR_RES_CM_IDN, |
| 1202 | 1215 | }, |
| 1203 | 1216 | [RDMA_RESTRACK_CQ] = { |
| 1204 | − .nldev_cmd = RDMA_NLDEV_CMD_RES_CQ_GET, | |
| 1205 | 1217 | .nldev_attr = RDMA_NLDEV_ATTR_RES_CQ, |
| 1206 | 1218 | .flags = NLDEV_PER_DEV, |
| 1207 | 1219 | .entry = RDMA_NLDEV_ATTR_RES_CQ_ENTRY, |
| 1208 | 1220 | .id = RDMA_NLDEV_ATTR_RES_CQN, |
| 1209 | 1221 | }, |
| 1210 | 1222 | [RDMA_RESTRACK_MR] = { |
| 1211 | − .nldev_cmd = RDMA_NLDEV_CMD_RES_MR_GET, | |
| 1212 | 1223 | .nldev_attr = RDMA_NLDEV_ATTR_RES_MR, |
| 1213 | 1224 | .flags = NLDEV_PER_DEV, |
| 1214 | 1225 | .entry = RDMA_NLDEV_ATTR_RES_MR_ENTRY, |
| 1215 | 1226 | .id = RDMA_NLDEV_ATTR_RES_MRN, |
| 1216 | 1227 | }, |
| 1217 | 1228 | [RDMA_RESTRACK_PD] = { |
| 1218 | − .nldev_cmd = RDMA_NLDEV_CMD_RES_PD_GET, | |
| 1219 | 1229 | .nldev_attr = RDMA_NLDEV_ATTR_RES_PD, |
| 1220 | 1230 | .flags = NLDEV_PER_DEV, |
| 1221 | 1231 | .entry = RDMA_NLDEV_ATTR_RES_PD_ENTRY, |
| 1222 | 1232 | .id = RDMA_NLDEV_ATTR_RES_PDN, |
| 1223 | 1233 | }, |
| 1224 | 1234 | [RDMA_RESTRACK_COUNTER] = { |
| 1225 | − .nldev_cmd = RDMA_NLDEV_CMD_STAT_GET, | |
| 1226 | 1235 | .nldev_attr = RDMA_NLDEV_ATTR_STAT_COUNTER, |
| 1227 | 1236 | .entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY, |
| 1228 | 1237 | .id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID, |
| @@ -1281,7 +1290,8 @@ static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh, · 52 unmodified lines | ||
| 1281 | 1290 | } |
| 1282 | 1291 | |
| 1283 | 1292 | nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, |
| 1284 | − RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, fe->nldev_cmd), | |
| 1293 | + RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, | |
| 1294 | + RDMA_NL_GET_OP(nlh->nlmsg_type)), | |
| 1285 | 1295 | 0, 0); |
| 1286 | 1296 | |
| 1287 | 1297 | if (fill_nldev_handle(msg, device)) { |
| @@ -1359,7 +1369,8 @@ static int res_get_common_dumpit(struct sk_buff *skb, · 71 unmodified lines | ||
| 1359 | 1369 | } |
| 1360 | 1370 | |
| 1361 | 1371 | nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, |
| 1362 | − RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, fe->nldev_cmd), | |
| 1372 | + RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, | |
| 1373 | + RDMA_NL_GET_OP(cb->nlh->nlmsg_type)), | |
| 1363 | 1374 | 0, NLM_F_MULTI); |
| 1364 | 1375 | |
| 1365 | 1376 | if (fill_nldev_handle(skb, device)) { |
| @@ -1441,26 +1452,29 @@ err_index: · 75 unmodified lines | ||
| 1441 | 1452 | return ret; |
| 1442 | 1453 | } |
| 1443 | 1454 | |
| 1444 | −#define RES_GET_FUNCS(name, type) \ | |
| 1445 | − static int nldev_res_get_##name##_dumpit(struct sk_buff *skb, \ | |
| 1455 | +#define RES_GET_FUNCS(name, type) \ | |
| 1456 | + static int nldev_res_get_##name##_dumpit(struct sk_buff *skb, \ | |
| 1446 | 1457 | struct netlink_callback *cb) \ |
| 1447 | − { \ | |
| 1448 | − return res_get_common_dumpit(skb, cb, type, \ | |
| 1449 | − fill_res_##name##_entry); \ | |
| 1450 | − } \ | |
| 1451 | − static int nldev_res_get_##name##_doit(struct sk_buff *skb, \ | |
| 1452 | − struct nlmsghdr *nlh, \ | |
| 1458 | + { \ | |
| 1459 | + return res_get_common_dumpit(skb, cb, type, \ | |
| 1460 | + fill_res_##name##_entry); \ | |
| 1461 | + } \ | |
| 1462 | + static int nldev_res_get_##name##_doit(struct sk_buff *skb, \ | |
| 1463 | + struct nlmsghdr *nlh, \ | |
| 1453 | 1464 | struct netlink_ext_ack *extack) \ |
| 1454 | − { \ | |
| 1455 | − return res_get_common_doit(skb, nlh, extack, type, \ | |
| 1456 | − fill_res_##name##_entry); \ | |
| 1465 | + { \ | |
| 1466 | + return res_get_common_doit(skb, nlh, extack, type, \ | |
| 1467 | + fill_res_##name##_entry); \ | |
| 1457 | 1468 | } |
| 1458 | 1469 | |
| 1459 | 1470 | RES_GET_FUNCS(qp, RDMA_RESTRACK_QP); |
| 1471 | +RES_GET_FUNCS(qp_raw, RDMA_RESTRACK_QP); | |
| 1460 | 1472 | RES_GET_FUNCS(cm_id, RDMA_RESTRACK_CM_ID); |
| 1461 | 1473 | RES_GET_FUNCS(cq, RDMA_RESTRACK_CQ); |
| 1474 | +RES_GET_FUNCS(cq_raw, RDMA_RESTRACK_CQ); | |
| 1462 | 1475 | RES_GET_FUNCS(pd, RDMA_RESTRACK_PD); |
| 1463 | 1476 | RES_GET_FUNCS(mr, RDMA_RESTRACK_MR); |
| 1477 | +RES_GET_FUNCS(mr_raw, RDMA_RESTRACK_MR); | |
| 1464 | 1478 | RES_GET_FUNCS(counter, RDMA_RESTRACK_COUNTER); |
| 1465 | 1479 | |
| 1466 | 1480 | static LIST_HEAD(link_ops); |
| @@ -2145,6 +2159,21 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = { · 678 unmodified lines | ||
| 2145 | 2159 | .doit = nldev_stat_del_doit, |
| 2146 | 2160 | .flags = RDMA_NL_ADMIN_PERM, |
| 2147 | 2161 | }, |
| 2162 | + [RDMA_NLDEV_CMD_RES_QP_GET_RAW] = { | |
| 2163 | + .doit = nldev_res_get_qp_raw_doit, | |
| 2164 | + .dump = nldev_res_get_qp_raw_dumpit, | |
| 2165 | + .flags = RDMA_NL_ADMIN_PERM, | |
| 2166 | + }, | |
| 2167 | + [RDMA_NLDEV_CMD_RES_CQ_GET_RAW] = { | |
| 2168 | + .doit = nldev_res_get_cq_raw_doit, | |
| 2169 | + .dump = nldev_res_get_cq_raw_dumpit, | |
| 2170 | + .flags = RDMA_NL_ADMIN_PERM, | |
| 2171 | + }, | |
| 2172 | + [RDMA_NLDEV_CMD_RES_MR_GET_RAW] = { | |
| 2173 | + .doit = nldev_res_get_mr_raw_doit, | |
| 2174 | + .dump = nldev_res_get_mr_raw_dumpit, | |
| 2175 | + .flags = RDMA_NL_ADMIN_PERM, | |
| 2176 | + }, | |
| 2148 | 2177 | }; |
| 2149 | 2178 | |
| 2150 | 2179 | void __init nldev_init(void) |
| @@ -58,7 +58,7 @@ struct ib_port { · 57 unmodified lines | ||
| 58 | 58 | struct ib_device *ibdev; |
| 59 | 59 | struct gid_attr_group *gid_attr_group; |
| 60 | 60 | struct attribute_group gid_group; |
| 61 | − struct attribute_group pkey_group; | |
| 61 | + struct attribute_group *pkey_group; | |
| 62 | 62 | struct attribute_group *pma_table; |
| 63 | 63 | struct attribute_group *hw_stats_ag; |
| 64 | 64 | struct rdma_hw_stats *hw_stats; |
| @@ -681,11 +681,16 @@ static void ib_port_release(struct kobject *kobj) · 616 unmodified lines | ||
| 681 | 681 | kfree(p->gid_group.attrs); |
| 682 | 682 | } |
| 683 | 683 | |
| 684 | − if (p->pkey_group.attrs) { | |
| 685 | − for (i = 0; (a = p->pkey_group.attrs[i]); ++i) | |
| 686 | − kfree(a); | |
| 684 | + if (p->pkey_group) { | |
| 685 | + if (p->pkey_group->attrs) { | |
| 686 | + for (i = 0; (a = p->pkey_group->attrs[i]); ++i) | |
| 687 | + kfree(a); | |
| 688 | + | |
| 689 | + kfree(p->pkey_group->attrs); | |
| 690 | + } | |
| 687 | 691 | |
| 688 | − kfree(p->pkey_group.attrs); | |
| 692 | + kfree(p->pkey_group); | |
| 693 | + p->pkey_group = NULL; | |
| 689 | 694 | } |
| 690 | 695 | |
| 691 | 696 | kfree(p); |
| @@ -1118,17 +1123,26 @@ static int add_port(struct ib_core_device *coredev, int port_num) · 426 unmodified lines | ||
| 1118 | 1123 | if (ret) |
| 1119 | 1124 | goto err_free_gid_type; |
| 1120 | 1125 | |
| 1121 | − p->pkey_group.name = "pkeys"; | |
| 1122 | − p->pkey_group.attrs = alloc_group_attrs(show_port_pkey, | |
| 1123 | − attr.pkey_tbl_len); | |
| 1124 | − if (!p->pkey_group.attrs) { | |
| 1125 | − ret = -ENOMEM; | |
| 1126 | − goto err_remove_gid_type; | |
| 1126 | + if (attr.pkey_tbl_len) { | |
| 1127 | + p->pkey_group = kzalloc(sizeof(*p->pkey_group), GFP_KERNEL); | |
| 1128 | + if (!p->pkey_group) { | |
| 1129 | + ret = -ENOMEM; | |
| 1130 | + goto err_remove_gid_type; | |
| 1131 | + } | |
| 1132 | + | |
| 1133 | + p->pkey_group->name = "pkeys"; | |
| 1134 | + p->pkey_group->attrs = alloc_group_attrs(show_port_pkey, | |
| 1135 | + attr.pkey_tbl_len); | |
| 1136 | + if (!p->pkey_group->attrs) { | |
| 1137 | + ret = -ENOMEM; | |
| 1138 | + goto err_free_pkey_group; | |
| 1139 | + } | |
| 1140 | + | |
| 1141 | + ret = sysfs_create_group(&p->kobj, p->pkey_group); | |
| 1142 | + if (ret) | |
| 1143 | + goto err_free_pkey; | |
| 1127 | 1144 | } |
| 1128 | 1145 | |
| 1129 | − ret = sysfs_create_group(&p->kobj, &p->pkey_group); | |
| 1130 | − if (ret) | |
| 1131 | − goto err_free_pkey; | |
| 1132 | 1146 | |
| 1133 | 1147 | if (device->ops.init_port && is_full_dev) { |
| 1134 | 1148 | ret = device->ops.init_port(device, port_num, &p->kobj); |
| @@ -1150,14 +1164,20 @@ static int add_port(struct ib_core_device *coredev, int port_num) · 15 unmodified lines | ||
| 1150 | 1164 | return 0; |
| 1151 | 1165 | |
| 1152 | 1166 | err_remove_pkey: |
| 1153 | − sysfs_remove_group(&p->kobj, &p->pkey_group); | |
| 1167 | + if (p->pkey_group) | |
| 1168 | + sysfs_remove_group(&p->kobj, p->pkey_group); | |
| 1154 | 1169 | |
| 1155 | 1170 | err_free_pkey: |
| 1156 | − for (i = 0; i < attr.pkey_tbl_len; ++i) | |
| 1157 | − kfree(p->pkey_group.attrs[i]); | |
| 1171 | + if (p->pkey_group) { | |
| 1172 | + for (i = 0; i < attr.pkey_tbl_len; ++i) | |
| 1173 | + kfree(p->pkey_group->attrs[i]); | |
| 1174 | + | |
| 1175 | + kfree(p->pkey_group->attrs); | |
| 1176 | + p->pkey_group->attrs = NULL; | |
| 1177 | + } | |
| 1158 | 1178 | |
| 1159 | − kfree(p->pkey_group.attrs); | |
| 1160 | − p->pkey_group.attrs = NULL; | |
| 1179 | +err_free_pkey_group: | |
| 1180 | + kfree(p->pkey_group); | |
| 1161 | 1181 | |
| 1162 | 1182 | err_remove_gid_type: |
| 1163 | 1183 | sysfs_remove_group(&p->gid_attr_group->kobj, |
| @@ -1317,7 +1337,8 @@ void ib_free_port_attrs(struct ib_core_device *coredev) · 153 unmodified lines | ||
| 1317 | 1337 | |
| 1318 | 1338 | if (port->pma_table) |
| 1319 | 1339 | sysfs_remove_group(p, port->pma_table); |
| 1320 | − sysfs_remove_group(p, &port->pkey_group); | |
| 1340 | + if (port->pkey_group) | |
| 1341 | + sysfs_remove_group(p, port->pkey_group); | |
| 1321 | 1342 | sysfs_remove_group(p, &port->gid_group); |
| 1322 | 1343 | sysfs_remove_group(&port->gid_attr_group->kobj, |
| 1323 | 1344 | &port->gid_attr_group->ndev); |
| @@ -9,6 +9,4 @@ · 8 unmodified lines | ||
| 9 | 9 | |
| 10 | 10 | #define CREATE_TRACE_POINTS |
| 11 | 11 | |
| 12 | −#include <rdma/ib_verbs.h> | |
| 13 | − | |
| 14 | 12 | #include <trace/events/rdma_core.h> |
| @@ -261,6 +261,7 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, · 260 unmodified lines | ||
| 261 | 261 | sg = umem->sg_head.sgl; |
| 262 | 262 | |
| 263 | 263 | while (npages) { |
| 264 | + cond_resched(); | |
| 264 | 265 | ret = pin_user_pages_fast(cur_base, |
| 265 | 266 | min_t(unsigned long, npages, |
| 266 | 267 | PAGE_SIZE / |
| @@ -152,6 +152,7 @@ EXPORT_SYMBOL(ib_umem_odp_alloc_implicit); · 151 unmodified lines | ||
| 152 | 152 | * ib_alloc_implicit_odp_umem() |
| 153 | 153 | * @addr: The starting userspace VA |
| 154 | 154 | * @size: The length of the userspace VA |
| 155 | + * @ops: MMU interval ops, currently only @invalidate | |
| 155 | 156 | */ |
| 156 | 157 | struct ib_umem_odp * |
| 157 | 158 | ib_umem_odp_alloc_child(struct ib_umem_odp *root, unsigned long addr, |
| @@ -213,6 +214,7 @@ EXPORT_SYMBOL(ib_umem_odp_alloc_child); · 55 unmodified lines | ||
| 213 | 214 | * @addr: userspace virtual address to start at |
| 214 | 215 | * @size: length of region to pin |
| 215 | 216 | * @access: IB_ACCESS_xxx flags for memory being pinned |
| 217 | + * @ops: MMU interval ops, currently only @invalidate | |
| 216 | 218 | * |
| 217 | 219 | * The driver should use when the access flags indicate ODP memory. It avoids |
| 218 | 220 | * pinning, instead, stores the mm for future page fault handling in |
| @@ -415,8 +415,8 @@ static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs) · 414 unmodified lines | ||
| 415 | 415 | |
| 416 | 416 | static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) |
| 417 | 417 | { |
| 418 | + struct ib_uverbs_alloc_pd_resp resp = {}; | |
| 418 | 419 | struct ib_uverbs_alloc_pd cmd; |
| 419 | − struct ib_uverbs_alloc_pd_resp resp; | |
| 420 | 420 | struct ib_uobject *uobj; |
| 421 | 421 | struct ib_pd *pd; |
| 422 | 422 | int ret; |
| @@ -438,29 +438,20 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) · 15 unmodified lines | ||
| 438 | 438 | |
| 439 | 439 | pd->device = ib_dev; |
| 440 | 440 | pd->uobject = uobj; |
| 441 | − pd->__internal_mr = NULL; | |
| 442 | 441 | atomic_set(&pd->usecnt, 0); |
| 443 | 442 | pd->res.type = RDMA_RESTRACK_PD; |
| 444 | 443 | |
| 445 | 444 | ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata); |
| 446 | 445 | if (ret) |
| 447 | 446 | goto err_alloc; |
| 448 | − | |
| 449 | − uobj->object = pd; | |
| 450 | − memset(&resp, 0, sizeof resp); | |
| 451 | − resp.pd_handle = uobj->id; | |
| 452 | 447 | rdma_restrack_uadd(&pd->res); |
| 453 | 448 | |
| 454 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 455 | − if (ret) | |
| 456 | − goto err_copy; | |
| 449 | + uobj->object = pd; | |
| 450 | + uobj_finalize_uobj_create(uobj, attrs); | |
| 457 | 451 | |
| 458 | − rdma_alloc_commit_uobject(uobj, attrs); | |
| 459 | − return 0; | |
| 452 | + resp.pd_handle = uobj->id; | |
| 453 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 460 | 454 | |
| 461 | −err_copy: | |
| 462 | − ib_dealloc_pd_user(pd, uverbs_get_cleared_udata(attrs)); | |
| 463 | − pd = NULL; | |
| 464 | 455 | err_alloc: |
| 465 | 456 | kfree(pd); |
| 466 | 457 | err: |
| @@ -568,15 +559,15 @@ static void xrcd_table_delete(struct ib_uverbs_device *dev, · 101 unmodified lines | ||
| 568 | 559 | static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) |
| 569 | 560 | { |
| 570 | 561 | struct ib_uverbs_device *ibudev = attrs->ufile->device; |
| 562 | + struct ib_uverbs_open_xrcd_resp resp = {}; | |
| 571 | 563 | struct ib_uverbs_open_xrcd cmd; |
| 572 | − struct ib_uverbs_open_xrcd_resp resp; | |
| 573 | 564 | struct ib_uxrcd_object *obj; |
| 574 | 565 | struct ib_xrcd *xrcd = NULL; |
| 575 | − struct fd f = {NULL, 0}; | |
| 576 | 566 | struct inode *inode = NULL; |
| 577 | − int ret = 0; | |
| 578 | 567 | int new_xrcd = 0; |
| 579 | 568 | struct ib_device *ib_dev; |
| 569 | + struct fd f = {}; | |
| 570 | + int ret; | |
| 580 | 571 | |
| 581 | 572 | ret = uverbs_request(attrs, &cmd, sizeof(cmd)); |
| 582 | 573 | if (ret) |
| @@ -614,24 +605,16 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) · 31 unmodified lines | ||
| 614 | 605 | } |
| 615 | 606 | |
| 616 | 607 | if (!xrcd) { |
| 617 | − xrcd = ib_dev->ops.alloc_xrcd(ib_dev, &attrs->driver_udata); | |
| 608 | + xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata); | |
| 618 | 609 | if (IS_ERR(xrcd)) { |
| 619 | 610 | ret = PTR_ERR(xrcd); |
| 620 | 611 | goto err; |
| 621 | 612 | } |
| 622 | − | |
| 623 | − xrcd->inode = inode; | |
| 624 | − xrcd->device = ib_dev; | |
| 625 | − atomic_set(&xrcd->usecnt, 0); | |
| 626 | − mutex_init(&xrcd->tgt_qp_mutex); | |
| 627 | − INIT_LIST_HEAD(&xrcd->tgt_qp_list); | |
| 628 | 613 | new_xrcd = 1; |
| 629 | 614 | } |
| 630 | 615 | |
| 631 | 616 | atomic_set(&obj->refcnt, 0); |
| 632 | 617 | obj->uobject.object = xrcd; |
| 633 | − memset(&resp, 0, sizeof resp); | |
| 634 | − resp.xrcd_handle = obj->uobject.id; | |
| 635 | 618 | |
| 636 | 619 | if (inode) { |
| 637 | 620 | if (new_xrcd) { |
| @@ -643,27 +626,17 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) · 5 unmodified lines | ||
| 643 | 626 | atomic_inc(&xrcd->usecnt); |
| 644 | 627 | } |
| 645 | 628 | |
| 646 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 647 | − if (ret) | |
| 648 | − goto err_copy; | |
| 649 | − | |
| 650 | 629 | if (f.file) |
| 651 | 630 | fdput(f); |
| 652 | 631 | |
| 653 | 632 | mutex_unlock(&ibudev->xrcd_tree_mutex); |
| 633 | + uobj_finalize_uobj_create(&obj->uobject, attrs); | |
| 654 | 634 | |
| 655 | − rdma_alloc_commit_uobject(&obj->uobject, attrs); | |
| 656 | − return 0; | |
| 657 | − | |
| 658 | −err_copy: | |
| 659 | − if (inode) { | |
| 660 | − if (new_xrcd) | |
| 661 | − xrcd_table_delete(ibudev, inode); | |
| 662 | − atomic_dec(&xrcd->usecnt); | |
| 663 | − } | |
| 635 | + resp.xrcd_handle = obj->uobject.id; | |
| 636 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 664 | 637 | |
| 665 | 638 | err_dealloc_xrcd: |
| 666 | − ib_dealloc_xrcd(xrcd, uverbs_get_cleared_udata(attrs)); | |
| 639 | + ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs)); | |
| 667 | 640 | |
| 668 | 641 | err: |
| 669 | 642 | uobj_alloc_abort(&obj->uobject, attrs); |
| @@ -701,7 +674,7 @@ int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd, · 31 unmodified lines | ||
| 701 | 674 | if (inode && !atomic_dec_and_test(&xrcd->usecnt)) |
| 702 | 675 | return 0; |
| 703 | 676 | |
| 704 | − ret = ib_dealloc_xrcd(xrcd, &attrs->driver_udata); | |
| 677 | + ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata); | |
| 705 | 678 | |
| 706 | 679 | if (ib_is_destroy_retryable(ret, why, uobject)) { |
| 707 | 680 | atomic_inc(&xrcd->usecnt); |
| @@ -716,8 +689,8 @@ int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd, · 8 unmodified lines | ||
| 716 | 689 | |
| 717 | 690 | static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) |
| 718 | 691 | { |
| 692 | + struct ib_uverbs_reg_mr_resp resp = {}; | |
| 719 | 693 | struct ib_uverbs_reg_mr cmd; |
| 720 | − struct ib_uverbs_reg_mr_resp resp; | |
| 721 | 694 | struct ib_uobject *uobj; |
| 722 | 695 | struct ib_pd *pd; |
| 723 | 696 | struct ib_mr *mr; |
| @@ -770,30 +743,20 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) · 46 unmodified lines | ||
| 770 | 743 | mr->uobject = uobj; |
| 771 | 744 | atomic_inc(&pd->usecnt); |
| 772 | 745 | mr->res.type = RDMA_RESTRACK_MR; |
| 746 | + mr->iova = cmd.hca_va; | |
| 773 | 747 | rdma_restrack_uadd(&mr->res); |
| 774 | 748 | |
| 775 | 749 | uobj->object = mr; |
| 776 | − | |
| 777 | − memset(&resp, 0, sizeof resp); | |
| 778 | − resp.lkey = mr->lkey; | |
| 779 | − resp.rkey = mr->rkey; | |
| 780 | − resp.mr_handle = uobj->id; | |
| 781 | − | |
| 782 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 783 | − if (ret) | |
| 784 | − goto err_copy; | |
| 785 | − | |
| 786 | 750 | uobj_put_obj_read(pd); |
| 751 | + uobj_finalize_uobj_create(uobj, attrs); | |
| 787 | 752 | |
| 788 | − rdma_alloc_commit_uobject(uobj, attrs); | |
| 789 | − return 0; | |
| 790 | − | |
| 791 | −err_copy: | |
| 792 | − ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs)); | |
| 753 | + resp.lkey = mr->lkey; | |
| 754 | + resp.rkey = mr->rkey; | |
| 755 | + resp.mr_handle = uobj->id; | |
| 756 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 793 | 757 | |
| 794 | 758 | err_put: |
| 795 | 759 | uobj_put_obj_read(pd); |
| 796 | − | |
| 797 | 760 | err_free: |
| 798 | 761 | uobj_alloc_abort(uobj, attrs); |
| 799 | 762 | return ret; |
| @@ -861,6 +824,9 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) · 61 unmodified lines | ||
| 861 | 824 | atomic_dec(&old_pd->usecnt); |
| 862 | 825 | } |
| 863 | 826 | |
| 827 | + if (cmd.flags & IB_MR_REREG_TRANS) | |
| 828 | + mr->iova = cmd.hca_va; | |
| 829 | + | |
| 864 | 830 | memset(&resp, 0, sizeof(resp)); |
| 865 | 831 | resp.lkey = mr->lkey; |
| 866 | 832 | resp.rkey = mr->rkey; |
| @@ -930,21 +896,13 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) · 63 unmodified lines | ||
| 930 | 896 | atomic_inc(&pd->usecnt); |
| 931 | 897 | |
| 932 | 898 | uobj->object = mw; |
| 899 | + uobj_put_obj_read(pd); | |
| 900 | + uobj_finalize_uobj_create(uobj, attrs); | |
| 933 | 901 | |
| 934 | − memset(&resp, 0, sizeof(resp)); | |
| 935 | − resp.rkey = mw->rkey; | |
| 902 | + resp.rkey = mw->rkey; | |
| 936 | 903 | resp.mw_handle = uobj->id; |
| 904 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 937 | 905 | |
| 938 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 939 | − if (ret) | |
| 940 | − goto err_copy; | |
| 941 | − | |
| 942 | − uobj_put_obj_read(pd); | |
| 943 | − rdma_alloc_commit_uobject(uobj, attrs); | |
| 944 | − return 0; | |
| 945 | − | |
| 946 | −err_copy: | |
| 947 | − uverbs_dealloc_mw(mw); | |
| 948 | 906 | err_put: |
| 949 | 907 | uobj_put_obj_read(pd); |
| 950 | 908 | err_free: |
| @@ -981,40 +939,33 @@ static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs) · 30 unmodified lines | ||
| 981 | 939 | if (IS_ERR(uobj)) |
| 982 | 940 | return PTR_ERR(uobj); |
| 983 | 941 | |
| 984 | − resp.fd = uobj->id; | |
| 985 | − | |
| 986 | 942 | ev_file = container_of(uobj, struct ib_uverbs_completion_event_file, |
| 987 | 943 | uobj); |
| 988 | 944 | ib_uverbs_init_event_queue(&ev_file->ev_queue); |
| 945 | + uobj_finalize_uobj_create(uobj, attrs); | |
| 989 | 946 | |
| 990 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 991 | − if (ret) { | |
| 992 | − uobj_alloc_abort(uobj, attrs); | |
| 993 | − return ret; | |
| 994 | − } | |
| 995 | − | |
| 996 | − rdma_alloc_commit_uobject(uobj, attrs); | |
| 997 | − return 0; | |
| 947 | + resp.fd = uobj->id; | |
| 948 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 998 | 949 | } |
| 999 | 950 | |
| 1000 | −static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, | |
| 1001 | − struct ib_uverbs_ex_create_cq *cmd) | |
| 951 | +static int create_cq(struct uverbs_attr_bundle *attrs, | |
| 952 | + struct ib_uverbs_ex_create_cq *cmd) | |
| 1002 | 953 | { |
| 1003 | 954 | struct ib_ucq_object *obj; |
| 1004 | 955 | struct ib_uverbs_completion_event_file *ev_file = NULL; |
| 1005 | 956 | struct ib_cq *cq; |
| 1006 | 957 | int ret; |
| 1007 | − struct ib_uverbs_ex_create_cq_resp resp; | |
| 958 | + struct ib_uverbs_ex_create_cq_resp resp = {}; | |
| 1008 | 959 | struct ib_cq_init_attr attr = {}; |
| 1009 | 960 | struct ib_device *ib_dev; |
| 1010 | 961 | |
| 1011 | 962 | if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors) |
| 1012 | − return ERR_PTR(-EINVAL); | |
| 963 | + return -EINVAL; | |
| 1013 | 964 | |
| 1014 | 965 | obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs, |
| 1015 | 966 | &ib_dev); |
| 1016 | 967 | if (IS_ERR(obj)) |
| 1017 | − return obj; | |
| 968 | + return PTR_ERR(obj); | |
| 1018 | 969 | |
| 1019 | 970 | if (cmd->comp_channel >= 0) { |
| 1020 | 971 | ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs); |
| @@ -1043,53 +994,38 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, · 22 unmodified lines | ||
| 1043 | 994 | cq->event_handler = ib_uverbs_cq_event_handler; |
| 1044 | 995 | cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; |
| 1045 | 996 | atomic_set(&cq->usecnt, 0); |
| 997 | + cq->res.type = RDMA_RESTRACK_CQ; | |
| 1046 | 998 | |
| 1047 | 999 | ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata); |
| 1048 | 1000 | if (ret) |
| 1049 | 1001 | goto err_free; |
| 1002 | + rdma_restrack_uadd(&cq->res); | |
| 1050 | 1003 | |
| 1051 | 1004 | obj->uevent.uobject.object = cq; |
| 1052 | 1005 | obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file); |
| 1053 | 1006 | if (obj->uevent.event_file) |
| 1054 | 1007 | uverbs_uobject_get(&obj->uevent.event_file->uobj); |
| 1008 | + uobj_finalize_uobj_create(&obj->uevent.uobject, attrs); | |
| 1055 | 1009 | |
| 1056 | − memset(&resp, 0, sizeof resp); | |
| 1057 | 1010 | resp.base.cq_handle = obj->uevent.uobject.id; |
| 1058 | − resp.base.cqe = cq->cqe; | |
| 1011 | + resp.base.cqe = cq->cqe; | |
| 1059 | 1012 | resp.response_length = uverbs_response_length(attrs, sizeof(resp)); |
| 1013 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 1060 | 1014 | |
| 1061 | − cq->res.type = RDMA_RESTRACK_CQ; | |
| 1062 | − rdma_restrack_uadd(&cq->res); | |
| 1063 | − | |
| 1064 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 1065 | − if (ret) | |
| 1066 | − goto err_cb; | |
| 1067 | − | |
| 1068 | − rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); | |
| 1069 | − return obj; | |
| 1070 | − | |
| 1071 | −err_cb: | |
| 1072 | − if (obj->uevent.event_file) | |
| 1073 | − uverbs_uobject_put(&obj->uevent.event_file->uobj); | |
| 1074 | − ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs)); | |
| 1075 | − cq = NULL; | |
| 1076 | 1015 | err_free: |
| 1077 | 1016 | kfree(cq); |
| 1078 | 1017 | err_file: |
| 1079 | 1018 | if (ev_file) |
| 1080 | 1019 | ib_uverbs_release_ucq(ev_file, obj); |
| 1081 | − | |
| 1082 | 1020 | err: |
| 1083 | 1021 | uobj_alloc_abort(&obj->uevent.uobject, attrs); |
| 1084 | − | |
| 1085 | − return ERR_PTR(ret); | |
| 1022 | + return ret; | |
| 1086 | 1023 | } |
| 1087 | 1024 | |
| 1088 | 1025 | static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs) |
| 1089 | 1026 | { |
| 1090 | 1027 | struct ib_uverbs_create_cq cmd; |
| 1091 | 1028 | struct ib_uverbs_ex_create_cq cmd_ex; |
| 1092 | − struct ib_ucq_object *obj; | |
| 1093 | 1029 | int ret; |
| 1094 | 1030 | |
| 1095 | 1031 | ret = uverbs_request(attrs, &cmd, sizeof(cmd)); |
| @@ -1102,14 +1038,12 @@ static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs) · 6 unmodified lines | ||
| 1102 | 1038 | cmd_ex.comp_vector = cmd.comp_vector; |
| 1103 | 1039 | cmd_ex.comp_channel = cmd.comp_channel; |
| 1104 | 1040 | |
| 1105 | − obj = create_cq(attrs, &cmd_ex); | |
| 1106 | − return PTR_ERR_OR_ZERO(obj); | |
| 1041 | + return create_cq(attrs, &cmd_ex); | |
| 1107 | 1042 | } |
| 1108 | 1043 | |
| 1109 | 1044 | static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs) |
| 1110 | 1045 | { |
| 1111 | 1046 | struct ib_uverbs_ex_create_cq cmd; |
| 1112 | − struct ib_ucq_object *obj; | |
| 1113 | 1047 | int ret; |
| 1114 | 1048 | |
| 1115 | 1049 | ret = uverbs_request(attrs, &cmd, sizeof(cmd)); |
| @@ -1122,8 +1056,7 @@ static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs) · 6 unmodified lines | ||
| 1122 | 1056 | if (cmd.reserved) |
| 1123 | 1057 | return -EINVAL; |
| 1124 | 1058 | |
| 1125 | − obj = create_cq(attrs, &cmd); | |
| 1126 | − return PTR_ERR_OR_ZERO(obj); | |
| 1059 | + return create_cq(attrs, &cmd); | |
| 1127 | 1060 | } |
| 1128 | 1061 | |
| 1129 | 1062 | static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs) |
| @@ -1131,7 +1064,7 @@ static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs) · 1 unmodified line | ||
| 1131 | 1064 | struct ib_uverbs_resize_cq cmd; |
| 1132 | 1065 | struct ib_uverbs_resize_cq_resp resp = {}; |
| 1133 | 1066 | struct ib_cq *cq; |
| 1134 | − int ret = -EINVAL; | |
| 1067 | + int ret; | |
| 1135 | 1068 | |
| 1136 | 1069 | ret = uverbs_request(attrs, &cmd, sizeof(cmd)); |
| 1137 | 1070 | if (ret) |
| @@ -1298,7 +1231,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs, · 160 unmodified lines | ||
| 1298 | 1231 | struct ib_srq *srq = NULL; |
| 1299 | 1232 | struct ib_qp *qp; |
| 1300 | 1233 | struct ib_qp_init_attr attr = {}; |
| 1301 | − struct ib_uverbs_ex_create_qp_resp resp; | |
| 1234 | + struct ib_uverbs_ex_create_qp_resp resp = {}; | |
| 1302 | 1235 | int ret; |
| 1303 | 1236 | struct ib_rwq_ind_table *ind_tbl = NULL; |
| 1304 | 1237 | bool has_sq = true; |
| @@ -1468,20 +1401,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, · 163 unmodified lines | ||
| 1468 | 1401 | if (obj->uevent.event_file) |
| 1469 | 1402 | uverbs_uobject_get(&obj->uevent.event_file->uobj); |
| 1470 | 1403 | |
| 1471 | − memset(&resp, 0, sizeof resp); | |
| 1472 | − resp.base.qpn = qp->qp_num; | |
| 1473 | − resp.base.qp_handle = obj->uevent.uobject.id; | |
| 1474 | − resp.base.max_recv_sge = attr.cap.max_recv_sge; | |
| 1475 | − resp.base.max_send_sge = attr.cap.max_send_sge; | |
| 1476 | − resp.base.max_recv_wr = attr.cap.max_recv_wr; | |
| 1477 | − resp.base.max_send_wr = attr.cap.max_send_wr; | |
| 1478 | − resp.base.max_inline_data = attr.cap.max_inline_data; | |
| 1479 | − resp.response_length = uverbs_response_length(attrs, sizeof(resp)); | |
| 1480 | − | |
| 1481 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 1482 | − if (ret) | |
| 1483 | − goto err_uevent; | |
| 1484 | − | |
| 1485 | 1404 | if (xrcd) { |
| 1486 | 1405 | obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, |
| 1487 | 1406 | uobject); |
| @@ -1502,12 +1421,18 @@ static int create_qp(struct uverbs_attr_bundle *attrs, · 14 unmodified lines | ||
| 1502 | 1421 | UVERBS_LOOKUP_READ); |
| 1503 | 1422 | if (ind_tbl) |
| 1504 | 1423 | uobj_put_obj_read(ind_tbl); |
| 1424 | + uobj_finalize_uobj_create(&obj->uevent.uobject, attrs); | |
| 1425 | + | |
| 1426 | + resp.base.qpn = qp->qp_num; | |
| 1427 | + resp.base.qp_handle = obj->uevent.uobject.id; | |
| 1428 | + resp.base.max_recv_sge = attr.cap.max_recv_sge; | |
| 1429 | + resp.base.max_send_sge = attr.cap.max_send_sge; | |
| 1430 | + resp.base.max_recv_wr = attr.cap.max_recv_wr; | |
| 1431 | + resp.base.max_send_wr = attr.cap.max_send_wr; | |
| 1432 | + resp.base.max_inline_data = attr.cap.max_inline_data; | |
| 1433 | + resp.response_length = uverbs_response_length(attrs, sizeof(resp)); | |
| 1434 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 1505 | 1435 | |
| 1506 | − rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); | |
| 1507 | − return 0; | |
| 1508 | −err_uevent: | |
| 1509 | − if (obj->uevent.event_file) | |
| 1510 | − uverbs_uobject_put(&obj->uevent.event_file->uobj); | |
| 1511 | 1436 | err_cb: |
| 1512 | 1437 | ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); |
| 1513 | 1438 | |
| @@ -1580,14 +1505,14 @@ static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs) · 66 unmodified lines | ||
| 1580 | 1505 | |
| 1581 | 1506 | static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) |
| 1582 | 1507 | { |
| 1508 | + struct ib_uverbs_create_qp_resp resp = {}; | |
| 1583 | 1509 | struct ib_uverbs_open_qp cmd; |
| 1584 | − struct ib_uverbs_create_qp_resp resp; | |
| 1585 | 1510 | struct ib_uqp_object *obj; |
| 1586 | 1511 | struct ib_xrcd *xrcd; |
| 1587 | − struct ib_uobject *xrcd_uobj; | |
| 1588 | 1512 | struct ib_qp *qp; |
| 1589 | 1513 | struct ib_qp_open_attr attr = {}; |
| 1590 | 1514 | int ret; |
| 1515 | + struct ib_uobject *xrcd_uobj; | |
| 1591 | 1516 | struct ib_device *ib_dev; |
| 1592 | 1517 | |
| 1593 | 1518 | ret = uverbs_request(attrs, &cmd, sizeof(cmd)); |
| @@ -1627,24 +1552,16 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) · 33 unmodified lines | ||
| 1627 | 1552 | obj->uevent.uobject.object = qp; |
| 1628 | 1553 | obj->uevent.uobject.user_handle = cmd.user_handle; |
| 1629 | 1554 | |
| 1630 | − memset(&resp, 0, sizeof resp); | |
| 1631 | − resp.qpn = qp->qp_num; | |
| 1632 | − resp.qp_handle = obj->uevent.uobject.id; | |
| 1633 | − | |
| 1634 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 1635 | − if (ret) | |
| 1636 | − goto err_destroy; | |
| 1637 | − | |
| 1638 | 1555 | obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); |
| 1639 | 1556 | atomic_inc(&obj->uxrcd->refcnt); |
| 1640 | 1557 | qp->uobject = obj; |
| 1641 | 1558 | uobj_put_read(xrcd_uobj); |
| 1559 | + uobj_finalize_uobj_create(&obj->uevent.uobject, attrs); | |
| 1642 | 1560 | |
| 1643 | − rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); | |
| 1644 | − return 0; | |
| 1561 | + resp.qpn = qp->qp_num; | |
| 1562 | + resp.qp_handle = obj->uevent.uobject.id; | |
| 1563 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 1645 | 1564 | |
| 1646 | −err_destroy: | |
| 1647 | − ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); | |
| 1648 | 1565 | err_xrcd: |
| 1649 | 1566 | uobj_put_read(xrcd_uobj); |
| 1650 | 1567 | err_put: |
| @@ -1980,7 +1897,7 @@ static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs) · 329 unmodified lines | ||
| 1980 | 1897 | * Last bit is reserved for extending the attr_mask by |
| 1981 | 1898 | * using another field. |
| 1982 | 1899 | */ |
| 1983 | − BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31)); | |
| 1900 | + BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1ULL << 31)); | |
| 1984 | 1901 | |
| 1985 | 1902 | if (cmd.base.attr_mask & |
| 1986 | 1903 | ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1)) |
| @@ -2480,24 +2397,14 @@ static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs) · 493 unmodified lines | ||
| 2480 | 2397 | ah->uobject = uobj; |
| 2481 | 2398 | uobj->user_handle = cmd.user_handle; |
| 2482 | 2399 | uobj->object = ah; |
| 2483 | − | |
| 2484 | − resp.ah_handle = uobj->id; | |
| 2485 | − | |
| 2486 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 2487 | − if (ret) | |
| 2488 | − goto err_copy; | |
| 2489 | − | |
| 2490 | 2400 | uobj_put_obj_read(pd); |
| 2491 | − rdma_alloc_commit_uobject(uobj, attrs); | |
| 2492 | − return 0; | |
| 2401 | + uobj_finalize_uobj_create(uobj, attrs); | |
| 2493 | 2402 | |
| 2494 | −err_copy: | |
| 2495 | − rdma_destroy_ah_user(ah, RDMA_DESTROY_AH_SLEEPABLE, | |
| 2496 | − uverbs_get_cleared_udata(attrs)); | |
| 2403 | + resp.ah_handle = uobj->id; | |
| 2404 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 2497 | 2405 | |
| 2498 | 2406 | err_put: |
| 2499 | 2407 | uobj_put_obj_read(pd); |
| 2500 | − | |
| 2501 | 2408 | err: |
| 2502 | 2409 | uobj_alloc_abort(uobj, attrs); |
| 2503 | 2410 | return ret; |
| @@ -2989,26 +2896,18 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) · 485 unmodified lines | ||
| 2989 | 2896 | if (obj->uevent.event_file) |
| 2990 | 2897 | uverbs_uobject_get(&obj->uevent.event_file->uobj); |
| 2991 | 2898 | |
| 2992 | − memset(&resp, 0, sizeof(resp)); | |
| 2899 | + uobj_put_obj_read(pd); | |
| 2900 | + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, | |
| 2901 | + UVERBS_LOOKUP_READ); | |
| 2902 | + uobj_finalize_uobj_create(&obj->uevent.uobject, attrs); | |
| 2903 | + | |
| 2993 | 2904 | resp.wq_handle = obj->uevent.uobject.id; |
| 2994 | 2905 | resp.max_sge = wq_init_attr.max_sge; |
| 2995 | 2906 | resp.max_wr = wq_init_attr.max_wr; |
| 2996 | 2907 | resp.wqn = wq->wq_num; |
| 2997 | 2908 | resp.response_length = uverbs_response_length(attrs, sizeof(resp)); |
| 2998 | − err = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 2999 | − if (err) | |
| 3000 | − goto err_copy; | |
| 3001 | − | |
| 3002 | − uobj_put_obj_read(pd); | |
| 3003 | − rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, | |
| 3004 | − UVERBS_LOOKUP_READ); | |
| 3005 | − rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); | |
| 3006 | − return 0; | |
| 2909 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 3007 | 2910 | |
| 3008 | −err_copy: | |
| 3009 | − if (obj->uevent.event_file) | |
| 3010 | − uverbs_uobject_put(&obj->uevent.event_file->uobj); | |
| 3011 | − ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs)); | |
| 3012 | 2911 | err_put_cq: |
| 3013 | 2912 | rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, |
| 3014 | 2913 | UVERBS_LOOKUP_READ); |
| @@ -3093,7 +2992,7 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) · 78 unmodified lines | ||
| 3093 | 2992 | struct ib_wq **wqs = NULL; |
| 3094 | 2993 | u32 *wqs_handles = NULL; |
| 3095 | 2994 | struct ib_wq *wq = NULL; |
| 3096 | − int i, j, num_read_wqs; | |
| 2995 | + int i, num_read_wqs; | |
| 3097 | 2996 | u32 num_wq_handles; |
| 3098 | 2997 | struct uverbs_req_iter iter; |
| 3099 | 2998 | struct ib_device *ib_dev; |
| @@ -3139,6 +3038,7 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) · 39 unmodified lines | ||
| 3139 | 3038 | } |
| 3140 | 3039 | |
| 3141 | 3040 | wqs[num_read_wqs] = wq; |
| 3041 | + atomic_inc(&wqs[num_read_wqs]->usecnt); | |
| 3142 | 3042 | } |
| 3143 | 3043 | |
| 3144 | 3044 | uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev); |
| @@ -3166,33 +3066,24 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) · 21 unmodified lines | ||
| 3166 | 3066 | atomic_set(&rwq_ind_tbl->usecnt, 0); |
| 3167 | 3067 | |
| 3168 | 3068 | for (i = 0; i < num_wq_handles; i++) |
| 3169 | − atomic_inc(&wqs[i]->usecnt); | |
| 3069 | + rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject, | |
| 3070 | + UVERBS_LOOKUP_READ); | |
| 3071 | + kfree(wqs_handles); | |
| 3072 | + uobj_finalize_uobj_create(uobj, attrs); | |
| 3170 | 3073 | |
| 3171 | 3074 | resp.ind_tbl_handle = uobj->id; |
| 3172 | 3075 | resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; |
| 3173 | 3076 | resp.response_length = uverbs_response_length(attrs, sizeof(resp)); |
| 3077 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 3174 | 3078 | |
| 3175 | − err = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 3176 | − if (err) | |
| 3177 | − goto err_copy; | |
| 3178 | − | |
| 3179 | − kfree(wqs_handles); | |
| 3180 | − | |
| 3181 | − for (j = 0; j < num_read_wqs; j++) | |
| 3182 | − rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject, | |
| 3183 | − UVERBS_LOOKUP_READ); | |
| 3184 | − | |
| 3185 | − rdma_alloc_commit_uobject(uobj, attrs); | |
| 3186 | − return 0; | |
| 3187 | − | |
| 3188 | −err_copy: | |
| 3189 | − ib_destroy_rwq_ind_table(rwq_ind_tbl); | |
| 3190 | 3079 | err_uobj: |
| 3191 | 3080 | uobj_alloc_abort(uobj, attrs); |
| 3192 | 3081 | put_wqs: |
| 3193 | − for (j = 0; j < num_read_wqs; j++) | |
| 3194 | − rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject, | |
| 3082 | + for (i = 0; i < num_read_wqs; i++) { | |
| 3083 | + rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject, | |
| 3195 | 3084 | UVERBS_LOOKUP_READ); |
| 3085 | + atomic_dec(&wqs[i]->usecnt); | |
| 3086 | + } | |
| 3196 | 3087 | err_free: |
| 3197 | 3088 | kfree(wqs_handles); |
| 3198 | 3089 | kfree(wqs); |
| @@ -3218,7 +3109,7 @@ static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs) · 19 unmodified lines | ||
| 3218 | 3109 | static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) |
| 3219 | 3110 | { |
| 3220 | 3111 | struct ib_uverbs_create_flow cmd; |
| 3221 | − struct ib_uverbs_create_flow_resp resp; | |
| 3112 | + struct ib_uverbs_create_flow_resp resp = {}; | |
| 3222 | 3113 | struct ib_uobject *uobj; |
| 3223 | 3114 | struct ib_flow *flow_id; |
| 3224 | 3115 | struct ib_uverbs_flow_attr *kern_flow_attr; |
| @@ -3351,23 +3242,17 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) · 126 unmodified lines | ||
| 3351 | 3242 | |
| 3352 | 3243 | ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res); |
| 3353 | 3244 | |
| 3354 | − memset(&resp, 0, sizeof(resp)); | |
| 3355 | − resp.flow_handle = uobj->id; | |
| 3356 | − | |
| 3357 | − err = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 3358 | − if (err) | |
| 3359 | − goto err_copy; | |
| 3360 | − | |
| 3361 | 3245 | rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, |
| 3362 | 3246 | UVERBS_LOOKUP_READ); |
| 3363 | 3247 | kfree(flow_attr); |
| 3248 | + | |
| 3364 | 3249 | if (cmd.flow_attr.num_of_specs) |
| 3365 | 3250 | kfree(kern_flow_attr); |
| 3366 | − rdma_alloc_commit_uobject(uobj, attrs); | |
| 3367 | − return 0; | |
| 3368 | −err_copy: | |
| 3369 | − if (!qp->device->ops.destroy_flow(flow_id)) | |
| 3370 | − atomic_dec(&qp->usecnt); | |
| 3251 | + uobj_finalize_uobj_create(uobj, attrs); | |
| 3252 | + | |
| 3253 | + resp.flow_handle = uobj->id; | |
| 3254 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 3255 | + | |
| 3371 | 3256 | err_free: |
| 3372 | 3257 | ib_uverbs_flow_resources_free(uflow_res); |
| 3373 | 3258 | err_free_flow_attr: |
| @@ -3402,13 +3287,13 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, · 28 unmodified lines | ||
| 3402 | 3287 | struct ib_uverbs_create_xsrq *cmd, |
| 3403 | 3288 | struct ib_udata *udata) |
| 3404 | 3289 | { |
| 3405 | − struct ib_uverbs_create_srq_resp resp; | |
| 3290 | + struct ib_uverbs_create_srq_resp resp = {}; | |
| 3406 | 3291 | struct ib_usrq_object *obj; |
| 3407 | 3292 | struct ib_pd *pd; |
| 3408 | 3293 | struct ib_srq *srq; |
| 3409 | − struct ib_uobject *xrcd_uobj; | |
| 3410 | 3294 | struct ib_srq_init_attr attr; |
| 3411 | 3295 | int ret; |
| 3296 | + struct ib_uobject *xrcd_uobj; | |
| 3412 | 3297 | struct ib_device *ib_dev; |
| 3413 | 3298 | |
| 3414 | 3299 | obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs, |
| @@ -3473,17 +3358,9 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, · 58 unmodified lines | ||
| 3473 | 3358 | if (obj->uevent.event_file) |
| 3474 | 3359 | uverbs_uobject_get(&obj->uevent.event_file->uobj); |
| 3475 | 3360 | |
| 3476 | − memset(&resp, 0, sizeof resp); | |
| 3477 | − resp.srq_handle = obj->uevent.uobject.id; | |
| 3478 | − resp.max_wr = attr.attr.max_wr; | |
| 3479 | − resp.max_sge = attr.attr.max_sge; | |
| 3480 | 3361 | if (cmd->srq_type == IB_SRQT_XRC) |
| 3481 | 3362 | resp.srqn = srq->ext.xrc.srq_num; |
| 3482 | 3363 | |
| 3483 | − ret = uverbs_response(attrs, &resp, sizeof(resp)); | |
| 3484 | − if (ret) | |
| 3485 | − goto err_copy; | |
| 3486 | − | |
| 3487 | 3364 | if (cmd->srq_type == IB_SRQT_XRC) |
| 3488 | 3365 | uobj_put_read(xrcd_uobj); |
| 3489 | 3366 | |
| @@ -3492,13 +3369,13 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, · 2 unmodified lines | ||
| 3492 | 3369 | UVERBS_LOOKUP_READ); |
| 3493 | 3370 | |
| 3494 | 3371 | uobj_put_obj_read(pd); |
| 3495 | − rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); | |
| 3496 | − return 0; | |
| 3372 | + uobj_finalize_uobj_create(&obj->uevent.uobject, attrs); | |
| 3373 | + | |
| 3374 | + resp.srq_handle = obj->uevent.uobject.id; | |
| 3375 | + resp.max_wr = attr.attr.max_wr; | |
| 3376 | + resp.max_sge = attr.attr.max_sge; | |
| 3377 | + return uverbs_response(attrs, &resp, sizeof(resp)); | |
| 3497 | 3378 | |
| 3498 | −err_copy: | |
| 3499 | − if (obj->uevent.event_file) | |
| 3500 | − uverbs_uobject_put(&obj->uevent.event_file->uobj); | |
| 3501 | − ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs)); | |
| 3502 | 3379 | err_put_pd: |
| 3503 | 3380 | uobj_put_obj_read(pd); |
| 3504 | 3381 | err_put_cq: |
| @@ -790,6 +790,7 @@ int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, · 789 unmodified lines | ||
| 790 | 790 | } |
| 791 | 791 | return uverbs_copy_to(bundle, idx, from, size); |
| 792 | 792 | } |
| 793 | +EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero); | |
| 793 | 794 | |
| 794 | 795 | /* Once called an abort will call through to the type's destroy_hw() */ |
| 795 | 796 | void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, |
| @@ -601,6 +601,7 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, · 600 unmodified lines | ||
| 601 | 601 | memset(bundle.attr_present, 0, sizeof(bundle.attr_present)); |
| 602 | 602 | bundle.ufile = file; |
| 603 | 603 | bundle.context = NULL; /* only valid if bundle has uobject */ |
| 604 | + bundle.uobject = NULL; | |
| 604 | 605 | if (!method_elm->is_ex) { |
| 605 | 606 | size_t in_len = hdr.in_words * 4 - sizeof(hdr); |
| 606 | 607 | size_t out_len = hdr.out_words * 4; |
| @@ -664,6 +665,9 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, · 57 unmodified lines | ||
| 664 | 665 | } |
| 665 | 666 | |
| 666 | 667 | ret = method_elm->handler(&bundle); |
| 668 | + if (bundle.uobject) | |
| 669 | + uverbs_finalize_object(bundle.uobject, UVERBS_ACCESS_NEW, true, | |
| 670 | + !ret, &bundle); | |
| 667 | 671 | out_unlock: |
| 668 | 672 | srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); |
| 669 | 673 | return (ret) ? : count; |
| @@ -46,7 +46,9 @@ static int uverbs_free_counters(struct ib_uobject *uobject, · 45 unmodified lines | ||
| 46 | 46 | if (ret) |
| 47 | 47 | return ret; |
| 48 | 48 | |
| 49 | − return counters->device->ops.destroy_counters(counters); | |
| 49 | + counters->device->ops.destroy_counters(counters); | |
| 50 | + kfree(counters); | |
| 51 | + return 0; | |
| 50 | 52 | } |
| 51 | 53 | |
| 52 | 54 | static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)( |
| @@ -66,20 +68,19 @@ static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)( · 13 unmodified lines | ||
| 66 | 68 | if (!ib_dev->ops.create_counters) |
| 67 | 69 | return -EOPNOTSUPP; |
| 68 | 70 | |
| 69 | − counters = ib_dev->ops.create_counters(ib_dev, attrs); | |
| 70 | − if (IS_ERR(counters)) { | |
| 71 | − ret = PTR_ERR(counters); | |
| 72 | − goto err_create_counters; | |
| 73 | − } | |
| 71 | + counters = rdma_zalloc_drv_obj(ib_dev, ib_counters); | |
| 72 | + if (!counters) | |
| 73 | + return -ENOMEM; | |
| 74 | 74 | |
| 75 | 75 | counters->device = ib_dev; |
| 76 | 76 | counters->uobject = uobj; |
| 77 | 77 | uobj->object = counters; |
| 78 | 78 | atomic_set(&counters->usecnt, 0); |
| 79 | 79 | |
| 80 | − return 0; | |
| 80 | + ret = ib_dev->ops.create_counters(counters, attrs); | |
| 81 | + if (ret) | |
| 82 | + kfree(counters); | |
| 81 | 83 | |
| 82 | −err_create_counters: | |
| 83 | 84 | return ret; |
| 84 | 85 | } |
| 85 | 86 | |
| @@ -207,11 +207,8 @@ DECLARE_UVERBS_NAMED_METHOD( · 206 unmodified lines | ||
| 207 | 207 | DECLARE_UVERBS_NAMED_OBJECT( |
| 208 | 208 | UVERBS_OBJECT_CQ, |
| 209 | 209 | UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), uverbs_free_cq), |
| 210 | − | |
| 211 | −#if IS_ENABLED(CONFIG_INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI) | |
| 212 | 210 | &UVERBS_METHOD(UVERBS_METHOD_CQ_CREATE), |
| 213 | 211 | &UVERBS_METHOD(UVERBS_METHOD_CQ_DESTROY) |
| 214 | −#endif | |
| 215 | 212 | ); |
| 216 | 213 | |
| 217 | 214 | const struct uapi_definition uverbs_def_obj_cq[] = { |
| @@ -38,7 +38,12 @@ static int UVERBS_HANDLER(UVERBS_METHOD_INVOKE_WRITE)( · 37 unmodified lines | ||
| 38 | 38 | attrs->ucore.outlen < method_elm->resp_size) |
| 39 | 39 | return -ENOSPC; |
| 40 | 40 | |
| 41 | − return method_elm->handler(attrs); | |
| 41 | + attrs->uobject = NULL; | |
| 42 | + rc = method_elm->handler(attrs); | |
| 43 | + if (attrs->uobject) | |
| 44 | + uverbs_finalize_object(attrs->uobject, UVERBS_ACCESS_NEW, true, | |
| 45 | + !rc, attrs); | |
| 46 | + return rc; | |
| 42 | 47 | } |
| 43 | 48 | |
| 44 | 49 | DECLARE_UVERBS_NAMED_METHOD(UVERBS_METHOD_INVOKE_WRITE, |
| @@ -229,6 +234,37 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( · 184 unmodified lines | ||
| 229 | 234 | return 0; |
| 230 | 235 | } |
| 231 | 236 | |
| 237 | +static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_CONTEXT)( | |
| 238 | + struct uverbs_attr_bundle *attrs) | |
| 239 | +{ | |
| 240 | + u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS; | |
| 241 | + struct ib_ucontext *ucontext; | |
| 242 | + struct ib_device *ib_dev; | |
| 243 | + u32 num_comp; | |
| 244 | + int ret; | |
| 245 | + | |
| 246 | + ucontext = ib_uverbs_get_ucontext(attrs); | |
| 247 | + if (IS_ERR(ucontext)) | |
| 248 | + return PTR_ERR(ucontext); | |
| 249 | + ib_dev = ucontext->device; | |
| 250 | + | |
| 251 | + if (!ib_dev->ops.query_ucontext) | |
| 252 | + return -EOPNOTSUPP; | |
| 253 | + | |
| 254 | + num_comp = attrs->ufile->device->num_comp_vectors; | |
| 255 | + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS, | |
| 256 | + &num_comp, sizeof(num_comp)); | |
| 257 | + if (IS_UVERBS_COPY_ERR(ret)) | |
| 258 | + return ret; | |
| 259 | + | |
| 260 | + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT, | |
| 261 | + &core_support, sizeof(core_support)); | |
| 262 | + if (IS_UVERBS_COPY_ERR(ret)) | |
| 263 | + return ret; | |
| 264 | + | |
| 265 | + return ucontext->device->ops.query_ucontext(ucontext, attrs); | |
| 266 | +} | |
| 267 | + | |
| 232 | 268 | DECLARE_UVERBS_NAMED_METHOD( |
| 233 | 269 | UVERBS_METHOD_GET_CONTEXT, |
| 234 | 270 | UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, |
| @@ -237,6 +273,13 @@ DECLARE_UVERBS_NAMED_METHOD( · 2 unmodified lines | ||
| 237 | 273 | UVERBS_ATTR_TYPE(u64), UA_OPTIONAL), |
| 238 | 274 | UVERBS_ATTR_UHW()); |
| 239 | 275 | |
| 276 | +DECLARE_UVERBS_NAMED_METHOD( | |
| 277 | + UVERBS_METHOD_QUERY_CONTEXT, | |
| 278 | + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS, | |
| 279 | + UVERBS_ATTR_TYPE(u32), UA_OPTIONAL), | |
| 280 | + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT, | |
| 281 | + UVERBS_ATTR_TYPE(u64), UA_OPTIONAL)); | |
| 282 | + | |
| 240 | 283 | DECLARE_UVERBS_NAMED_METHOD( |
| 241 | 284 | UVERBS_METHOD_INFO_HANDLES, |
| 242 | 285 | /* Also includes any device specific object ids */ |
| @@ -260,7 +303,8 @@ DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, · 17 unmodified lines | ||
| 260 | 303 | &UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT), |
| 261 | 304 | &UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE), |
| 262 | 305 | &UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES), |
| 263 | − &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT)); | |
| 306 | + &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT), | |
| 307 | + &UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT)); | |
| 264 | 308 | |
| 265 | 309 | const struct uapi_definition uverbs_def_obj_device[] = { |
| 266 | 310 | UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE), |
| @@ -69,7 +69,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_ADVISE_MR)( · 68 unmodified lines | ||
| 69 | 69 | |
| 70 | 70 | num_sge = uverbs_attr_ptr_get_array_size( |
| 71 | 71 | attrs, UVERBS_ATTR_ADVISE_MR_SGE_LIST, sizeof(struct ib_sge)); |
| 72 | − if (num_sge < 0) | |
| 72 | + if (num_sge <= 0) | |
| 73 | 73 | return num_sge; |
| 74 | 74 | |
| 75 | 75 | sg_list = uverbs_attr_get_alloced_ptr(attrs, |
| @@ -148,6 +148,36 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)( · 72 unmodified lines | ||
| 148 | 148 | return ret; |
| 149 | 149 | } |
| 150 | 150 | |
| 151 | +static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_MR)( | |
| 152 | + struct uverbs_attr_bundle *attrs) | |
| 153 | +{ | |
| 154 | + struct ib_mr *mr = | |
| 155 | + uverbs_attr_get_obj(attrs, UVERBS_ATTR_QUERY_MR_HANDLE); | |
| 156 | + int ret; | |
| 157 | + | |
| 158 | + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_LKEY, &mr->lkey, | |
| 159 | + sizeof(mr->lkey)); | |
| 160 | + if (ret) | |
| 161 | + return ret; | |
| 162 | + | |
| 163 | + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_RKEY, | |
| 164 | + &mr->rkey, sizeof(mr->rkey)); | |
| 165 | + | |
| 166 | + if (ret) | |
| 167 | + return ret; | |
| 168 | + | |
| 169 | + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_LENGTH, | |
| 170 | + &mr->length, sizeof(mr->length)); | |
| 171 | + | |
| 172 | + if (ret) | |
| 173 | + return ret; | |
| 174 | + | |
| 175 | + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_IOVA, | |
| 176 | + &mr->iova, sizeof(mr->iova)); | |
| 177 | + | |
| 178 | + return IS_UVERBS_COPY_ERR(ret) ? ret : 0; | |
| 179 | +} | |
| 180 | + | |
| 151 | 181 | DECLARE_UVERBS_NAMED_METHOD( |
| 152 | 182 | UVERBS_METHOD_ADVISE_MR, |
| 153 | 183 | UVERBS_ATTR_IDR(UVERBS_ATTR_ADVISE_MR_PD_HANDLE, |
| @@ -165,6 +195,25 @@ DECLARE_UVERBS_NAMED_METHOD( · 11 unmodified lines | ||
| 165 | 195 | UA_MANDATORY, |
| 166 | 196 | UA_ALLOC_AND_COPY)); |
| 167 | 197 | |
| 198 | +DECLARE_UVERBS_NAMED_METHOD( | |
| 199 | + UVERBS_METHOD_QUERY_MR, | |
| 200 | + UVERBS_ATTR_IDR(UVERBS_ATTR_QUERY_MR_HANDLE, | |
| 201 | + UVERBS_OBJECT_MR, | |
| 202 | + UVERBS_ACCESS_READ, | |
| 203 | + UA_MANDATORY), | |
| 204 | + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_RKEY, | |
| 205 | + UVERBS_ATTR_TYPE(u32), | |
| 206 | + UA_MANDATORY), | |
| 207 | + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_LKEY, | |
| 208 | + UVERBS_ATTR_TYPE(u32), | |
| 209 | + UA_MANDATORY), | |
| 210 | + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_LENGTH, | |
| 211 | + UVERBS_ATTR_TYPE(u64), | |
| 212 | + UA_MANDATORY), | |
| 213 | + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_IOVA, | |
| 214 | + UVERBS_ATTR_TYPE(u64), | |
| 215 | + UA_OPTIONAL)); | |
| 216 | + | |
| 168 | 217 | DECLARE_UVERBS_NAMED_METHOD( |
| 169 | 218 | UVERBS_METHOD_DM_MR_REG, |
| 170 | 219 | UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_HANDLE, |
| @@ -206,7 +255,8 @@ DECLARE_UVERBS_NAMED_OBJECT( · 35 unmodified lines | ||
| 206 | 255 | UVERBS_TYPE_ALLOC_IDR(uverbs_free_mr), |
| 207 | 256 | &UVERBS_METHOD(UVERBS_METHOD_DM_MR_REG), |
| 208 | 257 | &UVERBS_METHOD(UVERBS_METHOD_MR_DESTROY), |
| 209 | − &UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR)); | |
| 258 | + &UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR), | |
| 259 | + &UVERBS_METHOD(UVERBS_METHOD_QUERY_MR)); | |
| 210 | 260 | |
| 211 | 261 | const struct uapi_definition uverbs_def_obj_mr[] = { |
| 212 | 262 | UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MR, |
| @@ -988,8 +988,8 @@ EXPORT_SYMBOL(rdma_destroy_ah_user); · 987 unmodified lines | ||
| 988 | 988 | * @srq_init_attr: A list of initial attributes required to create the |
| 989 | 989 | * SRQ. If SRQ creation succeeds, then the attributes are updated to |
| 990 | 990 | * the actual capabilities of the created SRQ. |
| 991 | − * @uobject - uobject pointer if this is not a kernel SRQ | |
| 992 | − * @udata - udata pointer if this is not a kernel SRQ | |
| 991 | + * @uobject: uobject pointer if this is not a kernel SRQ | |
| 992 | + * @udata: udata pointer if this is not a kernel SRQ | |
| 993 | 993 | * |
| 994 | 994 | * srq_attr->max_wr and srq_attr->max_sge are read the determine the |
| 995 | 995 | * requested size of the SRQ, and set to the actual values allocated |
| @@ -1090,13 +1090,6 @@ static void __ib_shared_qp_event_handler(struct ib_event *event, void *context) · 94 unmodified lines | ||
| 1090 | 1090 | spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags); |
| 1091 | 1091 | } |
| 1092 | 1092 | |
| 1093 | −static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp) | |
| 1094 | −{ | |
| 1095 | − mutex_lock(&xrcd->tgt_qp_mutex); | |
| 1096 | − list_add(&qp->xrcd_list, &xrcd->tgt_qp_list); | |
| 1097 | − mutex_unlock(&xrcd->tgt_qp_mutex); | |
| 1098 | −} | |
| 1099 | − | |
| 1100 | 1093 | static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp, |
| 1101 | 1094 | void (*event_handler)(struct ib_event *, void *), |
| 1102 | 1095 | void *qp_context) |
| @@ -1139,16 +1132,15 @@ struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd, · 36 unmodified lines | ||
| 1139 | 1132 | if (qp_open_attr->qp_type != IB_QPT_XRC_TGT) |
| 1140 | 1133 | return ERR_PTR(-EINVAL); |
| 1141 | 1134 | |
| 1142 | − qp = ERR_PTR(-EINVAL); | |
| 1143 | − mutex_lock(&xrcd->tgt_qp_mutex); | |
| 1144 | − list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) { | |
| 1145 | − if (real_qp->qp_num == qp_open_attr->qp_num) { | |
| 1146 | − qp = __ib_open_qp(real_qp, qp_open_attr->event_handler, | |
| 1147 | − qp_open_attr->qp_context); | |
| 1148 | − break; | |
| 1149 | − } | |
| 1135 | + down_read(&xrcd->tgt_qps_rwsem); | |
| 1136 | + real_qp = xa_load(&xrcd->tgt_qps, qp_open_attr->qp_num); | |
| 1137 | + if (!real_qp) { | |
| 1138 | + up_read(&xrcd->tgt_qps_rwsem); | |
| 1139 | + return ERR_PTR(-EINVAL); | |
| 1150 | 1140 | } |
| 1151 | − mutex_unlock(&xrcd->tgt_qp_mutex); | |
| 1141 | + qp = __ib_open_qp(real_qp, qp_open_attr->event_handler, | |
| 1142 | + qp_open_attr->qp_context); | |
| 1143 | + up_read(&xrcd->tgt_qps_rwsem); | |
| 1152 | 1144 | return qp; |
| 1153 | 1145 | } |
| 1154 | 1146 | EXPORT_SYMBOL(ib_open_qp); |
| @@ -1157,6 +1149,7 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp, · 2 unmodified lines | ||
| 1157 | 1149 | struct ib_qp_init_attr *qp_init_attr) |
| 1158 | 1150 | { |
| 1159 | 1151 | struct ib_qp *real_qp = qp; |
| 1152 | + int err; | |
| 1160 | 1153 | |
| 1161 | 1154 | qp->event_handler = __ib_shared_qp_event_handler; |
| 1162 | 1155 | qp->qp_context = qp; |
| @@ -1172,7 +1165,12 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp, · 9 unmodified lines | ||
| 1172 | 1165 | if (IS_ERR(qp)) |
| 1173 | 1166 | return qp; |
| 1174 | 1167 | |
| 1175 | − __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp); | |
| 1168 | + err = xa_err(xa_store(&qp_init_attr->xrcd->tgt_qps, real_qp->qp_num, | |
| 1169 | + real_qp, GFP_KERNEL)); | |
| 1170 | + if (err) { | |
| 1171 | + ib_close_qp(qp); | |
| 1172 | + return ERR_PTR(err); | |
| 1173 | + } | |
| 1176 | 1174 | return qp; |
| 1177 | 1175 | } |
| 1178 | 1176 | |
| @@ -1712,7 +1710,7 @@ static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, · 533 unmodified lines | ||
| 1712 | 1710 | if (!(rdma_protocol_ib(qp->device, |
| 1713 | 1711 | attr->alt_ah_attr.port_num) && |
| 1714 | 1712 | rdma_protocol_ib(qp->device, port))) { |
| 1715 | − ret = EINVAL; | |
| 1713 | + ret = -EINVAL; | |
| 1716 | 1714 | goto out; |
| 1717 | 1715 | } |
| 1718 | 1716 | } |
| @@ -1887,21 +1885,18 @@ static int __ib_destroy_shared_qp(struct ib_qp *qp) · 168 unmodified lines | ||
| 1887 | 1885 | |
| 1888 | 1886 | real_qp = qp->real_qp; |
| 1889 | 1887 | xrcd = real_qp->xrcd; |
| 1890 | − | |
| 1891 | − mutex_lock(&xrcd->tgt_qp_mutex); | |
| 1888 | + down_write(&xrcd->tgt_qps_rwsem); | |
| 1892 | 1889 | ib_close_qp(qp); |
| 1893 | 1890 | if (atomic_read(&real_qp->usecnt) == 0) |
| 1894 | − list_del(&real_qp->xrcd_list); | |
| 1891 | + xa_erase(&xrcd->tgt_qps, real_qp->qp_num); | |
| 1895 | 1892 | else |
| 1896 | 1893 | real_qp = NULL; |
| 1897 | − mutex_unlock(&xrcd->tgt_qp_mutex); | |
| 1894 | + up_write(&xrcd->tgt_qps_rwsem); | |
| 1898 | 1895 | |
| 1899 | 1896 | if (real_qp) { |
| 1900 | 1897 | ret = ib_destroy_qp(real_qp); |
| 1901 | 1898 | if (!ret) |
| 1902 | 1899 | atomic_dec(&xrcd->usecnt); |
| 1903 | − else | |
| 1904 | − __ib_insert_xrcd_qp(xrcd, real_qp); | |
| 1905 | 1900 | } |
| 1906 | 1901 | |
| 1907 | 1902 | return 0; |
| @@ -2077,6 +2072,9 @@ int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, · 169 unmodified lines | ||
| 2077 | 2072 | if (!pd->device->ops.advise_mr) |
| 2078 | 2073 | return -EOPNOTSUPP; |
| 2079 | 2074 | |
| 2075 | + if (!num_sge) | |
| 2076 | + return 0; | |
| 2077 | + | |
| 2080 | 2078 | return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge, |
| 2081 | 2079 | NULL); |
| 2082 | 2080 | } |
| @@ -2104,11 +2102,10 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata) · 21 unmodified lines | ||
| 2104 | 2102 | EXPORT_SYMBOL(ib_dereg_mr_user); |
| 2105 | 2103 | |
| 2106 | 2104 | /** |
| 2107 | − * ib_alloc_mr_user() - Allocates a memory region | |
| 2105 | + * ib_alloc_mr() - Allocates a memory region | |
| 2108 | 2106 | * @pd: protection domain associated with the region |
| 2109 | 2107 | * @mr_type: memory region type |
| 2110 | 2108 | * @max_num_sg: maximum sg entries available for registration. |
| 2111 | − * @udata: user data or null for kernel objects | |
| 2112 | 2109 | * |
| 2113 | 2110 | * Notes: |
| 2114 | 2111 | * Memory registeration page/sg lists must not exceed max_num_sg. |
| @@ -2116,8 +2113,8 @@ EXPORT_SYMBOL(ib_dereg_mr_user); · 1 unmodified line | ||
| 2116 | 2113 | * max_num_sg * used_page_size. |
| 2117 | 2114 | * |
| 2118 | 2115 | */ |
| 2119 | −struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type, | |
| 2120 | − u32 max_num_sg, struct ib_udata *udata) | |
| 2116 | +struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, | |
| 2117 | + u32 max_num_sg) | |
| 2121 | 2118 | { |
| 2122 | 2119 | struct ib_mr *mr; |
| 2123 | 2120 | |
| @@ -2132,25 +2129,26 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type, · 8 unmodified lines | ||
| 2132 | 2129 | goto out; |
| 2133 | 2130 | } |
| 2134 | 2131 | |
| 2135 | − mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata); | |
| 2136 | − if (!IS_ERR(mr)) { | |
| 2137 | − mr->device = pd->device; | |
| 2138 | − mr->pd = pd; | |
| 2139 | − mr->dm = NULL; | |
| 2140 | − mr->uobject = NULL; | |
| 2141 | − atomic_inc(&pd->usecnt); | |
| 2142 | − mr->need_inval = false; | |
| 2143 | − mr->res.type = RDMA_RESTRACK_MR; | |
| 2144 | − rdma_restrack_kadd(&mr->res); | |
| 2145 | − mr->type = mr_type; | |
| 2146 | − mr->sig_attrs = NULL; | |
| 2147 | − } | |
| 2132 | + mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg); | |
| 2133 | + if (IS_ERR(mr)) | |
| 2134 | + goto out; | |
| 2135 | + | |
| 2136 | + mr->device = pd->device; | |
| 2137 | + mr->pd = pd; | |
| 2138 | + mr->dm = NULL; | |
| 2139 | + mr->uobject = NULL; | |
| 2140 | + atomic_inc(&pd->usecnt); | |
| 2141 | + mr->need_inval = false; | |
| 2142 | + mr->res.type = RDMA_RESTRACK_MR; | |
| 2143 | + rdma_restrack_kadd(&mr->res); | |
| 2144 | + mr->type = mr_type; | |
| 2145 | + mr->sig_attrs = NULL; | |
| 2148 | 2146 | |
| 2149 | 2147 | out: |
| 2150 | 2148 | trace_mr_alloc(pd, mr_type, max_num_sg, mr); |
| 2151 | 2149 | return mr; |
| 2152 | 2150 | } |
| 2153 | −EXPORT_SYMBOL(ib_alloc_mr_user); | |
| 2151 | +EXPORT_SYMBOL(ib_alloc_mr); | |
| 2154 | 2152 | |
| 2155 | 2153 | /** |
| 2156 | 2154 | * ib_alloc_mr_integrity() - Allocates an integrity memory region |
| @@ -2288,45 +2286,57 @@ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) · 131 unmodified lines | ||
| 2288 | 2286 | } |
| 2289 | 2287 | EXPORT_SYMBOL(ib_detach_mcast); |
| 2290 | 2288 | |
| 2291 | −struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller) | |
| 2289 | +/** | |
| 2290 | + * ib_alloc_xrcd_user - Allocates an XRC domain. | |
| 2291 | + * @device: The device on which to allocate the XRC domain. | |
| 2292 | + * @inode: inode to connect XRCD | |
| 2293 | + * @udata: Valid user data or NULL for kernel object | |
| 2294 | + */ | |
| 2295 | +struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device, | |
| 2296 | + struct inode *inode, struct ib_udata *udata) | |
| 2292 | 2297 | { |
| 2293 | 2298 | struct ib_xrcd *xrcd; |
| 2299 | + int ret; | |
| 2294 | 2300 | |
| 2295 | 2301 | if (!device->ops.alloc_xrcd) |
| 2296 | 2302 | return ERR_PTR(-EOPNOTSUPP); |
| 2297 | 2303 | |
| 2298 | − xrcd = device->ops.alloc_xrcd(device, NULL); | |
| 2299 | − if (!IS_ERR(xrcd)) { | |
| 2300 | − xrcd->device = device; | |
| 2301 | − xrcd->inode = NULL; | |
| 2302 | − atomic_set(&xrcd->usecnt, 0); | |
| 2303 | − mutex_init(&xrcd->tgt_qp_mutex); | |
| 2304 | − INIT_LIST_HEAD(&xrcd->tgt_qp_list); | |
| 2305 | − } | |
| 2304 | + xrcd = rdma_zalloc_drv_obj(device, ib_xrcd); | |
| 2305 | + if (!xrcd) | |
| 2306 | + return ERR_PTR(-ENOMEM); | |
| 2306 | 2307 | |
| 2308 | + xrcd->device = device; | |
| 2309 | + xrcd->inode = inode; | |
| 2310 | + atomic_set(&xrcd->usecnt, 0); | |
| 2311 | + init_rwsem(&xrcd->tgt_qps_rwsem); | |
| 2312 | + xa_init(&xrcd->tgt_qps); | |
| 2313 | + | |
| 2314 | + ret = device->ops.alloc_xrcd(xrcd, udata); | |
| 2315 | + if (ret) | |
| 2316 | + goto err; | |
| 2307 | 2317 | return xrcd; |
| 2318 | +err: | |
| 2319 | + kfree(xrcd); | |
| 2320 | + return ERR_PTR(ret); | |
| 2308 | 2321 | } |
| 2309 | −EXPORT_SYMBOL(__ib_alloc_xrcd); | |
| 2322 | +EXPORT_SYMBOL(ib_alloc_xrcd_user); | |
| 2310 | 2323 | |
| 2311 | −int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) | |
| 2324 | +/** | |
| 2325 | + * ib_dealloc_xrcd_user - Deallocates an XRC domain. | |
| 2326 | + * @xrcd: The XRC domain to deallocate. | |
| 2327 | + * @udata: Valid user data or NULL for kernel object | |
| 2328 | + */ | |
| 2329 | +int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata) | |
| 2312 | 2330 | { |
| 2313 | − struct ib_qp *qp; | |
| 2314 | − int ret; | |
| 2315 | − | |
| 2316 | 2331 | if (atomic_read(&xrcd->usecnt)) |
| 2317 | 2332 | return -EBUSY; |
| 2318 | 2333 | |
| 2319 | − while (!list_empty(&xrcd->tgt_qp_list)) { | |
| 2320 | − qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list); | |
| 2321 | − ret = ib_destroy_qp(qp); | |
| 2322 | − if (ret) | |
| 2323 | − return ret; | |
| 2324 | − } | |
| 2325 | − mutex_destroy(&xrcd->tgt_qp_mutex); | |
| 2326 | − | |
| 2327 | − return xrcd->device->ops.dealloc_xrcd(xrcd, udata); | |
| 2334 | + WARN_ON(!xa_empty(&xrcd->tgt_qps)); | |
| 2335 | + xrcd->device->ops.dealloc_xrcd(xrcd, udata); | |
| 2336 | + kfree(xrcd); | |
| 2337 | + return 0; | |
| 2328 | 2338 | } |
| 2329 | −EXPORT_SYMBOL(ib_dealloc_xrcd); | |
| 2339 | +EXPORT_SYMBOL(ib_dealloc_xrcd_user); | |
| 2330 | 2340 | |
| 2331 | 2341 | /** |
| 2332 | 2342 | * ib_create_wq - Creates a WQ associated with the specified protection |
| @@ -2409,45 +2419,6 @@ int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, · 76 unmodified lines | ||
| 2409 | 2419 | } |
| 2410 | 2420 | EXPORT_SYMBOL(ib_modify_wq); |
| 2411 | 2421 | |
| 2412 | −/* | |
| 2413 | − * ib_create_rwq_ind_table - Creates a RQ Indirection Table. | |
| 2414 | − * @device: The device on which to create the rwq indirection table. | |
| 2415 | − * @ib_rwq_ind_table_init_attr: A list of initial attributes required to | |
| 2416 | − * create the Indirection Table. | |
| 2417 | − * | |
| 2418 | − * Note: The life time of ib_rwq_ind_table_init_attr->ind_tbl is not less | |
| 2419 | − * than the created ib_rwq_ind_table object and the caller is responsible | |
| 2420 | − * for its memory allocation/free. | |
| 2421 | − */ | |
| 2422 | −struct ib_rwq_ind_table *ib_create_rwq_ind_table(struct ib_device *device, | |
| 2423 | − struct ib_rwq_ind_table_init_attr *init_attr) | |
| 2424 | −{ | |
| 2425 | − struct ib_rwq_ind_table *rwq_ind_table; | |
| 2426 | − int i; | |
| 2427 | − u32 table_size; | |
| 2428 | − | |
| 2429 | − if (!device->ops.create_rwq_ind_table) | |
| 2430 | − return ERR_PTR(-EOPNOTSUPP); | |
| 2431 | − | |
| 2432 | − table_size = (1 << init_attr->log_ind_tbl_size); | |
| 2433 | − rwq_ind_table = device->ops.create_rwq_ind_table(device, | |
| 2434 | − init_attr, NULL); | |
| 2435 | − if (IS_ERR(rwq_ind_table)) | |
| 2436 | − return rwq_ind_table; | |
| 2437 | − | |
| 2438 | − rwq_ind_table->ind_tbl = init_attr->ind_tbl; | |
| 2439 | − rwq_ind_table->log_ind_tbl_size = init_attr->log_ind_tbl_size; | |
| 2440 | − rwq_ind_table->device = device; | |
| 2441 | − rwq_ind_table->uobject = NULL; | |
| 2442 | − atomic_set(&rwq_ind_table->usecnt, 0); | |
| 2443 | − | |
| 2444 | − for (i = 0; i < table_size; i++) | |
| 2445 | − atomic_inc(&rwq_ind_table->ind_tbl[i]->usecnt); | |
| 2446 | − | |
| 2447 | − return rwq_ind_table; | |
| 2448 | −} | |
| 2449 | −EXPORT_SYMBOL(ib_create_rwq_ind_table); | |
| 2450 | − | |
| 2451 | 2422 | /* |
| 2452 | 2423 | * ib_destroy_rwq_ind_table - Destroys the specified Indirection Table. |
| 2453 | 2424 | * @wq_ind_table: The Indirection Table to destroy. |
| @@ -842,16 +842,79 @@ static u8 __from_ib_qp_type(enum ib_qp_type type) · 841 unmodified lines | ||
| 842 | 842 | } |
| 843 | 843 | } |
| 844 | 844 | |
| 845 | +static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp, | |
| 846 | + int rsge, int max) | |
| 847 | +{ | |
| 848 | + if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) | |
| 849 | + rsge = max; | |
| 850 | + return bnxt_re_get_rwqe_size(rsge); | |
| 851 | +} | |
| 852 | + | |
| 853 | +static u16 bnxt_re_get_wqe_size(int ilsize, int nsge) | |
| 854 | +{ | |
| 855 | + u16 wqe_size, calc_ils; | |
| 856 | + | |
| 857 | + wqe_size = bnxt_re_get_swqe_size(nsge); | |
| 858 | + if (ilsize) { | |
| 859 | + calc_ils = sizeof(struct sq_send_hdr) + ilsize; | |
| 860 | + wqe_size = max_t(u16, calc_ils, wqe_size); | |
| 861 | + wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr)); | |
| 862 | + } | |
| 863 | + return wqe_size; | |
| 864 | +} | |
| 865 | + | |
| 866 | +static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, | |
| 867 | + struct ib_qp_init_attr *init_attr) | |
| 868 | +{ | |
| 869 | + struct bnxt_qplib_dev_attr *dev_attr; | |
| 870 | + struct bnxt_qplib_qp *qplqp; | |
| 871 | + struct bnxt_re_dev *rdev; | |
| 872 | + struct bnxt_qplib_q *sq; | |
| 873 | + int align, ilsize; | |
| 874 | + | |
| 875 | + rdev = qp->rdev; | |
| 876 | + qplqp = &qp->qplib_qp; | |
| 877 | + sq = &qplqp->sq; | |
| 878 | + dev_attr = &rdev->dev_attr; | |
| 879 | + | |
| 880 | + align = sizeof(struct sq_send_hdr); | |
| 881 | + ilsize = ALIGN(init_attr->cap.max_inline_data, align); | |
| 882 | + | |
| 883 | + sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge); | |
| 884 | + if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges)) | |
| 885 | + return -EINVAL; | |
| 886 | + /* For gen p4 and gen p5 backward compatibility mode | |
| 887 | + * wqe size is fixed to 128 bytes | |
| 888 | + */ | |
| 889 | + if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) && | |
| 890 | + qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) | |
| 891 | + sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges); | |
| 892 | + | |
| 893 | + if (init_attr->cap.max_inline_data) { | |
| 894 | + qplqp->max_inline_data = sq->wqe_size - | |
| 895 | + sizeof(struct sq_send_hdr); | |
| 896 | + init_attr->cap.max_inline_data = qplqp->max_inline_data; | |
| 897 | + if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) | |
| 898 | + sq->max_sge = qplqp->max_inline_data / | |
| 899 | + sizeof(struct sq_sge); | |
| 900 | + } | |
| 901 | + | |
| 902 | + return 0; | |
| 903 | +} | |
| 904 | + | |
| 845 | 905 | static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, |
| 846 | 906 | struct bnxt_re_qp *qp, struct ib_udata *udata) |
| 847 | 907 | { |
| 908 | + struct bnxt_qplib_qp *qplib_qp; | |
| 909 | + struct bnxt_re_ucontext *cntx; | |
| 848 | 910 | struct bnxt_re_qp_req ureq; |
| 849 | − struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; | |
| 850 | − struct ib_umem *umem; | |
| 851 | 911 | int bytes = 0, psn_sz; |
| 852 | − struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context( | |
| 853 | − udata, struct bnxt_re_ucontext, ib_uctx); | |
| 912 | + struct ib_umem *umem; | |
| 913 | + int psn_nume; | |
| 854 | 914 | |
| 915 | + qplib_qp = &qp->qplib_qp; | |
| 916 | + cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, | |
| 917 | + ib_uctx); | |
| 855 | 918 | if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) |
| 856 | 919 | return -EFAULT; |
| 857 | 920 | |
| @@ -859,10 +922,15 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, · 1 unmodified line | ||
| 859 | 922 | /* Consider mapping PSN search memory only for RC QPs. */ |
| 860 | 923 | if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { |
| 861 | 924 | psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ? |
| 862 | − sizeof(struct sq_psn_search_ext) : | |
| 863 | − sizeof(struct sq_psn_search); | |
| 864 | − bytes += (qplib_qp->sq.max_wqe * psn_sz); | |
| 925 | + sizeof(struct sq_psn_search_ext) : | |
| 926 | + sizeof(struct sq_psn_search); | |
| 927 | + psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? | |
| 928 | + qplib_qp->sq.max_wqe : | |
| 929 | + ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / | |
| 930 | + sizeof(struct bnxt_qplib_sge)); | |
| 931 | + bytes += (psn_nume * psn_sz); | |
| 865 | 932 | } |
| 933 | + | |
| 866 | 934 | bytes = PAGE_ALIGN(bytes); |
| 867 | 935 | umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes, |
| 868 | 936 | IB_ACCESS_LOCAL_WRITE); |
| @@ -975,7 +1043,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp · 106 unmodified lines | ||
| 975 | 1043 | qp->qplib_qp.sig_type = true; |
| 976 | 1044 | |
| 977 | 1045 | /* Shadow QP SQ depth should be same as QP1 RQ depth */ |
| 978 | − qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(); | |
| 1046 | + qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6); | |
| 979 | 1047 | qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; |
| 980 | 1048 | qp->qplib_qp.sq.max_sge = 2; |
| 981 | 1049 | /* Q full delta can be 1 since it is internal QP */ |
| @@ -986,7 +1054,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp · 4 unmodified lines | ||
| 986 | 1054 | qp->qplib_qp.scq = qp1_qp->scq; |
| 987 | 1055 | qp->qplib_qp.rcq = qp1_qp->rcq; |
| 988 | 1056 | |
| 989 | − qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(); | |
| 1057 | + qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6); | |
| 990 | 1058 | qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; |
| 991 | 1059 | qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; |
| 992 | 1060 | /* Q full delta can be 1 since it is internal QP */ |
| @@ -1041,19 +1109,21 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, · 48 unmodified lines | ||
| 1041 | 1109 | qplqp->srq = &srq->qplib_srq; |
| 1042 | 1110 | rq->max_wqe = 0; |
| 1043 | 1111 | } else { |
| 1044 | − rq->wqe_size = bnxt_re_get_rwqe_size(); | |
| 1112 | + rq->max_sge = init_attr->cap.max_recv_sge; | |
| 1113 | + if (rq->max_sge > dev_attr->max_qp_sges) | |
| 1114 | + rq->max_sge = dev_attr->max_qp_sges; | |
| 1115 | + init_attr->cap.max_recv_sge = rq->max_sge; | |
| 1116 | + rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge, | |
| 1117 | + dev_attr->max_qp_sges); | |
| 1045 | 1118 | /* Allocate 1 more than what's provided so posting max doesn't |
| 1046 | 1119 | * mean empty. |
| 1047 | 1120 | */ |
| 1048 | 1121 | entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1); |
| 1049 | 1122 | rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); |
| 1050 | − rq->q_full_delta = rq->max_wqe - init_attr->cap.max_recv_wr; | |
| 1051 | − rq->max_sge = init_attr->cap.max_recv_sge; | |
| 1052 | − if (rq->max_sge > dev_attr->max_qp_sges) | |
| 1053 | − rq->max_sge = dev_attr->max_qp_sges; | |
| 1123 | + rq->q_full_delta = 0; | |
| 1124 | + rq->sg_info.pgsize = PAGE_SIZE; | |
| 1125 | + rq->sg_info.pgshft = PAGE_SHIFT; | |
| 1054 | 1126 | } |
| 1055 | − rq->sg_info.pgsize = PAGE_SIZE; | |
| 1056 | − rq->sg_info.pgshft = PAGE_SHIFT; | |
| 1057 | 1127 | |
| 1058 | 1128 | return 0; |
| 1059 | 1129 | } |
| @@ -1068,41 +1138,48 @@ static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) · 8 unmodified lines | ||
| 1068 | 1138 | qplqp = &qp->qplib_qp; |
| 1069 | 1139 | dev_attr = &rdev->dev_attr; |
| 1070 | 1140 | |
| 1071 | − qplqp->rq.max_sge = dev_attr->max_qp_sges; | |
| 1072 | − if (qplqp->rq.max_sge > dev_attr->max_qp_sges) | |
| 1141 | + if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) { | |
| 1073 | 1142 | qplqp->rq.max_sge = dev_attr->max_qp_sges; |
| 1074 | − qplqp->rq.max_sge = 6; | |
| 1143 | + if (qplqp->rq.max_sge > dev_attr->max_qp_sges) | |
| 1144 | + qplqp->rq.max_sge = dev_attr->max_qp_sges; | |
| 1145 | + qplqp->rq.max_sge = 6; | |
| 1146 | + } | |
| 1075 | 1147 | } |
| 1076 | 1148 | |
| 1077 | −static void bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, | |
| 1078 | − struct ib_qp_init_attr *init_attr, | |
| 1079 | − struct ib_udata *udata) | |
| 1149 | +static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, | |
| 1150 | + struct ib_qp_init_attr *init_attr, | |
| 1151 | + struct ib_udata *udata) | |
| 1080 | 1152 | { |
| 1081 | 1153 | struct bnxt_qplib_dev_attr *dev_attr; |
| 1082 | 1154 | struct bnxt_qplib_qp *qplqp; |
| 1083 | 1155 | struct bnxt_re_dev *rdev; |
| 1084 | 1156 | struct bnxt_qplib_q *sq; |
| 1085 | 1157 | int entries; |
| 1158 | + int diff; | |
| 1159 | + int rc; | |
| 1086 | 1160 | |
| 1087 | 1161 | rdev = qp->rdev; |
| 1088 | 1162 | qplqp = &qp->qplib_qp; |
| 1089 | 1163 | sq = &qplqp->sq; |
| 1090 | 1164 | dev_attr = &rdev->dev_attr; |
| 1091 | 1165 | |
| 1092 | − sq->wqe_size = bnxt_re_get_swqe_size(); | |
| 1093 | 1166 | sq->max_sge = init_attr->cap.max_send_sge; |
| 1094 | − if (sq->max_sge > dev_attr->max_qp_sges) | |
| 1167 | + if (sq->max_sge > dev_attr->max_qp_sges) { | |
| 1095 | 1168 | sq->max_sge = dev_attr->max_qp_sges; |
| 1096 | − /* | |
| 1097 | − * Change the SQ depth if user has requested minimum using | |
| 1098 | − * configfs. Only supported for kernel consumers | |
| 1099 | − */ | |
| 1169 | + init_attr->cap.max_send_sge = sq->max_sge; | |
| 1170 | + } | |
| 1171 | + | |
| 1172 | + rc = bnxt_re_setup_swqe_size(qp, init_attr); | |
| 1173 | + if (rc) | |
| 1174 | + return rc; | |
| 1175 | + | |
| 1100 | 1176 | entries = init_attr->cap.max_send_wr; |
| 1101 | 1177 | /* Allocate 128 + 1 more than what's provided */ |
| 1102 | − entries = roundup_pow_of_two(entries + BNXT_QPLIB_RESERVED_QP_WRS + 1); | |
| 1103 | − sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + | |
| 1104 | − BNXT_QPLIB_RESERVED_QP_WRS + 1); | |
| 1105 | − sq->q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1; | |
| 1178 | + diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ? | |
| 1179 | + 0 : BNXT_QPLIB_RESERVED_QP_WRS; | |
| 1180 | + entries = roundup_pow_of_two(entries + diff + 1); | |
| 1181 | + sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); | |
| 1182 | + sq->q_full_delta = diff + 1; | |
| 1106 | 1183 | /* |
| 1107 | 1184 | * Reserving one slot for Phantom WQE. Application can |
| 1108 | 1185 | * post one extra entry in this case. But allowing this to avoid |
| @@ -1111,6 +1188,8 @@ static void bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, · 2 unmodified lines | ||
| 1111 | 1188 | qplqp->sq.q_full_delta -= 1; |
| 1112 | 1189 | qplqp->sq.sg_info.pgsize = PAGE_SIZE; |
| 1113 | 1190 | qplqp->sq.sg_info.pgshft = PAGE_SHIFT; |
| 1191 | + | |
| 1192 | + return 0; | |
| 1114 | 1193 | } |
| 1115 | 1194 | |
| 1116 | 1195 | static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, |
| @@ -1125,13 +1204,16 @@ static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, · 8 unmodified lines | ||
| 1125 | 1204 | qplqp = &qp->qplib_qp; |
| 1126 | 1205 | dev_attr = &rdev->dev_attr; |
| 1127 | 1206 | |
| 1128 | − entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1); | |
| 1129 | − qplqp->sq.max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); | |
| 1130 | − qplqp->sq.q_full_delta = qplqp->sq.max_wqe - | |
| 1131 | − init_attr->cap.max_send_wr; | |
| 1132 | − qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ | |
| 1133 | − if (qplqp->sq.max_sge > dev_attr->max_qp_sges) | |
| 1134 | − qplqp->sq.max_sge = dev_attr->max_qp_sges; | |
| 1207 | + if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) { | |
| 1208 | + entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1); | |
| 1209 | + qplqp->sq.max_wqe = min_t(u32, entries, | |
| 1210 | + dev_attr->max_qp_wqes + 1); | |
| 1211 | + qplqp->sq.q_full_delta = qplqp->sq.max_wqe - | |
| 1212 | + init_attr->cap.max_send_wr; | |
| 1213 | + qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ | |
| 1214 | + if (qplqp->sq.max_sge > dev_attr->max_qp_sges) | |
| 1215 | + qplqp->sq.max_sge = dev_attr->max_qp_sges; | |
| 1216 | + } | |
| 1135 | 1217 | } |
| 1136 | 1218 | |
| 1137 | 1219 | static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, |
| @@ -1183,6 +1265,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, · 45 unmodified lines | ||
| 1183 | 1265 | goto out; |
| 1184 | 1266 | } |
| 1185 | 1267 | qplqp->type = (u8)qptype; |
| 1268 | + qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode; | |
| 1186 | 1269 | |
| 1187 | 1270 | if (init_attr->qp_type == IB_QPT_RC) { |
| 1188 | 1271 | qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; |
| @@ -1226,7 +1309,9 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, · 37 unmodified lines | ||
| 1226 | 1309 | bnxt_re_adjust_gsi_rq_attr(qp); |
| 1227 | 1310 | |
| 1228 | 1311 | /* Setup SQ */ |
| 1229 | − bnxt_re_init_sq_attr(qp, init_attr, udata); | |
| 1312 | + rc = bnxt_re_init_sq_attr(qp, init_attr, udata); | |
| 1313 | + if (rc) | |
| 1314 | + goto out; | |
| 1230 | 1315 | if (init_attr->qp_type == IB_QPT_GSI) |
| 1231 | 1316 | bnxt_re_adjust_gsi_sq_attr(qp, init_attr); |
| 1232 | 1317 | |
| @@ -1574,8 +1659,9 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, · 341 unmodified lines | ||
| 1574 | 1659 | entries = dev_attr->max_srq_wqes + 1; |
| 1575 | 1660 | srq->qplib_srq.max_wqe = entries; |
| 1576 | 1661 | |
| 1577 | − srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(); | |
| 1578 | 1662 | srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; |
| 1663 | + srq->qplib_srq.wqe_size = | |
| 1664 | + bnxt_re_get_rwqe_size(srq->qplib_srq.max_sge); | |
| 1579 | 1665 | srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; |
| 1580 | 1666 | srq->srq_limit = srq_init_attr->attr.srq_limit; |
| 1581 | 1667 | srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id; |
| @@ -3569,7 +3655,7 @@ int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, · 1987 unmodified lines | ||
| 3569 | 3655 | } |
| 3570 | 3656 | |
| 3571 | 3657 | struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, |
| 3572 | − u32 max_num_sg, struct ib_udata *udata) | |
| 3658 | + u32 max_num_sg) | |
| 3573 | 3659 | { |
| 3574 | 3660 | struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); |
| 3575 | 3661 | struct bnxt_re_dev *rdev = pd->rdev; |
| @@ -136,14 +136,14 @@ struct bnxt_re_ucontext { · 135 unmodified lines | ||
| 136 | 136 | spinlock_t sh_lock; /* protect shpg */ |
| 137 | 137 | }; |
| 138 | 138 | |
| 139 | −static inline u16 bnxt_re_get_swqe_size(void) | |
| 139 | +static inline u16 bnxt_re_get_swqe_size(int nsge) | |
| 140 | 140 | { |
| 141 | − return sizeof(struct sq_send); | |
| 141 | + return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge); | |
| 142 | 142 | } |
| 143 | 143 | |
| 144 | −static inline u16 bnxt_re_get_rwqe_size(void) | |
| 144 | +static inline u16 bnxt_re_get_rwqe_size(int nsge) | |
| 145 | 145 | { |
| 146 | − return sizeof(struct rq_wqe); | |
| 146 | + return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge)); | |
| 147 | 147 | } |
| 148 | 148 | |
| 149 | 149 | int bnxt_re_query_device(struct ib_device *ibdev, |
| @@ -201,7 +201,7 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); · 51 unmodified lines | ||
| 201 | 201 | int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, |
| 202 | 202 | unsigned int *sg_offset); |
| 203 | 203 | struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type, |
| 204 | − u32 max_num_sg, struct ib_udata *udata); | |
| 204 | + u32 max_num_sg); | |
| 205 | 205 | int bnxt_re_dereg_mr(struct ib_mr *mr, struct ib_udata *udata); |
| 206 | 206 | struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, |
| 207 | 207 | struct ib_udata *udata); |
| @@ -82,6 +82,15 @@ static void bnxt_re_remove_device(struct bnxt_re_dev *rdev); · 81 unmodified lines | ||
| 82 | 82 | static void bnxt_re_dealloc_driver(struct ib_device *ib_dev); |
| 83 | 83 | static void bnxt_re_stop_irq(void *handle); |
| 84 | 84 | |
| 85 | +static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode) | |
| 86 | +{ | |
| 87 | + struct bnxt_qplib_chip_ctx *cctx; | |
| 88 | + | |
| 89 | + cctx = rdev->chip_ctx; | |
| 90 | + cctx->modes.wqe_mode = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ? | |
| 91 | + mode : BNXT_QPLIB_WQE_MODE_STATIC; | |
| 92 | +} | |
| 93 | + | |
| 85 | 94 | static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) |
| 86 | 95 | { |
| 87 | 96 | struct bnxt_qplib_chip_ctx *chip_ctx; |
| @@ -97,7 +106,7 @@ static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) · 9 unmodified lines | ||
| 97 | 106 | kfree(chip_ctx); |
| 98 | 107 | } |
| 99 | 108 | |
| 100 | −static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev) | |
| 109 | +static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode) | |
| 101 | 110 | { |
| 102 | 111 | struct bnxt_qplib_chip_ctx *chip_ctx; |
| 103 | 112 | struct bnxt_en_dev *en_dev; |
| @@ -117,6 +126,7 @@ static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev) · 13 unmodified lines | ||
| 117 | 126 | rdev->qplib_res.cctx = rdev->chip_ctx; |
| 118 | 127 | rdev->rcfw.res = &rdev->qplib_res; |
| 119 | 128 | |
| 129 | + bnxt_re_set_drv_mode(rdev, wqe_mode); | |
| 120 | 130 | return 0; |
| 121 | 131 | } |
| 122 | 132 | |
| @@ -1386,7 +1396,7 @@ static void bnxt_re_worker(struct work_struct *work) · 1263 unmodified lines | ||
| 1386 | 1396 | schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); |
| 1387 | 1397 | } |
| 1388 | 1398 | |
| 1389 | −static int bnxt_re_dev_init(struct bnxt_re_dev *rdev) | |
| 1399 | +static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode) | |
| 1390 | 1400 | { |
| 1391 | 1401 | struct bnxt_qplib_creq_ctx *creq; |
| 1392 | 1402 | struct bnxt_re_ring_attr rattr; |
| @@ -1406,7 +1416,7 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev) · 13 unmodified lines | ||
| 1406 | 1416 | } |
| 1407 | 1417 | set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); |
| 1408 | 1418 | |
| 1409 | − rc = bnxt_re_setup_chip_ctx(rdev); | |
| 1419 | + rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode); | |
| 1410 | 1420 | if (rc) { |
| 1411 | 1421 | ibdev_err(&rdev->ibdev, "Failed to get chip context\n"); |
| 1412 | 1422 | return -EINVAL; |
| @@ -1585,7 +1595,7 @@ static void bnxt_re_remove_device(struct bnxt_re_dev *rdev) · 172 unmodified lines | ||
| 1585 | 1595 | } |
| 1586 | 1596 | |
| 1587 | 1597 | static int bnxt_re_add_device(struct bnxt_re_dev **rdev, |
| 1588 | − struct net_device *netdev) | |
| 1598 | + struct net_device *netdev, u8 wqe_mode) | |
| 1589 | 1599 | { |
| 1590 | 1600 | int rc; |
| 1591 | 1601 | |
| @@ -1599,7 +1609,7 @@ static int bnxt_re_add_device(struct bnxt_re_dev **rdev, · 7 unmodified lines | ||
| 1599 | 1609 | } |
| 1600 | 1610 | |
| 1601 | 1611 | pci_dev_get((*rdev)->en_dev->pdev); |
| 1602 | − rc = bnxt_re_dev_init(*rdev); | |
| 1612 | + rc = bnxt_re_dev_init(*rdev, wqe_mode); | |
| 1603 | 1613 | if (rc) { |
| 1604 | 1614 | pci_dev_put((*rdev)->en_dev->pdev); |
| 1605 | 1615 | bnxt_re_dev_unreg(*rdev); |
| @@ -1711,7 +1721,8 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, · 105 unmodified lines | ||
| 1711 | 1721 | case NETDEV_REGISTER: |
| 1712 | 1722 | if (rdev) |
| 1713 | 1723 | break; |
| 1714 | − rc = bnxt_re_add_device(&rdev, real_dev); | |
| 1724 | + rc = bnxt_re_add_device(&rdev, real_dev, | |
| 1725 | + BNXT_QPLIB_WQE_MODE_STATIC); | |
| 1715 | 1726 | if (!rc) |
| 1716 | 1727 | sch_work = true; |
| 1717 | 1728 | release = false; |
| @@ -178,11 +178,11 @@ static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, · 177 unmodified lines | ||
| 178 | 178 | |
| 179 | 179 | if (qp->rq_hdr_buf) |
| 180 | 180 | dma_free_coherent(&res->pdev->dev, |
| 181 | − rq->hwq.max_elements * qp->rq_hdr_buf_size, | |
| 181 | + rq->max_wqe * qp->rq_hdr_buf_size, | |
| 182 | 182 | qp->rq_hdr_buf, qp->rq_hdr_buf_map); |
| 183 | 183 | if (qp->sq_hdr_buf) |
| 184 | 184 | dma_free_coherent(&res->pdev->dev, |
| 185 | − sq->hwq.max_elements * qp->sq_hdr_buf_size, | |
| 185 | + sq->max_wqe * qp->sq_hdr_buf_size, | |
| 186 | 186 | qp->sq_hdr_buf, qp->sq_hdr_buf_map); |
| 187 | 187 | qp->rq_hdr_buf = NULL; |
| 188 | 188 | qp->sq_hdr_buf = NULL; |
| @@ -199,10 +199,9 @@ static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, · 10 unmodified lines | ||
| 199 | 199 | struct bnxt_qplib_q *sq = &qp->sq; |
| 200 | 200 | int rc = 0; |
| 201 | 201 | |
| 202 | − if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { | |
| 202 | + if (qp->sq_hdr_buf_size && sq->max_wqe) { | |
| 203 | 203 | qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, |
| 204 | − sq->hwq.max_elements * | |
| 205 | − qp->sq_hdr_buf_size, | |
| 204 | + sq->max_wqe * qp->sq_hdr_buf_size, | |
| 206 | 205 | &qp->sq_hdr_buf_map, GFP_KERNEL); |
| 207 | 206 | if (!qp->sq_hdr_buf) { |
| 208 | 207 | rc = -ENOMEM; |
| @@ -212,9 +211,9 @@ static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, · 3 unmodified lines | ||
| 212 | 211 | } |
| 213 | 212 | } |
| 214 | 213 | |
| 215 | − if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { | |
| 214 | + if (qp->rq_hdr_buf_size && rq->max_wqe) { | |
| 216 | 215 | qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, |
| 217 | − rq->hwq.max_elements * | |
| 216 | + rq->max_wqe * | |
| 218 | 217 | qp->rq_hdr_buf_size, |
| 219 | 218 | &qp->rq_hdr_buf_map, |
| 220 | 219 | GFP_KERNEL); |
| @@ -661,6 +660,7 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, · 440 unmodified lines | ||
| 661 | 660 | srq->dbinfo.hwq = &srq->hwq; |
| 662 | 661 | srq->dbinfo.xid = srq->id; |
| 663 | 662 | srq->dbinfo.db = srq->dpi->dbr; |
| 663 | + srq->dbinfo.max_slot = 1; | |
| 664 | 664 | srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem; |
| 665 | 665 | if (srq->threshold) |
| 666 | 666 | bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); |
| @@ -784,6 +784,28 @@ done: · 117 unmodified lines | ||
| 784 | 784 | } |
| 785 | 785 | |
| 786 | 786 | /* QP */ |
| 787 | + | |
| 788 | +static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que) | |
| 789 | +{ | |
| 790 | + int rc = 0; | |
| 791 | + int indx; | |
| 792 | + | |
| 793 | + que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL); | |
| 794 | + if (!que->swq) { | |
| 795 | + rc = -ENOMEM; | |
| 796 | + goto out; | |
| 797 | + } | |
| 798 | + | |
| 799 | + que->swq_start = 0; | |
| 800 | + que->swq_last = que->max_wqe - 1; | |
| 801 | + for (indx = 0; indx < que->max_wqe; indx++) | |
| 802 | + que->swq[indx].next_idx = indx + 1; | |
| 803 | + que->swq[que->swq_last].next_idx = 0; /* Make it circular */ | |
| 804 | + que->swq_last = 0; | |
| 805 | +out: | |
| 806 | + return rc; | |
| 807 | +} | |
| 808 | + | |
| 787 | 809 | int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) |
| 788 | 810 | { |
| 789 | 811 | struct bnxt_qplib_hwq_attr hwq_attr = {}; |
| @@ -808,71 +830,63 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 18 unmodified lines | ||
| 808 | 830 | /* SQ */ |
| 809 | 831 | hwq_attr.res = res; |
| 810 | 832 | hwq_attr.sginfo = &sq->sg_info; |
| 811 | − hwq_attr.depth = sq->max_wqe; | |
| 812 | − hwq_attr.stride = sq->wqe_size; | |
| 833 | + hwq_attr.stride = sizeof(struct sq_sge); | |
| 834 | + hwq_attr.depth = bnxt_qplib_get_depth(sq); | |
| 813 | 835 | hwq_attr.type = HWQ_TYPE_QUEUE; |
| 814 | 836 | rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); |
| 815 | 837 | if (rc) |
| 816 | 838 | goto exit; |
| 817 | 839 | |
| 818 | − sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); | |
| 819 | − if (!sq->swq) { | |
| 820 | − rc = -ENOMEM; | |
| 840 | + rc = bnxt_qplib_alloc_init_swq(sq); | |
| 841 | + if (rc) | |
| 821 | 842 | goto fail_sq; |
| 822 | − } | |
| 843 | + | |
| 844 | + req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); | |
| 823 | 845 | pbl = &sq->hwq.pbl[PBL_LVL_0]; |
| 824 | 846 | req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); |
| 825 | 847 | pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << |
| 826 | 848 | CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT); |
| 827 | 849 | pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK); |
| 828 | 850 | req.sq_pg_size_sq_lvl = pg_sz_lvl; |
| 851 | + req.sq_fwo_sq_sge = | |
| 852 | + cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << | |
| 853 | + CMDQ_CREATE_QP1_SQ_SGE_SFT); | |
| 854 | + req.scq_cid = cpu_to_le32(qp->scq->id); | |
| 829 | 855 | |
| 830 | − if (qp->scq) | |
| 831 | − req.scq_cid = cpu_to_le32(qp->scq->id); | |
| 832 | 856 | /* RQ */ |
| 833 | 857 | if (rq->max_wqe) { |
| 834 | 858 | hwq_attr.res = res; |
| 835 | 859 | hwq_attr.sginfo = &rq->sg_info; |
| 836 | − hwq_attr.stride = rq->wqe_size; | |
| 837 | − hwq_attr.depth = qp->rq.max_wqe; | |
| 860 | + hwq_attr.stride = sizeof(struct sq_sge); | |
| 861 | + hwq_attr.depth = bnxt_qplib_get_depth(rq); | |
| 838 | 862 | hwq_attr.type = HWQ_TYPE_QUEUE; |
| 839 | 863 | rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); |
| 840 | 864 | if (rc) |
| 841 | − goto fail_sq; | |
| 842 | − | |
| 843 | − rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), | |
| 844 | − GFP_KERNEL); | |
| 845 | − if (!rq->swq) { | |
| 846 | − rc = -ENOMEM; | |
| 865 | + goto sq_swq; | |
| 866 | + rc = bnxt_qplib_alloc_init_swq(rq); | |
| 867 | + if (rc) | |
| 847 | 868 | goto fail_rq; |
| 848 | − } | |
| 869 | + req.rq_size = cpu_to_le32(rq->max_wqe); | |
| 849 | 870 | pbl = &rq->hwq.pbl[PBL_LVL_0]; |
| 850 | 871 | req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); |
| 851 | 872 | pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << |
| 852 | 873 | CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT); |
| 853 | 874 | pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK); |
| 854 | 875 | req.rq_pg_size_rq_lvl = pg_sz_lvl; |
| 855 | − if (qp->rcq) | |
| 856 | − req.rcq_cid = cpu_to_le32(qp->rcq->id); | |
| 876 | + req.rq_fwo_rq_sge = | |
| 877 | + cpu_to_le16((rq->max_sge & | |
| 878 | + CMDQ_CREATE_QP1_RQ_SGE_MASK) << | |
| 879 | + CMDQ_CREATE_QP1_RQ_SGE_SFT); | |
| 857 | 880 | } |
| 881 | + req.rcq_cid = cpu_to_le32(qp->rcq->id); | |
| 858 | 882 | /* Header buffer - allow hdr_buf pass in */ |
| 859 | 883 | rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); |
| 860 | 884 | if (rc) { |
| 861 | 885 | rc = -ENOMEM; |
| 862 | − goto fail; | |
| 886 | + goto rq_rwq; | |
| 863 | 887 | } |
| 864 | 888 | qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; |
| 865 | 889 | req.qp_flags = cpu_to_le32(qp_flags); |
| 866 | − req.sq_size = cpu_to_le32(sq->hwq.max_elements); | |
| 867 | − req.rq_size = cpu_to_le32(rq->hwq.max_elements); | |
| 868 | − | |
| 869 | − req.sq_fwo_sq_sge = | |
| 870 | − cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << | |
| 871 | − CMDQ_CREATE_QP1_SQ_SGE_SFT); | |
| 872 | − req.rq_fwo_rq_sge = | |
| 873 | − cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << | |
| 874 | − CMDQ_CREATE_QP1_RQ_SGE_SFT); | |
| 875 | − | |
| 876 | 890 | req.pd_id = cpu_to_le32(qp->pd->id); |
| 877 | 891 | |
| 878 | 892 | rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, |
| @@ -886,10 +900,12 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 7 unmodified lines | ||
| 886 | 900 | sq->dbinfo.hwq = &sq->hwq; |
| 887 | 901 | sq->dbinfo.xid = qp->id; |
| 888 | 902 | sq->dbinfo.db = qp->dpi->dbr; |
| 903 | + sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode); | |
| 889 | 904 | if (rq->max_wqe) { |
| 890 | 905 | rq->dbinfo.hwq = &rq->hwq; |
| 891 | 906 | rq->dbinfo.xid = qp->id; |
| 892 | 907 | rq->dbinfo.db = qp->dpi->dbr; |
| 908 | + rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size); | |
| 893 | 909 | } |
| 894 | 910 | rcfw->qp_tbl[qp->id].qp_id = qp->id; |
| 895 | 911 | rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; |
| @@ -898,12 +914,14 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 2 unmodified lines | ||
| 898 | 914 | |
| 899 | 915 | fail: |
| 900 | 916 | bnxt_qplib_free_qp_hdr_buf(res, qp); |
| 917 | +rq_rwq: | |
| 918 | + kfree(rq->swq); | |
| 901 | 919 | fail_rq: |
| 902 | 920 | bnxt_qplib_free_hwq(res, &rq->hwq); |
| 903 | − kfree(rq->swq); | |
| 921 | +sq_swq: | |
| 922 | + kfree(sq->swq); | |
| 904 | 923 | fail_sq: |
| 905 | 924 | bnxt_qplib_free_hwq(res, &sq->hwq); |
| 906 | − kfree(sq->swq); | |
| 907 | 925 | exit: |
| 908 | 926 | return rc; |
| 909 | 927 | } |
| @@ -912,26 +930,18 @@ static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size) · 2 unmodified lines | ||
| 912 | 930 | { |
| 913 | 931 | struct bnxt_qplib_hwq *hwq; |
| 914 | 932 | struct bnxt_qplib_q *sq; |
| 915 | − u64 fpsne, psne, psn_pg; | |
| 916 | − u16 indx_pad = 0, indx; | |
| 917 | − u16 pg_num, pg_indx; | |
| 918 | − u64 *page; | |
| 933 | + u64 fpsne, psn_pg; | |
| 934 | + u16 indx_pad = 0; | |
| 919 | 935 | |
| 920 | 936 | sq = &qp->sq; |
| 921 | 937 | hwq = &sq->hwq; |
| 922 | − | |
| 923 | − fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->max_elements, &psn_pg); | |
| 938 | + fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg); | |
| 924 | 939 | if (!IS_ALIGNED(fpsne, PAGE_SIZE)) |
| 925 | 940 | indx_pad = ALIGN(fpsne, PAGE_SIZE) / size; |
| 926 | 941 | |
| 927 | − page = (u64 *)psn_pg; | |
| 928 | − for (indx = 0; indx < hwq->max_elements; indx++) { | |
| 929 | − pg_num = (indx + indx_pad) / (PAGE_SIZE / size); | |
| 930 | − pg_indx = (indx + indx_pad) % (PAGE_SIZE / size); | |
| 931 | − psne = page[pg_num] + pg_indx * size; | |
| 932 | − sq->swq[indx].psn_ext = (struct sq_psn_search_ext *)psne; | |
| 933 | − sq->swq[indx].psn_search = (struct sq_psn_search *)psne; | |
| 934 | − } | |
| 942 | + hwq->pad_pgofft = indx_pad; | |
| 943 | + hwq->pad_pg = (u64 *)psn_pg; | |
| 944 | + hwq->pad_stride = size; | |
| 935 | 945 | } |
| 936 | 946 | |
| 937 | 947 | int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) |
| @@ -944,12 +954,12 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 6 unmodified lines | ||
| 944 | 954 | struct creq_create_qp_resp resp; |
| 945 | 955 | int rc, req_size, psn_sz = 0; |
| 946 | 956 | struct bnxt_qplib_hwq *xrrq; |
| 947 | − u16 cmd_flags = 0, max_ssge; | |
| 948 | 957 | struct bnxt_qplib_pbl *pbl; |
| 949 | 958 | struct cmdq_create_qp req; |
| 959 | + u16 cmd_flags = 0; | |
| 950 | 960 | u32 qp_flags = 0; |
| 951 | 961 | u8 pg_sz_lvl; |
| 952 | − u16 max_rsge; | |
| 962 | + u16 nsge; | |
| 953 | 963 | |
| 954 | 964 | RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); |
| 955 | 965 | |
| @@ -967,97 +977,78 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 11 unmodified lines | ||
| 967 | 977 | |
| 968 | 978 | hwq_attr.res = res; |
| 969 | 979 | hwq_attr.sginfo = &sq->sg_info; |
| 970 | − hwq_attr.stride = sq->wqe_size; | |
| 971 | − hwq_attr.depth = sq->max_wqe; | |
| 980 | + hwq_attr.stride = sizeof(struct sq_sge); | |
| 981 | + hwq_attr.depth = bnxt_qplib_get_depth(sq); | |
| 972 | 982 | hwq_attr.aux_stride = psn_sz; |
| 973 | − hwq_attr.aux_depth = hwq_attr.depth; | |
| 983 | + hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode); | |
| 974 | 984 | hwq_attr.type = HWQ_TYPE_QUEUE; |
| 975 | 985 | rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); |
| 976 | 986 | if (rc) |
| 977 | 987 | goto exit; |
| 978 | 988 | |
| 979 | − sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); | |
| 980 | − if (!sq->swq) { | |
| 981 | − rc = -ENOMEM; | |
| 989 | + rc = bnxt_qplib_alloc_init_swq(sq); | |
| 990 | + if (rc) | |
| 982 | 991 | goto fail_sq; |
| 983 | − } | |
| 984 | 992 | |
| 985 | 993 | if (psn_sz) |
| 986 | 994 | bnxt_qplib_init_psn_ptr(qp, psn_sz); |
| 987 | 995 | |
| 996 | + req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); | |
| 988 | 997 | pbl = &sq->hwq.pbl[PBL_LVL_0]; |
| 989 | 998 | req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); |
| 990 | 999 | pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << |
| 991 | 1000 | CMDQ_CREATE_QP_SQ_PG_SIZE_SFT); |
| 992 | 1001 | pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK); |
| 993 | 1002 | req.sq_pg_size_sq_lvl = pg_sz_lvl; |
| 994 | − | |
| 995 | − if (qp->scq) | |
| 996 | − req.scq_cid = cpu_to_le32(qp->scq->id); | |
| 1003 | + req.sq_fwo_sq_sge = | |
| 1004 | + cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) << | |
| 1005 | + CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); | |
| 1006 | + req.scq_cid = cpu_to_le32(qp->scq->id); | |
| 997 | 1007 | |
| 998 | 1008 | /* RQ */ |
| 999 | − if (rq->max_wqe) { | |
| 1009 | + if (!qp->srq) { | |
| 1000 | 1010 | hwq_attr.res = res; |
| 1001 | 1011 | hwq_attr.sginfo = &rq->sg_info; |
| 1002 | − hwq_attr.stride = rq->wqe_size; | |
| 1003 | − hwq_attr.depth = rq->max_wqe; | |
| 1012 | + hwq_attr.stride = sizeof(struct sq_sge); | |
| 1013 | + hwq_attr.depth = bnxt_qplib_get_depth(rq); | |
| 1004 | 1014 | hwq_attr.aux_stride = 0; |
| 1005 | 1015 | hwq_attr.aux_depth = 0; |
| 1006 | 1016 | hwq_attr.type = HWQ_TYPE_QUEUE; |
| 1007 | 1017 | rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); |
| 1008 | 1018 | if (rc) |
| 1009 | − goto fail_sq; | |
| 1010 | − | |
| 1011 | − rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), | |
| 1012 | − GFP_KERNEL); | |
| 1013 | − if (!rq->swq) { | |
| 1014 | − rc = -ENOMEM; | |
| 1019 | + goto sq_swq; | |
| 1020 | + rc = bnxt_qplib_alloc_init_swq(rq); | |
| 1021 | + if (rc) | |
| 1015 | 1022 | goto fail_rq; |
| 1016 | − } | |
| 1023 | + | |
| 1024 | + req.rq_size = cpu_to_le32(rq->max_wqe); | |
| 1017 | 1025 | pbl = &rq->hwq.pbl[PBL_LVL_0]; |
| 1018 | 1026 | req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); |
| 1019 | 1027 | pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << |
| 1020 | 1028 | CMDQ_CREATE_QP_RQ_PG_SIZE_SFT); |
| 1021 | 1029 | pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK); |
| 1022 | 1030 | req.rq_pg_size_rq_lvl = pg_sz_lvl; |
| 1031 | + nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? | |
| 1032 | + 6 : rq->max_sge; | |
| 1033 | + req.rq_fwo_rq_sge = | |
| 1034 | + cpu_to_le16(((nsge & | |
| 1035 | + CMDQ_CREATE_QP_RQ_SGE_MASK) << | |
| 1036 | + CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); | |
| 1023 | 1037 | } else { |
| 1024 | 1038 | /* SRQ */ |
| 1025 | − if (qp->srq) { | |
| 1026 | − qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; | |
| 1027 | − req.srq_cid = cpu_to_le32(qp->srq->id); | |
| 1028 | − } | |
| 1039 | + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; | |
| 1040 | + req.srq_cid = cpu_to_le32(qp->srq->id); | |
| 1029 | 1041 | } |
| 1030 | − | |
| 1031 | − if (qp->rcq) | |
| 1032 | − req.rcq_cid = cpu_to_le32(qp->rcq->id); | |
| 1042 | + req.rcq_cid = cpu_to_le32(qp->rcq->id); | |
| 1033 | 1043 | |
| 1034 | 1044 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; |
| 1035 | 1045 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; |
| 1036 | 1046 | if (qp->sig_type) |
| 1037 | 1047 | qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; |
| 1048 | + if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) | |
| 1049 | + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED; | |
| 1038 | 1050 | req.qp_flags = cpu_to_le32(qp_flags); |
| 1039 | 1051 | |
| 1040 | − req.sq_size = cpu_to_le32(sq->hwq.max_elements); | |
| 1041 | − req.rq_size = cpu_to_le32(rq->hwq.max_elements); | |
| 1042 | − qp->sq_hdr_buf = NULL; | |
| 1043 | − qp->rq_hdr_buf = NULL; | |
| 1044 | − | |
| 1045 | − rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); | |
| 1046 | − if (rc) | |
| 1047 | − goto fail_rq; | |
| 1048 | − | |
| 1049 | − /* CTRL-22434: Irrespective of the requested SGE count on the SQ | |
| 1050 | − * always create the QP with max send sges possible if the requested | |
| 1051 | − * inline size is greater than 0. | |
| 1052 | − */ | |
| 1053 | − max_ssge = qp->max_inline_data ? 6 : sq->max_sge; | |
| 1054 | − req.sq_fwo_sq_sge = cpu_to_le16( | |
| 1055 | − ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) | |
| 1056 | − << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); | |
| 1057 | − max_rsge = bnxt_qplib_is_chip_gen_p5(res->cctx) ? 6 : rq->max_sge; | |
| 1058 | − req.rq_fwo_rq_sge = cpu_to_le16( | |
| 1059 | − ((max_rsge & CMDQ_CREATE_QP_RQ_SGE_MASK) | |
| 1060 | − << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); | |
| 1061 | 1052 | /* ORRQ and IRRQ */ |
| 1062 | 1053 | if (psn_sz) { |
| 1063 | 1054 | xrrq = &qp->orrq; |
| @@ -1078,7 +1069,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 14 unmodified lines | ||
| 1078 | 1069 | hwq_attr.type = HWQ_TYPE_CTX; |
| 1079 | 1070 | rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); |
| 1080 | 1071 | if (rc) |
| 1081 | − goto fail_buf_free; | |
| 1072 | + goto rq_swq; | |
| 1082 | 1073 | pbl = &xrrq->pbl[PBL_LVL_0]; |
| 1083 | 1074 | req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); |
| 1084 | 1075 | |
| @@ -1113,30 +1104,29 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 28 unmodified lines | ||
| 1113 | 1104 | sq->dbinfo.hwq = &sq->hwq; |
| 1114 | 1105 | sq->dbinfo.xid = qp->id; |
| 1115 | 1106 | sq->dbinfo.db = qp->dpi->dbr; |
| 1107 | + sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode); | |
| 1116 | 1108 | if (rq->max_wqe) { |
| 1117 | 1109 | rq->dbinfo.hwq = &rq->hwq; |
| 1118 | 1110 | rq->dbinfo.xid = qp->id; |
| 1119 | 1111 | rq->dbinfo.db = qp->dpi->dbr; |
| 1112 | + rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size); | |
| 1120 | 1113 | } |
| 1121 | 1114 | rcfw->qp_tbl[qp->id].qp_id = qp->id; |
| 1122 | 1115 | rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; |
| 1123 | 1116 | |
| 1124 | 1117 | return 0; |
| 1125 | − | |
| 1126 | 1118 | fail: |
| 1127 | − if (qp->irrq.max_elements) | |
| 1128 | − bnxt_qplib_free_hwq(res, &qp->irrq); | |
| 1119 | + bnxt_qplib_free_hwq(res, &qp->irrq); | |
| 1129 | 1120 | fail_orrq: |
| 1130 | − if (qp->orrq.max_elements) | |
| 1131 | − bnxt_qplib_free_hwq(res, &qp->orrq); | |
| 1132 | −fail_buf_free: | |
| 1133 | − bnxt_qplib_free_qp_hdr_buf(res, qp); | |
| 1121 | + bnxt_qplib_free_hwq(res, &qp->orrq); | |
| 1122 | +rq_swq: | |
| 1123 | + kfree(rq->swq); | |
| 1134 | 1124 | fail_rq: |
| 1135 | 1125 | bnxt_qplib_free_hwq(res, &rq->hwq); |
| 1136 | − kfree(rq->swq); | |
| 1126 | +sq_swq: | |
| 1127 | + kfree(sq->swq); | |
| 1137 | 1128 | fail_sq: |
| 1138 | 1129 | bnxt_qplib_free_hwq(res, &sq->hwq); |
| 1139 | − kfree(sq->swq); | |
| 1140 | 1130 | exit: |
| 1141 | 1131 | return rc; |
| 1142 | 1132 | } |
| @@ -1512,7 +1502,7 @@ void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, · 369 unmodified lines | ||
| 1512 | 1502 | memset(sge, 0, sizeof(*sge)); |
| 1513 | 1503 | |
| 1514 | 1504 | if (qp->sq_hdr_buf) { |
| 1515 | − sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
| 1505 | + sw_prod = sq->swq_start; | |
| 1516 | 1506 | sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + |
| 1517 | 1507 | sw_prod * qp->sq_hdr_buf_size); |
| 1518 | 1508 | sge->lkey = 0xFFFFFFFF; |
| @@ -1526,7 +1516,7 @@ u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) · 7 unmodified lines | ||
| 1526 | 1516 | { |
| 1527 | 1517 | struct bnxt_qplib_q *rq = &qp->rq; |
| 1528 | 1518 | |
| 1529 | − return HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1519 | + return rq->swq_start; | |
| 1530 | 1520 | } |
| 1531 | 1521 | |
| 1532 | 1522 | dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) |
| @@ -1543,7 +1533,7 @@ void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, · 10 unmodified lines | ||
| 1543 | 1533 | memset(sge, 0, sizeof(*sge)); |
| 1544 | 1534 | |
| 1545 | 1535 | if (qp->rq_hdr_buf) { |
| 1546 | − sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1536 | + sw_prod = rq->swq_start; | |
| 1547 | 1537 | sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + |
| 1548 | 1538 | sw_prod * qp->rq_hdr_buf_size); |
| 1549 | 1539 | sge->lkey = 0xFFFFFFFF; |
| @@ -1562,6 +1552,8 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, · 12 unmodified lines | ||
| 1562 | 1552 | u32 flg_npsn; |
| 1563 | 1553 | u32 op_spsn; |
| 1564 | 1554 | |
| 1555 | + if (!swq->psn_search) | |
| 1556 | + return; | |
| 1565 | 1557 | psns = swq->psn_search; |
| 1566 | 1558 | psns_ext = swq->psn_ext; |
| 1567 | 1559 | |
| @@ -1575,12 +1567,122 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, · 7 unmodified lines | ||
| 1575 | 1567 | if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) { |
| 1576 | 1568 | psns_ext->opcode_start_psn = cpu_to_le32(op_spsn); |
| 1577 | 1569 | psns_ext->flags_next_psn = cpu_to_le32(flg_npsn); |
| 1570 | + psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx); | |
| 1578 | 1571 | } else { |
| 1579 | 1572 | psns->opcode_start_psn = cpu_to_le32(op_spsn); |
| 1580 | 1573 | psns->flags_next_psn = cpu_to_le32(flg_npsn); |
| 1581 | 1574 | } |
| 1582 | 1575 | } |
| 1583 | 1576 | |
| 1577 | +static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp, | |
| 1578 | + struct bnxt_qplib_swqe *wqe, | |
| 1579 | + u16 *idx) | |
| 1580 | +{ | |
| 1581 | + struct bnxt_qplib_hwq *hwq; | |
| 1582 | + int len, t_len, offt; | |
| 1583 | + bool pull_dst = true; | |
| 1584 | + void *il_dst = NULL; | |
| 1585 | + void *il_src = NULL; | |
| 1586 | + int t_cplen, cplen; | |
| 1587 | + int indx; | |
| 1588 | + | |
| 1589 | + hwq = &qp->sq.hwq; | |
| 1590 | + t_len = 0; | |
| 1591 | + for (indx = 0; indx < wqe->num_sge; indx++) { | |
| 1592 | + len = wqe->sg_list[indx].size; | |
| 1593 | + il_src = (void *)wqe->sg_list[indx].addr; | |
| 1594 | + t_len += len; | |
| 1595 | + if (t_len > qp->max_inline_data) | |
| 1596 | + goto bad; | |
| 1597 | + while (len) { | |
| 1598 | + if (pull_dst) { | |
| 1599 | + pull_dst = false; | |
| 1600 | + il_dst = bnxt_qplib_get_prod_qe(hwq, *idx); | |
| 1601 | + (*idx)++; | |
| 1602 | + t_cplen = 0; | |
| 1603 | + offt = 0; | |
| 1604 | + } | |
| 1605 | + cplen = min_t(int, len, sizeof(struct sq_sge)); | |
| 1606 | + cplen = min_t(int, cplen, | |
| 1607 | + (sizeof(struct sq_sge) - offt)); | |
| 1608 | + memcpy(il_dst, il_src, cplen); | |
| 1609 | + t_cplen += cplen; | |
| 1610 | + il_src += cplen; | |
| 1611 | + il_dst += cplen; | |
| 1612 | + offt += cplen; | |
| 1613 | + len -= cplen; | |
| 1614 | + if (t_cplen == sizeof(struct sq_sge)) | |
| 1615 | + pull_dst = true; | |
| 1616 | + } | |
| 1617 | + } | |
| 1618 | + | |
| 1619 | + return t_len; | |
| 1620 | +bad: | |
| 1621 | + return -ENOMEM; | |
| 1622 | +} | |
| 1623 | + | |
| 1624 | +static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq, | |
| 1625 | + struct bnxt_qplib_sge *ssge, | |
| 1626 | + u16 nsge, u16 *idx) | |
| 1627 | +{ | |
| 1628 | + struct sq_sge *dsge; | |
| 1629 | + int indx, len = 0; | |
| 1630 | + | |
| 1631 | + for (indx = 0; indx < nsge; indx++, (*idx)++) { | |
| 1632 | + dsge = bnxt_qplib_get_prod_qe(hwq, *idx); | |
| 1633 | + dsge->va_or_pa = cpu_to_le64(ssge[indx].addr); | |
| 1634 | + dsge->l_key = cpu_to_le32(ssge[indx].lkey); | |
| 1635 | + dsge->size = cpu_to_le32(ssge[indx].size); | |
| 1636 | + len += ssge[indx].size; | |
| 1637 | + } | |
| 1638 | + | |
| 1639 | + return len; | |
| 1640 | +} | |
| 1641 | + | |
| 1642 | +static u16 bnxt_qplib_required_slots(struct bnxt_qplib_qp *qp, | |
| 1643 | + struct bnxt_qplib_swqe *wqe, | |
| 1644 | + u16 *wqe_sz, u16 *qdf, u8 mode) | |
| 1645 | +{ | |
| 1646 | + u32 ilsize, bytes; | |
| 1647 | + u16 nsge; | |
| 1648 | + u16 slot; | |
| 1649 | + | |
| 1650 | + nsge = wqe->num_sge; | |
| 1651 | + /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */ | |
| 1652 | + bytes = sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge); | |
| 1653 | + if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { | |
| 1654 | + ilsize = bnxt_qplib_calc_ilsize(wqe, qp->max_inline_data); | |
| 1655 | + bytes = ALIGN(ilsize, sizeof(struct sq_sge)); | |
| 1656 | + bytes += sizeof(struct sq_send_hdr); | |
| 1657 | + } | |
| 1658 | + | |
| 1659 | + *qdf = __xlate_qfd(qp->sq.q_full_delta, bytes); | |
| 1660 | + slot = bytes >> 4; | |
| 1661 | + *wqe_sz = slot; | |
| 1662 | + if (mode == BNXT_QPLIB_WQE_MODE_STATIC) | |
| 1663 | + slot = 8; | |
| 1664 | + return slot; | |
| 1665 | +} | |
| 1666 | + | |
| 1667 | +static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_q *sq, | |
| 1668 | + struct bnxt_qplib_swq *swq) | |
| 1669 | +{ | |
| 1670 | + struct bnxt_qplib_hwq *hwq; | |
| 1671 | + u32 pg_num, pg_indx; | |
| 1672 | + void *buff; | |
| 1673 | + u32 tail; | |
| 1674 | + | |
| 1675 | + hwq = &sq->hwq; | |
| 1676 | + if (!hwq->pad_pg) | |
| 1677 | + return; | |
| 1678 | + tail = swq->slot_idx / sq->dbinfo.max_slot; | |
| 1679 | + pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride); | |
| 1680 | + pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride); | |
| 1681 | + buff = (void *)(hwq->pad_pg[pg_num] + pg_indx * hwq->pad_stride); | |
| 1682 | + swq->psn_ext = buff; | |
| 1683 | + swq->psn_search = buff; | |
| 1684 | +} | |
| 1685 | + | |
| 1584 | 1686 | void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) |
| 1585 | 1687 | { |
| 1586 | 1688 | struct bnxt_qplib_q *sq = &qp->sq; |
| @@ -1594,88 +1696,84 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 7 unmodified lines | ||
| 1594 | 1696 | struct bnxt_qplib_nq_work *nq_work = NULL; |
| 1595 | 1697 | int i, rc = 0, data_len = 0, pkt_num = 0; |
| 1596 | 1698 | struct bnxt_qplib_q *sq = &qp->sq; |
| 1597 | − struct sq_send *hw_sq_send_hdr; | |
| 1699 | + struct bnxt_qplib_hwq *hwq; | |
| 1598 | 1700 | struct bnxt_qplib_swq *swq; |
| 1599 | 1701 | bool sch_handler = false; |
| 1600 | − struct sq_sge *hw_sge; | |
| 1601 | − u8 wqe_size16; | |
| 1702 | + u16 wqe_sz, qdf = 0; | |
| 1703 | + void *base_hdr; | |
| 1704 | + void *ext_hdr; | |
| 1602 | 1705 | __le32 temp32; |
| 1603 | − u32 sw_prod; | |
| 1706 | + u32 wqe_idx; | |
| 1707 | + u32 slots; | |
| 1708 | + u16 idx; | |
| 1604 | 1709 | |
| 1605 | − if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { | |
| 1606 | − if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
| 1607 | − sch_handler = true; | |
| 1608 | − dev_dbg(&sq->hwq.pdev->dev, | |
| 1609 | − "%s Error QP. Scheduling for poll_cq\n", | |
| 1610 | − __func__); | |
| 1611 | − goto queue_err; | |
| 1612 | − } | |
| 1710 | + hwq = &sq->hwq; | |
| 1711 | + if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS && | |
| 1712 | + qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
| 1713 | + dev_err(&hwq->pdev->dev, | |
| 1714 | + "QPLIB: FP: QP (0x%x) is in the 0x%x state", | |
| 1715 | + qp->id, qp->state); | |
| 1716 | + rc = -EINVAL; | |
| 1717 | + goto done; | |
| 1613 | 1718 | } |
| 1614 | 1719 | |
| 1615 | − if (bnxt_qplib_queue_full(sq)) { | |
| 1616 | − dev_err(&sq->hwq.pdev->dev, | |
| 1720 | + slots = bnxt_qplib_required_slots(qp, wqe, &wqe_sz, &qdf, qp->wqe_mode); | |
| 1721 | + if (bnxt_qplib_queue_full(sq, slots + qdf)) { | |
| 1722 | + dev_err(&hwq->pdev->dev, | |
| 1617 | 1723 | "prod = %#x cons = %#x qdepth = %#x delta = %#x\n", |
| 1618 | − sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements, | |
| 1619 | − sq->q_full_delta); | |
| 1724 | + hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta); | |
| 1620 | 1725 | rc = -ENOMEM; |
| 1621 | 1726 | goto done; |
| 1622 | 1727 | } |
| 1623 | − sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
| 1624 | − swq = &sq->swq[sw_prod]; | |
| 1728 | + | |
| 1729 | + swq = bnxt_qplib_get_swqe(sq, &wqe_idx); | |
| 1730 | + bnxt_qplib_pull_psn_buff(sq, swq); | |
| 1731 | + | |
| 1732 | + idx = 0; | |
| 1733 | + swq->slot_idx = hwq->prod; | |
| 1734 | + swq->slots = slots; | |
| 1625 | 1735 | swq->wr_id = wqe->wr_id; |
| 1626 | 1736 | swq->type = wqe->type; |
| 1627 | 1737 | swq->flags = wqe->flags; |
| 1738 | + swq->start_psn = sq->psn & BTH_PSN_MASK; | |
| 1628 | 1739 | if (qp->sig_type) |
| 1629 | 1740 | swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; |
| 1630 | − swq->start_psn = sq->psn & BTH_PSN_MASK; | |
| 1631 | − | |
| 1632 | − hw_sq_send_hdr = bnxt_qplib_get_qe(&sq->hwq, sw_prod, NULL); | |
| 1633 | − memset(hw_sq_send_hdr, 0, sq->wqe_size); | |
| 1634 | 1741 | |
| 1635 | − if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { | |
| 1636 | − /* Copy the inline data */ | |
| 1637 | − if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { | |
| 1638 | − dev_warn(&sq->hwq.pdev->dev, | |
| 1639 | − "Inline data length > 96 detected\n"); | |
| 1640 | − data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH; | |
| 1641 | − } else { | |
| 1642 | − data_len = wqe->inline_len; | |
| 1643 | − } | |
| 1644 | − memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); | |
| 1645 | − wqe_size16 = (data_len + 15) >> 4; | |
| 1646 | − } else { | |
| 1647 | − for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data; | |
| 1648 | − i < wqe->num_sge; i++, hw_sge++) { | |
| 1649 | − hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); | |
| 1650 | − hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); | |
| 1651 | − hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); | |
| 1652 | − data_len += wqe->sg_list[i].size; | |
| 1653 | − } | |
| 1654 | − /* Each SGE entry = 1 WQE size16 */ | |
| 1655 | − wqe_size16 = wqe->num_sge; | |
| 1656 | − /* HW requires wqe size has room for atleast one SGE even if | |
| 1657 | − * none was supplied by ULP | |
| 1658 | − */ | |
| 1659 | − if (!wqe->num_sge) | |
| 1660 | − wqe_size16++; | |
| 1742 | + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
| 1743 | + sch_handler = true; | |
| 1744 | + dev_dbg(&hwq->pdev->dev, | |
| 1745 | + "%s Error QP. Scheduling for poll_cq\n", __func__); | |
| 1746 | + goto queue_err; | |
| 1661 | 1747 | } |
| 1662 | 1748 | |
| 1749 | + base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); | |
| 1750 | + ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); | |
| 1751 | + memset(base_hdr, 0, sizeof(struct sq_sge)); | |
| 1752 | + memset(ext_hdr, 0, sizeof(struct sq_sge)); | |
| 1753 | + | |
| 1754 | + if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) | |
| 1755 | + /* Copy the inline data */ | |
| 1756 | + data_len = bnxt_qplib_put_inline(qp, wqe, &idx); | |
| 1757 | + else | |
| 1758 | + data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, | |
| 1759 | + &idx); | |
| 1760 | + if (data_len < 0) | |
| 1761 | + goto queue_err; | |
| 1663 | 1762 | /* Specifics */ |
| 1664 | 1763 | switch (wqe->type) { |
| 1665 | 1764 | case BNXT_QPLIB_SWQE_TYPE_SEND: |
| 1666 | 1765 | if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { |
| 1766 | + struct sq_send_raweth_qp1_hdr *sqe = base_hdr; | |
| 1767 | + struct sq_raw_ext_hdr *ext_sqe = ext_hdr; | |
| 1667 | 1768 | /* Assemble info for Raw Ethertype QPs */ |
| 1668 | − struct sq_send_raweth_qp1 *sqe = | |
| 1669 | − (struct sq_send_raweth_qp1 *)hw_sq_send_hdr; | |
| 1670 | 1769 | |
| 1671 | 1770 | sqe->wqe_type = wqe->type; |
| 1672 | 1771 | sqe->flags = wqe->flags; |
| 1673 | − sqe->wqe_size = wqe_size16 + | |
| 1674 | − ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
| 1772 | + sqe->wqe_size = wqe_sz; | |
| 1675 | 1773 | sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); |
| 1676 | 1774 | sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); |
| 1677 | 1775 | sqe->length = cpu_to_le32(data_len); |
| 1678 | − sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & | |
| 1776 | + ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & | |
| 1679 | 1777 | SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << |
| 1680 | 1778 | SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); |
| 1681 | 1779 | |
| @@ -1685,27 +1783,24 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 3 unmodified lines | ||
| 1685 | 1783 | case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: |
| 1686 | 1784 | case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: |
| 1687 | 1785 | { |
| 1688 | − struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr; | |
| 1786 | + struct sq_ud_ext_hdr *ext_sqe = ext_hdr; | |
| 1787 | + struct sq_send_hdr *sqe = base_hdr; | |
| 1689 | 1788 | |
| 1690 | 1789 | sqe->wqe_type = wqe->type; |
| 1691 | 1790 | sqe->flags = wqe->flags; |
| 1692 | − sqe->wqe_size = wqe_size16 + | |
| 1693 | − ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
| 1694 | − sqe->inv_key_or_imm_data = cpu_to_le32( | |
| 1695 | − wqe->send.inv_key); | |
| 1791 | + sqe->wqe_size = wqe_sz; | |
| 1792 | + sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key); | |
| 1696 | 1793 | if (qp->type == CMDQ_CREATE_QP_TYPE_UD || |
| 1697 | 1794 | qp->type == CMDQ_CREATE_QP_TYPE_GSI) { |
| 1698 | 1795 | sqe->q_key = cpu_to_le32(wqe->send.q_key); |
| 1699 | − sqe->dst_qp = cpu_to_le32( | |
| 1700 | − wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); | |
| 1701 | 1796 | sqe->length = cpu_to_le32(data_len); |
| 1702 | − sqe->avid = cpu_to_le32(wqe->send.avid & | |
| 1703 | − SQ_SEND_AVID_MASK); | |
| 1704 | 1797 | sq->psn = (sq->psn + 1) & BTH_PSN_MASK; |
| 1798 | + ext_sqe->dst_qp = cpu_to_le32(wqe->send.dst_qp & | |
| 1799 | + SQ_SEND_DST_QP_MASK); | |
| 1800 | + ext_sqe->avid = cpu_to_le32(wqe->send.avid & | |
| 1801 | + SQ_SEND_AVID_MASK); | |
| 1705 | 1802 | } else { |
| 1706 | 1803 | sqe->length = cpu_to_le32(data_len); |
| 1707 | − sqe->dst_qp = 0; | |
| 1708 | − sqe->avid = 0; | |
| 1709 | 1804 | if (qp->mtu) |
| 1710 | 1805 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; |
| 1711 | 1806 | if (!pkt_num) |
| @@ -1718,16 +1813,16 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 6 unmodified lines | ||
| 1718 | 1813 | case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: |
| 1719 | 1814 | case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: |
| 1720 | 1815 | { |
| 1721 | − struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr; | |
| 1816 | + struct sq_rdma_ext_hdr *ext_sqe = ext_hdr; | |
| 1817 | + struct sq_rdma_hdr *sqe = base_hdr; | |
| 1722 | 1818 | |
| 1723 | 1819 | sqe->wqe_type = wqe->type; |
| 1724 | 1820 | sqe->flags = wqe->flags; |
| 1725 | − sqe->wqe_size = wqe_size16 + | |
| 1726 | − ((offsetof(typeof(*sqe), data) + 15) >> 4); | |
| 1821 | + sqe->wqe_size = wqe_sz; | |
| 1727 | 1822 | sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); |
| 1728 | 1823 | sqe->length = cpu_to_le32((u32)data_len); |
| 1729 | − sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); | |
| 1730 | − sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); | |
| 1824 | + ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); | |
| 1825 | + ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); | |
| 1731 | 1826 | if (qp->mtu) |
| 1732 | 1827 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; |
| 1733 | 1828 | if (!pkt_num) |
| @@ -1738,14 +1833,15 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 4 unmodified lines | ||
| 1738 | 1833 | case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: |
| 1739 | 1834 | case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: |
| 1740 | 1835 | { |
| 1741 | − struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr; | |
| 1836 | + struct sq_atomic_ext_hdr *ext_sqe = ext_hdr; | |
| 1837 | + struct sq_atomic_hdr *sqe = base_hdr; | |
| 1742 | 1838 | |
| 1743 | 1839 | sqe->wqe_type = wqe->type; |
| 1744 | 1840 | sqe->flags = wqe->flags; |
| 1745 | 1841 | sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); |
| 1746 | 1842 | sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); |
| 1747 | − sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); | |
| 1748 | − sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); | |
| 1843 | + ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); | |
| 1844 | + ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); | |
| 1749 | 1845 | if (qp->mtu) |
| 1750 | 1846 | pkt_num = (data_len + qp->mtu - 1) / qp->mtu; |
| 1751 | 1847 | if (!pkt_num) |
| @@ -1755,8 +1851,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 3 unmodified lines | ||
| 1755 | 1851 | } |
| 1756 | 1852 | case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: |
| 1757 | 1853 | { |
| 1758 | − struct sq_localinvalidate *sqe = | |
| 1759 | − (struct sq_localinvalidate *)hw_sq_send_hdr; | |
| 1854 | + struct sq_localinvalidate *sqe = base_hdr; | |
| 1760 | 1855 | |
| 1761 | 1856 | sqe->wqe_type = wqe->type; |
| 1762 | 1857 | sqe->flags = wqe->flags; |
| @@ -1766,7 +1861,8 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 3 unmodified lines | ||
| 1766 | 1861 | } |
| 1767 | 1862 | case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: |
| 1768 | 1863 | { |
| 1769 | − struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr; | |
| 1864 | + struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr; | |
| 1865 | + struct sq_fr_pmr_hdr *sqe = base_hdr; | |
| 1770 | 1866 | |
| 1771 | 1867 | sqe->wqe_type = wqe->type; |
| 1772 | 1868 | sqe->flags = wqe->flags; |
| @@ -1790,14 +1886,15 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 17 unmodified lines | ||
| 1790 | 1886 | wqe->frmr.pbl_ptr[i] = cpu_to_le64( |
| 1791 | 1887 | wqe->frmr.page_list[i] | |
| 1792 | 1888 | PTU_PTE_VALID); |
| 1793 | − sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); | |
| 1794 | − sqe->va = cpu_to_le64(wqe->frmr.va); | |
| 1889 | + ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); | |
| 1890 | + ext_sqe->va = cpu_to_le64(wqe->frmr.va); | |
| 1795 | 1891 | |
| 1796 | 1892 | break; |
| 1797 | 1893 | } |
| 1798 | 1894 | case BNXT_QPLIB_SWQE_TYPE_BIND_MW: |
| 1799 | 1895 | { |
| 1800 | − struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr; | |
| 1896 | + struct sq_bind_ext_hdr *ext_sqe = ext_hdr; | |
| 1897 | + struct sq_bind_hdr *sqe = base_hdr; | |
| 1801 | 1898 | |
| 1802 | 1899 | sqe->wqe_type = wqe->type; |
| 1803 | 1900 | sqe->flags = wqe->flags; |
| @@ -1806,9 +1903,8 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 2 unmodified lines | ||
| 1806 | 1903 | (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); |
| 1807 | 1904 | sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); |
| 1808 | 1905 | sqe->l_key = cpu_to_le32(wqe->bind.r_key); |
| 1809 | − sqe->va = cpu_to_le64(wqe->bind.va); | |
| 1810 | − temp32 = cpu_to_le32(wqe->bind.length); | |
| 1811 | − memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); | |
| 1906 | + ext_sqe->va = cpu_to_le64(wqe->bind.va); | |
| 1907 | + ext_sqe->length_lo = cpu_to_le32(wqe->bind.length); | |
| 1812 | 1908 | break; |
| 1813 | 1909 | } |
| 1814 | 1910 | default: |
| @@ -1817,23 +1913,11 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 2 unmodified lines | ||
| 1817 | 1913 | goto done; |
| 1818 | 1914 | } |
| 1819 | 1915 | swq->next_psn = sq->psn & BTH_PSN_MASK; |
| 1820 | − if (qp->type == CMDQ_CREATE_QP_TYPE_RC) | |
| 1821 | − bnxt_qplib_fill_psn_search(qp, wqe, swq); | |
| 1916 | + bnxt_qplib_fill_psn_search(qp, wqe, swq); | |
| 1822 | 1917 | queue_err: |
| 1823 | − if (sch_handler) { | |
| 1824 | − /* Store the ULP info in the software structures */ | |
| 1825 | − sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
| 1826 | − swq = &sq->swq[sw_prod]; | |
| 1827 | − swq->wr_id = wqe->wr_id; | |
| 1828 | − swq->type = wqe->type; | |
| 1829 | − swq->flags = wqe->flags; | |
| 1830 | − if (qp->sig_type) | |
| 1831 | − swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; | |
| 1832 | − swq->start_psn = sq->psn & BTH_PSN_MASK; | |
| 1833 | − } | |
| 1834 | − sq->hwq.prod++; | |
| 1918 | + bnxt_qplib_swq_mod_start(sq, wqe_idx); | |
| 1919 | + bnxt_qplib_hwq_incr_prod(hwq, swq->slots); | |
| 1835 | 1920 | qp->wqe_cnt++; |
| 1836 | − | |
| 1837 | 1921 | done: |
| 1838 | 1922 | if (sch_handler) { |
| 1839 | 1923 | nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); |
| @@ -1843,7 +1927,7 @@ done: · 3 unmodified lines | ||
| 1843 | 1927 | INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); |
| 1844 | 1928 | queue_work(qp->scq->nq->cqn_wq, &nq_work->work); |
| 1845 | 1929 | } else { |
| 1846 | − dev_err(&sq->hwq.pdev->dev, | |
| 1930 | + dev_err(&hwq->pdev->dev, | |
| 1847 | 1931 | "FP: Failed to allocate SQ nq_work!\n"); |
| 1848 | 1932 | rc = -ENOMEM; |
| 1849 | 1933 | } |
| @@ -1863,58 +1947,65 @@ int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, · 13 unmodified lines | ||
| 1863 | 1947 | { |
| 1864 | 1948 | struct bnxt_qplib_nq_work *nq_work = NULL; |
| 1865 | 1949 | struct bnxt_qplib_q *rq = &qp->rq; |
| 1950 | + struct rq_wqe_hdr *base_hdr; | |
| 1951 | + struct rq_ext_hdr *ext_hdr; | |
| 1952 | + struct bnxt_qplib_hwq *hwq; | |
| 1953 | + struct bnxt_qplib_swq *swq; | |
| 1866 | 1954 | bool sch_handler = false; |
| 1867 | − struct sq_sge *hw_sge; | |
| 1868 | − struct rq_wqe *rqe; | |
| 1869 | − int i, rc = 0; | |
| 1870 | − u32 sw_prod; | |
| 1955 | + u16 wqe_sz, idx; | |
| 1956 | + u32 wqe_idx; | |
| 1957 | + int rc = 0; | |
| 1871 | 1958 | |
| 1872 | − if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
| 1873 | − sch_handler = true; | |
| 1874 | − dev_dbg(&rq->hwq.pdev->dev, | |
| 1875 | − "%s: Error QP. Scheduling for poll_cq\n", __func__); | |
| 1876 | − goto queue_err; | |
| 1959 | + hwq = &rq->hwq; | |
| 1960 | + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { | |
| 1961 | + dev_err(&hwq->pdev->dev, | |
| 1962 | + "QPLIB: FP: QP (0x%x) is in the 0x%x state", | |
| 1963 | + qp->id, qp->state); | |
| 1964 | + rc = -EINVAL; | |
| 1965 | + goto done; | |
| 1877 | 1966 | } |
| 1878 | − if (bnxt_qplib_queue_full(rq)) { | |
| 1879 | − dev_err(&rq->hwq.pdev->dev, | |
| 1967 | + | |
| 1968 | + if (bnxt_qplib_queue_full(rq, rq->dbinfo.max_slot)) { | |
| 1969 | + dev_err(&hwq->pdev->dev, | |
| 1880 | 1970 | "FP: QP (0x%x) RQ is full!\n", qp->id); |
| 1881 | 1971 | rc = -EINVAL; |
| 1882 | 1972 | goto done; |
| 1883 | 1973 | } |
| 1884 | − sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1885 | − rq->swq[sw_prod].wr_id = wqe->wr_id; | |
| 1886 | 1974 | |
| 1887 | − rqe = bnxt_qplib_get_qe(&rq->hwq, sw_prod, NULL); | |
| 1888 | − memset(rqe, 0, rq->wqe_size); | |
| 1975 | + swq = bnxt_qplib_get_swqe(rq, &wqe_idx); | |
| 1976 | + swq->wr_id = wqe->wr_id; | |
| 1977 | + swq->slots = rq->dbinfo.max_slot; | |
| 1889 | 1978 | |
| 1890 | − /* Calculate wqe_size16 and data_len */ | |
| 1891 | − for (i = 0, hw_sge = (struct sq_sge *)rqe->data; | |
| 1892 | − i < wqe->num_sge; i++, hw_sge++) { | |
| 1893 | − hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); | |
| 1894 | − hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); | |
| 1895 | − hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); | |
| 1979 | + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
| 1980 | + sch_handler = true; | |
| 1981 | + dev_dbg(&hwq->pdev->dev, | |
| 1982 | + "%s: Error QP. Scheduling for poll_cq\n", __func__); | |
| 1983 | + goto queue_err; | |
| 1896 | 1984 | } |
| 1897 | − rqe->wqe_type = wqe->type; | |
| 1898 | − rqe->flags = wqe->flags; | |
| 1899 | − rqe->wqe_size = wqe->num_sge + | |
| 1900 | − ((offsetof(typeof(*rqe), data) + 15) >> 4); | |
| 1901 | − /* HW requires wqe size has room for atleast one SGE even if none | |
| 1902 | − * was supplied by ULP | |
| 1903 | − */ | |
| 1904 | − if (!wqe->num_sge) | |
| 1905 | − rqe->wqe_size++; | |
| 1906 | − | |
| 1907 | − /* Supply the rqe->wr_id index to the wr_id_tbl for now */ | |
| 1908 | − rqe->wr_id[0] = cpu_to_le32(sw_prod); | |
| 1909 | 1985 | |
| 1986 | + idx = 0; | |
| 1987 | + base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); | |
| 1988 | + ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); | |
| 1989 | + memset(base_hdr, 0, sizeof(struct sq_sge)); | |
| 1990 | + memset(ext_hdr, 0, sizeof(struct sq_sge)); | |
| 1991 | + wqe_sz = (sizeof(struct rq_wqe_hdr) + | |
| 1992 | + wqe->num_sge * sizeof(struct sq_sge)) >> 4; | |
| 1993 | + bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx); | |
| 1994 | + if (!wqe->num_sge) { | |
| 1995 | + struct sq_sge *sge; | |
| 1996 | + | |
| 1997 | + sge = bnxt_qplib_get_prod_qe(hwq, idx++); | |
| 1998 | + sge->size = 0; | |
| 1999 | + wqe_sz++; | |
| 2000 | + } | |
| 2001 | + base_hdr->wqe_type = wqe->type; | |
| 2002 | + base_hdr->flags = wqe->flags; | |
| 2003 | + base_hdr->wqe_size = wqe_sz; | |
| 2004 | + base_hdr->wr_id[0] = cpu_to_le32(wqe_idx); | |
| 1910 | 2005 | queue_err: |
| 1911 | − if (sch_handler) { | |
| 1912 | − /* Store the ULP info in the software structures */ | |
| 1913 | − sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 1914 | − rq->swq[sw_prod].wr_id = wqe->wr_id; | |
| 1915 | − } | |
| 1916 | − | |
| 1917 | − rq->hwq.prod++; | |
| 2006 | + bnxt_qplib_swq_mod_start(rq, wqe_idx); | |
| 2007 | + bnxt_qplib_hwq_incr_prod(hwq, swq->slots); | |
| 2008 | +done: | |
| 1918 | 2009 | if (sch_handler) { |
| 1919 | 2010 | nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); |
| 1920 | 2011 | if (nq_work) { |
| @@ -1923,12 +2014,12 @@ queue_err: · 2 unmodified lines | ||
| 1923 | 2014 | INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); |
| 1924 | 2015 | queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); |
| 1925 | 2016 | } else { |
| 1926 | − dev_err(&rq->hwq.pdev->dev, | |
| 2017 | + dev_err(&hwq->pdev->dev, | |
| 1927 | 2018 | "FP: Failed to allocate RQ nq_work!\n"); |
| 1928 | 2019 | rc = -ENOMEM; |
| 1929 | 2020 | } |
| 1930 | 2021 | } |
| 1931 | −done: | |
| 2022 | + | |
| 1932 | 2023 | return rc; |
| 1933 | 2024 | } |
| 1934 | 2025 | |
| @@ -2026,20 +2117,19 @@ int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) · 91 unmodified lines | ||
| 2026 | 2117 | static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, |
| 2027 | 2118 | struct bnxt_qplib_cqe **pcqe, int *budget) |
| 2028 | 2119 | { |
| 2029 | − u32 sw_prod, sw_cons; | |
| 2030 | 2120 | struct bnxt_qplib_cqe *cqe; |
| 2121 | + u32 start, last; | |
| 2031 | 2122 | int rc = 0; |
| 2032 | 2123 | |
| 2033 | 2124 | /* Now complete all outstanding SQEs with FLUSHED_ERR */ |
| 2034 | − sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); | |
| 2125 | + start = sq->swq_start; | |
| 2035 | 2126 | cqe = *pcqe; |
| 2036 | 2127 | while (*budget) { |
| 2037 | − sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | |
| 2038 | − if (sw_cons == sw_prod) { | |
| 2128 | + last = sq->swq_last; | |
| 2129 | + if (start == last) | |
| 2039 | 2130 | break; |
| 2040 | − } | |
| 2041 | 2131 | /* Skip the FENCE WQE completions */ |
| 2042 | − if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) { | |
| 2132 | + if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) { | |
| 2043 | 2133 | bnxt_qplib_cancel_phantom_processing(qp); |
| 2044 | 2134 | goto skip_compl; |
| 2045 | 2135 | } |
| @@ -2047,16 +2137,17 @@ static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, · 1 unmodified line | ||
| 2047 | 2137 | cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; |
| 2048 | 2138 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; |
| 2049 | 2139 | cqe->qp_handle = (u64)(unsigned long)qp; |
| 2050 | − cqe->wr_id = sq->swq[sw_cons].wr_id; | |
| 2140 | + cqe->wr_id = sq->swq[last].wr_id; | |
| 2051 | 2141 | cqe->src_qp = qp->id; |
| 2052 | − cqe->type = sq->swq[sw_cons].type; | |
| 2142 | + cqe->type = sq->swq[last].type; | |
| 2053 | 2143 | cqe++; |
| 2054 | 2144 | (*budget)--; |
| 2055 | 2145 | skip_compl: |
| 2056 | − sq->hwq.cons++; | |
| 2146 | + bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots); | |
| 2147 | + sq->swq_last = sq->swq[last].next_idx; | |
| 2057 | 2148 | } |
| 2058 | 2149 | *pcqe = cqe; |
| 2059 | − if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) | |
| 2150 | + if (!(*budget) && sq->swq_last != start) | |
| 2060 | 2151 | /* Out of budget */ |
| 2061 | 2152 | rc = -EAGAIN; |
| 2062 | 2153 | |
| @@ -2067,9 +2158,9 @@ static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, · 4 unmodified lines | ||
| 2067 | 2158 | struct bnxt_qplib_cqe **pcqe, int *budget) |
| 2068 | 2159 | { |
| 2069 | 2160 | struct bnxt_qplib_cqe *cqe; |
| 2070 | − u32 sw_prod, sw_cons; | |
| 2071 | − int rc = 0; | |
| 2161 | + u32 start, last; | |
| 2072 | 2162 | int opcode = 0; |
| 2163 | + int rc = 0; | |
| 2073 | 2164 | |
| 2074 | 2165 | switch (qp->type) { |
| 2075 | 2166 | case CMDQ_CREATE_QP1_TYPE_GSI: |
| @@ -2085,24 +2176,25 @@ static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, · 9 unmodified lines | ||
| 2085 | 2176 | } |
| 2086 | 2177 | |
| 2087 | 2178 | /* Flush the rest of the RQ */ |
| 2088 | − sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); | |
| 2179 | + start = rq->swq_start; | |
| 2089 | 2180 | cqe = *pcqe; |
| 2090 | 2181 | while (*budget) { |
| 2091 | − sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); | |
| 2092 | − if (sw_cons == sw_prod) | |
| 2182 | + last = rq->swq_last; | |
| 2183 | + if (last == start) | |
| 2093 | 2184 | break; |
| 2094 | 2185 | memset(cqe, 0, sizeof(*cqe)); |
| 2095 | 2186 | cqe->status = |
| 2096 | 2187 | CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; |
| 2097 | 2188 | cqe->opcode = opcode; |
| 2098 | 2189 | cqe->qp_handle = (unsigned long)qp; |
| 2099 | − cqe->wr_id = rq->swq[sw_cons].wr_id; | |
| 2190 | + cqe->wr_id = rq->swq[last].wr_id; | |
| 2100 | 2191 | cqe++; |
| 2101 | 2192 | (*budget)--; |
| 2102 | − rq->hwq.cons++; | |
| 2193 | + bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots); | |
| 2194 | + rq->swq_last = rq->swq[last].next_idx; | |
| 2103 | 2195 | } |
| 2104 | 2196 | *pcqe = cqe; |
| 2105 | − if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) | |
| 2197 | + if (!*budget && rq->swq_last != start) | |
| 2106 | 2198 | /* Out of budget */ |
| 2107 | 2199 | rc = -EAGAIN; |
| 2108 | 2200 | |
| @@ -2125,7 +2217,7 @@ void bnxt_qplib_mark_qp_error(void *qp_handle) · 16 unmodified lines | ||
| 2125 | 2217 | * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 |
| 2126 | 2218 | */ |
| 2127 | 2219 | static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, |
| 2128 | − u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons) | |
| 2220 | + u32 cq_cons, u32 swq_last, u32 cqe_sq_cons) | |
| 2129 | 2221 | { |
| 2130 | 2222 | u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx; |
| 2131 | 2223 | struct bnxt_qplib_q *sq = &qp->sq; |
| @@ -2138,7 +2230,7 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, · 6 unmodified lines | ||
| 2138 | 2230 | |
| 2139 | 2231 | /* Normal mode */ |
| 2140 | 2232 | /* Check for the psn_search marking before completing */ |
| 2141 | − swq = &sq->swq[sw_sq_cons]; | |
| 2233 | + swq = &sq->swq[swq_last]; | |
| 2142 | 2234 | if (swq->psn_search && |
| 2143 | 2235 | le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { |
| 2144 | 2236 | /* Unmark */ |
| @@ -2147,7 +2239,7 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, · 2 unmodified lines | ||
| 2147 | 2239 | & ~0x80000000); |
| 2148 | 2240 | dev_dbg(&cq->hwq.pdev->dev, |
| 2149 | 2241 | "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", |
| 2150 | − cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); | |
| 2242 | + cq_cons, qp->id, swq_last, cqe_sq_cons); | |
| 2151 | 2243 | sq->condition = true; |
| 2152 | 2244 | sq->send_phantom = true; |
| 2153 | 2245 | |
| @@ -2184,9 +2276,10 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, · 30 unmodified lines | ||
| 2184 | 2276 | le64_to_cpu |
| 2185 | 2277 | (peek_req_hwcqe->qp_handle)); |
| 2186 | 2278 | peek_sq = &peek_qp->sq; |
| 2187 | − peek_sq_cons_idx = HWQ_CMP(le16_to_cpu( | |
| 2188 | − peek_req_hwcqe->sq_cons_idx) - 1 | |
| 2189 | − , &sq->hwq); | |
| 2279 | + peek_sq_cons_idx = | |
| 2280 | + ((le16_to_cpu( | |
| 2281 | + peek_req_hwcqe->sq_cons_idx) | |
| 2282 | + - 1) % sq->max_wqe); | |
| 2190 | 2283 | /* If the hwcqe's sq's wr_id matches */ |
| 2191 | 2284 | if (peek_sq == sq && |
| 2192 | 2285 | sq->swq[peek_sq_cons_idx].wr_id == |
| @@ -2214,7 +2307,7 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, · 21 unmodified lines | ||
| 2214 | 2307 | } |
| 2215 | 2308 | dev_err(&cq->hwq.pdev->dev, |
| 2216 | 2309 | "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n", |
| 2217 | − cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); | |
| 2310 | + cq_cons, qp->id, swq_last, cqe_sq_cons); | |
| 2218 | 2311 | rc = -EINVAL; |
| 2219 | 2312 | } |
| 2220 | 2313 | out: |
| @@ -2226,11 +2319,11 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 5 unmodified lines | ||
| 2226 | 2319 | struct bnxt_qplib_cqe **pcqe, int *budget, |
| 2227 | 2320 | u32 cq_cons, struct bnxt_qplib_qp **lib_qp) |
| 2228 | 2321 | { |
| 2229 | − u32 sw_sq_cons, cqe_sq_cons; | |
| 2230 | 2322 | struct bnxt_qplib_swq *swq; |
| 2231 | 2323 | struct bnxt_qplib_cqe *cqe; |
| 2232 | 2324 | struct bnxt_qplib_qp *qp; |
| 2233 | 2325 | struct bnxt_qplib_q *sq; |
| 2326 | + u32 cqe_sq_cons; | |
| 2234 | 2327 | int rc = 0; |
| 2235 | 2328 | |
| 2236 | 2329 | qp = (struct bnxt_qplib_qp *)((unsigned long) |
| @@ -2242,14 +2335,7 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 5 unmodified lines | ||
| 2242 | 2335 | } |
| 2243 | 2336 | sq = &qp->sq; |
| 2244 | 2337 | |
| 2245 | − cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); | |
| 2246 | − if (cqe_sq_cons > sq->hwq.max_elements) { | |
| 2247 | − dev_err(&cq->hwq.pdev->dev, | |
| 2248 | − "FP: CQ Process req reported sq_cons_idx 0x%x which exceeded max 0x%x\n", | |
| 2249 | − cqe_sq_cons, sq->hwq.max_elements); | |
| 2250 | − return -EINVAL; | |
| 2251 | − } | |
| 2252 | − | |
| 2338 | + cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe; | |
| 2253 | 2339 | if (qp->sq.flushed) { |
| 2254 | 2340 | dev_dbg(&cq->hwq.pdev->dev, |
| 2255 | 2341 | "%s: QP in Flush QP = %p\n", __func__, qp); |
| @@ -2261,12 +2347,11 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 5 unmodified lines | ||
| 2261 | 2347 | */ |
| 2262 | 2348 | cqe = *pcqe; |
| 2263 | 2349 | while (*budget) { |
| 2264 | − sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | |
| 2265 | − if (sw_sq_cons == cqe_sq_cons) | |
| 2350 | + if (sq->swq_last == cqe_sq_cons) | |
| 2266 | 2351 | /* Done */ |
| 2267 | 2352 | break; |
| 2268 | 2353 | |
| 2269 | − swq = &sq->swq[sw_sq_cons]; | |
| 2354 | + swq = &sq->swq[sq->swq_last]; | |
| 2270 | 2355 | memset(cqe, 0, sizeof(*cqe)); |
| 2271 | 2356 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; |
| 2272 | 2357 | cqe->qp_handle = (u64)(unsigned long)qp; |
| @@ -2280,12 +2365,12 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 7 unmodified lines | ||
| 2280 | 2365 | * of the request being signaled or not, it must complete with |
| 2281 | 2366 | * the hwcqe error status |
| 2282 | 2367 | */ |
| 2283 | − if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons && | |
| 2368 | + if (swq->next_idx == cqe_sq_cons && | |
| 2284 | 2369 | hwcqe->status != CQ_REQ_STATUS_OK) { |
| 2285 | 2370 | cqe->status = hwcqe->status; |
| 2286 | 2371 | dev_err(&cq->hwq.pdev->dev, |
| 2287 | 2372 | "FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n", |
| 2288 | − sw_sq_cons, cqe->wr_id, cqe->status); | |
| 2373 | + sq->swq_last, cqe->wr_id, cqe->status); | |
| 2289 | 2374 | cqe++; |
| 2290 | 2375 | (*budget)--; |
| 2291 | 2376 | bnxt_qplib_mark_qp_error(qp); |
| @@ -2293,7 +2378,7 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 1 unmodified line | ||
| 2293 | 2378 | bnxt_qplib_add_flush_qp(qp); |
| 2294 | 2379 | } else { |
| 2295 | 2380 | /* Before we complete, do WA 9060 */ |
| 2296 | − if (do_wa9060(qp, cq, cq_cons, sw_sq_cons, | |
| 2381 | + if (do_wa9060(qp, cq, cq_cons, sq->swq_last, | |
| 2297 | 2382 | cqe_sq_cons)) { |
| 2298 | 2383 | *lib_qp = qp; |
| 2299 | 2384 | goto out; |
| @@ -2305,13 +2390,14 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 5 unmodified lines | ||
| 2305 | 2390 | } |
| 2306 | 2391 | } |
| 2307 | 2392 | skip: |
| 2308 | − sq->hwq.cons++; | |
| 2393 | + bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots); | |
| 2394 | + sq->swq_last = swq->next_idx; | |
| 2309 | 2395 | if (sq->single) |
| 2310 | 2396 | break; |
| 2311 | 2397 | } |
| 2312 | 2398 | out: |
| 2313 | 2399 | *pcqe = cqe; |
| 2314 | − if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) { | |
| 2400 | + if (sq->swq_last != cqe_sq_cons) { | |
| 2315 | 2401 | /* Out of budget */ |
| 2316 | 2402 | rc = -EAGAIN; |
| 2317 | 2403 | goto done; |
| @@ -2386,17 +2472,23 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, · 68 unmodified lines | ||
| 2386 | 2472 | (*budget)--; |
| 2387 | 2473 | *pcqe = cqe; |
| 2388 | 2474 | } else { |
| 2475 | + struct bnxt_qplib_swq *swq; | |
| 2476 | + | |
| 2389 | 2477 | rq = &qp->rq; |
| 2390 | − if (wr_id_idx >= rq->hwq.max_elements) { | |
| 2478 | + if (wr_id_idx > (rq->max_wqe - 1)) { | |
| 2391 | 2479 | dev_err(&cq->hwq.pdev->dev, |
| 2392 | 2480 | "FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n", |
| 2393 | − wr_id_idx, rq->hwq.max_elements); | |
| 2481 | + wr_id_idx, rq->max_wqe); | |
| 2394 | 2482 | return -EINVAL; |
| 2395 | 2483 | } |
| 2396 | − cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
| 2484 | + if (wr_id_idx != rq->swq_last) | |
| 2485 | + return -EINVAL; | |
| 2486 | + swq = &rq->swq[rq->swq_last]; | |
| 2487 | + cqe->wr_id = swq->wr_id; | |
| 2397 | 2488 | cqe++; |
| 2398 | 2489 | (*budget)--; |
| 2399 | − rq->hwq.cons++; | |
| 2490 | + bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); | |
| 2491 | + rq->swq_last = swq->next_idx; | |
| 2400 | 2492 | *pcqe = cqe; |
| 2401 | 2493 | |
| 2402 | 2494 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { |
| @@ -2467,18 +2559,24 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, · 64 unmodified lines | ||
| 2467 | 2559 | (*budget)--; |
| 2468 | 2560 | *pcqe = cqe; |
| 2469 | 2561 | } else { |
| 2562 | + struct bnxt_qplib_swq *swq; | |
| 2563 | + | |
| 2470 | 2564 | rq = &qp->rq; |
| 2471 | − if (wr_id_idx >= rq->hwq.max_elements) { | |
| 2565 | + if (wr_id_idx > (rq->max_wqe - 1)) { | |
| 2472 | 2566 | dev_err(&cq->hwq.pdev->dev, |
| 2473 | 2567 | "FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n", |
| 2474 | − wr_id_idx, rq->hwq.max_elements); | |
| 2568 | + wr_id_idx, rq->max_wqe); | |
| 2475 | 2569 | return -EINVAL; |
| 2476 | 2570 | } |
| 2477 | 2571 | |
| 2478 | − cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
| 2572 | + if (rq->swq_last != wr_id_idx) | |
| 2573 | + return -EINVAL; | |
| 2574 | + swq = &rq->swq[rq->swq_last]; | |
| 2575 | + cqe->wr_id = swq->wr_id; | |
| 2479 | 2576 | cqe++; |
| 2480 | 2577 | (*budget)--; |
| 2481 | − rq->hwq.cons++; | |
| 2578 | + bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); | |
| 2579 | + rq->swq_last = swq->next_idx; | |
| 2482 | 2580 | *pcqe = cqe; |
| 2483 | 2581 | |
| 2484 | 2582 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { |
| @@ -2569,17 +2667,23 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, · 84 unmodified lines | ||
| 2569 | 2667 | (*budget)--; |
| 2570 | 2668 | *pcqe = cqe; |
| 2571 | 2669 | } else { |
| 2670 | + struct bnxt_qplib_swq *swq; | |
| 2671 | + | |
| 2572 | 2672 | rq = &qp->rq; |
| 2573 | − if (wr_id_idx >= rq->hwq.max_elements) { | |
| 2673 | + if (wr_id_idx > (rq->max_wqe - 1)) { | |
| 2574 | 2674 | dev_err(&cq->hwq.pdev->dev, |
| 2575 | 2675 | "FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n", |
| 2576 | − wr_id_idx, rq->hwq.max_elements); | |
| 2676 | + wr_id_idx, rq->max_wqe); | |
| 2577 | 2677 | return -EINVAL; |
| 2578 | 2678 | } |
| 2579 | − cqe->wr_id = rq->swq[wr_id_idx].wr_id; | |
| 2679 | + if (rq->swq_last != wr_id_idx) | |
| 2680 | + return -EINVAL; | |
| 2681 | + swq = &rq->swq[rq->swq_last]; | |
| 2682 | + cqe->wr_id = swq->wr_id; | |
| 2580 | 2683 | cqe++; |
| 2581 | 2684 | (*budget)--; |
| 2582 | − rq->hwq.cons++; | |
| 2685 | + bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); | |
| 2686 | + rq->swq_last = swq->next_idx; | |
| 2583 | 2687 | *pcqe = cqe; |
| 2584 | 2688 | |
| 2585 | 2689 | if (hwcqe->status != CQ_RES_RC_STATUS_OK) { |
| @@ -2601,7 +2705,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, · 15 unmodified lines | ||
| 2601 | 2705 | struct bnxt_qplib_qp *qp; |
| 2602 | 2706 | struct bnxt_qplib_q *sq, *rq; |
| 2603 | 2707 | struct bnxt_qplib_cqe *cqe; |
| 2604 | − u32 sw_cons = 0, cqe_cons; | |
| 2708 | + u32 swq_last = 0, cqe_cons; | |
| 2605 | 2709 | int rc = 0; |
| 2606 | 2710 | |
| 2607 | 2711 | /* Check the Status */ |
| @@ -2627,13 +2731,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, · 19 unmodified lines | ||
| 2627 | 2731 | cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); |
| 2628 | 2732 | if (cqe_cons == 0xFFFF) |
| 2629 | 2733 | goto do_rq; |
| 2630 | − | |
| 2631 | − if (cqe_cons > sq->hwq.max_elements) { | |
| 2632 | − dev_err(&cq->hwq.pdev->dev, | |
| 2633 | − "FP: CQ Process terminal reported sq_cons_idx 0x%x which exceeded max 0x%x\n", | |
| 2634 | − cqe_cons, sq->hwq.max_elements); | |
| 2635 | − goto do_rq; | |
| 2636 | − } | |
| 2734 | + cqe_cons %= sq->max_wqe; | |
| 2637 | 2735 | |
| 2638 | 2736 | if (qp->sq.flushed) { |
| 2639 | 2737 | dev_dbg(&cq->hwq.pdev->dev, |
| @@ -2647,24 +2745,25 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, · 7 unmodified lines | ||
| 2647 | 2745 | */ |
| 2648 | 2746 | cqe = *pcqe; |
| 2649 | 2747 | while (*budget) { |
| 2650 | − sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); | |
| 2651 | − if (sw_cons == cqe_cons) | |
| 2748 | + swq_last = sq->swq_last; | |
| 2749 | + if (swq_last == cqe_cons) | |
| 2652 | 2750 | break; |
| 2653 | − if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { | |
| 2751 | + if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { | |
| 2654 | 2752 | memset(cqe, 0, sizeof(*cqe)); |
| 2655 | 2753 | cqe->status = CQ_REQ_STATUS_OK; |
| 2656 | 2754 | cqe->opcode = CQ_BASE_CQE_TYPE_REQ; |
| 2657 | 2755 | cqe->qp_handle = (u64)(unsigned long)qp; |
| 2658 | 2756 | cqe->src_qp = qp->id; |
| 2659 | − cqe->wr_id = sq->swq[sw_cons].wr_id; | |
| 2660 | − cqe->type = sq->swq[sw_cons].type; | |
| 2757 | + cqe->wr_id = sq->swq[swq_last].wr_id; | |
| 2758 | + cqe->type = sq->swq[swq_last].type; | |
| 2661 | 2759 | cqe++; |
| 2662 | 2760 | (*budget)--; |
| 2663 | 2761 | } |
| 2664 | − sq->hwq.cons++; | |
| 2762 | + bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots); | |
| 2763 | + sq->swq_last = sq->swq[swq_last].next_idx; | |
| 2665 | 2764 | } |
| 2666 | 2765 | *pcqe = cqe; |
| 2667 | − if (!(*budget) && sw_cons != cqe_cons) { | |
| 2766 | + if (!(*budget) && swq_last != cqe_cons) { | |
| 2668 | 2767 | /* Out of budget */ |
| 2669 | 2768 | rc = -EAGAIN; |
| 2670 | 2769 | goto sq_done; |
| @@ -2676,10 +2775,10 @@ do_rq: · 5 unmodified lines | ||
| 2676 | 2775 | cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); |
| 2677 | 2776 | if (cqe_cons == 0xFFFF) { |
| 2678 | 2777 | goto done; |
| 2679 | − } else if (cqe_cons > rq->hwq.max_elements) { | |
| 2778 | + } else if (cqe_cons > rq->max_wqe - 1) { | |
| 2680 | 2779 | dev_err(&cq->hwq.pdev->dev, |
| 2681 | 2780 | "FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n", |
| 2682 | − cqe_cons, rq->hwq.max_elements); | |
| 2781 | + cqe_cons, rq->max_wqe); | |
| 2683 | 2782 | goto done; |
| 2684 | 2783 | } |
| 2685 | 2784 | |
| @@ -39,6 +39,51 @@ · 38 unmodified lines | ||
| 39 | 39 | #ifndef __BNXT_QPLIB_FP_H__ |
| 40 | 40 | #define __BNXT_QPLIB_FP_H__ |
| 41 | 41 | |
| 42 | +/* Few helper structures temporarily defined here | |
| 43 | + * should get rid of these when roce_hsi.h is updated | |
| 44 | + * in original code base | |
| 45 | + */ | |
| 46 | +struct sq_ud_ext_hdr { | |
| 47 | + __le32 dst_qp; | |
| 48 | + __le32 avid; | |
| 49 | + __le64 rsvd; | |
| 50 | +}; | |
| 51 | + | |
| 52 | +struct sq_raw_ext_hdr { | |
| 53 | + __le32 cfa_meta; | |
| 54 | + __le32 rsvd0; | |
| 55 | + __le64 rsvd1; | |
| 56 | +}; | |
| 57 | + | |
| 58 | +struct sq_rdma_ext_hdr { | |
| 59 | + __le64 remote_va; | |
| 60 | + __le32 remote_key; | |
| 61 | + __le32 rsvd; | |
| 62 | +}; | |
| 63 | + | |
| 64 | +struct sq_atomic_ext_hdr { | |
| 65 | + __le64 swap_data; | |
| 66 | + __le64 cmp_data; | |
| 67 | +}; | |
| 68 | + | |
| 69 | +struct sq_fr_pmr_ext_hdr { | |
| 70 | + __le64 pblptr; | |
| 71 | + __le64 va; | |
| 72 | +}; | |
| 73 | + | |
| 74 | +struct sq_bind_ext_hdr { | |
| 75 | + __le64 va; | |
| 76 | + __le32 length_lo; | |
| 77 | + __le32 length_hi; | |
| 78 | +}; | |
| 79 | + | |
| 80 | +struct rq_ext_hdr { | |
| 81 | + __le64 rsvd1; | |
| 82 | + __le64 rsvd2; | |
| 83 | +}; | |
| 84 | + | |
| 85 | +/* Helper structures end */ | |
| 86 | + | |
| 42 | 87 | struct bnxt_qplib_srq { |
| 43 | 88 | struct bnxt_qplib_pd *pd; |
| 44 | 89 | struct bnxt_qplib_dpi *dpi; |
| @@ -74,6 +119,8 @@ struct bnxt_qplib_swq { · 29 unmodified lines | ||
| 74 | 119 | u8 flags; |
| 75 | 120 | u32 start_psn; |
| 76 | 121 | u32 next_psn; |
| 122 | + u32 slot_idx; | |
| 123 | + u8 slots; | |
| 77 | 124 | struct sq_psn_search *psn_search; |
| 78 | 125 | struct sq_psn_search_ext *psn_ext; |
| 79 | 126 | }; |
| @@ -213,6 +260,8 @@ struct bnxt_qplib_q { · 133 unmodified lines | ||
| 213 | 260 | u32 phantom_cqe_cnt; |
| 214 | 261 | u32 next_cq_cons; |
| 215 | 262 | bool flushed; |
| 263 | + u32 swq_start; | |
| 264 | + u32 swq_last; | |
| 216 | 265 | }; |
| 217 | 266 | |
| 218 | 267 | struct bnxt_qplib_qp { |
| @@ -224,9 +273,10 @@ struct bnxt_qplib_qp { · 5 unmodified lines | ||
| 224 | 273 | u32 id; |
| 225 | 274 | u8 type; |
| 226 | 275 | u8 sig_type; |
| 227 | − u32 modify_flags; | |
| 276 | + u8 wqe_mode; | |
| 228 | 277 | u8 state; |
| 229 | 278 | u8 cur_qp_state; |
| 279 | + u64 modify_flags; | |
| 230 | 280 | u32 max_inline_data; |
| 231 | 281 | u32 mtu; |
| 232 | 282 | u8 path_mtu; |
| @@ -300,11 +350,18 @@ struct bnxt_qplib_qp { · 67 unmodified lines | ||
| 300 | 350 | (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ |
| 301 | 351 | !((raw_cons) & (cp_bit))) |
| 302 | 352 | |
| 303 | −static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *qplib_q) | |
| 353 | +static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *que, | |
| 354 | + u8 slots) | |
| 304 | 355 | { |
| 305 | − return HWQ_CMP((qplib_q->hwq.prod + qplib_q->q_full_delta), | |
| 306 | − &qplib_q->hwq) == HWQ_CMP(qplib_q->hwq.cons, | |
| 307 | − &qplib_q->hwq); | |
| 356 | + struct bnxt_qplib_hwq *hwq; | |
| 357 | + int avail; | |
| 358 | + | |
| 359 | + hwq = &que->hwq; | |
| 360 | + /* False full is possible, retrying post-send makes sense */ | |
| 361 | + avail = hwq->cons - hwq->prod; | |
| 362 | + if (hwq->cons <= hwq->prod) | |
| 363 | + avail += hwq->depth; | |
| 364 | + return avail <= slots; | |
| 308 | 365 | } |
| 309 | 366 | |
| 310 | 367 | struct bnxt_qplib_cqe { |
| @@ -489,4 +546,64 @@ int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, · 178 unmodified lines | ||
| 489 | 546 | struct bnxt_qplib_cqe *cqe, |
| 490 | 547 | int num_cqes); |
| 491 | 548 | void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); |
| 549 | + | |
| 550 | +static inline void *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, u32 *swq_idx) | |
| 551 | +{ | |
| 552 | + u32 idx; | |
| 553 | + | |
| 554 | + idx = que->swq_start; | |
| 555 | + if (swq_idx) | |
| 556 | + *swq_idx = idx; | |
| 557 | + return &que->swq[idx]; | |
| 558 | +} | |
| 559 | + | |
| 560 | +static inline void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx) | |
| 561 | +{ | |
| 562 | + que->swq_start = que->swq[idx].next_idx; | |
| 563 | +} | |
| 564 | + | |
| 565 | +static inline u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que) | |
| 566 | +{ | |
| 567 | + return (que->wqe_size * que->max_wqe) / sizeof(struct sq_sge); | |
| 568 | +} | |
| 569 | + | |
| 570 | +static inline u32 bnxt_qplib_set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode) | |
| 571 | +{ | |
| 572 | + return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? | |
| 573 | + que->max_wqe : bnxt_qplib_get_depth(que); | |
| 574 | +} | |
| 575 | + | |
| 576 | +static inline u32 bnxt_qplib_set_sq_max_slot(u8 wqe_mode) | |
| 577 | +{ | |
| 578 | + return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? | |
| 579 | + sizeof(struct sq_send) / sizeof(struct sq_sge) : 1; | |
| 580 | +} | |
| 581 | + | |
| 582 | +static inline u32 bnxt_qplib_set_rq_max_slot(u32 wqe_size) | |
| 583 | +{ | |
| 584 | + return (wqe_size / sizeof(struct sq_sge)); | |
| 585 | +} | |
| 586 | + | |
| 587 | +static inline u16 __xlate_qfd(u16 delta, u16 wqe_bytes) | |
| 588 | +{ | |
| 589 | + /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128 | |
| 590 | + * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128. | |
| 591 | + * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512. | |
| 592 | + * when 8916 is disabled. | |
| 593 | + */ | |
| 594 | + return (delta * wqe_bytes) / sizeof(struct sq_sge); | |
| 595 | +} | |
| 596 | + | |
| 597 | +static inline u16 bnxt_qplib_calc_ilsize(struct bnxt_qplib_swqe *wqe, u16 max) | |
| 598 | +{ | |
| 599 | + u16 size = 0; | |
| 600 | + int indx; | |
| 601 | + | |
| 602 | + for (indx = 0; indx < wqe->num_sge; indx++) | |
| 603 | + size += wqe->sg_list[indx].size; | |
| 604 | + if (size > max) | |
| 605 | + size = max; | |
| 606 | + | |
| 607 | + return size; | |
| 608 | +} | |
| 492 | 609 | #endif /* __BNXT_QPLIB_FP_H__ */ |
| @@ -41,6 +41,28 @@ · 40 unmodified lines | ||
| 41 | 41 | |
| 42 | 42 | extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; |
| 43 | 43 | |
| 44 | +#define CHIP_NUM_57508 0x1750 | |
| 45 | +#define CHIP_NUM_57504 0x1751 | |
| 46 | +#define CHIP_NUM_57502 0x1752 | |
| 47 | + | |
| 48 | +enum bnxt_qplib_wqe_mode { | |
| 49 | + BNXT_QPLIB_WQE_MODE_STATIC = 0x00, | |
| 50 | + BNXT_QPLIB_WQE_MODE_VARIABLE = 0x01, | |
| 51 | + BNXT_QPLIB_WQE_MODE_INVALID = 0x02 | |
| 52 | +}; | |
| 53 | + | |
| 54 | +struct bnxt_qplib_drv_modes { | |
| 55 | + u8 wqe_mode; | |
| 56 | + /* Other modes to follow here */ | |
| 57 | +}; | |
| 58 | + | |
| 59 | +struct bnxt_qplib_chip_ctx { | |
| 60 | + u16 chip_num; | |
| 61 | + u8 chip_rev; | |
| 62 | + u8 chip_metal; | |
| 63 | + struct bnxt_qplib_drv_modes modes; | |
| 64 | +}; | |
| 65 | + | |
| 44 | 66 | #define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *)) |
| 45 | 67 | #define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1) |
| 46 | 68 | #define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG) |
| @@ -141,6 +163,9 @@ struct bnxt_qplib_hwq { · 94 unmodified lines | ||
| 141 | 163 | u32 cons; /* raw */ |
| 142 | 164 | u8 cp_bit; |
| 143 | 165 | u8 is_user; |
| 166 | + u64 *pad_pg; | |
| 167 | + u32 pad_stride; | |
| 168 | + u32 pad_pgofft; | |
| 144 | 169 | }; |
| 145 | 170 | |
| 146 | 171 | struct bnxt_qplib_db_info { |
| @@ -148,6 +173,7 @@ struct bnxt_qplib_db_info { · 1 unmodified line | ||
| 148 | 173 | void __iomem *priv_db; |
| 149 | 174 | struct bnxt_qplib_hwq *hwq; |
| 150 | 175 | u32 xid; |
| 176 | + u32 max_slot; | |
| 151 | 177 | }; |
| 152 | 178 | |
| 153 | 179 | /* Tables */ |
| @@ -230,16 +256,6 @@ struct bnxt_qplib_ctx { · 76 unmodified lines | ||
| 230 | 256 | u64 hwrm_intf_ver; |
| 231 | 257 | }; |
| 232 | 258 | |
| 233 | −struct bnxt_qplib_chip_ctx { | |
| 234 | − u16 chip_num; | |
| 235 | − u8 chip_rev; | |
| 236 | − u8 chip_metal; | |
| 237 | −}; | |
| 238 | − | |
| 239 | −#define CHIP_NUM_57508 0x1750 | |
| 240 | −#define CHIP_NUM_57504 0x1751 | |
| 241 | −#define CHIP_NUM_57502 0x1752 | |
| 242 | − | |
| 243 | 259 | struct bnxt_qplib_res { |
| 244 | 260 | struct pci_dev *pdev; |
| 245 | 261 | struct bnxt_qplib_chip_ctx *cctx; |
| @@ -317,6 +333,14 @@ static inline void *bnxt_qplib_get_qe(struct bnxt_qplib_hwq *hwq, · 71 unmodified lines | ||
| 317 | 333 | return (void *)(hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx); |
| 318 | 334 | } |
| 319 | 335 | |
| 336 | +static inline void *bnxt_qplib_get_prod_qe(struct bnxt_qplib_hwq *hwq, u32 idx) | |
| 337 | +{ | |
| 338 | + idx += hwq->prod; | |
| 339 | + if (idx >= hwq->depth) | |
| 340 | + idx -= hwq->depth; | |
| 341 | + return bnxt_qplib_get_qe(hwq, idx, NULL); | |
| 342 | +} | |
| 343 | + | |
| 320 | 344 | #define to_bnxt_qplib(ptr, type, member) \ |
| 321 | 345 | container_of(ptr, type, member) |
| 322 | 346 | |
| @@ -351,6 +375,17 @@ int bnxt_qplib_alloc_ctx(struct bnxt_qplib_res *res, · 28 unmodified lines | ||
| 351 | 375 | struct bnxt_qplib_ctx *ctx, |
| 352 | 376 | bool virt_fn, bool is_p5); |
| 353 | 377 | |
| 378 | +static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_hwq *hwq, u32 cnt) | |
| 379 | +{ | |
| 380 | + hwq->prod = (hwq->prod + cnt) % hwq->depth; | |
| 381 | +} | |
| 382 | + | |
| 383 | +static inline void bnxt_qplib_hwq_incr_cons(struct bnxt_qplib_hwq *hwq, | |
| 384 | + u32 cnt) | |
| 385 | +{ | |
| 386 | + hwq->cons = (hwq->cons + cnt) % hwq->depth; | |
| 387 | +} | |
| 388 | + | |
| 354 | 389 | static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info, |
| 355 | 390 | bool arm) |
| 356 | 391 | { |
| @@ -383,8 +418,7 @@ static inline void bnxt_qplib_ring_prod_db(struct bnxt_qplib_db_info *info, · 26 unmodified lines | ||
| 383 | 418 | |
| 384 | 419 | key = (info->xid & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | type; |
| 385 | 420 | key <<= 32; |
| 386 | − key |= (info->hwq->prod & (info->hwq->max_elements - 1)) & | |
| 387 | − DBC_DBC_INDEX_MASK; | |
| 421 | + key |= ((info->hwq->prod / info->max_slot)) & DBC_DBC_INDEX_MASK; | |
| 388 | 422 | writeq(key, info->db); |
| 389 | 423 | } |
| 390 | 424 | |
| @@ -1126,6 +1126,7 @@ struct cmdq_create_qp { · 1125 unmodified lines | ||
| 1126 | 1126 | #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL |
| 1127 | 1127 | #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL |
| 1128 | 1128 | #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL |
| 1129 | + #define CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED 0x10UL | |
| 1129 | 1130 | u8 type; |
| 1130 | 1131 | #define CMDQ_CREATE_QP_TYPE_RC 0x2UL |
| 1131 | 1132 | #define CMDQ_CREATE_QP_TYPE_UD 0x4UL |
| @@ -980,7 +980,7 @@ int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len); · 979 unmodified lines | ||
| 980 | 980 | void c4iw_qp_add_ref(struct ib_qp *qp); |
| 981 | 981 | void c4iw_qp_rem_ref(struct ib_qp *qp); |
| 982 | 982 | struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 983 | − u32 max_num_sg, struct ib_udata *udata); | |
| 983 | + u32 max_num_sg); | |
| 984 | 984 | int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, |
| 985 | 985 | unsigned int *sg_offset); |
| 986 | 986 | int c4iw_dealloc_mw(struct ib_mw *mw); |
| @@ -1053,8 +1053,9 @@ int c4iw_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, · 66 unmodified lines | ||
| 1053 | 1053 | const struct ib_recv_wr **bad_wr); |
| 1054 | 1054 | struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp); |
| 1055 | 1055 | |
| 1056 | −typedef int c4iw_restrack_func(struct sk_buff *msg, | |
| 1057 | − struct rdma_restrack_entry *res); | |
| 1058 | −extern c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX]; | |
| 1056 | +int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr); | |
| 1057 | +int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq); | |
| 1058 | +int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp); | |
| 1059 | +int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, struct rdma_cm_id *cm_id); | |
| 1059 | 1060 | |
| 1060 | 1061 | #endif |
| @@ -399,7 +399,6 @@ static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag) · 398 unmodified lines | ||
| 399 | 399 | mmid = stag >> 8; |
| 400 | 400 | mhp->ibmr.rkey = mhp->ibmr.lkey = stag; |
| 401 | 401 | mhp->ibmr.length = mhp->attr.len; |
| 402 | − mhp->ibmr.iova = mhp->attr.va_fbo; | |
| 403 | 402 | mhp->ibmr.page_size = 1U << (mhp->attr.page_size + 12); |
| 404 | 403 | pr_debug("mmid 0x%x mhp %p\n", mmid, mhp); |
| 405 | 404 | return xa_insert_irq(&mhp->rhp->mrs, mmid, mhp, GFP_KERNEL); |
| @@ -691,7 +690,7 @@ int c4iw_dealloc_mw(struct ib_mw *mw) · 285 unmodified lines | ||
| 691 | 690 | } |
| 692 | 691 | |
| 693 | 692 | struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 694 | − u32 max_num_sg, struct ib_udata *udata) | |
| 693 | + u32 max_num_sg) | |
| 695 | 694 | { |
| 696 | 695 | struct c4iw_dev *rhp; |
| 697 | 696 | struct c4iw_pd *php; |
| @@ -236,14 +236,6 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata) · 235 unmodified lines | ||
| 236 | 236 | return 0; |
| 237 | 237 | } |
| 238 | 238 | |
| 239 | −static int c4iw_query_pkey(struct ib_device *ibdev, u8 port, u16 index, | |
| 240 | − u16 *pkey) | |
| 241 | −{ | |
| 242 | − pr_debug("ibdev %p\n", ibdev); | |
| 243 | − *pkey = 0; | |
| 244 | − return 0; | |
| 245 | −} | |
| 246 | − | |
| 247 | 239 | static int c4iw_query_gid(struct ib_device *ibdev, u8 port, int index, |
| 248 | 240 | union ib_gid *gid) |
| 249 | 241 | { |
| @@ -317,7 +309,6 @@ static int c4iw_query_port(struct ib_device *ibdev, u8 port, · 67 unmodified lines | ||
| 317 | 309 | IB_PORT_DEVICE_MGMT_SUP | |
| 318 | 310 | IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; |
| 319 | 311 | props->gid_tbl_len = 1; |
| 320 | − props->pkey_tbl_len = 1; | |
| 321 | 312 | props->max_msg_sz = -1; |
| 322 | 313 | |
| 323 | 314 | return ret; |
| @@ -439,7 +430,6 @@ static int c4iw_port_immutable(struct ib_device *ibdev, u8 port_num, · 115 unmodified lines | ||
| 439 | 430 | if (err) |
| 440 | 431 | return err; |
| 441 | 432 | |
| 442 | − immutable->pkey_tbl_len = attr.pkey_tbl_len; | |
| 443 | 433 | immutable->gid_tbl_len = attr.gid_tbl_len; |
| 444 | 434 | |
| 445 | 435 | return 0; |
| @@ -458,13 +448,6 @@ static void get_dev_fw_str(struct ib_device *dev, char *str) · 12 unmodified lines | ||
| 458 | 448 | FW_HDR_FW_VER_BUILD_G(c4iw_dev->rdev.lldi.fw_vers)); |
| 459 | 449 | } |
| 460 | 450 | |
| 461 | −static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res) | |
| 462 | −{ | |
| 463 | − return (res->type < ARRAY_SIZE(c4iw_restrack_funcs) && | |
| 464 | − c4iw_restrack_funcs[res->type]) ? | |
| 465 | − c4iw_restrack_funcs[res->type](msg, res) : 0; | |
| 466 | −} | |
| 467 | − | |
| 468 | 451 | static const struct ib_device_ops c4iw_dev_ops = { |
| 469 | 452 | .owner = THIS_MODULE, |
| 470 | 453 | .driver_id = RDMA_DRIVER_CXGB4, |
| @@ -485,7 +468,9 @@ static const struct ib_device_ops c4iw_dev_ops = { · 14 unmodified lines | ||
| 485 | 468 | .destroy_cq = c4iw_destroy_cq, |
| 486 | 469 | .destroy_qp = c4iw_destroy_qp, |
| 487 | 470 | .destroy_srq = c4iw_destroy_srq, |
| 488 | − .fill_res_entry = fill_res_entry, | |
| 471 | + .fill_res_cq_entry = c4iw_fill_res_cq_entry, | |
| 472 | + .fill_res_cm_id_entry = c4iw_fill_res_cm_id_entry, | |
| 473 | + .fill_res_mr_entry = c4iw_fill_res_mr_entry, | |
| 489 | 474 | .get_dev_fw_str = get_dev_fw_str, |
| 490 | 475 | .get_dma_mr = c4iw_get_dma_mr, |
| 491 | 476 | .get_hw_stats = c4iw_get_mib, |
| @@ -508,7 +493,6 @@ static const struct ib_device_ops c4iw_dev_ops = { · 16 unmodified lines | ||
| 508 | 493 | .post_srq_recv = c4iw_post_srq_recv, |
| 509 | 494 | .query_device = c4iw_query_device, |
| 510 | 495 | .query_gid = c4iw_query_gid, |
| 511 | − .query_pkey = c4iw_query_pkey, | |
| 512 | 496 | .query_port = c4iw_query_port, |
| 513 | 497 | .query_qp = c4iw_ib_query_qp, |
| 514 | 498 | .reg_user_mr = c4iw_reg_user_mr, |
| @@ -134,10 +134,8 @@ err: · 133 unmodified lines | ||
| 134 | 134 | return -EMSGSIZE; |
| 135 | 135 | } |
| 136 | 136 | |
| 137 | −static int fill_res_qp_entry(struct sk_buff *msg, | |
| 138 | − struct rdma_restrack_entry *res) | |
| 137 | +int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp) | |
| 139 | 138 | { |
| 140 | − struct ib_qp *ibqp = container_of(res, struct ib_qp, res); | |
| 141 | 139 | struct t4_swsqe *fsp = NULL, *lsp = NULL; |
| 142 | 140 | struct c4iw_qp *qhp = to_c4iw_qp(ibqp); |
| 143 | 141 | u16 first_sq_idx = 0, last_sq_idx = 0; |
| @@ -195,10 +193,9 @@ union union_ep { · 51 unmodified lines | ||
| 195 | 193 | struct c4iw_ep ep; |
| 196 | 194 | }; |
| 197 | 195 | |
| 198 | −static int fill_res_ep_entry(struct sk_buff *msg, | |
| 199 | − struct rdma_restrack_entry *res) | |
| 196 | +int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, | |
| 197 | + struct rdma_cm_id *cm_id) | |
| 200 | 198 | { |
| 201 | − struct rdma_cm_id *cm_id = rdma_res_to_id(res); | |
| 202 | 199 | struct nlattr *table_attr; |
| 203 | 200 | struct c4iw_ep_common *epcp; |
| 204 | 201 | struct c4iw_listen_ep *listen_ep = NULL; |
| @@ -372,10 +369,8 @@ err: · 167 unmodified lines | ||
| 372 | 369 | return -EMSGSIZE; |
| 373 | 370 | } |
| 374 | 371 | |
| 375 | −static int fill_res_cq_entry(struct sk_buff *msg, | |
| 376 | − struct rdma_restrack_entry *res) | |
| 372 | +int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq) | |
| 377 | 373 | { |
| 378 | − struct ib_cq *ibcq = container_of(res, struct ib_cq, res); | |
| 379 | 374 | struct c4iw_cq *chp = to_c4iw_cq(ibcq); |
| 380 | 375 | struct nlattr *table_attr; |
| 381 | 376 | struct t4_cqe hwcqes[2]; |
| @@ -433,10 +428,8 @@ err: · 51 unmodified lines | ||
| 433 | 428 | return -EMSGSIZE; |
| 434 | 429 | } |
| 435 | 430 | |
| 436 | −static int fill_res_mr_entry(struct sk_buff *msg, | |
| 437 | − struct rdma_restrack_entry *res) | |
| 431 | +int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr) | |
| 438 | 432 | { |
| 439 | − struct ib_mr *ibmr = container_of(res, struct ib_mr, res); | |
| 440 | 433 | struct c4iw_mr *mhp = to_c4iw_mr(ibmr); |
| 441 | 434 | struct c4iw_dev *dev = mhp->rhp; |
| 442 | 435 | u32 stag = mhp->attr.stag; |
| @@ -492,10 +485,3 @@ err_cancel_table: · 49 unmodified lines | ||
| 492 | 485 | err: |
| 493 | 486 | return -EMSGSIZE; |
| 494 | 487 | } |
| 495 | − | |
| 496 | −c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX] = { | |
| 497 | − [RDMA_RESTRACK_QP] = fill_res_qp_entry, | |
| 498 | − [RDMA_RESTRACK_CM_ID] = fill_res_ep_entry, | |
| 499 | − [RDMA_RESTRACK_CQ] = fill_res_cq_entry, | |
| 500 | − [RDMA_RESTRACK_MR] = fill_res_mr_entry, | |
| 501 | −}; | |
| @@ -606,8 +606,8 @@ struct efa_admin_feature_queue_attr_desc { · 605 unmodified lines | ||
| 606 | 606 | /* Number of sub-CQs to be created for each CQ */ |
| 607 | 607 | u16 sub_cqs_per_cq; |
| 608 | 608 | |
| 609 | − /* MBZ */ | |
| 610 | − u16 reserved; | |
| 609 | + /* Minimum number of WQEs per SQ */ | |
| 610 | + u16 min_sq_depth; | |
| 611 | 611 | |
| 612 | 612 | /* Maximum number of SGEs (buffers) allowed for a single send WQE */ |
| 613 | 613 | u16 max_wr_send_sges; |
| @@ -632,6 +632,17 @@ struct efa_admin_feature_queue_attr_desc { · 18 unmodified lines | ||
| 632 | 632 | |
| 633 | 633 | /* Maximum number of SGEs for a single RDMA read WQE */ |
| 634 | 634 | u16 max_wr_rdma_sges; |
| 635 | + | |
| 636 | + /* | |
| 637 | + * Maximum number of bytes that can be written to SQ between two | |
| 638 | + * consecutive doorbells (in units of 64B). Driver must ensure that only | |
| 639 | + * complete WQEs are written to queue before issuing a doorbell. | |
| 640 | + * Examples: max_tx_batch=16 and WQE size = 64B, means up to 16 WQEs can | |
| 641 | + * be written to SQ between two consecutive doorbells. max_tx_batch=11 | |
| 642 | + * and WQE size = 128B, means up to 5 WQEs can be written to SQ between | |
| 643 | + * two consecutive doorbells. Zero means unlimited. | |
| 644 | + */ | |
| 645 | + u16 max_tx_batch; | |
| 635 | 646 | }; |
| 636 | 647 | |
| 637 | 648 | struct efa_admin_feature_aenq_desc { |
| @@ -480,6 +480,8 @@ int efa_com_get_device_attr(struct efa_com_dev *edev, · 479 unmodified lines | ||
| 480 | 480 | result->max_llq_size = resp.u.queue_attr.max_llq_size; |
| 481 | 481 | result->sub_cqs_per_cq = resp.u.queue_attr.sub_cqs_per_cq; |
| 482 | 482 | result->max_wr_rdma_sge = resp.u.queue_attr.max_wr_rdma_sges; |
| 483 | + result->max_tx_batch = resp.u.queue_attr.max_tx_batch; | |
| 484 | + result->min_sq_depth = resp.u.queue_attr.min_sq_depth; | |
| 483 | 485 | |
| 484 | 486 | err = efa_com_get_feature(edev, &resp, EFA_ADMIN_NETWORK_ATTR); |
| 485 | 487 | if (err) { |
| @@ -127,6 +127,8 @@ struct efa_com_get_device_attr_result { · 126 unmodified lines | ||
| 127 | 127 | u16 max_sq_sge; |
| 128 | 128 | u16 max_rq_sge; |
| 129 | 129 | u16 max_wr_rdma_sge; |
| 130 | + u16 max_tx_batch; | |
| 131 | + u16 min_sq_depth; | |
| 130 | 132 | u8 db_bar; |
| 131 | 133 | }; |
| 132 | 134 | |
| @@ -12,10 +12,12 @@ · 11 unmodified lines | ||
| 12 | 12 | |
| 13 | 13 | #include "efa.h" |
| 14 | 14 | |
| 15 | −#define PCI_DEV_ID_EFA_VF 0xefa0 | |
| 15 | +#define PCI_DEV_ID_EFA0_VF 0xefa0 | |
| 16 | +#define PCI_DEV_ID_EFA1_VF 0xefa1 | |
| 16 | 17 | |
| 17 | 18 | static const struct pci_device_id efa_pci_tbl[] = { |
| 18 | − { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA_VF) }, | |
| 19 | + { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) }, | |
| 20 | + { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) }, | |
| 19 | 21 | { } |
| 20 | 22 | }; |
| 21 | 23 | |
| @@ -1502,11 +1502,39 @@ static int efa_dealloc_uar(struct efa_dev *dev, u16 uarn) · 1501 unmodified lines | ||
| 1502 | 1502 | return efa_com_dealloc_uar(&dev->edev, ¶ms); |
| 1503 | 1503 | } |
| 1504 | 1504 | |
| 1505 | +#define EFA_CHECK_USER_COMP(_dev, _comp_mask, _attr, _mask, _attr_str) \ | |
| 1506 | + (_attr_str = (!(_dev)->dev_attr._attr || ((_comp_mask) & (_mask))) ? \ | |
| 1507 | + NULL : #_attr) | |
| 1508 | + | |
| 1509 | +static int efa_user_comp_handshake(const struct ib_ucontext *ibucontext, | |
| 1510 | + const struct efa_ibv_alloc_ucontext_cmd *cmd) | |
| 1511 | +{ | |
| 1512 | + struct efa_dev *dev = to_edev(ibucontext->device); | |
| 1513 | + char *attr_str; | |
| 1514 | + | |
| 1515 | + if (EFA_CHECK_USER_COMP(dev, cmd->comp_mask, max_tx_batch, | |
| 1516 | + EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH, attr_str)) | |
| 1517 | + goto err; | |
| 1518 | + | |
| 1519 | + if (EFA_CHECK_USER_COMP(dev, cmd->comp_mask, min_sq_depth, | |
| 1520 | + EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR, | |
| 1521 | + attr_str)) | |
| 1522 | + goto err; | |
| 1523 | + | |
| 1524 | + return 0; | |
| 1525 | + | |
| 1526 | +err: | |
| 1527 | + ibdev_dbg(&dev->ibdev, "Userspace handshake failed for %s attribute\n", | |
| 1528 | + attr_str); | |
| 1529 | + return -EOPNOTSUPP; | |
| 1530 | +} | |
| 1531 | + | |
| 1505 | 1532 | int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) |
| 1506 | 1533 | { |
| 1507 | 1534 | struct efa_ucontext *ucontext = to_eucontext(ibucontext); |
| 1508 | 1535 | struct efa_dev *dev = to_edev(ibucontext->device); |
| 1509 | 1536 | struct efa_ibv_alloc_ucontext_resp resp = {}; |
| 1537 | + struct efa_ibv_alloc_ucontext_cmd cmd = {}; | |
| 1510 | 1538 | struct efa_com_alloc_uar_result result; |
| 1511 | 1539 | int err; |
| 1512 | 1540 | |
| @@ -1515,6 +1543,18 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) · 2 unmodified lines | ||
| 1515 | 1543 | * we will ack input fields in our response. |
| 1516 | 1544 | */ |
| 1517 | 1545 | |
| 1546 | + err = ib_copy_from_udata(&cmd, udata, | |
| 1547 | + min(sizeof(cmd), udata->inlen)); | |
| 1548 | + if (err) { | |
| 1549 | + ibdev_dbg(&dev->ibdev, | |
| 1550 | + "Cannot copy udata for alloc_ucontext\n"); | |
| 1551 | + goto err_out; | |
| 1552 | + } | |
| 1553 | + | |
| 1554 | + err = efa_user_comp_handshake(ibucontext, &cmd); | |
| 1555 | + if (err) | |
| 1556 | + goto err_out; | |
| 1557 | + | |
| 1518 | 1558 | err = efa_com_alloc_uar(&dev->edev, &result); |
| 1519 | 1559 | if (err) |
| 1520 | 1560 | goto err_out; |
| @@ -1526,6 +1566,8 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) · 5 unmodified lines | ||
| 1526 | 1566 | resp.sub_cqs_per_cq = dev->dev_attr.sub_cqs_per_cq; |
| 1527 | 1567 | resp.inline_buf_size = dev->dev_attr.inline_buf_size; |
| 1528 | 1568 | resp.max_llq_size = dev->dev_attr.max_llq_size; |
| 1569 | + resp.max_tx_batch = dev->dev_attr.max_tx_batch; | |
| 1570 | + resp.min_sq_wr = dev->dev_attr.min_sq_depth; | |
| 1529 | 1571 | |
| 1530 | 1572 | if (udata && udata->outlen) { |
| 1531 | 1573 | err = ib_copy_to_udata(udata, &resp, |
| @@ -7317,11 +7317,11 @@ static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width) · 7316 unmodified lines | ||
| 7317 | 7317 | case 1: return OPA_LINK_WIDTH_1X; |
| 7318 | 7318 | case 2: return OPA_LINK_WIDTH_2X; |
| 7319 | 7319 | case 3: return OPA_LINK_WIDTH_3X; |
| 7320 | + case 4: return OPA_LINK_WIDTH_4X; | |
| 7320 | 7321 | default: |
| 7321 | 7322 | dd_dev_info(dd, "%s: invalid width %d, using 4\n", |
| 7322 | 7323 | __func__, width); |
| 7323 | − /* fall through */ | |
| 7324 | − case 4: return OPA_LINK_WIDTH_4X; | |
| 7324 | + return OPA_LINK_WIDTH_4X; | |
| 7325 | 7325 | } |
| 7326 | 7326 | } |
| 7327 | 7327 | |
| @@ -7376,12 +7376,13 @@ static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width, · 48 unmodified lines | ||
| 7376 | 7376 | case 0: |
| 7377 | 7377 | dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G; |
| 7378 | 7378 | break; |
| 7379 | + case 1: | |
| 7380 | + dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G; | |
| 7381 | + break; | |
| 7379 | 7382 | default: |
| 7380 | 7383 | dd_dev_err(dd, |
| 7381 | 7384 | "%s: unexpected max rate %d, using 25Gb\n", |
| 7382 | 7385 | __func__, (int)max_rate); |
| 7383 | − /* fall through */ | |
| 7384 | − case 1: | |
| 7385 | 7386 | dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G; |
| 7386 | 7387 | break; |
| 7387 | 7388 | } |
| @@ -12878,11 +12879,6 @@ bail: · 5490 unmodified lines | ||
| 12878 | 12879 | static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate) |
| 12879 | 12880 | { |
| 12880 | 12881 | switch (chip_lstate) { |
| 12881 | − default: | |
| 12882 | − dd_dev_err(dd, | |
| 12883 | − "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n", | |
| 12884 | − chip_lstate); | |
| 12885 | − /* fall through */ | |
| 12886 | 12882 | case LSTATE_DOWN: |
| 12887 | 12883 | return IB_PORT_DOWN; |
| 12888 | 12884 | case LSTATE_INIT: |
| @@ -12891,6 +12887,11 @@ static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate) · 2 unmodified lines | ||
| 12891 | 12887 | return IB_PORT_ARMED; |
| 12892 | 12888 | case LSTATE_ACTIVE: |
| 12893 | 12889 | return IB_PORT_ACTIVE; |
| 12890 | + default: | |
| 12891 | + dd_dev_err(dd, | |
| 12892 | + "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n", | |
| 12893 | + chip_lstate); | |
| 12894 | + return IB_PORT_DOWN; | |
| 12894 | 12895 | } |
| 12895 | 12896 | } |
| 12896 | 12897 | |
| @@ -12898,10 +12899,6 @@ u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate) · 1 unmodified line | ||
| 12898 | 12899 | { |
| 12899 | 12900 | /* look at the HFI meta-states only */ |
| 12900 | 12901 | switch (chip_pstate & 0xf0) { |
| 12901 | − default: | |
| 12902 | − dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n", | |
| 12903 | − chip_pstate); | |
| 12904 | − /* fall through */ | |
| 12905 | 12902 | case PLS_DISABLED: |
| 12906 | 12903 | return IB_PORTPHYSSTATE_DISABLED; |
| 12907 | 12904 | case PLS_OFFLINE: |
| @@ -12914,6 +12911,10 @@ u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate) · 6 unmodified lines | ||
| 12914 | 12911 | return IB_PORTPHYSSTATE_LINKUP; |
| 12915 | 12912 | case PLS_PHYTEST: |
| 12916 | 12913 | return IB_PORTPHYSSTATE_PHY_TEST; |
| 12914 | + default: | |
| 12915 | + dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n", | |
| 12916 | + chip_pstate); | |
| 12917 | + return IB_PORTPHYSSTATE_DISABLED; | |
| 12917 | 12918 | } |
| 12918 | 12919 | } |
| 12919 | 12920 | |
| @@ -1868,11 +1868,8 @@ int parse_platform_config(struct hfi1_devdata *dd) · 1867 unmodified lines | ||
| 1868 | 1868 | 2; |
| 1869 | 1869 | break; |
| 1870 | 1870 | case PLATFORM_CONFIG_RX_PRESET_TABLE: |
| 1871 | − /* fall through */ | |
| 1872 | 1871 | case PLATFORM_CONFIG_TX_PRESET_TABLE: |
| 1873 | − /* fall through */ | |
| 1874 | 1872 | case PLATFORM_CONFIG_QSFP_ATTEN_TABLE: |
| 1875 | − /* fall through */ | |
| 1876 | 1873 | case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE: |
| 1877 | 1874 | pcfgcache->config_tables[table_type].num_table = |
| 1878 | 1875 | table_length_dwords; |
| @@ -1890,15 +1887,10 @@ int parse_platform_config(struct hfi1_devdata *dd) · 11 unmodified lines | ||
| 1890 | 1887 | /* metadata table */ |
| 1891 | 1888 | switch (table_type) { |
| 1892 | 1889 | case PLATFORM_CONFIG_SYSTEM_TABLE: |
| 1893 | − /* fall through */ | |
| 1894 | 1890 | case PLATFORM_CONFIG_PORT_TABLE: |
| 1895 | − /* fall through */ | |
| 1896 | 1891 | case PLATFORM_CONFIG_RX_PRESET_TABLE: |
| 1897 | − /* fall through */ | |
| 1898 | 1892 | case PLATFORM_CONFIG_TX_PRESET_TABLE: |
| 1899 | − /* fall through */ | |
| 1900 | 1893 | case PLATFORM_CONFIG_QSFP_ATTEN_TABLE: |
| 1901 | − /* fall through */ | |
| 1902 | 1894 | case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE: |
| 1903 | 1895 | break; |
| 1904 | 1896 | default: |
| @@ -2027,15 +2019,10 @@ static int get_platform_fw_field_metadata(struct hfi1_devdata *dd, int table, · 122 unmodified lines | ||
| 2027 | 2019 | |
| 2028 | 2020 | switch (table) { |
| 2029 | 2021 | case PLATFORM_CONFIG_SYSTEM_TABLE: |
| 2030 | − /* fall through */ | |
| 2031 | 2022 | case PLATFORM_CONFIG_PORT_TABLE: |
| 2032 | − /* fall through */ | |
| 2033 | 2023 | case PLATFORM_CONFIG_RX_PRESET_TABLE: |
| 2034 | − /* fall through */ | |
| 2035 | 2024 | case PLATFORM_CONFIG_TX_PRESET_TABLE: |
| 2036 | − /* fall through */ | |
| 2037 | 2025 | case PLATFORM_CONFIG_QSFP_ATTEN_TABLE: |
| 2038 | − /* fall through */ | |
| 2039 | 2026 | case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE: |
| 2040 | 2027 | if (field && field < platform_config_table_limits[table]) |
| 2041 | 2028 | src_ptr = |
| @@ -2138,11 +2125,8 @@ int get_platform_config_field(struct hfi1_devdata *dd, · 96 unmodified lines | ||
| 2138 | 2125 | pcfgcache->config_tables[table_type].table; |
| 2139 | 2126 | break; |
| 2140 | 2127 | case PLATFORM_CONFIG_RX_PRESET_TABLE: |
| 2141 | − /* fall through */ | |
| 2142 | 2128 | case PLATFORM_CONFIG_TX_PRESET_TABLE: |
| 2143 | − /* fall through */ | |
| 2144 | 2129 | case PLATFORM_CONFIG_QSFP_ATTEN_TABLE: |
| 2145 | − /* fall through */ | |
| 2146 | 2130 | case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE: |
| 2147 | 2131 | src_ptr = pcfgcache->config_tables[table_type].table; |
| 2148 | 2132 | |
| @@ -721,7 +721,7 @@ static int check_mkey(struct hfi1_ibport *ibp, struct ib_mad_hdr *mad, · 720 unmodified lines | ||
| 721 | 721 | /* Bad mkey not a violation below level 2 */ |
| 722 | 722 | if (ibp->rvp.mkeyprot < 2) |
| 723 | 723 | break; |
| 724 | − /* fall through */ | |
| 724 | + fallthrough; | |
| 725 | 725 | case IB_MGMT_METHOD_SET: |
| 726 | 726 | case IB_MGMT_METHOD_TRAP_REPRESS: |
| 727 | 727 | if (ibp->rvp.mkey_violations != 0xFFFF) |
| @@ -1272,7 +1272,7 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp, · 544 unmodified lines | ||
| 1272 | 1272 | case IB_PORT_NOP: |
| 1273 | 1273 | if (phys_state == IB_PORTPHYSSTATE_NOP) |
| 1274 | 1274 | break; |
| 1275 | − /* FALLTHROUGH */ | |
| 1275 | + fallthrough; | |
| 1276 | 1276 | case IB_PORT_DOWN: |
| 1277 | 1277 | if (phys_state == IB_PORTPHYSSTATE_NOP) { |
| 1278 | 1278 | link_state = HLS_DN_DOWNDEF; |
| @@ -2300,7 +2300,6 @@ static int __subn_set_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data, · 1021 unmodified lines | ||
| 2300 | 2300 | * can be changed from the default values |
| 2301 | 2301 | */ |
| 2302 | 2302 | case OPA_VLARB_PREEMPT_ELEMENTS: |
| 2303 | − /* FALLTHROUGH */ | |
| 2304 | 2303 | case OPA_VLARB_PREEMPT_MATRIX: |
| 2305 | 2304 | smp->status |= IB_SMP_UNSUP_METH_ATTR; |
| 2306 | 2305 | break; |
| @@ -4170,7 +4169,7 @@ static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am, · 1863 unmodified lines | ||
| 4170 | 4169 | return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; |
| 4171 | 4170 | if (ibp->rvp.port_cap_flags & IB_PORT_SM) |
| 4172 | 4171 | return IB_MAD_RESULT_SUCCESS; |
| 4173 | − /* FALLTHROUGH */ | |
| 4172 | + fallthrough; | |
| 4174 | 4173 | default: |
| 4175 | 4174 | smp->status |= IB_SMP_UNSUP_METH_ATTR; |
| 4176 | 4175 | ret = reply((struct ib_mad_hdr *)smp); |
| @@ -4240,7 +4239,7 @@ static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am, · 63 unmodified lines | ||
| 4240 | 4239 | return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; |
| 4241 | 4240 | if (ibp->rvp.port_cap_flags & IB_PORT_SM) |
| 4242 | 4241 | return IB_MAD_RESULT_SUCCESS; |
| 4243 | − /* FALLTHROUGH */ | |
| 4242 | + fallthrough; | |
| 4244 | 4243 | default: |
| 4245 | 4244 | smp->status |= IB_SMP_UNSUP_METH_ATTR; |
| 4246 | 4245 | ret = reply((struct ib_mad_hdr *)smp); |
| @@ -306,7 +306,7 @@ int pcie_speeds(struct hfi1_devdata *dd) · 305 unmodified lines | ||
| 306 | 306 | ret = pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &linkcap); |
| 307 | 307 | if (ret) { |
| 308 | 308 | dd_dev_err(dd, "Unable to read from PCI config\n"); |
| 309 | − return ret; | |
| 309 | + return pcibios_err_to_errno(ret); | |
| 310 | 310 | } |
| 311 | 311 | |
| 312 | 312 | if ((linkcap & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_8_0GB) { |
| @@ -334,10 +334,14 @@ int pcie_speeds(struct hfi1_devdata *dd) · 21 unmodified lines | ||
| 334 | 334 | return 0; |
| 335 | 335 | } |
| 336 | 336 | |
| 337 | −/* restore command and BARs after a reset has wiped them out */ | |
| 337 | +/** | |
| 338 | + * Restore command and BARs after a reset has wiped them out | |
| 339 | + * | |
| 340 | + * Returns 0 on success, otherwise a negative error value | |
| 341 | + */ | |
| 338 | 342 | int restore_pci_variables(struct hfi1_devdata *dd) |
| 339 | 343 | { |
| 340 | − int ret = 0; | |
| 344 | + int ret; | |
| 341 | 345 | |
| 342 | 346 | ret = pci_write_config_word(dd->pcidev, PCI_COMMAND, dd->pci_command); |
| 343 | 347 | if (ret) |
| @@ -386,13 +390,17 @@ int restore_pci_variables(struct hfi1_devdata *dd) · 42 unmodified lines | ||
| 386 | 390 | |
| 387 | 391 | error: |
| 388 | 392 | dd_dev_err(dd, "Unable to write to PCI config\n"); |
| 389 | − return ret; | |
| 393 | + return pcibios_err_to_errno(ret); | |
| 390 | 394 | } |
| 391 | 395 | |
| 392 | −/* Save BARs and command to rewrite after device reset */ | |
| 396 | +/** | |
| 397 | + * Save BARs and command to rewrite after device reset | |
| 398 | + * | |
| 399 | + * Returns 0 on success, otherwise a negative error value | |
| 400 | + */ | |
| 393 | 401 | int save_pci_variables(struct hfi1_devdata *dd) |
| 394 | 402 | { |
| 395 | − int ret = 0; | |
| 403 | + int ret; | |
| 396 | 404 | |
| 397 | 405 | ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0, |
| 398 | 406 | &dd->pcibar0); |
| @@ -441,7 +449,7 @@ int save_pci_variables(struct hfi1_devdata *dd) · 42 unmodified lines | ||
| 441 | 449 | |
| 442 | 450 | error: |
| 443 | 451 | dd_dev_err(dd, "Unable to read from PCI config\n"); |
| 444 | − return ret; | |
| 452 | + return pcibios_err_to_errno(ret); | |
| 445 | 453 | } |
| 446 | 454 | |
| 447 | 455 | /* |
| @@ -86,7 +86,7 @@ void pio_send_control(struct hfi1_devdata *dd, int op) · 85 unmodified lines | ||
| 86 | 86 | switch (op) { |
| 87 | 87 | case PSC_GLOBAL_ENABLE: |
| 88 | 88 | reg |= SEND_CTRL_SEND_ENABLE_SMASK; |
| 89 | − /* Fall through */ | |
| 89 | + fallthrough; | |
| 90 | 90 | case PSC_DATA_VL_ENABLE: |
| 91 | 91 | mask = 0; |
| 92 | 92 | for (i = 0; i < ARRAY_SIZE(dd->vld); i++) |
| @@ -191,22 +191,22 @@ static inline void jcopy(u8 *dest, const u8 *src, u32 n) · 190 unmodified lines | ||
| 191 | 191 | switch (n) { |
| 192 | 192 | case 7: |
| 193 | 193 | *dest++ = *src++; |
| 194 | − /* fall through */ | |
| 194 | + fallthrough; | |
| 195 | 195 | case 6: |
| 196 | 196 | *dest++ = *src++; |
| 197 | − /* fall through */ | |
| 197 | + fallthrough; | |
| 198 | 198 | case 5: |
| 199 | 199 | *dest++ = *src++; |
| 200 | − /* fall through */ | |
| 200 | + fallthrough; | |
| 201 | 201 | case 4: |
| 202 | 202 | *dest++ = *src++; |
| 203 | − /* fall through */ | |
| 203 | + fallthrough; | |
| 204 | 204 | case 3: |
| 205 | 205 | *dest++ = *src++; |
| 206 | − /* fall through */ | |
| 206 | + fallthrough; | |
| 207 | 207 | case 2: |
| 208 | 208 | *dest++ = *src++; |
| 209 | − /* fall through */ | |
| 209 | + fallthrough; | |
| 210 | 210 | case 1: |
| 211 | 211 | *dest++ = *src++; |
| 212 | 212 | /* fall through */ |
| @@ -668,8 +668,8 @@ static u8 aoc_low_power_setting(struct hfi1_pportdata *ppd) · 667 unmodified lines | ||
| 668 | 668 | |
| 669 | 669 | /* active optical cables only */ |
| 670 | 670 | switch ((cache[QSFP_MOD_TECH_OFFS] & 0xF0) >> 4) { |
| 671 | − case 0x0 ... 0x9: /* fallthrough */ | |
| 672 | − case 0xC: /* fallthrough */ | |
| 671 | + case 0x0 ... 0x9: fallthrough; | |
| 672 | + case 0xC: fallthrough; | |
| 673 | 673 | case 0xE: |
| 674 | 674 | /* active AOC */ |
| 675 | 675 | power_class = get_qsfp_power_class(cache[QSFP_MOD_PWR_OFFS]); |
| @@ -899,8 +899,8 @@ static int tune_qsfp(struct hfi1_pportdata *ppd, · 223 unmodified lines | ||
| 899 | 899 | |
| 900 | 900 | *ptr_tuning_method = OPA_PASSIVE_TUNING; |
| 901 | 901 | break; |
| 902 | − case 0x0 ... 0x9: /* fallthrough */ | |
| 903 | − case 0xC: /* fallthrough */ | |
| 902 | + case 0x0 ... 0x9: fallthrough; | |
| 903 | + case 0xC: fallthrough; | |
| 904 | 904 | case 0xE: |
| 905 | 905 | ret = tune_active_qsfp(ppd, ptr_tx_preset, ptr_rx_preset, |
| 906 | 906 | ptr_total_atten); |
| @@ -909,7 +909,7 @@ static int tune_qsfp(struct hfi1_pportdata *ppd, · 2 unmodified lines | ||
| 909 | 909 | |
| 910 | 910 | *ptr_tuning_method = OPA_ACTIVE_TUNING; |
| 911 | 911 | break; |
| 912 | − case 0xD: /* fallthrough */ | |
| 912 | + case 0xD: fallthrough; | |
| 913 | 913 | case 0xF: |
| 914 | 914 | default: |
| 915 | 915 | dd_dev_warn(ppd->dd, "%s: Unknown/unsupported cable\n", |
| @@ -312,7 +312,7 @@ int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) · 311 unmodified lines | ||
| 312 | 312 | switch (qp->ibqp.qp_type) { |
| 313 | 313 | case IB_QPT_RC: |
| 314 | 314 | hfi1_setup_tid_rdma_wqe(qp, wqe); |
| 315 | − /* fall through */ | |
| 315 | + fallthrough; | |
| 316 | 316 | case IB_QPT_UC: |
| 317 | 317 | if (wqe->length > 0x80000000U) |
| 318 | 318 | return -EINVAL; |
| @@ -112,20 +112,6 @@ static inline void clear_ahg(struct rvt_qp *qp) · 111 unmodified lines | ||
| 112 | 112 | qp->s_ahgidx = -1; |
| 113 | 113 | } |
| 114 | 114 | |
| 115 | −/** | |
| 116 | − * hfi1_create_qp - create a queue pair for a device | |
| 117 | − * @ibpd: the protection domain who's device we create the queue pair for | |
| 118 | − * @init_attr: the attributes of the queue pair | |
| 119 | − * @udata: user data for libibverbs.so | |
| 120 | − * | |
| 121 | − * Returns the queue pair on success, otherwise returns an errno. | |
| 122 | − * | |
| 123 | − * Called by the ib_create_qp() core verbs function. | |
| 124 | − */ | |
| 125 | −struct ib_qp *hfi1_create_qp(struct ib_pd *ibpd, | |
| 126 | − struct ib_qp_init_attr *init_attr, | |
| 127 | − struct ib_udata *udata); | |
| 128 | − | |
| 129 | 115 | /** |
| 130 | 116 | * hfi1_qp_wakeup - wake up on the indicated event |
| 131 | 117 | * @qp: the QP |
| @@ -231,7 +231,7 @@ static int i2c_bus_write(struct hfi1_devdata *dd, struct hfi1_i2c_bus *i2c, · 230 unmodified lines | ||
| 231 | 231 | break; |
| 232 | 232 | case 2: |
| 233 | 233 | offset_bytes[1] = (offset >> 8) & 0xff; |
| 234 | − /* fall through */ | |
| 234 | + fallthrough; | |
| 235 | 235 | case 1: |
| 236 | 236 | num_msgs = 2; |
| 237 | 237 | offset_bytes[0] = offset & 0xff; |
| @@ -279,7 +279,7 @@ static int i2c_bus_read(struct hfi1_devdata *dd, struct hfi1_i2c_bus *bus, · 41 unmodified lines | ||
| 279 | 279 | break; |
| 280 | 280 | case 2: |
| 281 | 281 | offset_bytes[1] = (offset >> 8) & 0xff; |
| 282 | − /* fall through */ | |
| 282 | + fallthrough; | |
| 283 | 283 | case 1: |
| 284 | 284 | num_msgs = 2; |
| 285 | 285 | offset_bytes[0] = offset & 0xff; |
| @@ -141,7 +141,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp, · 140 unmodified lines | ||
| 141 | 141 | case OP(RDMA_READ_RESPONSE_ONLY): |
| 142 | 142 | e = &qp->s_ack_queue[qp->s_tail_ack_queue]; |
| 143 | 143 | release_rdma_sge_mr(e); |
| 144 | − /* FALLTHROUGH */ | |
| 144 | + fallthrough; | |
| 145 | 145 | case OP(ATOMIC_ACKNOWLEDGE): |
| 146 | 146 | /* |
| 147 | 147 | * We can increment the tail pointer now that the last |
| @@ -160,7 +160,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp, · 12 unmodified lines | ||
| 160 | 160 | qp->s_acked_ack_queue = next; |
| 161 | 161 | qp->s_tail_ack_queue = next; |
| 162 | 162 | trace_hfi1_rsp_make_rc_ack(qp, e->psn); |
| 163 | − /* FALLTHROUGH */ | |
| 163 | + fallthrough; | |
| 164 | 164 | case OP(SEND_ONLY): |
| 165 | 165 | case OP(ACKNOWLEDGE): |
| 166 | 166 | /* Check for no next entry in the queue. */ |
| @@ -267,7 +267,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp, · 100 unmodified lines | ||
| 267 | 267 | |
| 268 | 268 | case OP(RDMA_READ_RESPONSE_FIRST): |
| 269 | 269 | qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); |
| 270 | − /* FALLTHROUGH */ | |
| 270 | + fallthrough; | |
| 271 | 271 | case OP(RDMA_READ_RESPONSE_MIDDLE): |
| 272 | 272 | ps->s_txreq->ss = &qp->s_ack_rdma_sge; |
| 273 | 273 | ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr; |
| @@ -881,8 +881,7 @@ no_flow_control: · 607 unmodified lines | ||
| 881 | 881 | goto bail; |
| 882 | 882 | } |
| 883 | 883 | qp->s_num_rd_atomic++; |
| 884 | − | |
| 885 | − /* FALLTHROUGH */ | |
| 884 | + fallthrough; | |
| 886 | 885 | case IB_WR_OPFN: |
| 887 | 886 | if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) |
| 888 | 887 | qp->s_lsn++; |
| @@ -946,10 +945,10 @@ no_flow_control: · 57 unmodified lines | ||
| 946 | 945 | * See restart_rc(). |
| 947 | 946 | */ |
| 948 | 947 | qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); |
| 949 | − /* FALLTHROUGH */ | |
| 948 | + fallthrough; | |
| 950 | 949 | case OP(SEND_FIRST): |
| 951 | 950 | qp->s_state = OP(SEND_MIDDLE); |
| 952 | − /* FALLTHROUGH */ | |
| 951 | + fallthrough; | |
| 953 | 952 | case OP(SEND_MIDDLE): |
| 954 | 953 | bth2 = mask_psn(qp->s_psn++); |
| 955 | 954 | ss = &qp->s_sge; |
| @@ -991,10 +990,10 @@ no_flow_control: · 35 unmodified lines | ||
| 991 | 990 | * See restart_rc(). |
| 992 | 991 | */ |
| 993 | 992 | qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); |
| 994 | − /* FALLTHROUGH */ | |
| 993 | + fallthrough; | |
| 995 | 994 | case OP(RDMA_WRITE_FIRST): |
| 996 | 995 | qp->s_state = OP(RDMA_WRITE_MIDDLE); |
| 997 | − /* FALLTHROUGH */ | |
| 996 | + fallthrough; | |
| 998 | 997 | case OP(RDMA_WRITE_MIDDLE): |
| 999 | 998 | bth2 = mask_psn(qp->s_psn++); |
| 1000 | 999 | ss = &qp->s_sge; |
| @@ -2901,7 +2900,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) · 1900 unmodified lines | ||
| 2901 | 2900 | if (!ret) |
| 2902 | 2901 | goto rnr_nak; |
| 2903 | 2902 | qp->r_rcv_len = 0; |
| 2904 | − /* FALLTHROUGH */ | |
| 2903 | + fallthrough; | |
| 2905 | 2904 | case OP(SEND_MIDDLE): |
| 2906 | 2905 | case OP(RDMA_WRITE_MIDDLE): |
| 2907 | 2906 | send_middle: |
| @@ -2941,7 +2940,7 @@ send_middle: · 33 unmodified lines | ||
| 2941 | 2940 | goto no_immediate_data; |
| 2942 | 2941 | if (opcode == OP(SEND_ONLY_WITH_INVALIDATE)) |
| 2943 | 2942 | goto send_last_inv; |
| 2944 | − /* FALLTHROUGH -- for SEND_ONLY_WITH_IMMEDIATE */ | |
| 2943 | + fallthrough; /* for SEND_ONLY_WITH_IMMEDIATE */ | |
| 2945 | 2944 | case OP(SEND_LAST_WITH_IMMEDIATE): |
| 2946 | 2945 | send_last_imm: |
| 2947 | 2946 | wc.ex.imm_data = ohdr->u.imm_data; |
| @@ -2957,7 +2956,7 @@ send_last_inv: · 9 unmodified lines | ||
| 2957 | 2956 | goto send_last; |
| 2958 | 2957 | case OP(RDMA_WRITE_LAST): |
| 2959 | 2958 | copy_last = rvt_is_user_qp(qp); |
| 2960 | − /* fall through */ | |
| 2959 | + fallthrough; | |
| 2961 | 2960 | case OP(SEND_LAST): |
| 2962 | 2961 | no_immediate_data: |
| 2963 | 2962 | wc.wc_flags = 0; |
| @@ -3010,7 +3009,7 @@ send_last: · 46 unmodified lines | ||
| 3010 | 3009 | |
| 3011 | 3010 | case OP(RDMA_WRITE_ONLY): |
| 3012 | 3011 | copy_last = rvt_is_user_qp(qp); |
| 3013 | − /* fall through */ | |
| 3012 | + fallthrough; | |
| 3014 | 3013 | case OP(RDMA_WRITE_FIRST): |
| 3015 | 3014 | case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): |
| 3016 | 3015 | if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) |
| @@ -2584,7 +2584,7 @@ static void __sdma_process_event(struct sdma_engine *sde, · 2583 unmodified lines | ||
| 2584 | 2584 | * 7220, e.g. |
| 2585 | 2585 | */ |
| 2586 | 2586 | ss->go_s99_running = 1; |
| 2587 | − /* fall through -- and start dma engine */ | |
| 2587 | + fallthrough; /* and start dma engine */ | |
| 2588 | 2588 | case sdma_event_e10_go_hw_start: |
| 2589 | 2589 | /* This reference means the state machine is started */ |
| 2590 | 2590 | sdma_get(&sde->state); |
| @@ -2726,7 +2726,6 @@ static void __sdma_process_event(struct sdma_engine *sde, · 135 unmodified lines | ||
| 2726 | 2726 | case sdma_event_e70_go_idle: |
| 2727 | 2727 | break; |
| 2728 | 2728 | case sdma_event_e85_link_down: |
| 2729 | − /* fall through */ | |
| 2730 | 2729 | case sdma_event_e80_hw_freeze: |
| 2731 | 2730 | sdma_set_state(sde, sdma_state_s80_hw_freeze); |
| 2732 | 2731 | atomic_dec(&sde->dd->sdma_unfreeze_count); |
| @@ -3007,7 +3006,7 @@ static void __sdma_process_event(struct sdma_engine *sde, · 274 unmodified lines | ||
| 3007 | 3006 | case sdma_event_e60_hw_halted: |
| 3008 | 3007 | need_progress = 1; |
| 3009 | 3008 | sdma_err_progress_check_schedule(sde); |
| 3010 | − /* fall through */ | |
| 3009 | + fallthrough; | |
| 3011 | 3010 | case sdma_event_e90_sw_halted: |
| 3012 | 3011 | /* |
| 3013 | 3012 | * SW initiated halt does not perform engines |
| @@ -3021,7 +3020,7 @@ static void __sdma_process_event(struct sdma_engine *sde, · 7 unmodified lines | ||
| 3021 | 3020 | break; |
| 3022 | 3021 | case sdma_event_e85_link_down: |
| 3023 | 3022 | ss->go_s99_running = 0; |
| 3024 | − /* fall through */ | |
| 3023 | + fallthrough; | |
| 3025 | 3024 | case sdma_event_e80_hw_freeze: |
| 3026 | 3025 | sdma_set_state(sde, sdma_state_s80_hw_freeze); |
| 3027 | 3026 | atomic_dec(&sde->dd->sdma_unfreeze_count); |
| @@ -3252,7 +3251,7 @@ void _sdma_txreq_ahgadd( · 224 unmodified lines | ||
| 3252 | 3251 | tx->num_desc++; |
| 3253 | 3252 | tx->descs[2].qw[0] = 0; |
| 3254 | 3253 | tx->descs[2].qw[1] = 0; |
| 3255 | − /* FALLTHROUGH */ | |
| 3254 | + fallthrough; | |
| 3256 | 3255 | case SDMA_AHG_APPLY_UPDATE2: |
| 3257 | 3256 | tx->num_desc++; |
| 3258 | 3257 | tx->descs[1].qw[0] = 0; |
| @@ -3227,7 +3227,7 @@ bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) · 3226 unmodified lines | ||
| 3227 | 3227 | case IB_WR_RDMA_READ: |
| 3228 | 3228 | if (prev->wr.opcode != IB_WR_TID_RDMA_WRITE) |
| 3229 | 3229 | break; |
| 3230 | − /* fall through */ | |
| 3230 | + fallthrough; | |
| 3231 | 3231 | case IB_WR_TID_RDMA_READ: |
| 3232 | 3232 | switch (prev->wr.opcode) { |
| 3233 | 3233 | case IB_WR_RDMA_READ: |
| @@ -5067,7 +5067,7 @@ int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps) · 1833 unmodified lines | ||
| 5067 | 5067 | if (priv->s_state == TID_OP(WRITE_REQ)) |
| 5068 | 5068 | hfi1_tid_rdma_restart_req(qp, wqe, &bth2); |
| 5069 | 5069 | priv->s_state = TID_OP(WRITE_DATA); |
| 5070 | − /* fall through */ | |
| 5070 | + fallthrough; | |
| 5071 | 5071 | |
| 5072 | 5072 | case TID_OP(WRITE_DATA): |
| 5073 | 5073 | /* |
| @@ -216,7 +216,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) · 215 unmodified lines | ||
| 216 | 216 | |
| 217 | 217 | case OP(SEND_FIRST): |
| 218 | 218 | qp->s_state = OP(SEND_MIDDLE); |
| 219 | − /* FALLTHROUGH */ | |
| 219 | + fallthrough; | |
| 220 | 220 | case OP(SEND_MIDDLE): |
| 221 | 221 | len = qp->s_len; |
| 222 | 222 | if (len > pmtu) { |
| @@ -241,7 +241,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) · 18 unmodified lines | ||
| 241 | 241 | |
| 242 | 242 | case OP(RDMA_WRITE_FIRST): |
| 243 | 243 | qp->s_state = OP(RDMA_WRITE_MIDDLE); |
| 244 | − /* FALLTHROUGH */ | |
| 244 | + fallthrough; | |
| 245 | 245 | case OP(RDMA_WRITE_MIDDLE): |
| 246 | 246 | len = qp->s_len; |
| 247 | 247 | if (len > pmtu) { |
| @@ -414,7 +414,7 @@ send_first: · 166 unmodified lines | ||
| 414 | 414 | goto no_immediate_data; |
| 415 | 415 | else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE)) |
| 416 | 416 | goto send_last_imm; |
| 417 | − /* FALLTHROUGH */ | |
| 417 | + fallthrough; | |
| 418 | 418 | case OP(SEND_MIDDLE): |
| 419 | 419 | /* Check for invalid length PMTU or posted rwqe len. */ |
| 420 | 420 | /* |
| @@ -515,7 +515,7 @@ rdma_first: · 94 unmodified lines | ||
| 515 | 515 | wc.ex.imm_data = ohdr->u.rc.imm_data; |
| 516 | 516 | goto rdma_last_imm; |
| 517 | 517 | } |
| 518 | − /* FALLTHROUGH */ | |
| 518 | + fallthrough; | |
| 519 | 519 | case OP(RDMA_WRITE_MIDDLE): |
| 520 | 520 | /* Check for invalid length PMTU or posted rwqe len. */ |
| 521 | 521 | if (unlikely(tlen != (hdrsize + pmtu + 4))) |
| @@ -37,9 +37,8 @@ · 36 unmodified lines | ||
| 37 | 37 | |
| 38 | 38 | #define DRV_NAME "hns_roce" |
| 39 | 39 | |
| 40 | −/* hip08 is a pci device, it includes two version according pci version id */ | |
| 41 | −#define PCI_REVISION_ID_HIP08_A 0x20 | |
| 42 | −#define PCI_REVISION_ID_HIP08_B 0x21 | |
| 40 | +/* hip08 is a pci device */ | |
| 41 | +#define PCI_REVISION_ID_HIP08 0x21 | |
| 43 | 42 | |
| 44 | 43 | #define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6') |
| 45 | 44 | |
| @@ -348,20 +347,22 @@ struct hns_roce_buf_attr { · 302 unmodified lines | ||
| 348 | 347 | bool mtt_only; /* only alloc buffer-required MTT memory */ |
| 349 | 348 | }; |
| 350 | 349 | |
| 350 | +struct hns_roce_hem_cfg { | |
| 351 | + dma_addr_t root_ba; /* root BA table's address */ | |
| 352 | + bool is_direct; /* addressing without BA table */ | |
| 353 | + unsigned int ba_pg_shift; /* BA table page shift */ | |
| 354 | + unsigned int buf_pg_shift; /* buffer page shift */ | |
| 355 | + unsigned int buf_pg_count; /* buffer page count */ | |
| 356 | + struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION]; | |
| 357 | + int region_count; | |
| 358 | +}; | |
| 359 | + | |
| 351 | 360 | /* memory translate region */ |
| 352 | 361 | struct hns_roce_mtr { |
| 353 | 362 | struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */ |
| 354 | 363 | struct ib_umem *umem; /* user space buffer */ |
| 355 | 364 | struct hns_roce_buf *kmem; /* kernel space buffer */ |
| 356 | − struct { | |
| 357 | − dma_addr_t root_ba; /* root BA table's address */ | |
| 358 | − bool is_direct; /* addressing without BA table */ | |
| 359 | − unsigned int ba_pg_shift; /* BA table page shift */ | |
| 360 | − unsigned int buf_pg_shift; /* buffer page shift */ | |
| 361 | − int buf_pg_count; /* buffer page count */ | |
| 362 | − struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION]; | |
| 363 | − unsigned int region_count; | |
| 364 | − } hem_cfg; /* config for hardware addressing */ | |
| 365 | + struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */ | |
| 365 | 366 | }; |
| 366 | 367 | |
| 367 | 368 | struct hns_roce_mw { |
| @@ -1192,7 +1193,7 @@ int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length, · 824 unmodified lines | ||
| 1192 | 1193 | u64 virt_addr, int mr_access_flags, struct ib_pd *pd, |
| 1193 | 1194 | struct ib_udata *udata); |
| 1194 | 1195 | struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 1195 | − u32 max_num_sg, struct ib_udata *udata); | |
| 1196 | + u32 max_num_sg); | |
| 1196 | 1197 | int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, |
| 1197 | 1198 | unsigned int *sg_offset); |
| 1198 | 1199 | int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); |
| @@ -1267,6 +1268,6 @@ void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev); · 68 unmodified lines | ||
| 1267 | 1268 | int hns_roce_init(struct hns_roce_dev *hr_dev); |
| 1268 | 1269 | void hns_roce_exit(struct hns_roce_dev *hr_dev); |
| 1269 | 1270 | |
| 1270 | −int hns_roce_fill_res_entry(struct sk_buff *msg, | |
| 1271 | − struct rdma_restrack_entry *res); | |
| 1271 | +int hns_roce_fill_res_cq_entry(struct sk_buff *msg, | |
| 1272 | + struct ib_cq *ib_cq); | |
| 1272 | 1273 | #endif /* _HNS_ROCE_DEVICE_H */ |
| @@ -2483,7 +2483,6 @@ static int find_wqe_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 2482 unmodified lines | ||
| 2483 | 2483 | u64 *sq_ba, u64 *rq_ba, dma_addr_t *bt_ba) |
| 2484 | 2484 | { |
| 2485 | 2485 | struct ib_device *ibdev = &hr_dev->ib_dev; |
| 2486 | − int rq_pa_start; | |
| 2487 | 2486 | int count; |
| 2488 | 2487 | |
| 2489 | 2488 | count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, sq_ba, 1, bt_ba); |
| @@ -2491,9 +2490,9 @@ static int find_wqe_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 1 unmodified line | ||
| 2491 | 2490 | ibdev_err(ibdev, "Failed to find SQ ba\n"); |
| 2492 | 2491 | return -ENOBUFS; |
| 2493 | 2492 | } |
| 2494 | − rq_pa_start = hr_qp->rq.offset >> hr_qp->mtr.hem_cfg.buf_pg_shift; | |
| 2495 | − count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, rq_pa_start, rq_ba, 1, | |
| 2496 | − NULL); | |
| 2493 | + | |
| 2494 | + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, rq_ba, | |
| 2495 | + 1, NULL); | |
| 2497 | 2496 | if (!count) { |
| 2498 | 2497 | ibdev_err(ibdev, "Failed to find RQ ba\n"); |
| 2499 | 2498 | return -ENOBUFS; |
| @@ -91,10 +91,11 @@ static u32 to_hr_opcode(u32 ib_opcode) · 90 unmodified lines | ||
| 91 | 91 | } |
| 92 | 92 | |
| 93 | 93 | static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, |
| 94 | − void *wqe, const struct ib_reg_wr *wr) | |
| 94 | + const struct ib_reg_wr *wr) | |
| 95 | 95 | { |
| 96 | + struct hns_roce_wqe_frmr_seg *fseg = | |
| 97 | + (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); | |
| 96 | 98 | struct hns_roce_mr *mr = to_hr_mr(wr->mr); |
| 97 | − struct hns_roce_wqe_frmr_seg *fseg = wqe; | |
| 98 | 99 | u64 pbl_ba; |
| 99 | 100 | |
| 100 | 101 | /* use ib_access_flags */ |
| @@ -128,14 +129,16 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, · 27 unmodified lines | ||
| 128 | 129 | V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0); |
| 129 | 130 | } |
| 130 | 131 | |
| 131 | −static void set_atomic_seg(const struct ib_send_wr *wr, void *wqe, | |
| 132 | +static void set_atomic_seg(const struct ib_send_wr *wr, | |
| 132 | 133 | struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, |
| 133 | 134 | unsigned int valid_num_sge) |
| 134 | 135 | { |
| 135 | − struct hns_roce_wqe_atomic_seg *aseg; | |
| 136 | + struct hns_roce_v2_wqe_data_seg *dseg = | |
| 137 | + (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); | |
| 138 | + struct hns_roce_wqe_atomic_seg *aseg = | |
| 139 | + (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg); | |
| 136 | 140 | |
| 137 | − set_data_seg_v2(wqe, wr->sg_list); | |
| 138 | − aseg = wqe + sizeof(struct hns_roce_v2_wqe_data_seg); | |
| 141 | + set_data_seg_v2(dseg, wr->sg_list); | |
| 139 | 142 | |
| 140 | 143 | if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { |
| 141 | 144 | aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap); |
| @@ -143,7 +146,7 @@ static void set_atomic_seg(const struct ib_send_wr *wr, void *wqe, · 1 unmodified line | ||
| 143 | 146 | } else { |
| 144 | 147 | aseg->fetchadd_swap_data = |
| 145 | 148 | cpu_to_le64(atomic_wr(wr)->compare_add); |
| 146 | − aseg->cmp_data = 0; | |
| 149 | + aseg->cmp_data = 0; | |
| 147 | 150 | } |
| 148 | 151 | |
| 149 | 152 | roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, |
| @@ -176,13 +179,15 @@ static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, · 26 unmodified lines | ||
| 176 | 179 | |
| 177 | 180 | static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, |
| 178 | 181 | struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, |
| 179 | − void *wqe, unsigned int *sge_ind, | |
| 182 | + unsigned int *sge_ind, | |
| 180 | 183 | unsigned int valid_num_sge) |
| 181 | 184 | { |
| 182 | 185 | struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); |
| 183 | − struct hns_roce_v2_wqe_data_seg *dseg = wqe; | |
| 186 | + struct hns_roce_v2_wqe_data_seg *dseg = | |
| 187 | + (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); | |
| 184 | 188 | struct ib_device *ibdev = &hr_dev->ib_dev; |
| 185 | 189 | struct hns_roce_qp *qp = to_hr_qp(ibqp); |
| 190 | + void *wqe = dseg; | |
| 186 | 191 | int j = 0; |
| 187 | 192 | int i; |
| 188 | 193 | |
| @@ -438,7 +443,6 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, · 249 unmodified lines | ||
| 438 | 443 | roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S, |
| 439 | 444 | owner_bit); |
| 440 | 445 | |
| 441 | − wqe += sizeof(struct hns_roce_v2_rc_send_wqe); | |
| 442 | 446 | switch (wr->opcode) { |
| 443 | 447 | case IB_WR_RDMA_READ: |
| 444 | 448 | case IB_WR_RDMA_WRITE: |
| @@ -451,7 +455,7 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, · 6 unmodified lines | ||
| 451 | 455 | rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); |
| 452 | 456 | break; |
| 453 | 457 | case IB_WR_REG_MR: |
| 454 | − set_frmr_seg(rc_sq_wqe, wqe, reg_wr(wr)); | |
| 458 | + set_frmr_seg(rc_sq_wqe, reg_wr(wr)); | |
| 455 | 459 | break; |
| 456 | 460 | case IB_WR_ATOMIC_CMP_AND_SWP: |
| 457 | 461 | case IB_WR_ATOMIC_FETCH_AND_ADD: |
| @@ -468,10 +472,10 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, · 10 unmodified lines | ||
| 468 | 472 | |
| 469 | 473 | if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || |
| 470 | 474 | wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) |
| 471 | − set_atomic_seg(wr, wqe, rc_sq_wqe, valid_num_sge); | |
| 475 | + set_atomic_seg(wr, rc_sq_wqe, valid_num_sge); | |
| 472 | 476 | else if (wr->opcode != IB_WR_REG_MR) |
| 473 | 477 | ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe, |
| 474 | − wqe, &curr_idx, valid_num_sge); | |
| 478 | + &curr_idx, valid_num_sge); | |
| 475 | 479 | |
| 476 | 480 | *sge_idx = curr_idx; |
| 477 | 481 | |
| @@ -1510,8 +1514,6 @@ static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) · 1032 unmodified lines | ||
| 1510 | 1514 | |
| 1511 | 1515 | req_a = (struct hns_roce_vf_res_a *)desc[0].data; |
| 1512 | 1516 | req_b = (struct hns_roce_vf_res_b *)desc[1].data; |
| 1513 | − memset(req_a, 0, sizeof(*req_a)); | |
| 1514 | − memset(req_b, 0, sizeof(*req_b)); | |
| 1515 | 1517 | for (i = 0; i < 2; i++) { |
| 1516 | 1518 | hns_roce_cmq_setup_basic_desc(&desc[i], |
| 1517 | 1519 | HNS_ROCE_OPC_ALLOC_VF_RES, false); |
| @@ -1744,27 +1746,25 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) · 226 unmodified lines | ||
| 1744 | 1746 | caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR; |
| 1745 | 1747 | caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE; |
| 1746 | 1748 | |
| 1747 | − if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) { | |
| 1748 | − caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW | | |
| 1749 | − HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR | | |
| 1750 | − HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; | |
| 1749 | + caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW | | |
| 1750 | + HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR | | |
| 1751 | + HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; | |
| 1751 | 1752 | |
| 1752 | − caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM; | |
| 1753 | − caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; | |
| 1754 | − caps->qpc_timer_ba_pg_sz = 0; | |
| 1755 | − caps->qpc_timer_buf_pg_sz = 0; | |
| 1756 | − caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; | |
| 1757 | − caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM; | |
| 1758 | − caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; | |
| 1759 | − caps->cqc_timer_ba_pg_sz = 0; | |
| 1760 | − caps->cqc_timer_buf_pg_sz = 0; | |
| 1761 | − caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; | |
| 1753 | + caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM; | |
| 1754 | + caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; | |
| 1755 | + caps->qpc_timer_ba_pg_sz = 0; | |
| 1756 | + caps->qpc_timer_buf_pg_sz = 0; | |
| 1757 | + caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; | |
| 1758 | + caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM; | |
| 1759 | + caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; | |
| 1760 | + caps->cqc_timer_ba_pg_sz = 0; | |
| 1761 | + caps->cqc_timer_buf_pg_sz = 0; | |
| 1762 | + caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; | |
| 1762 | 1763 | |
| 1763 | − caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ; | |
| 1764 | − caps->sccc_ba_pg_sz = 0; | |
| 1765 | − caps->sccc_buf_pg_sz = 0; | |
| 1766 | − caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; | |
| 1767 | − } | |
| 1764 | + caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ; | |
| 1765 | + caps->sccc_ba_pg_sz = 0; | |
| 1766 | + caps->sccc_buf_pg_sz = 0; | |
| 1767 | + caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; | |
| 1768 | 1768 | } |
| 1769 | 1769 | |
| 1770 | 1770 | static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num, |
| @@ -1995,20 +1995,18 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) · 224 unmodified lines | ||
| 1995 | 1995 | caps->srqc_bt_num, &caps->srqc_buf_pg_sz, |
| 1996 | 1996 | &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC); |
| 1997 | 1997 | |
| 1998 | − if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) { | |
| 1999 | − caps->sccc_hop_num = ctx_hop_num; | |
| 2000 | − caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; | |
| 2001 | − caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; | |
| 1998 | + caps->sccc_hop_num = ctx_hop_num; | |
| 1999 | + caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; | |
| 2000 | + caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; | |
| 2002 | 2001 | |
| 2003 | − calc_pg_sz(caps->num_qps, caps->sccc_entry_sz, | |
| 2004 | − caps->sccc_hop_num, caps->sccc_bt_num, | |
| 2005 | − &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz, | |
| 2006 | − HEM_TYPE_SCCC); | |
| 2007 | − calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz, | |
| 2008 | − caps->cqc_timer_hop_num, caps->cqc_timer_bt_num, | |
| 2009 | − &caps->cqc_timer_buf_pg_sz, | |
| 2010 | − &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER); | |
| 2011 | − } | |
| 2002 | + calc_pg_sz(caps->num_qps, caps->sccc_entry_sz, | |
| 2003 | + caps->sccc_hop_num, caps->sccc_bt_num, | |
| 2004 | + &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz, | |
| 2005 | + HEM_TYPE_SCCC); | |
| 2006 | + calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz, | |
| 2007 | + caps->cqc_timer_hop_num, caps->cqc_timer_bt_num, | |
| 2008 | + &caps->cqc_timer_buf_pg_sz, | |
| 2009 | + &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER); | |
| 2012 | 2010 | |
| 2013 | 2011 | calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num, |
| 2014 | 2012 | 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); |
| @@ -2055,22 +2053,19 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) · 40 unmodified lines | ||
| 2055 | 2053 | return ret; |
| 2056 | 2054 | } |
| 2057 | 2055 | |
| 2058 | − if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) { | |
| 2059 | − ret = hns_roce_query_pf_timer_resource(hr_dev); | |
| 2060 | − if (ret) { | |
| 2061 | − dev_err(hr_dev->dev, | |
| 2062 | − "Query pf timer resource fail, ret = %d.\n", | |
| 2063 | − ret); | |
| 2064 | − return ret; | |
| 2065 | − } | |
| 2056 | + ret = hns_roce_query_pf_timer_resource(hr_dev); | |
| 2057 | + if (ret) { | |
| 2058 | + dev_err(hr_dev->dev, | |
| 2059 | + "failed to query pf timer resource, ret = %d.\n", ret); | |
| 2060 | + return ret; | |
| 2061 | + } | |
| 2066 | 2062 | |
| 2067 | − ret = hns_roce_set_vf_switch_param(hr_dev, 0); | |
| 2068 | − if (ret) { | |
| 2069 | − dev_err(hr_dev->dev, | |
| 2070 | − "Set function switch param fail, ret = %d.\n", | |
| 2071 | − ret); | |
| 2072 | − return ret; | |
| 2073 | − } | |
| 2063 | + ret = hns_roce_set_vf_switch_param(hr_dev, 0); | |
| 2064 | + if (ret) { | |
| 2065 | + dev_err(hr_dev->dev, | |
| 2066 | + "failed to set function switch param, ret = %d.\n", | |
| 2067 | + ret); | |
| 2068 | + return ret; | |
| 2074 | 2069 | } |
| 2075 | 2070 | |
| 2076 | 2071 | hr_dev->vendor_part_id = hr_dev->pci_dev->device; |
| @@ -2336,8 +2331,7 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) · 259 unmodified lines | ||
| 2336 | 2331 | { |
| 2337 | 2332 | struct hns_roce_v2_priv *priv = hr_dev->priv; |
| 2338 | 2333 | |
| 2339 | − if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) | |
| 2340 | − hns_roce_function_clear(hr_dev); | |
| 2334 | + hns_roce_function_clear(hr_dev); | |
| 2341 | 2335 | |
| 2342 | 2336 | hns_roce_free_link_table(hr_dev, &priv->tpq); |
| 2343 | 2337 | hns_roce_free_link_table(hr_dev, &priv->tsq); |
| @@ -3053,6 +3047,7 @@ static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, · 709 unmodified lines | ||
| 3053 | 3047 | IB_WC_RETRY_EXC_ERR }, |
| 3054 | 3048 | { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR }, |
| 3055 | 3049 | { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR }, |
| 3050 | + { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR} | |
| 3056 | 3051 | }; |
| 3057 | 3052 | |
| 3058 | 3053 | u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M, |
| @@ -3074,6 +3069,14 @@ static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, · 15 unmodified lines | ||
| 3074 | 3069 | print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe, |
| 3075 | 3070 | sizeof(*cqe), false); |
| 3076 | 3071 | |
| 3072 | + /* | |
| 3073 | + * For hns ROCEE, GENERAL_ERR is an error type that is not defined in | |
| 3074 | + * the standard protocol, the driver must ignore it and needn't to set | |
| 3075 | + * the QP to an error state. | |
| 3076 | + */ | |
| 3077 | + if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR) | |
| 3078 | + return; | |
| 3079 | + | |
| 3077 | 3080 | /* |
| 3078 | 3081 | * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets |
| 3079 | 3082 | * into errored mode. Hence, as a workaround to this hardware |
| @@ -3170,51 +3173,51 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 90 unmodified lines | ||
| 3170 | 3173 | /* SQ corresponding to CQE */ |
| 3171 | 3174 | switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, |
| 3172 | 3175 | V2_CQE_BYTE_4_OPCODE_S) & 0x1f) { |
| 3173 | − case HNS_ROCE_SQ_OPCODE_SEND: | |
| 3176 | + case HNS_ROCE_V2_WQE_OP_SEND: | |
| 3174 | 3177 | wc->opcode = IB_WC_SEND; |
| 3175 | 3178 | break; |
| 3176 | − case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV: | |
| 3179 | + case HNS_ROCE_V2_WQE_OP_SEND_WITH_INV: | |
| 3177 | 3180 | wc->opcode = IB_WC_SEND; |
| 3178 | 3181 | break; |
| 3179 | − case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM: | |
| 3182 | + case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM: | |
| 3180 | 3183 | wc->opcode = IB_WC_SEND; |
| 3181 | 3184 | wc->wc_flags |= IB_WC_WITH_IMM; |
| 3182 | 3185 | break; |
| 3183 | − case HNS_ROCE_SQ_OPCODE_RDMA_READ: | |
| 3186 | + case HNS_ROCE_V2_WQE_OP_RDMA_READ: | |
| 3184 | 3187 | wc->opcode = IB_WC_RDMA_READ; |
| 3185 | 3188 | wc->byte_len = le32_to_cpu(cqe->byte_cnt); |
| 3186 | 3189 | break; |
| 3187 | − case HNS_ROCE_SQ_OPCODE_RDMA_WRITE: | |
| 3190 | + case HNS_ROCE_V2_WQE_OP_RDMA_WRITE: | |
| 3188 | 3191 | wc->opcode = IB_WC_RDMA_WRITE; |
| 3189 | 3192 | break; |
| 3190 | − case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM: | |
| 3193 | + case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM: | |
| 3191 | 3194 | wc->opcode = IB_WC_RDMA_WRITE; |
| 3192 | 3195 | wc->wc_flags |= IB_WC_WITH_IMM; |
| 3193 | 3196 | break; |
| 3194 | − case HNS_ROCE_SQ_OPCODE_LOCAL_INV: | |
| 3197 | + case HNS_ROCE_V2_WQE_OP_LOCAL_INV: | |
| 3195 | 3198 | wc->opcode = IB_WC_LOCAL_INV; |
| 3196 | 3199 | wc->wc_flags |= IB_WC_WITH_INVALIDATE; |
| 3197 | 3200 | break; |
| 3198 | − case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP: | |
| 3201 | + case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP: | |
| 3199 | 3202 | wc->opcode = IB_WC_COMP_SWAP; |
| 3200 | 3203 | wc->byte_len = 8; |
| 3201 | 3204 | break; |
| 3202 | − case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD: | |
| 3205 | + case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD: | |
| 3203 | 3206 | wc->opcode = IB_WC_FETCH_ADD; |
| 3204 | 3207 | wc->byte_len = 8; |
| 3205 | 3208 | break; |
| 3206 | − case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP: | |
| 3209 | + case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP: | |
| 3207 | 3210 | wc->opcode = IB_WC_MASKED_COMP_SWAP; |
| 3208 | 3211 | wc->byte_len = 8; |
| 3209 | 3212 | break; |
| 3210 | − case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD: | |
| 3213 | + case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD: | |
| 3211 | 3214 | wc->opcode = IB_WC_MASKED_FETCH_ADD; |
| 3212 | 3215 | wc->byte_len = 8; |
| 3213 | 3216 | break; |
| 3214 | − case HNS_ROCE_SQ_OPCODE_FAST_REG_WR: | |
| 3217 | + case HNS_ROCE_V2_WQE_OP_FAST_REG_PMR: | |
| 3215 | 3218 | wc->opcode = IB_WC_REG_MR; |
| 3216 | 3219 | break; |
| 3217 | − case HNS_ROCE_SQ_OPCODE_BIND_MW: | |
| 3220 | + case HNS_ROCE_V2_WQE_OP_BIND_MW: | |
| 3218 | 3221 | wc->opcode = IB_WC_REG_MR; |
| 3219 | 3222 | break; |
| 3220 | 3223 | default: |
| @@ -3374,11 +3377,33 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, · 153 unmodified lines | ||
| 3374 | 3377 | return op + step_idx; |
| 3375 | 3378 | } |
| 3376 | 3379 | |
| 3380 | +static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba, | |
| 3381 | + u32 hem_type, int step_idx) | |
| 3382 | +{ | |
| 3383 | + struct hns_roce_cmd_mailbox *mailbox; | |
| 3384 | + int ret; | |
| 3385 | + int op; | |
| 3386 | + | |
| 3387 | + op = get_op_for_set_hem(hr_dev, hem_type, step_idx); | |
| 3388 | + if (op < 0) | |
| 3389 | + return 0; | |
| 3390 | + | |
| 3391 | + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); | |
| 3392 | + if (IS_ERR(mailbox)) | |
| 3393 | + return PTR_ERR(mailbox); | |
| 3394 | + | |
| 3395 | + ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj, | |
| 3396 | + 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS); | |
| 3397 | + | |
| 3398 | + hns_roce_free_cmd_mailbox(hr_dev, mailbox); | |
| 3399 | + | |
| 3400 | + return ret; | |
| 3401 | +} | |
| 3402 | + | |
| 3377 | 3403 | static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, |
| 3378 | 3404 | struct hns_roce_hem_table *table, int obj, |
| 3379 | 3405 | int step_idx) |
| 3380 | 3406 | { |
| 3381 | − struct hns_roce_cmd_mailbox *mailbox; | |
| 3382 | 3407 | struct hns_roce_hem_iter iter; |
| 3383 | 3408 | struct hns_roce_hem_mhop mhop; |
| 3384 | 3409 | struct hns_roce_hem *hem; |
| @@ -3390,7 +3415,6 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, · 5 unmodified lines | ||
| 3390 | 3415 | u64 bt_ba = 0; |
| 3391 | 3416 | u32 chunk_ba_num; |
| 3392 | 3417 | u32 hop_num; |
| 3393 | − int op; | |
| 3394 | 3418 | |
| 3395 | 3419 | if (!hns_roce_check_whether_mhop(hr_dev, table->type)) |
| 3396 | 3420 | return 0; |
| @@ -3412,14 +3436,6 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, · 15 unmodified lines | ||
| 3412 | 3436 | hem_idx = i; |
| 3413 | 3437 | } |
| 3414 | 3438 | |
| 3415 | − op = get_op_for_set_hem(hr_dev, table->type, step_idx); | |
| 3416 | − if (op == -EINVAL) | |
| 3417 | − return 0; | |
| 3418 | − | |
| 3419 | − mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); | |
| 3420 | − if (IS_ERR(mailbox)) | |
| 3421 | − return PTR_ERR(mailbox); | |
| 3422 | − | |
| 3423 | 3439 | if (table->type == HEM_TYPE_SCCC) |
| 3424 | 3440 | obj = mhop.l0_idx; |
| 3425 | 3441 | |
| @@ -3428,11 +3444,8 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, · 2 unmodified lines | ||
| 3428 | 3444 | for (hns_roce_hem_first(hem, &iter); |
| 3429 | 3445 | !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { |
| 3430 | 3446 | bt_ba = hns_roce_hem_addr(&iter); |
| 3431 | − | |
| 3432 | − /* configure the ba, tag, and op */ | |
| 3433 | − ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, | |
| 3434 | − obj, 0, op, | |
| 3435 | − HNS_ROCE_CMD_TIMEOUT_MSECS); | |
| 3447 | + ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, | |
| 3448 | + step_idx); | |
| 3436 | 3449 | } |
| 3437 | 3450 | } else { |
| 3438 | 3451 | if (step_idx == 0) |
| @@ -3440,12 +3453,9 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, · 1 unmodified line | ||
| 3440 | 3453 | else if (step_idx == 1 && hop_num == 2) |
| 3441 | 3454 | bt_ba = table->bt_l1_dma_addr[l1_idx]; |
| 3442 | 3455 | |
| 3443 | − /* configure the ba, tag, and op */ | |
| 3444 | − ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj, | |
| 3445 | − 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS); | |
| 3456 | + ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx); | |
| 3446 | 3457 | } |
| 3447 | 3458 | |
| 3448 | − hns_roce_free_cmd_mailbox(hr_dev, mailbox); | |
| 3449 | 3459 | return ret; |
| 3450 | 3460 | } |
| 3451 | 3461 | |
| @@ -3745,51 +3755,23 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, · 293 unmodified lines | ||
| 3745 | 3755 | } |
| 3746 | 3756 | } |
| 3747 | 3757 | |
| 3748 | −static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev, | |
| 3749 | − struct hns_roce_qp *hr_qp, int mtt_cnt, | |
| 3750 | − u32 page_size) | |
| 3751 | −{ | |
| 3752 | − struct ib_device *ibdev = &hr_dev->ib_dev; | |
| 3753 | − | |
| 3754 | − if (hr_qp->rq.wqe_cnt < 1) | |
| 3755 | − return true; | |
| 3756 | − | |
| 3757 | − if (mtt_cnt < 1) { | |
| 3758 | − ibdev_err(ibdev, "failed to find RQWQE buf ba of QP(0x%lx)\n", | |
| 3759 | − hr_qp->qpn); | |
| 3760 | − return false; | |
| 3761 | − } | |
| 3762 | − | |
| 3763 | − if (mtt_cnt < MTT_MIN_COUNT && | |
| 3764 | − (hr_qp->rq.offset + page_size) < hr_qp->buff_size) { | |
| 3765 | − ibdev_err(ibdev, | |
| 3766 | − "failed to find next RQWQE buf ba of QP(0x%lx)\n", | |
| 3767 | − hr_qp->qpn); | |
| 3768 | − return false; | |
| 3769 | − } | |
| 3770 | − | |
| 3771 | − return true; | |
| 3772 | −} | |
| 3773 | − | |
| 3774 | 3758 | static int config_qp_rq_buf(struct hns_roce_dev *hr_dev, |
| 3775 | 3759 | struct hns_roce_qp *hr_qp, |
| 3776 | 3760 | struct hns_roce_v2_qp_context *context, |
| 3777 | 3761 | struct hns_roce_v2_qp_context *qpc_mask) |
| 3778 | 3762 | { |
| 3779 | − struct ib_qp *ibqp = &hr_qp->ibqp; | |
| 3780 | 3763 | u64 mtts[MTT_MIN_COUNT] = { 0 }; |
| 3781 | 3764 | u64 wqe_sge_ba; |
| 3782 | − u32 page_size; | |
| 3783 | 3765 | int count; |
| 3784 | 3766 | |
| 3785 | 3767 | /* Search qp buf's mtts */ |
| 3786 | − page_size = 1 << hr_qp->mtr.hem_cfg.buf_pg_shift; | |
| 3787 | − count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, | |
| 3788 | − hr_qp->rq.offset / page_size, mtts, | |
| 3768 | + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts, | |
| 3789 | 3769 | MTT_MIN_COUNT, &wqe_sge_ba); |
| 3790 | − if (!ibqp->srq) | |
| 3791 | − if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size)) | |
| 3792 | − return -EINVAL; | |
| 3770 | + if (hr_qp->rq.wqe_cnt && count < 1) { | |
| 3771 | + ibdev_err(&hr_dev->ib_dev, | |
| 3772 | + "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn); | |
| 3773 | + return -EINVAL; | |
| 3774 | + } | |
| 3793 | 3775 | |
| 3794 | 3776 | context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3); |
| 3795 | 3777 | qpc_mask->wqe_sge_ba = 0; |
| @@ -3891,7 +3873,6 @@ static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, · 95 unmodified lines | ||
| 3891 | 3873 | struct ib_device *ibdev = &hr_dev->ib_dev; |
| 3892 | 3874 | u64 sge_cur_blk = 0; |
| 3893 | 3875 | u64 sq_cur_blk = 0; |
| 3894 | − u32 page_size; | |
| 3895 | 3876 | int count; |
| 3896 | 3877 | |
| 3897 | 3878 | /* search qp buf's mtts */ |
| @@ -3902,9 +3883,8 @@ static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, · 4 unmodified lines | ||
| 3902 | 3883 | return -EINVAL; |
| 3903 | 3884 | } |
| 3904 | 3885 | if (hr_qp->sge.sge_cnt > 0) { |
| 3905 | − page_size = 1 << hr_qp->mtr.hem_cfg.buf_pg_shift; | |
| 3906 | 3886 | count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, |
| 3907 | − hr_qp->sge.offset / page_size, | |
| 3887 | + hr_qp->sge.offset, | |
| 3908 | 3888 | &sge_cur_blk, 1, NULL); |
| 3909 | 3889 | if (count < 1) { |
| 3910 | 3890 | ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n", |
| @@ -4265,12 +4245,13 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, · 354 unmodified lines | ||
| 4265 | 4245 | roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M, |
| 4266 | 4246 | V2_QPC_BYTE_24_HOP_LIMIT_S, 0); |
| 4267 | 4247 | |
| 4268 | − if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B && is_udp) | |
| 4248 | + if (is_udp) | |
| 4269 | 4249 | roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, |
| 4270 | 4250 | V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2); |
| 4271 | 4251 | else |
| 4272 | 4252 | roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, |
| 4273 | 4253 | V2_QPC_BYTE_24_TC_S, grh->traffic_class); |
| 4254 | + | |
| 4274 | 4255 | roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, |
| 4275 | 4256 | V2_QPC_BYTE_24_TC_S, 0); |
| 4276 | 4257 | roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, |
| @@ -4301,7 +4282,9 @@ static bool check_qp_state(enum ib_qp_state cur_state, · 24 unmodified lines | ||
| 4301 | 4282 | [IB_QPS_RTR] = { [IB_QPS_RESET] = true, |
| 4302 | 4283 | [IB_QPS_RTS] = true, |
| 4303 | 4284 | [IB_QPS_ERR] = true }, |
| 4304 | − [IB_QPS_RTS] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }, | |
| 4285 | + [IB_QPS_RTS] = { [IB_QPS_RESET] = true, | |
| 4286 | + [IB_QPS_RTS] = true, | |
| 4287 | + [IB_QPS_ERR] = true }, | |
| 4305 | 4288 | [IB_QPS_SQD] = {}, |
| 4306 | 4289 | [IB_QPS_SQE] = {}, |
| 4307 | 4290 | [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true } |
| @@ -179,26 +179,10 @@ enum { · 178 unmodified lines | ||
| 179 | 179 | HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD = 0x9, |
| 180 | 180 | HNS_ROCE_V2_WQE_OP_FAST_REG_PMR = 0xa, |
| 181 | 181 | HNS_ROCE_V2_WQE_OP_LOCAL_INV = 0xb, |
| 182 | − HNS_ROCE_V2_WQE_OP_BIND_MW_TYPE = 0xc, | |
| 182 | + HNS_ROCE_V2_WQE_OP_BIND_MW = 0xc, | |
| 183 | 183 | HNS_ROCE_V2_WQE_OP_MASK = 0x1f, |
| 184 | 184 | }; |
| 185 | 185 | |
| 186 | −enum { | |
| 187 | − HNS_ROCE_SQ_OPCODE_SEND = 0x0, | |
| 188 | − HNS_ROCE_SQ_OPCODE_SEND_WITH_INV = 0x1, | |
| 189 | − HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM = 0x2, | |
| 190 | − HNS_ROCE_SQ_OPCODE_RDMA_WRITE = 0x3, | |
| 191 | − HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM = 0x4, | |
| 192 | − HNS_ROCE_SQ_OPCODE_RDMA_READ = 0x5, | |
| 193 | − HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP = 0x6, | |
| 194 | − HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD = 0x7, | |
| 195 | − HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP = 0x8, | |
| 196 | − HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD = 0x9, | |
| 197 | − HNS_ROCE_SQ_OPCODE_FAST_REG_WR = 0xa, | |
| 198 | − HNS_ROCE_SQ_OPCODE_LOCAL_INV = 0xb, | |
| 199 | − HNS_ROCE_SQ_OPCODE_BIND_MW = 0xc, | |
| 200 | −}; | |
| 201 | − | |
| 202 | 186 | enum { |
| 203 | 187 | /* rq operations */ |
| 204 | 188 | HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM = 0x0, |
| @@ -230,6 +214,7 @@ enum { · 25 unmodified lines | ||
| 230 | 214 | HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR = 0x15, |
| 231 | 215 | HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR = 0x16, |
| 232 | 216 | HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR = 0x22, |
| 217 | + HNS_ROCE_CQE_V2_GENERAL_ERR = 0x23, | |
| 233 | 218 | |
| 234 | 219 | HNS_ROCE_V2_CQE_STATUS_MASK = 0xff, |
| 235 | 220 | }; |
| @@ -428,7 +428,7 @@ static const struct ib_device_ops hns_roce_dev_ops = { · 427 unmodified lines | ||
| 428 | 428 | .destroy_ah = hns_roce_destroy_ah, |
| 429 | 429 | .destroy_cq = hns_roce_destroy_cq, |
| 430 | 430 | .disassociate_ucontext = hns_roce_disassociate_ucontext, |
| 431 | − .fill_res_entry = hns_roce_fill_res_entry, | |
| 431 | + .fill_res_cq_entry = hns_roce_fill_res_cq_entry, | |
| 432 | 432 | .get_dma_mr = hns_roce_get_dma_mr, |
| 433 | 433 | .get_link_layer = hns_roce_get_link_layer, |
| 434 | 434 | .get_port_immutable = hns_roce_port_immutable, |
| @@ -415,7 +415,7 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) · 414 unmodified lines | ||
| 415 | 415 | } |
| 416 | 416 | |
| 417 | 417 | struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 418 | − u32 max_num_sg, struct ib_udata *udata) | |
| 418 | + u32 max_num_sg) | |
| 419 | 419 | { |
| 420 | 420 | struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); |
| 421 | 421 | struct device *dev = hr_dev->dev; |
| @@ -871,6 +871,15 @@ int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 449 unmodified lines | ||
| 871 | 871 | int err; |
| 872 | 872 | int i; |
| 873 | 873 | |
| 874 | + /* | |
| 875 | + * Only use the first page address as root ba when hopnum is 0, this | |
| 876 | + * is because the addresses of all pages are consecutive in this case. | |
| 877 | + */ | |
| 878 | + if (mtr->hem_cfg.is_direct) { | |
| 879 | + mtr->hem_cfg.root_ba = pages[0]; | |
| 880 | + return 0; | |
| 881 | + } | |
| 882 | + | |
| 874 | 883 | for (i = 0; i < mtr->hem_cfg.region_count; i++) { |
| 875 | 884 | r = &mtr->hem_cfg.region[i]; |
| 876 | 885 | if (r->offset + r->count > page_cnt) { |
| @@ -896,6 +905,8 @@ int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 19 unmodified lines | ||
| 896 | 905 | int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, |
| 897 | 906 | int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr) |
| 898 | 907 | { |
| 908 | + struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg; | |
| 909 | + int start_index; | |
| 899 | 910 | int mtt_count; |
| 900 | 911 | int total = 0; |
| 901 | 912 | __le64 *mtts; |
| @@ -907,26 +918,32 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 5 unmodified lines | ||
| 907 | 918 | goto done; |
| 908 | 919 | |
| 909 | 920 | /* no mtt memory in direct mode, so just return the buffer address */ |
| 910 | − if (mtr->hem_cfg.is_direct) { | |
| 911 | − npage = offset; | |
| 912 | − for (total = 0; total < mtt_max; total++, npage++) { | |
| 913 | − addr = mtr->hem_cfg.root_ba + | |
| 914 | − (npage << mtr->hem_cfg.buf_pg_shift); | |
| 915 | − | |
| 921 | + if (cfg->is_direct) { | |
| 922 | + start_index = offset >> HNS_HW_PAGE_SHIFT; | |
| 923 | + for (mtt_count = 0; mtt_count < cfg->region_count && | |
| 924 | + total < mtt_max; mtt_count++) { | |
| 925 | + npage = cfg->region[mtt_count].offset; | |
| 926 | + if (npage < start_index) | |
| 927 | + continue; | |
| 928 | + | |
| 929 | + addr = cfg->root_ba + (npage << HNS_HW_PAGE_SHIFT); | |
| 916 | 930 | if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) |
| 917 | 931 | mtt_buf[total] = to_hr_hw_page_addr(addr); |
| 918 | 932 | else |
| 919 | 933 | mtt_buf[total] = addr; |
| 934 | + | |
| 935 | + total++; | |
| 920 | 936 | } |
| 921 | 937 | |
| 922 | 938 | goto done; |
| 923 | 939 | } |
| 924 | 940 | |
| 941 | + start_index = offset >> cfg->buf_pg_shift; | |
| 925 | 942 | left = mtt_max; |
| 926 | 943 | while (left > 0) { |
| 927 | 944 | mtt_count = 0; |
| 928 | 945 | mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, |
| 929 | − offset + total, | |
| 946 | + start_index + total, | |
| 930 | 947 | &mtt_count, NULL); |
| 931 | 948 | if (!mtts || !mtt_count) |
| 932 | 949 | goto done; |
| @@ -939,104 +956,136 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 6 unmodified lines | ||
| 939 | 956 | |
| 940 | 957 | done: |
| 941 | 958 | if (base_addr) |
| 942 | − *base_addr = mtr->hem_cfg.root_ba; | |
| 959 | + *base_addr = cfg->root_ba; | |
| 943 | 960 | |
| 944 | 961 | return total; |
| 945 | 962 | } |
| 946 | 963 | |
| 947 | −/* convert buffer size to page index and page count */ | |
| 948 | −static unsigned int mtr_init_region(struct hns_roce_buf_attr *attr, | |
| 949 | − int page_cnt, | |
| 950 | − struct hns_roce_buf_region *regions, | |
| 951 | − int region_cnt, unsigned int page_shift) | |
| 964 | +static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev, | |
| 965 | + struct hns_roce_buf_attr *attr, | |
| 966 | + struct hns_roce_hem_cfg *cfg, | |
| 967 | + unsigned int *buf_page_shift) | |
| 952 | 968 | { |
| 953 | − unsigned int page_size = 1 << page_shift; | |
| 954 | − int max_region = attr->region_count; | |
| 955 | 969 | struct hns_roce_buf_region *r; |
| 956 | − unsigned int i = 0; | |
| 957 | − int page_idx = 0; | |
| 958 | − | |
| 959 | − for (; i < region_cnt && i < max_region && page_idx < page_cnt; i++) { | |
| 960 | − r = ®ions[i]; | |
| 961 | − r->hopnum = attr->region[i].hopnum == HNS_ROCE_HOP_NUM_0 ? | |
| 962 | − 0 : attr->region[i].hopnum; | |
| 963 | − r->offset = page_idx; | |
| 964 | − r->count = DIV_ROUND_UP(attr->region[i].size, page_size); | |
| 965 | − page_idx += r->count; | |
| 970 | + unsigned int page_shift = 0; | |
| 971 | + int page_cnt = 0; | |
| 972 | + size_t buf_size; | |
| 973 | + int region_cnt; | |
| 974 | + | |
| 975 | + if (cfg->is_direct) { | |
| 976 | + buf_size = cfg->buf_pg_count << cfg->buf_pg_shift; | |
| 977 | + page_cnt = DIV_ROUND_UP(buf_size, HNS_HW_PAGE_SIZE); | |
| 978 | + /* | |
| 979 | + * When HEM buffer use level-0 addressing, the page size equals | |
| 980 | + * the buffer size, and the the page size = 4K * 2^N. | |
| 981 | + */ | |
| 982 | + cfg->buf_pg_shift = HNS_HW_PAGE_SHIFT + order_base_2(page_cnt); | |
| 983 | + if (attr->region_count > 1) { | |
| 984 | + cfg->buf_pg_count = page_cnt; | |
| 985 | + page_shift = HNS_HW_PAGE_SHIFT; | |
| 986 | + } else { | |
| 987 | + cfg->buf_pg_count = 1; | |
| 988 | + page_shift = cfg->buf_pg_shift; | |
| 989 | + if (buf_size != 1 << page_shift) { | |
| 990 | + ibdev_err(&hr_dev->ib_dev, | |
| 991 | + "failed to check direct size %zu shift %d.\n", | |
| 992 | + buf_size, page_shift); | |
| 993 | + return -EINVAL; | |
| 994 | + } | |
| 995 | + } | |
| 996 | + } else { | |
| 997 | + page_shift = cfg->buf_pg_shift; | |
| 998 | + } | |
| 999 | + | |
| 1000 | + /* convert buffer size to page index and page count */ | |
| 1001 | + for (page_cnt = 0, region_cnt = 0; page_cnt < cfg->buf_pg_count && | |
| 1002 | + region_cnt < attr->region_count && | |
| 1003 | + region_cnt < ARRAY_SIZE(cfg->region); region_cnt++) { | |
| 1004 | + r = &cfg->region[region_cnt]; | |
| 1005 | + r->offset = page_cnt; | |
| 1006 | + buf_size = hr_hw_page_align(attr->region[region_cnt].size); | |
| 1007 | + r->count = DIV_ROUND_UP(buf_size, 1 << page_shift); | |
| 1008 | + page_cnt += r->count; | |
| 1009 | + r->hopnum = to_hr_hem_hopnum(attr->region[region_cnt].hopnum, | |
| 1010 | + r->count); | |
| 1011 | + } | |
| 1012 | + | |
| 1013 | + if (region_cnt < 1) { | |
| 1014 | + ibdev_err(&hr_dev->ib_dev, | |
| 1015 | + "failed to check mtr region count, pages = %d.\n", | |
| 1016 | + cfg->buf_pg_count); | |
| 1017 | + return -ENOBUFS; | |
| 966 | 1018 | } |
| 967 | 1019 | |
| 968 | − return i; | |
| 1020 | + cfg->region_count = region_cnt; | |
| 1021 | + *buf_page_shift = page_shift; | |
| 1022 | + | |
| 1023 | + return page_cnt; | |
| 969 | 1024 | } |
| 970 | 1025 | |
| 971 | 1026 | /** |
| 972 | 1027 | * hns_roce_mtr_create - Create hns memory translate region. |
| 973 | 1028 | * |
| 974 | 1029 | * @mtr: memory translate region |
| 975 | − * @init_attr: init attribute for creating mtr | |
| 976 | − * @page_shift: page shift for multi-hop base address table | |
| 1030 | + * @buf_attr: buffer attribute for creating mtr | |
| 1031 | + * @ba_page_shift: page shift for multi-hop base address table | |
| 977 | 1032 | * @udata: user space context, if it's NULL, means kernel space |
| 978 | 1033 | * @user_addr: userspace virtual address to start at |
| 979 | − * @buf_alloced: mtr has private buffer, true means need to alloc | |
| 980 | 1034 | */ |
| 981 | 1035 | int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, |
| 982 | 1036 | struct hns_roce_buf_attr *buf_attr, |
| 983 | − unsigned int page_shift, struct ib_udata *udata, | |
| 1037 | + unsigned int ba_page_shift, struct ib_udata *udata, | |
| 984 | 1038 | unsigned long user_addr) |
| 985 | 1039 | { |
| 1040 | + struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg; | |
| 986 | 1041 | struct ib_device *ibdev = &hr_dev->ib_dev; |
| 1042 | + unsigned int buf_page_shift = 0; | |
| 987 | 1043 | dma_addr_t *pages = NULL; |
| 988 | − int region_cnt = 0; | |
| 989 | 1044 | int all_pg_cnt; |
| 990 | 1045 | int get_pg_cnt; |
| 991 | − bool has_mtt; | |
| 992 | − int err = 0; | |
| 1046 | + int ret = 0; | |
| 1047 | + | |
| 1048 | + /* if disable mtt, all pages must in a continuous address range */ | |
| 1049 | + cfg->is_direct = !mtr_has_mtt(buf_attr); | |
| 993 | 1050 | |
| 994 | − has_mtt = mtr_has_mtt(buf_attr); | |
| 995 | 1051 | /* if buffer only need mtt, just init the hem cfg */ |
| 996 | 1052 | if (buf_attr->mtt_only) { |
| 997 | − mtr->hem_cfg.buf_pg_shift = buf_attr->page_shift; | |
| 998 | − mtr->hem_cfg.buf_pg_count = mtr_bufs_size(buf_attr) >> | |
| 999 | − buf_attr->page_shift; | |
| 1053 | + cfg->buf_pg_shift = buf_attr->page_shift; | |
| 1054 | + cfg->buf_pg_count = mtr_bufs_size(buf_attr) >> | |
| 1055 | + buf_attr->page_shift; | |
| 1000 | 1056 | mtr->umem = NULL; |
| 1001 | 1057 | mtr->kmem = NULL; |
| 1002 | 1058 | } else { |
| 1003 | − err = mtr_alloc_bufs(hr_dev, mtr, buf_attr, !has_mtt, udata, | |
| 1004 | − user_addr); | |
| 1005 | − if (err) { | |
| 1006 | − ibdev_err(ibdev, "Failed to alloc mtr bufs, err %d\n", | |
| 1007 | − err); | |
| 1008 | − return err; | |
| 1059 | + ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, cfg->is_direct, | |
| 1060 | + udata, user_addr); | |
| 1061 | + if (ret) { | |
| 1062 | + ibdev_err(ibdev, | |
| 1063 | + "failed to alloc mtr bufs, ret = %d.\n", ret); | |
| 1064 | + return ret; | |
| 1009 | 1065 | } |
| 1010 | 1066 | } |
| 1011 | 1067 | |
| 1012 | − /* alloc mtt memory */ | |
| 1013 | − all_pg_cnt = mtr->hem_cfg.buf_pg_count; | |
| 1014 | − hns_roce_hem_list_init(&mtr->hem_list); | |
| 1015 | − mtr->hem_cfg.is_direct = !has_mtt; | |
| 1016 | − mtr->hem_cfg.ba_pg_shift = page_shift; | |
| 1017 | − mtr->hem_cfg.region_count = 0; | |
| 1018 | − region_cnt = mtr_init_region(buf_attr, all_pg_cnt, | |
| 1019 | − mtr->hem_cfg.region, | |
| 1020 | − ARRAY_SIZE(mtr->hem_cfg.region), | |
| 1021 | − mtr->hem_cfg.buf_pg_shift); | |
| 1022 | − if (region_cnt < 1) { | |
| 1023 | − err = -ENOBUFS; | |
| 1024 | − ibdev_err(ibdev, "failed to init mtr region %d\n", region_cnt); | |
| 1068 | + all_pg_cnt = mtr_init_buf_cfg(hr_dev, buf_attr, cfg, &buf_page_shift); | |
| 1069 | + if (all_pg_cnt < 1) { | |
| 1070 | + ret = -ENOBUFS; | |
| 1071 | + ibdev_err(ibdev, "failed to init mtr buf cfg.\n"); | |
| 1025 | 1072 | goto err_alloc_bufs; |
| 1026 | 1073 | } |
| 1027 | 1074 | |
| 1028 | − mtr->hem_cfg.region_count = region_cnt; | |
| 1029 | − | |
| 1030 | − if (has_mtt) { | |
| 1031 | − err = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, | |
| 1032 | − mtr->hem_cfg.region, region_cnt, | |
| 1033 | − page_shift); | |
| 1034 | − if (err) { | |
| 1035 | − ibdev_err(ibdev, "Failed to request mtr hem, err %d\n", | |
| 1036 | − err); | |
| 1075 | + hns_roce_hem_list_init(&mtr->hem_list); | |
| 1076 | + if (!cfg->is_direct) { | |
| 1077 | + ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, | |
| 1078 | + cfg->region, cfg->region_count, | |
| 1079 | + ba_page_shift); | |
| 1080 | + if (ret) { | |
| 1081 | + ibdev_err(ibdev, "failed to request mtr hem, ret = %d.\n", | |
| 1082 | + ret); | |
| 1037 | 1083 | goto err_alloc_bufs; |
| 1038 | 1084 | } |
| 1039 | − mtr->hem_cfg.root_ba = mtr->hem_list.root_ba; | |
| 1085 | + cfg->root_ba = mtr->hem_list.root_ba; | |
| 1086 | + cfg->ba_pg_shift = ba_page_shift; | |
| 1087 | + } else { | |
| 1088 | + cfg->ba_pg_shift = cfg->buf_pg_shift; | |
| 1040 | 1089 | } |
| 1041 | 1090 | |
| 1042 | 1091 | /* no buffer to map */ |
| @@ -1046,31 +1095,26 @@ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 3 unmodified lines | ||
| 1046 | 1095 | /* alloc a tmp array to store buffer's dma address */ |
| 1047 | 1096 | pages = kvcalloc(all_pg_cnt, sizeof(dma_addr_t), GFP_KERNEL); |
| 1048 | 1097 | if (!pages) { |
| 1049 | − err = -ENOMEM; | |
| 1050 | − ibdev_err(ibdev, "Failed to alloc mtr page list %d\n", | |
| 1098 | + ret = -ENOMEM; | |
| 1099 | + ibdev_err(ibdev, "failed to alloc mtr page list %d.\n", | |
| 1051 | 1100 | all_pg_cnt); |
| 1052 | 1101 | goto err_alloc_hem_list; |
| 1053 | 1102 | } |
| 1054 | 1103 | |
| 1055 | 1104 | get_pg_cnt = mtr_get_pages(hr_dev, mtr, pages, all_pg_cnt, |
| 1056 | − mtr->hem_cfg.buf_pg_shift); | |
| 1105 | + buf_page_shift); | |
| 1057 | 1106 | if (get_pg_cnt != all_pg_cnt) { |
| 1058 | − ibdev_err(ibdev, "Failed to get mtr page %d != %d\n", | |
| 1107 | + ibdev_err(ibdev, "failed to get mtr page %d != %d.\n", | |
| 1059 | 1108 | get_pg_cnt, all_pg_cnt); |
| 1060 | − err = -ENOBUFS; | |
| 1109 | + ret = -ENOBUFS; | |
| 1061 | 1110 | goto err_alloc_page_list; |
| 1062 | 1111 | } |
| 1063 | 1112 | |
| 1064 | − if (!has_mtt) { | |
| 1065 | − mtr->hem_cfg.root_ba = pages[0]; | |
| 1066 | − } else { | |
| 1067 | − /* write buffer's dma address to BA table */ | |
| 1068 | − err = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt); | |
| 1069 | − if (err) { | |
| 1070 | − ibdev_err(ibdev, "Failed to map mtr pages, err %d\n", | |
| 1071 | − err); | |
| 1072 | − goto err_alloc_page_list; | |
| 1073 | − } | |
| 1113 | + /* write buffer's dma address to BA table */ | |
| 1114 | + ret = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt); | |
| 1115 | + if (ret) { | |
| 1116 | + ibdev_err(ibdev, "failed to map mtr pages, ret = %d.\n", ret); | |
| 1117 | + goto err_alloc_page_list; | |
| 1074 | 1118 | } |
| 1075 | 1119 | |
| 1076 | 1120 | /* drop tmp array */ |
| @@ -1082,7 +1126,7 @@ err_alloc_hem_list: · 5 unmodified lines | ||
| 1082 | 1126 | hns_roce_hem_list_release(hr_dev, &mtr->hem_list); |
| 1083 | 1127 | err_alloc_bufs: |
| 1084 | 1128 | mtr_free_bufs(hr_dev, mtr); |
| 1085 | − return err; | |
| 1129 | + return ret; | |
| 1086 | 1130 | } |
| 1087 | 1131 | |
| 1088 | 1132 | void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) |
| @@ -411,7 +411,6 @@ static int set_extend_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, · 410 unmodified lines | ||
| 411 | 411 | struct hns_roce_qp *hr_qp, |
| 412 | 412 | struct ib_qp_cap *cap) |
| 413 | 413 | { |
| 414 | − struct ib_device *ibdev = &hr_dev->ib_dev; | |
| 415 | 414 | u32 cnt; |
| 416 | 415 | |
| 417 | 416 | cnt = max(1U, cap->max_send_sge); |
| @@ -431,15 +430,6 @@ static int set_extend_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, · 13 unmodified lines | ||
| 431 | 430 | } else if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) { |
| 432 | 431 | cnt = roundup_pow_of_two(sq_wqe_cnt * |
| 433 | 432 | (hr_qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE)); |
| 434 | − | |
| 435 | − if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_A) { | |
| 436 | − if (cnt > hr_dev->caps.max_extend_sg) { | |
| 437 | − ibdev_err(ibdev, | |
| 438 | − "failed to check exSGE num, exSGE num = %d.\n", | |
| 439 | − cnt); | |
| 440 | − return -EINVAL; | |
| 441 | − } | |
| 442 | − } | |
| 443 | 433 | } else { |
| 444 | 434 | cnt = 0; |
| 445 | 435 | } |
| @@ -76,10 +76,9 @@ err: · 75 unmodified lines | ||
| 76 | 76 | return -EMSGSIZE; |
| 77 | 77 | } |
| 78 | 78 | |
| 79 | −static int hns_roce_fill_res_cq_entry(struct sk_buff *msg, | |
| 80 | − struct rdma_restrack_entry *res) | |
| 79 | +int hns_roce_fill_res_cq_entry(struct sk_buff *msg, | |
| 80 | + struct ib_cq *ib_cq) | |
| 81 | 81 | { |
| 82 | − struct ib_cq *ib_cq = container_of(res, struct ib_cq, res); | |
| 83 | 82 | struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); |
| 84 | 83 | struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); |
| 85 | 84 | struct hns_roce_v2_cq_context *context; |
| @@ -119,12 +118,3 @@ err: · 33 unmodified lines | ||
| 119 | 118 | kfree(context); |
| 120 | 119 | return ret; |
| 121 | 120 | } |
| 122 | − | |
| 123 | −int hns_roce_fill_res_entry(struct sk_buff *msg, | |
| 124 | − struct rdma_restrack_entry *res) | |
| 125 | −{ | |
| 126 | − if (res->type == RDMA_RESTRACK_CQ) | |
| 127 | − return hns_roce_fill_res_cq_entry(msg, res); | |
| 128 | − | |
| 129 | − return 0; | |
| 130 | −} | |
| @@ -101,7 +101,6 @@ static int i40iw_query_port(struct ib_device *ibdev, · 100 unmodified lines | ||
| 101 | 101 | props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | |
| 102 | 102 | IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; |
| 103 | 103 | props->gid_tbl_len = 1; |
| 104 | − props->pkey_tbl_len = 1; | |
| 105 | 104 | props->active_width = IB_WIDTH_4X; |
| 106 | 105 | props->active_speed = 1; |
| 107 | 106 | props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE; |
| @@ -1543,10 +1542,9 @@ static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr · 1435 unmodified lines | ||
| 1543 | 1542 | * @pd: ibpd pointer |
| 1544 | 1543 | * @mr_type: memory for stag registrion |
| 1545 | 1544 | * @max_num_sg: man number of pages |
| 1546 | − * @udata: user data or NULL for kernel objects | |
| 1547 | 1545 | */ |
| 1548 | 1546 | static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 1549 | − u32 max_num_sg, struct ib_udata *udata) | |
| 1547 | + u32 max_num_sg) | |
| 1550 | 1548 | { |
| 1551 | 1549 | struct i40iw_pd *iwpd = to_iwpd(pd); |
| 1552 | 1550 | struct i40iw_device *iwdev = to_iwdev(pd->device); |
| @@ -2460,7 +2458,6 @@ static int i40iw_port_immutable(struct ib_device *ibdev, u8 port_num, · 907 unmodified lines | ||
| 2460 | 2458 | if (err) |
| 2461 | 2459 | return err; |
| 2462 | 2460 | |
| 2463 | − immutable->pkey_tbl_len = attr.pkey_tbl_len; | |
| 2464 | 2461 | immutable->gid_tbl_len = attr.gid_tbl_len; |
| 2465 | 2462 | |
| 2466 | 2463 | return 0; |
| @@ -2616,22 +2613,6 @@ static int i40iw_query_gid(struct ib_device *ibdev, · 149 unmodified lines | ||
| 2616 | 2613 | return 0; |
| 2617 | 2614 | } |
| 2618 | 2615 | |
| 2619 | −/** | |
| 2620 | − * i40iw_query_pkey - Query partition key | |
| 2621 | − * @ibdev: device pointer from stack | |
| 2622 | − * @port: port number | |
| 2623 | − * @index: index of pkey | |
| 2624 | − * @pkey: pointer to store the pkey | |
| 2625 | − */ | |
| 2626 | −static int i40iw_query_pkey(struct ib_device *ibdev, | |
| 2627 | − u8 port, | |
| 2628 | − u16 index, | |
| 2629 | − u16 *pkey) | |
| 2630 | −{ | |
| 2631 | − *pkey = 0; | |
| 2632 | − return 0; | |
| 2633 | −} | |
| 2634 | − | |
| 2635 | 2616 | static const struct ib_device_ops i40iw_dev_ops = { |
| 2636 | 2617 | .owner = THIS_MODULE, |
| 2637 | 2618 | .driver_id = RDMA_DRIVER_I40IW, |
| @@ -2671,7 +2652,6 @@ static const struct ib_device_ops i40iw_dev_ops = { · 33 unmodified lines | ||
| 2671 | 2652 | .post_send = i40iw_post_send, |
| 2672 | 2653 | .query_device = i40iw_query_device, |
| 2673 | 2654 | .query_gid = i40iw_query_gid, |
| 2674 | − .query_pkey = i40iw_query_pkey, | |
| 2675 | 2655 | .query_port = i40iw_query_port, |
| 2676 | 2656 | .query_qp = i40iw_query_qp, |
| 2677 | 2657 | .reg_user_mr = i40iw_reg_user_mr, |
| @@ -1219,56 +1219,47 @@ static void mlx4_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) · 1218 unmodified lines | ||
| 1219 | 1219 | mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn); |
| 1220 | 1220 | } |
| 1221 | 1221 | |
| 1222 | −static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev, | |
| 1223 | − struct ib_udata *udata) | |
| 1222 | +static int mlx4_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) | |
| 1224 | 1223 | { |
| 1225 | − struct mlx4_ib_xrcd *xrcd; | |
| 1224 | + struct mlx4_ib_dev *dev = to_mdev(ibxrcd->device); | |
| 1225 | + struct mlx4_ib_xrcd *xrcd = to_mxrcd(ibxrcd); | |
| 1226 | 1226 | struct ib_cq_init_attr cq_attr = {}; |
| 1227 | 1227 | int err; |
| 1228 | 1228 | |
| 1229 | − if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) | |
| 1230 | − return ERR_PTR(-ENOSYS); | |
| 1231 | − | |
| 1232 | − xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL); | |
| 1233 | − if (!xrcd) | |
| 1234 | − return ERR_PTR(-ENOMEM); | |
| 1229 | + if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) | |
| 1230 | + return -EOPNOTSUPP; | |
| 1235 | 1231 | |
| 1236 | − err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn); | |
| 1232 | + err = mlx4_xrcd_alloc(dev->dev, &xrcd->xrcdn); | |
| 1237 | 1233 | if (err) |
| 1238 | − goto err1; | |
| 1234 | + return err; | |
| 1239 | 1235 | |
| 1240 | − xrcd->pd = ib_alloc_pd(ibdev, 0); | |
| 1236 | + xrcd->pd = ib_alloc_pd(ibxrcd->device, 0); | |
| 1241 | 1237 | if (IS_ERR(xrcd->pd)) { |
| 1242 | 1238 | err = PTR_ERR(xrcd->pd); |
| 1243 | 1239 | goto err2; |
| 1244 | 1240 | } |
| 1245 | 1241 | |
| 1246 | 1242 | cq_attr.cqe = 1; |
| 1247 | − xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr); | |
| 1243 | + xrcd->cq = ib_create_cq(ibxrcd->device, NULL, NULL, xrcd, &cq_attr); | |
| 1248 | 1244 | if (IS_ERR(xrcd->cq)) { |
| 1249 | 1245 | err = PTR_ERR(xrcd->cq); |
| 1250 | 1246 | goto err3; |
| 1251 | 1247 | } |
| 1252 | 1248 | |
| 1253 | − return &xrcd->ibxrcd; | |
| 1249 | + return 0; | |
| 1254 | 1250 | |
| 1255 | 1251 | err3: |
| 1256 | 1252 | ib_dealloc_pd(xrcd->pd); |
| 1257 | 1253 | err2: |
| 1258 | − mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn); | |
| 1259 | −err1: | |
| 1260 | − kfree(xrcd); | |
| 1261 | − return ERR_PTR(err); | |
| 1254 | + mlx4_xrcd_free(dev->dev, xrcd->xrcdn); | |
| 1255 | + return err; | |
| 1262 | 1256 | } |
| 1263 | 1257 | |
| 1264 | −static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) | |
| 1258 | +static void mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) | |
| 1265 | 1259 | { |
| 1266 | 1260 | ib_destroy_cq(to_mxrcd(xrcd)->cq); |
| 1267 | 1261 | ib_dealloc_pd(to_mxrcd(xrcd)->pd); |
| 1268 | 1262 | mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn); |
| 1269 | − kfree(xrcd); | |
| 1270 | − | |
| 1271 | − return 0; | |
| 1272 | 1263 | } |
| 1273 | 1264 | |
| 1274 | 1265 | static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid) |
| @@ -2607,6 +2598,8 @@ static const struct ib_device_ops mlx4_ib_dev_mw_ops = { · 1332 unmodified lines | ||
| 2607 | 2598 | static const struct ib_device_ops mlx4_ib_dev_xrc_ops = { |
| 2608 | 2599 | .alloc_xrcd = mlx4_ib_alloc_xrcd, |
| 2609 | 2600 | .dealloc_xrcd = mlx4_ib_dealloc_xrcd, |
| 2601 | + | |
| 2602 | + INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx4_ib_xrcd, ibxrcd), | |
| 2610 | 2603 | }; |
| 2611 | 2604 | |
| 2612 | 2605 | static const struct ib_device_ops mlx4_ib_dev_fs_ops = { |
| @@ -729,7 +729,7 @@ struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, · 728 unmodified lines | ||
| 729 | 729 | struct ib_udata *udata); |
| 730 | 730 | int mlx4_ib_dealloc_mw(struct ib_mw *mw); |
| 731 | 731 | struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 732 | − u32 max_num_sg, struct ib_udata *udata); | |
| 732 | + u32 max_num_sg); | |
| 733 | 733 | int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, |
| 734 | 734 | unsigned int *sg_offset); |
| 735 | 735 | int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); |
| @@ -439,7 +439,6 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · 438 unmodified lines | ||
| 439 | 439 | |
| 440 | 440 | mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; |
| 441 | 441 | mr->ibmr.length = length; |
| 442 | − mr->ibmr.iova = virt_addr; | |
| 443 | 442 | mr->ibmr.page_size = 1U << shift; |
| 444 | 443 | |
| 445 | 444 | return &mr->ibmr; |
| @@ -655,7 +654,7 @@ int mlx4_ib_dealloc_mw(struct ib_mw *ibmw) · 209 unmodified lines | ||
| 655 | 654 | } |
| 656 | 655 | |
| 657 | 656 | struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 658 | − u32 max_num_sg, struct ib_udata *udata) | |
| 657 | + u32 max_num_sg) | |
| 659 | 658 | { |
| 660 | 659 | struct mlx4_ib_dev *dev = to_mdev(pd->device); |
| 661 | 660 | struct mlx4_ib_mr *mr; |
| @@ -4,6 +4,7 @@ obj-$(CONFIG_MLX5_INFINIBAND) += mlx5_ib.o · 3 unmodified lines | ||
| 4 | 4 | mlx5_ib-y := ah.o \ |
| 5 | 5 | cmd.o \ |
| 6 | 6 | cong.o \ |
| 7 | + counters.o \ | |
| 7 | 8 | cq.o \ |
| 8 | 9 | doorbell.o \ |
| 9 | 10 | gsi.o \ |
| @@ -22,5 +23,6 @@ mlx5_ib-y := ah.o \ · 12 unmodified lines | ||
| 22 | 23 | mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o |
| 23 | 24 | mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o |
| 24 | 25 | mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o \ |
| 25 | − flow.o \ | |
| 26 | − qos.o | |
| 26 | + fs.o \ | |
| 27 | + qos.o \ | |
| 28 | + std_types.o | |
| @@ -148,18 +148,6 @@ void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length) · 147 unmodified lines | ||
| 148 | 148 | spin_unlock(&dm->lock); |
| 149 | 149 | } |
| 150 | 150 | |
| 151 | −int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out) | |
| 152 | −{ | |
| 153 | − u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; | |
| 154 | − int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); | |
| 155 | − | |
| 156 | − MLX5_SET(ppcnt_reg, in, local_port, 1); | |
| 157 | − | |
| 158 | − MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP); | |
| 159 | − return mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PPCNT, | |
| 160 | − 0, 0); | |
| 161 | −} | |
| 162 | − | |
| 163 | 151 | void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid) |
| 164 | 152 | { |
| 165 | 153 | u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {}; |
| @@ -41,7 +41,6 @@ int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey); · 40 unmodified lines | ||
| 41 | 41 | int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey); |
| 42 | 42 | int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, |
| 43 | 43 | void *out); |
| 44 | −int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out); | |
| 45 | 44 | int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr, |
| 46 | 45 | u64 length, u32 alignment); |
| 47 | 46 | void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length); |
| @@ -0,0 +1,709 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB | |
| 2 | +/* | |
| 3 | + * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#include "mlx5_ib.h" | |
| 7 | +#include <linux/mlx5/eswitch.h> | |
| 8 | +#include "counters.h" | |
| 9 | +#include "ib_rep.h" | |
| 10 | +#include "qp.h" | |
| 11 | + | |
| 12 | +struct mlx5_ib_counter { | |
| 13 | + const char *name; | |
| 14 | + size_t offset; | |
| 15 | +}; | |
| 16 | + | |
| 17 | +#define INIT_Q_COUNTER(_name) \ | |
| 18 | + { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)} | |
| 19 | + | |
| 20 | +static const struct mlx5_ib_counter basic_q_cnts[] = { | |
| 21 | + INIT_Q_COUNTER(rx_write_requests), | |
| 22 | + INIT_Q_COUNTER(rx_read_requests), | |
| 23 | + INIT_Q_COUNTER(rx_atomic_requests), | |
| 24 | + INIT_Q_COUNTER(out_of_buffer), | |
| 25 | +}; | |
| 26 | + | |
| 27 | +static const struct mlx5_ib_counter out_of_seq_q_cnts[] = { | |
| 28 | + INIT_Q_COUNTER(out_of_sequence), | |
| 29 | +}; | |
| 30 | + | |
| 31 | +static const struct mlx5_ib_counter retrans_q_cnts[] = { | |
| 32 | + INIT_Q_COUNTER(duplicate_request), | |
| 33 | + INIT_Q_COUNTER(rnr_nak_retry_err), | |
| 34 | + INIT_Q_COUNTER(packet_seq_err), | |
| 35 | + INIT_Q_COUNTER(implied_nak_seq_err), | |
| 36 | + INIT_Q_COUNTER(local_ack_timeout_err), | |
| 37 | +}; | |
| 38 | + | |
| 39 | +#define INIT_CONG_COUNTER(_name) \ | |
| 40 | + { .name = #_name, .offset = \ | |
| 41 | + MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)} | |
| 42 | + | |
| 43 | +static const struct mlx5_ib_counter cong_cnts[] = { | |
| 44 | + INIT_CONG_COUNTER(rp_cnp_ignored), | |
| 45 | + INIT_CONG_COUNTER(rp_cnp_handled), | |
| 46 | + INIT_CONG_COUNTER(np_ecn_marked_roce_packets), | |
| 47 | + INIT_CONG_COUNTER(np_cnp_sent), | |
| 48 | +}; | |
| 49 | + | |
| 50 | +static const struct mlx5_ib_counter extended_err_cnts[] = { | |
| 51 | + INIT_Q_COUNTER(resp_local_length_error), | |
| 52 | + INIT_Q_COUNTER(resp_cqe_error), | |
| 53 | + INIT_Q_COUNTER(req_cqe_error), | |
| 54 | + INIT_Q_COUNTER(req_remote_invalid_request), | |
| 55 | + INIT_Q_COUNTER(req_remote_access_errors), | |
| 56 | + INIT_Q_COUNTER(resp_remote_access_errors), | |
| 57 | + INIT_Q_COUNTER(resp_cqe_flush_error), | |
| 58 | + INIT_Q_COUNTER(req_cqe_flush_error), | |
| 59 | +}; | |
| 60 | + | |
| 61 | +static const struct mlx5_ib_counter roce_accl_cnts[] = { | |
| 62 | + INIT_Q_COUNTER(roce_adp_retrans), | |
| 63 | + INIT_Q_COUNTER(roce_adp_retrans_to), | |
| 64 | + INIT_Q_COUNTER(roce_slow_restart), | |
| 65 | + INIT_Q_COUNTER(roce_slow_restart_cnps), | |
| 66 | + INIT_Q_COUNTER(roce_slow_restart_trans), | |
| 67 | +}; | |
| 68 | + | |
| 69 | +#define INIT_EXT_PPCNT_COUNTER(_name) \ | |
| 70 | + { .name = #_name, .offset = \ | |
| 71 | + MLX5_BYTE_OFF(ppcnt_reg, \ | |
| 72 | + counter_set.eth_extended_cntrs_grp_data_layout._name##_high)} | |
| 73 | + | |
| 74 | +static const struct mlx5_ib_counter ext_ppcnt_cnts[] = { | |
| 75 | + INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated), | |
| 76 | +}; | |
| 77 | + | |
| 78 | +static int mlx5_ib_read_counters(struct ib_counters *counters, | |
| 79 | + struct ib_counters_read_attr *read_attr, | |
| 80 | + struct uverbs_attr_bundle *attrs) | |
| 81 | +{ | |
| 82 | + struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); | |
| 83 | + struct mlx5_read_counters_attr mread_attr = {}; | |
| 84 | + struct mlx5_ib_flow_counters_desc *desc; | |
| 85 | + int ret, i; | |
| 86 | + | |
| 87 | + mutex_lock(&mcounters->mcntrs_mutex); | |
| 88 | + if (mcounters->cntrs_max_index > read_attr->ncounters) { | |
| 89 | + ret = -EINVAL; | |
| 90 | + goto err_bound; | |
| 91 | + } | |
| 92 | + | |
| 93 | + mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64), | |
| 94 | + GFP_KERNEL); | |
| 95 | + if (!mread_attr.out) { | |
| 96 | + ret = -ENOMEM; | |
| 97 | + goto err_bound; | |
| 98 | + } | |
| 99 | + | |
| 100 | + mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl; | |
| 101 | + mread_attr.flags = read_attr->flags; | |
| 102 | + ret = mcounters->read_counters(counters->device, &mread_attr); | |
| 103 | + if (ret) | |
| 104 | + goto err_read; | |
| 105 | + | |
| 106 | + /* do the pass over the counters data array to assign according to the | |
| 107 | + * descriptions and indexing pairs | |
| 108 | + */ | |
| 109 | + desc = mcounters->counters_data; | |
| 110 | + for (i = 0; i < mcounters->ncounters; i++) | |
| 111 | + read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description]; | |
| 112 | + | |
| 113 | +err_read: | |
| 114 | + kfree(mread_attr.out); | |
| 115 | +err_bound: | |
| 116 | + mutex_unlock(&mcounters->mcntrs_mutex); | |
| 117 | + return ret; | |
| 118 | +} | |
| 119 | + | |
| 120 | +static void mlx5_ib_destroy_counters(struct ib_counters *counters) | |
| 121 | +{ | |
| 122 | + struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); | |
| 123 | + | |
| 124 | + mlx5_ib_counters_clear_description(counters); | |
| 125 | + if (mcounters->hw_cntrs_hndl) | |
| 126 | + mlx5_fc_destroy(to_mdev(counters->device)->mdev, | |
| 127 | + mcounters->hw_cntrs_hndl); | |
| 128 | +} | |
| 129 | + | |
| 130 | +static int mlx5_ib_create_counters(struct ib_counters *counters, | |
| 131 | + struct uverbs_attr_bundle *attrs) | |
| 132 | +{ | |
| 133 | + struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); | |
| 134 | + | |
| 135 | + mutex_init(&mcounters->mcntrs_mutex); | |
| 136 | + return 0; | |
| 137 | +} | |
| 138 | + | |
| 139 | + | |
| 140 | +static bool is_mdev_switchdev_mode(const struct mlx5_core_dev *mdev) | |
| 141 | +{ | |
| 142 | + return MLX5_ESWITCH_MANAGER(mdev) && | |
| 143 | + mlx5_ib_eswitch_mode(mdev->priv.eswitch) == | |
| 144 | + MLX5_ESWITCH_OFFLOADS; | |
| 145 | +} | |
| 146 | + | |
| 147 | +static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, | |
| 148 | + u8 port_num) | |
| 149 | +{ | |
| 150 | + return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts : | |
| 151 | + &dev->port[port_num].cnts; | |
| 152 | +} | |
| 153 | + | |
| 154 | +/** | |
| 155 | + * mlx5_ib_get_counters_id - Returns counters id to use for device+port | |
| 156 | + * @dev: Pointer to mlx5 IB device | |
| 157 | + * @port_num: Zero based port number | |
| 158 | + * | |
| 159 | + * mlx5_ib_get_counters_id() Returns counters set id to use for given | |
| 160 | + * device port combination in switchdev and non switchdev mode of the | |
| 161 | + * parent device. | |
| 162 | + */ | |
| 163 | +u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num) | |
| 164 | +{ | |
| 165 | + const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); | |
| 166 | + | |
| 167 | + return cnts->set_id; | |
| 168 | +} | |
| 169 | + | |
| 170 | +static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev, | |
| 171 | + u8 port_num) | |
| 172 | +{ | |
| 173 | + struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 174 | + const struct mlx5_ib_counters *cnts; | |
| 175 | + bool is_switchdev = is_mdev_switchdev_mode(dev->mdev); | |
| 176 | + | |
| 177 | + if ((is_switchdev && port_num) || (!is_switchdev && !port_num)) | |
| 178 | + return NULL; | |
| 179 | + | |
| 180 | + cnts = get_counters(dev, port_num - 1); | |
| 181 | + | |
| 182 | + return rdma_alloc_hw_stats_struct(cnts->names, | |
| 183 | + cnts->num_q_counters + | |
| 184 | + cnts->num_cong_counters + | |
| 185 | + cnts->num_ext_ppcnt_counters, | |
| 186 | + RDMA_HW_STATS_DEFAULT_LIFESPAN); | |
| 187 | +} | |
| 188 | + | |
| 189 | +static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, | |
| 190 | + const struct mlx5_ib_counters *cnts, | |
| 191 | + struct rdma_hw_stats *stats, | |
| 192 | + u16 set_id) | |
| 193 | +{ | |
| 194 | + u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; | |
| 195 | + u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; | |
| 196 | + __be32 val; | |
| 197 | + int ret, i; | |
| 198 | + | |
| 199 | + MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); | |
| 200 | + MLX5_SET(query_q_counter_in, in, counter_set_id, set_id); | |
| 201 | + ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); | |
| 202 | + if (ret) | |
| 203 | + return ret; | |
| 204 | + | |
| 205 | + for (i = 0; i < cnts->num_q_counters; i++) { | |
| 206 | + val = *(__be32 *)((void *)out + cnts->offsets[i]); | |
| 207 | + stats->value[i] = (u64)be32_to_cpu(val); | |
| 208 | + } | |
| 209 | + | |
| 210 | + return 0; | |
| 211 | +} | |
| 212 | + | |
| 213 | +static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev, | |
| 214 | + const struct mlx5_ib_counters *cnts, | |
| 215 | + struct rdma_hw_stats *stats) | |
| 216 | +{ | |
| 217 | + int offset = cnts->num_q_counters + cnts->num_cong_counters; | |
| 218 | + u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; | |
| 219 | + int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); | |
| 220 | + int ret, i; | |
| 221 | + void *out; | |
| 222 | + | |
| 223 | + out = kvzalloc(sz, GFP_KERNEL); | |
| 224 | + if (!out) | |
| 225 | + return -ENOMEM; | |
| 226 | + | |
| 227 | + MLX5_SET(ppcnt_reg, in, local_port, 1); | |
| 228 | + MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP); | |
| 229 | + ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT, | |
| 230 | + 0, 0); | |
| 231 | + if (ret) | |
| 232 | + goto free; | |
| 233 | + | |
| 234 | + for (i = 0; i < cnts->num_ext_ppcnt_counters; i++) | |
| 235 | + stats->value[i + offset] = | |
| 236 | + be64_to_cpup((__be64 *)(out + | |
| 237 | + cnts->offsets[i + offset])); | |
| 238 | +free: | |
| 239 | + kvfree(out); | |
| 240 | + return ret; | |
| 241 | +} | |
| 242 | + | |
| 243 | +static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, | |
| 244 | + struct rdma_hw_stats *stats, | |
| 245 | + u8 port_num, int index) | |
| 246 | +{ | |
| 247 | + struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 248 | + const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1); | |
| 249 | + struct mlx5_core_dev *mdev; | |
| 250 | + int ret, num_counters; | |
| 251 | + u8 mdev_port_num; | |
| 252 | + | |
| 253 | + if (!stats) | |
| 254 | + return -EINVAL; | |
| 255 | + | |
| 256 | + num_counters = cnts->num_q_counters + | |
| 257 | + cnts->num_cong_counters + | |
| 258 | + cnts->num_ext_ppcnt_counters; | |
| 259 | + | |
| 260 | + /* q_counters are per IB device, query the master mdev */ | |
| 261 | + ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id); | |
| 262 | + if (ret) | |
| 263 | + return ret; | |
| 264 | + | |
| 265 | + if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { | |
| 266 | + ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats); | |
| 267 | + if (ret) | |
| 268 | + return ret; | |
| 269 | + } | |
| 270 | + | |
| 271 | + if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { | |
| 272 | + mdev = mlx5_ib_get_native_port_mdev(dev, port_num, | |
| 273 | + &mdev_port_num); | |
| 274 | + if (!mdev) { | |
| 275 | + /* If port is not affiliated yet, its in down state | |
| 276 | + * which doesn't have any counters yet, so it would be | |
| 277 | + * zero. So no need to read from the HCA. | |
| 278 | + */ | |
| 279 | + goto done; | |
| 280 | + } | |
| 281 | + ret = mlx5_lag_query_cong_counters(dev->mdev, | |
| 282 | + stats->value + | |
| 283 | + cnts->num_q_counters, | |
| 284 | + cnts->num_cong_counters, | |
| 285 | + cnts->offsets + | |
| 286 | + cnts->num_q_counters); | |
| 287 | + | |
| 288 | + mlx5_ib_put_native_port_mdev(dev, port_num); | |
| 289 | + if (ret) | |
| 290 | + return ret; | |
| 291 | + } | |
| 292 | + | |
| 293 | +done: | |
| 294 | + return num_counters; | |
| 295 | +} | |
| 296 | + | |
| 297 | +static struct rdma_hw_stats * | |
| 298 | +mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) | |
| 299 | +{ | |
| 300 | + struct mlx5_ib_dev *dev = to_mdev(counter->device); | |
| 301 | + const struct mlx5_ib_counters *cnts = | |
| 302 | + get_counters(dev, counter->port - 1); | |
| 303 | + | |
| 304 | + return rdma_alloc_hw_stats_struct(cnts->names, | |
| 305 | + cnts->num_q_counters + | |
| 306 | + cnts->num_cong_counters + | |
| 307 | + cnts->num_ext_ppcnt_counters, | |
| 308 | + RDMA_HW_STATS_DEFAULT_LIFESPAN); | |
| 309 | +} | |
| 310 | + | |
| 311 | +static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) | |
| 312 | +{ | |
| 313 | + struct mlx5_ib_dev *dev = to_mdev(counter->device); | |
| 314 | + const struct mlx5_ib_counters *cnts = | |
| 315 | + get_counters(dev, counter->port - 1); | |
| 316 | + | |
| 317 | + return mlx5_ib_query_q_counters(dev->mdev, cnts, | |
| 318 | + counter->stats, counter->id); | |
| 319 | +} | |
| 320 | + | |
| 321 | +static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) | |
| 322 | +{ | |
| 323 | + struct mlx5_ib_dev *dev = to_mdev(counter->device); | |
| 324 | + u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; | |
| 325 | + | |
| 326 | + if (!counter->id) | |
| 327 | + return 0; | |
| 328 | + | |
| 329 | + MLX5_SET(dealloc_q_counter_in, in, opcode, | |
| 330 | + MLX5_CMD_OP_DEALLOC_Q_COUNTER); | |
| 331 | + MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id); | |
| 332 | + return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); | |
| 333 | +} | |
| 334 | + | |
| 335 | +static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, | |
| 336 | + struct ib_qp *qp) | |
| 337 | +{ | |
| 338 | + struct mlx5_ib_dev *dev = to_mdev(qp->device); | |
| 339 | + int err; | |
| 340 | + | |
| 341 | + if (!counter->id) { | |
| 342 | + u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; | |
| 343 | + u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; | |
| 344 | + | |
| 345 | + MLX5_SET(alloc_q_counter_in, in, opcode, | |
| 346 | + MLX5_CMD_OP_ALLOC_Q_COUNTER); | |
| 347 | + MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID); | |
| 348 | + err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); | |
| 349 | + if (err) | |
| 350 | + return err; | |
| 351 | + counter->id = | |
| 352 | + MLX5_GET(alloc_q_counter_out, out, counter_set_id); | |
| 353 | + } | |
| 354 | + | |
| 355 | + err = mlx5_ib_qp_set_counter(qp, counter); | |
| 356 | + if (err) | |
| 357 | + goto fail_set_counter; | |
| 358 | + | |
| 359 | + return 0; | |
| 360 | + | |
| 361 | +fail_set_counter: | |
| 362 | + mlx5_ib_counter_dealloc(counter); | |
| 363 | + counter->id = 0; | |
| 364 | + | |
| 365 | + return err; | |
| 366 | +} | |
| 367 | + | |
| 368 | +static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) | |
| 369 | +{ | |
| 370 | + return mlx5_ib_qp_set_counter(qp, NULL); | |
| 371 | +} | |
| 372 | + | |
| 373 | + | |
| 374 | +static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev, | |
| 375 | + const char **names, | |
| 376 | + size_t *offsets) | |
| 377 | +{ | |
| 378 | + int i; | |
| 379 | + int j = 0; | |
| 380 | + | |
| 381 | + for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) { | |
| 382 | + names[j] = basic_q_cnts[i].name; | |
| 383 | + offsets[j] = basic_q_cnts[i].offset; | |
| 384 | + } | |
| 385 | + | |
| 386 | + if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) { | |
| 387 | + for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) { | |
| 388 | + names[j] = out_of_seq_q_cnts[i].name; | |
| 389 | + offsets[j] = out_of_seq_q_cnts[i].offset; | |
| 390 | + } | |
| 391 | + } | |
| 392 | + | |
| 393 | + if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) { | |
| 394 | + for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) { | |
| 395 | + names[j] = retrans_q_cnts[i].name; | |
| 396 | + offsets[j] = retrans_q_cnts[i].offset; | |
| 397 | + } | |
| 398 | + } | |
| 399 | + | |
| 400 | + if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) { | |
| 401 | + for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) { | |
| 402 | + names[j] = extended_err_cnts[i].name; | |
| 403 | + offsets[j] = extended_err_cnts[i].offset; | |
| 404 | + } | |
| 405 | + } | |
| 406 | + | |
| 407 | + if (MLX5_CAP_GEN(dev->mdev, roce_accl)) { | |
| 408 | + for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) { | |
| 409 | + names[j] = roce_accl_cnts[i].name; | |
| 410 | + offsets[j] = roce_accl_cnts[i].offset; | |
| 411 | + } | |
| 412 | + } | |
| 413 | + | |
| 414 | + if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { | |
| 415 | + for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) { | |
| 416 | + names[j] = cong_cnts[i].name; | |
| 417 | + offsets[j] = cong_cnts[i].offset; | |
| 418 | + } | |
| 419 | + } | |
| 420 | + | |
| 421 | + if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { | |
| 422 | + for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) { | |
| 423 | + names[j] = ext_ppcnt_cnts[i].name; | |
| 424 | + offsets[j] = ext_ppcnt_cnts[i].offset; | |
| 425 | + } | |
| 426 | + } | |
| 427 | +} | |
| 428 | + | |
| 429 | + | |
| 430 | +static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, | |
| 431 | + struct mlx5_ib_counters *cnts) | |
| 432 | +{ | |
| 433 | + u32 num_counters; | |
| 434 | + | |
| 435 | + num_counters = ARRAY_SIZE(basic_q_cnts); | |
| 436 | + | |
| 437 | + if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) | |
| 438 | + num_counters += ARRAY_SIZE(out_of_seq_q_cnts); | |
| 439 | + | |
| 440 | + if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) | |
| 441 | + num_counters += ARRAY_SIZE(retrans_q_cnts); | |
| 442 | + | |
| 443 | + if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) | |
| 444 | + num_counters += ARRAY_SIZE(extended_err_cnts); | |
| 445 | + | |
| 446 | + if (MLX5_CAP_GEN(dev->mdev, roce_accl)) | |
| 447 | + num_counters += ARRAY_SIZE(roce_accl_cnts); | |
| 448 | + | |
| 449 | + cnts->num_q_counters = num_counters; | |
| 450 | + | |
| 451 | + if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { | |
| 452 | + cnts->num_cong_counters = ARRAY_SIZE(cong_cnts); | |
| 453 | + num_counters += ARRAY_SIZE(cong_cnts); | |
| 454 | + } | |
| 455 | + if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { | |
| 456 | + cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts); | |
| 457 | + num_counters += ARRAY_SIZE(ext_ppcnt_cnts); | |
| 458 | + } | |
| 459 | + cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL); | |
| 460 | + if (!cnts->names) | |
| 461 | + return -ENOMEM; | |
| 462 | + | |
| 463 | + cnts->offsets = kcalloc(num_counters, | |
| 464 | + sizeof(cnts->offsets), GFP_KERNEL); | |
| 465 | + if (!cnts->offsets) | |
| 466 | + goto err_names; | |
| 467 | + | |
| 468 | + return 0; | |
| 469 | + | |
| 470 | +err_names: | |
| 471 | + kfree(cnts->names); | |
| 472 | + cnts->names = NULL; | |
| 473 | + return -ENOMEM; | |
| 474 | +} | |
| 475 | + | |
| 476 | +static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev) | |
| 477 | +{ | |
| 478 | + u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; | |
| 479 | + int num_cnt_ports; | |
| 480 | + int i; | |
| 481 | + | |
| 482 | + num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; | |
| 483 | + | |
| 484 | + MLX5_SET(dealloc_q_counter_in, in, opcode, | |
| 485 | + MLX5_CMD_OP_DEALLOC_Q_COUNTER); | |
| 486 | + | |
| 487 | + for (i = 0; i < num_cnt_ports; i++) { | |
| 488 | + if (dev->port[i].cnts.set_id) { | |
| 489 | + MLX5_SET(dealloc_q_counter_in, in, counter_set_id, | |
| 490 | + dev->port[i].cnts.set_id); | |
| 491 | + mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); | |
| 492 | + } | |
| 493 | + kfree(dev->port[i].cnts.names); | |
| 494 | + kfree(dev->port[i].cnts.offsets); | |
| 495 | + } | |
| 496 | +} | |
| 497 | + | |
| 498 | +static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev) | |
| 499 | +{ | |
| 500 | + u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; | |
| 501 | + u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; | |
| 502 | + int num_cnt_ports; | |
| 503 | + int err = 0; | |
| 504 | + int i; | |
| 505 | + bool is_shared; | |
| 506 | + | |
| 507 | + MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); | |
| 508 | + is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0; | |
| 509 | + num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; | |
| 510 | + | |
| 511 | + for (i = 0; i < num_cnt_ports; i++) { | |
| 512 | + err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts); | |
| 513 | + if (err) | |
| 514 | + goto err_alloc; | |
| 515 | + | |
| 516 | + mlx5_ib_fill_counters(dev, dev->port[i].cnts.names, | |
| 517 | + dev->port[i].cnts.offsets); | |
| 518 | + | |
| 519 | + MLX5_SET(alloc_q_counter_in, in, uid, | |
| 520 | + is_shared ? MLX5_SHARED_RESOURCE_UID : 0); | |
| 521 | + | |
| 522 | + err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); | |
| 523 | + if (err) { | |
| 524 | + mlx5_ib_warn(dev, | |
| 525 | + "couldn't allocate queue counter for port %d, err %d\n", | |
| 526 | + i + 1, err); | |
| 527 | + goto err_alloc; | |
| 528 | + } | |
| 529 | + | |
| 530 | + dev->port[i].cnts.set_id = | |
| 531 | + MLX5_GET(alloc_q_counter_out, out, counter_set_id); | |
| 532 | + } | |
| 533 | + return 0; | |
| 534 | + | |
| 535 | +err_alloc: | |
| 536 | + mlx5_ib_dealloc_counters(dev); | |
| 537 | + return err; | |
| 538 | +} | |
| 539 | + | |
| 540 | +static int read_flow_counters(struct ib_device *ibdev, | |
| 541 | + struct mlx5_read_counters_attr *read_attr) | |
| 542 | +{ | |
| 543 | + struct mlx5_fc *fc = read_attr->hw_cntrs_hndl; | |
| 544 | + struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 545 | + | |
| 546 | + return mlx5_fc_query(dev->mdev, fc, | |
| 547 | + &read_attr->out[IB_COUNTER_PACKETS], | |
| 548 | + &read_attr->out[IB_COUNTER_BYTES]); | |
| 549 | +} | |
| 550 | + | |
| 551 | +/* flow counters currently expose two counters packets and bytes */ | |
| 552 | +#define FLOW_COUNTERS_NUM 2 | |
| 553 | +static int counters_set_description( | |
| 554 | + struct ib_counters *counters, enum mlx5_ib_counters_type counters_type, | |
| 555 | + struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters) | |
| 556 | +{ | |
| 557 | + struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); | |
| 558 | + u32 cntrs_max_index = 0; | |
| 559 | + int i; | |
| 560 | + | |
| 561 | + if (counters_type != MLX5_IB_COUNTERS_FLOW) | |
| 562 | + return -EINVAL; | |
| 563 | + | |
| 564 | + /* init the fields for the object */ | |
| 565 | + mcounters->type = counters_type; | |
| 566 | + mcounters->read_counters = read_flow_counters; | |
| 567 | + mcounters->counters_num = FLOW_COUNTERS_NUM; | |
| 568 | + mcounters->ncounters = ncounters; | |
| 569 | + /* each counter entry have both description and index pair */ | |
| 570 | + for (i = 0; i < ncounters; i++) { | |
| 571 | + if (desc_data[i].description > IB_COUNTER_BYTES) | |
| 572 | + return -EINVAL; | |
| 573 | + | |
| 574 | + if (cntrs_max_index <= desc_data[i].index) | |
| 575 | + cntrs_max_index = desc_data[i].index + 1; | |
| 576 | + } | |
| 577 | + | |
| 578 | + mutex_lock(&mcounters->mcntrs_mutex); | |
| 579 | + mcounters->counters_data = desc_data; | |
| 580 | + mcounters->cntrs_max_index = cntrs_max_index; | |
| 581 | + mutex_unlock(&mcounters->mcntrs_mutex); | |
| 582 | + | |
| 583 | + return 0; | |
| 584 | +} | |
| 585 | + | |
| 586 | +#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2)) | |
| 587 | +int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters, | |
| 588 | + struct mlx5_ib_create_flow *ucmd) | |
| 589 | +{ | |
| 590 | + struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters); | |
| 591 | + struct mlx5_ib_flow_counters_data *cntrs_data = NULL; | |
| 592 | + struct mlx5_ib_flow_counters_desc *desc_data = NULL; | |
| 593 | + bool hw_hndl = false; | |
| 594 | + int ret = 0; | |
| 595 | + | |
| 596 | + if (ucmd && ucmd->ncounters_data != 0) { | |
| 597 | + cntrs_data = ucmd->data; | |
| 598 | + if (cntrs_data->ncounters > MAX_COUNTERS_NUM) | |
| 599 | + return -EINVAL; | |
| 600 | + | |
| 601 | + desc_data = kcalloc(cntrs_data->ncounters, | |
| 602 | + sizeof(*desc_data), | |
| 603 | + GFP_KERNEL); | |
| 604 | + if (!desc_data) | |
| 605 | + return -ENOMEM; | |
| 606 | + | |
| 607 | + if (copy_from_user(desc_data, | |
| 608 | + u64_to_user_ptr(cntrs_data->counters_data), | |
| 609 | + sizeof(*desc_data) * cntrs_data->ncounters)) { | |
| 610 | + ret = -EFAULT; | |
| 611 | + goto free; | |
| 612 | + } | |
| 613 | + } | |
| 614 | + | |
| 615 | + if (!mcounters->hw_cntrs_hndl) { | |
| 616 | + mcounters->hw_cntrs_hndl = mlx5_fc_create( | |
| 617 | + to_mdev(ibcounters->device)->mdev, false); | |
| 618 | + if (IS_ERR(mcounters->hw_cntrs_hndl)) { | |
| 619 | + ret = PTR_ERR(mcounters->hw_cntrs_hndl); | |
| 620 | + goto free; | |
| 621 | + } | |
| 622 | + hw_hndl = true; | |
| 623 | + } | |
| 624 | + | |
| 625 | + if (desc_data) { | |
| 626 | + /* counters already bound to at least one flow */ | |
| 627 | + if (mcounters->cntrs_max_index) { | |
| 628 | + ret = -EINVAL; | |
| 629 | + goto free_hndl; | |
| 630 | + } | |
| 631 | + | |
| 632 | + ret = counters_set_description(ibcounters, | |
| 633 | + MLX5_IB_COUNTERS_FLOW, | |
| 634 | + desc_data, | |
| 635 | + cntrs_data->ncounters); | |
| 636 | + if (ret) | |
| 637 | + goto free_hndl; | |
| 638 | + | |
| 639 | + } else if (!mcounters->cntrs_max_index) { | |
| 640 | + /* counters not bound yet, must have udata passed */ | |
| 641 | + ret = -EINVAL; | |
| 642 | + goto free_hndl; | |
| 643 | + } | |
| 644 | + | |
| 645 | + return 0; | |
| 646 | + | |
| 647 | +free_hndl: | |
| 648 | + if (hw_hndl) { | |
| 649 | + mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev, | |
| 650 | + mcounters->hw_cntrs_hndl); | |
| 651 | + mcounters->hw_cntrs_hndl = NULL; | |
| 652 | + } | |
| 653 | +free: | |
| 654 | + kfree(desc_data); | |
| 655 | + return ret; | |
| 656 | +} | |
| 657 | + | |
| 658 | +void mlx5_ib_counters_clear_description(struct ib_counters *counters) | |
| 659 | +{ | |
| 660 | + struct mlx5_ib_mcounters *mcounters; | |
| 661 | + | |
| 662 | + if (!counters || atomic_read(&counters->usecnt) != 1) | |
| 663 | + return; | |
| 664 | + | |
| 665 | + mcounters = to_mcounters(counters); | |
| 666 | + | |
| 667 | + mutex_lock(&mcounters->mcntrs_mutex); | |
| 668 | + kfree(mcounters->counters_data); | |
| 669 | + mcounters->counters_data = NULL; | |
| 670 | + mcounters->cntrs_max_index = 0; | |
| 671 | + mutex_unlock(&mcounters->mcntrs_mutex); | |
| 672 | +} | |
| 673 | + | |
| 674 | +static const struct ib_device_ops hw_stats_ops = { | |
| 675 | + .alloc_hw_stats = mlx5_ib_alloc_hw_stats, | |
| 676 | + .get_hw_stats = mlx5_ib_get_hw_stats, | |
| 677 | + .counter_bind_qp = mlx5_ib_counter_bind_qp, | |
| 678 | + .counter_unbind_qp = mlx5_ib_counter_unbind_qp, | |
| 679 | + .counter_dealloc = mlx5_ib_counter_dealloc, | |
| 680 | + .counter_alloc_stats = mlx5_ib_counter_alloc_stats, | |
| 681 | + .counter_update_stats = mlx5_ib_counter_update_stats, | |
| 682 | +}; | |
| 683 | + | |
| 684 | +static const struct ib_device_ops counters_ops = { | |
| 685 | + .create_counters = mlx5_ib_create_counters, | |
| 686 | + .destroy_counters = mlx5_ib_destroy_counters, | |
| 687 | + .read_counters = mlx5_ib_read_counters, | |
| 688 | + | |
| 689 | + INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs), | |
| 690 | +}; | |
| 691 | + | |
| 692 | +int mlx5_ib_counters_init(struct mlx5_ib_dev *dev) | |
| 693 | +{ | |
| 694 | + ib_set_device_ops(&dev->ib_dev, &counters_ops); | |
| 695 | + | |
| 696 | + if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) | |
| 697 | + return 0; | |
| 698 | + | |
| 699 | + ib_set_device_ops(&dev->ib_dev, &hw_stats_ops); | |
| 700 | + return mlx5_ib_alloc_counters(dev); | |
| 701 | +} | |
| 702 | + | |
| 703 | +void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev) | |
| 704 | +{ | |
| 705 | + if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) | |
| 706 | + return; | |
| 707 | + | |
| 708 | + mlx5_ib_dealloc_counters(dev); | |
| 709 | +} | |
| @@ -0,0 +1,17 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 2 | +/* | |
| 3 | + * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#ifndef _MLX5_IB_COUNTERS_H | |
| 7 | +#define _MLX5_IB_COUNTERS_H | |
| 8 | + | |
| 9 | +#include "mlx5_ib.h" | |
| 10 | + | |
| 11 | +int mlx5_ib_counters_init(struct mlx5_ib_dev *dev); | |
| 12 | +void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev); | |
| 13 | +void mlx5_ib_counters_clear_description(struct ib_counters *counters); | |
| 14 | +int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters, | |
| 15 | + struct mlx5_ib_create_flow *ucmd); | |
| 16 | +u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num); | |
| 17 | +#endif /* _MLX5_IB_COUNTERS_H */ | |
| @@ -14,6 +14,7 @@ · 13 unmodified lines | ||
| 14 | 14 | #include <linux/mlx5/driver.h> |
| 15 | 15 | #include <linux/mlx5/fs.h> |
| 16 | 16 | #include "mlx5_ib.h" |
| 17 | +#include "devx.h" | |
| 17 | 18 | #include "qp.h" |
| 18 | 19 | #include <linux/xarray.h> |
| 19 | 20 | |
| @@ -89,22 +90,6 @@ struct devx_async_event_file { · 69 unmodified lines | ||
| 89 | 90 | u8 is_destroyed:1; |
| 90 | 91 | }; |
| 91 | 92 | |
| 92 | −#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) | |
| 93 | −struct devx_obj { | |
| 94 | − struct mlx5_ib_dev *ib_dev; | |
| 95 | − u64 obj_id; | |
| 96 | − u32 dinlen; /* destroy inbox length */ | |
| 97 | − u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; | |
| 98 | − u32 flags; | |
| 99 | − union { | |
| 100 | − struct mlx5_ib_devx_mr devx_mr; | |
| 101 | − struct mlx5_core_dct core_dct; | |
| 102 | − struct mlx5_core_cq core_cq; | |
| 103 | − u32 flow_counter_bulk_size; | |
| 104 | − }; | |
| 105 | − struct list_head event_sub; /* holds devx_event_subscription entries */ | |
| 106 | −}; | |
| 107 | − | |
| 108 | 93 | struct devx_umem { |
| 109 | 94 | struct mlx5_core_dev *mdev; |
| 110 | 95 | struct ib_umem *umem; |
| @@ -171,48 +156,6 @@ void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) · 60 unmodified lines | ||
| 171 | 156 | mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); |
| 172 | 157 | } |
| 173 | 158 | |
| 174 | −bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type) | |
| 175 | −{ | |
| 176 | − struct devx_obj *devx_obj = obj; | |
| 177 | − u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); | |
| 178 | − | |
| 179 | − switch (opcode) { | |
| 180 | − case MLX5_CMD_OP_DESTROY_TIR: | |
| 181 | − *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; | |
| 182 | − *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, | |
| 183 | − obj_id); | |
| 184 | − return true; | |
| 185 | − | |
| 186 | − case MLX5_CMD_OP_DESTROY_FLOW_TABLE: | |
| 187 | − *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; | |
| 188 | − *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, | |
| 189 | − table_id); | |
| 190 | − return true; | |
| 191 | − default: | |
| 192 | − return false; | |
| 193 | − } | |
| 194 | −} | |
| 195 | − | |
| 196 | −bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id) | |
| 197 | −{ | |
| 198 | − struct devx_obj *devx_obj = obj; | |
| 199 | − u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); | |
| 200 | − | |
| 201 | − if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { | |
| 202 | − | |
| 203 | − if (offset && offset >= devx_obj->flow_counter_bulk_size) | |
| 204 | − return false; | |
| 205 | − | |
| 206 | − *counter_id = MLX5_GET(dealloc_flow_counter_in, | |
| 207 | − devx_obj->dinbox, | |
| 208 | − flow_counter_id); | |
| 209 | − *counter_id += offset; | |
| 210 | − return true; | |
| 211 | − } | |
| 212 | − | |
| 213 | − return false; | |
| 214 | −} | |
| 215 | − | |
| 216 | 159 | static bool is_legacy_unaffiliated_event_num(u16 event_num) |
| 217 | 160 | { |
| 218 | 161 | switch (event_num) { |
| @@ -2419,17 +2362,24 @@ static int devx_event_notifier(struct notifier_block *nb, · 2200 unmodified lines | ||
| 2419 | 2362 | return NOTIFY_OK; |
| 2420 | 2363 | } |
| 2421 | 2364 | |
| 2422 | −void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) | |
| 2365 | +int mlx5_ib_devx_init(struct mlx5_ib_dev *dev) | |
| 2423 | 2366 | { |
| 2424 | 2367 | struct mlx5_devx_event_table *table = &dev->devx_event_table; |
| 2368 | + int uid; | |
| 2425 | 2369 | |
| 2426 | − xa_init(&table->event_xa); | |
| 2427 | − mutex_init(&table->event_xa_lock); | |
| 2428 | − MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY); | |
| 2429 | − mlx5_eq_notifier_register(dev->mdev, &table->devx_nb); | |
| 2370 | + uid = mlx5_ib_devx_create(dev, false); | |
| 2371 | + if (uid > 0) { | |
| 2372 | + dev->devx_whitelist_uid = uid; | |
| 2373 | + xa_init(&table->event_xa); | |
| 2374 | + mutex_init(&table->event_xa_lock); | |
| 2375 | + MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY); | |
| 2376 | + mlx5_eq_notifier_register(dev->mdev, &table->devx_nb); | |
| 2377 | + } | |
| 2378 | + | |
| 2379 | + return 0; | |
| 2430 | 2380 | } |
| 2431 | 2381 | |
| 2432 | −void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) | |
| 2382 | +void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev) | |
| 2433 | 2383 | { |
| 2434 | 2384 | struct mlx5_devx_event_table *table = &dev->devx_event_table; |
| 2435 | 2385 | struct devx_event_subscription *sub, *tmp; |
| @@ -2437,17 +2387,21 @@ void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) · 1 unmodified line | ||
| 2437 | 2387 | void *entry; |
| 2438 | 2388 | unsigned long id; |
| 2439 | 2389 | |
| 2440 | − mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb); | |
| 2441 | − mutex_lock(&dev->devx_event_table.event_xa_lock); | |
| 2442 | − xa_for_each(&table->event_xa, id, entry) { | |
| 2443 | − event = entry; | |
| 2444 | − list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list, | |
| 2445 | − xa_list) | |
| 2446 | − devx_cleanup_subscription(dev, sub); | |
| 2447 | − kfree(entry); | |
| 2390 | + if (dev->devx_whitelist_uid) { | |
| 2391 | + mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb); | |
| 2392 | + mutex_lock(&dev->devx_event_table.event_xa_lock); | |
| 2393 | + xa_for_each(&table->event_xa, id, entry) { | |
| 2394 | + event = entry; | |
| 2395 | + list_for_each_entry_safe( | |
| 2396 | + sub, tmp, &event->unaffiliated_list, xa_list) | |
| 2397 | + devx_cleanup_subscription(dev, sub); | |
| 2398 | + kfree(entry); | |
| 2399 | + } | |
| 2400 | + mutex_unlock(&dev->devx_event_table.event_xa_lock); | |
| 2401 | + xa_destroy(&table->event_xa); | |
| 2402 | + | |
| 2403 | + mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid); | |
| 2448 | 2404 | } |
| 2449 | − mutex_unlock(&dev->devx_event_table.event_xa_lock); | |
| 2450 | − xa_destroy(&table->event_xa); | |
| 2451 | 2405 | } |
| 2452 | 2406 | |
| 2453 | 2407 | static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf, |
| @@ -0,0 +1,45 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 2 | +/* | |
| 3 | + * Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved. | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#ifndef _MLX5_IB_DEVX_H | |
| 7 | +#define _MLX5_IB_DEVX_H | |
| 8 | + | |
| 9 | +#include "mlx5_ib.h" | |
| 10 | + | |
| 11 | +#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) | |
| 12 | +struct devx_obj { | |
| 13 | + struct mlx5_ib_dev *ib_dev; | |
| 14 | + u64 obj_id; | |
| 15 | + u32 dinlen; /* destroy inbox length */ | |
| 16 | + u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; | |
| 17 | + u32 flags; | |
| 18 | + union { | |
| 19 | + struct mlx5_ib_devx_mr devx_mr; | |
| 20 | + struct mlx5_core_dct core_dct; | |
| 21 | + struct mlx5_core_cq core_cq; | |
| 22 | + u32 flow_counter_bulk_size; | |
| 23 | + }; | |
| 24 | + struct list_head event_sub; /* holds devx_event_subscription entries */ | |
| 25 | +}; | |
| 26 | +#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) | |
| 27 | +int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user); | |
| 28 | +void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid); | |
| 29 | +int mlx5_ib_devx_init(struct mlx5_ib_dev *dev); | |
| 30 | +void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev); | |
| 31 | +#else | |
| 32 | +static inline int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user) | |
| 33 | +{ | |
| 34 | + return -EOPNOTSUPP; | |
| 35 | +} | |
| 36 | +static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {} | |
| 37 | +static inline int mlx5_ib_devx_init(struct mlx5_ib_dev *dev) | |
| 38 | +{ | |
| 39 | + return 0; | |
| 40 | +} | |
| 41 | +static inline void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev) | |
| 42 | +{ | |
| 43 | +} | |
| 44 | +#endif | |
| 45 | +#endif /* _MLX5_IB_DEVX_H */ | |
| @@ -1,765 +0,0 @@ | ||
| 1 | −// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB | |
| 2 | −/* | |
| 3 | − * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. | |
| 4 | − */ | |
| 5 | − | |
| 6 | −#include <rdma/ib_user_verbs.h> | |
| 7 | −#include <rdma/ib_verbs.h> | |
| 8 | −#include <rdma/uverbs_types.h> | |
| 9 | −#include <rdma/uverbs_ioctl.h> | |
| 10 | −#include <rdma/uverbs_std_types.h> | |
| 11 | −#include <rdma/mlx5_user_ioctl_cmds.h> | |
| 12 | −#include <rdma/mlx5_user_ioctl_verbs.h> | |
| 13 | −#include <rdma/ib_umem.h> | |
| 14 | −#include <linux/mlx5/driver.h> | |
| 15 | −#include <linux/mlx5/fs.h> | |
| 16 | −#include "mlx5_ib.h" | |
| 17 | − | |
| 18 | −#define UVERBS_MODULE_NAME mlx5_ib | |
| 19 | −#include <rdma/uverbs_named_ioctl.h> | |
| 20 | − | |
| 21 | −static int | |
| 22 | −mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, | |
| 23 | − enum mlx5_flow_namespace_type *namespace) | |
| 24 | −{ | |
| 25 | − switch (table_type) { | |
| 26 | − case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: | |
| 27 | − *namespace = MLX5_FLOW_NAMESPACE_BYPASS; | |
| 28 | − break; | |
| 29 | − case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: | |
| 30 | − *namespace = MLX5_FLOW_NAMESPACE_EGRESS; | |
| 31 | − break; | |
| 32 | − case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: | |
| 33 | − *namespace = MLX5_FLOW_NAMESPACE_FDB; | |
| 34 | − break; | |
| 35 | − case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: | |
| 36 | − *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; | |
| 37 | − break; | |
| 38 | − case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: | |
| 39 | − *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; | |
| 40 | − break; | |
| 41 | − default: | |
| 42 | − return -EINVAL; | |
| 43 | − } | |
| 44 | − | |
| 45 | − return 0; | |
| 46 | −} | |
| 47 | − | |
| 48 | −static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { | |
| 49 | − [MLX5_IB_FLOW_TYPE_NORMAL] = { | |
| 50 | − .type = UVERBS_ATTR_TYPE_PTR_IN, | |
| 51 | − .u.ptr = { | |
| 52 | − .len = sizeof(u16), /* data is priority */ | |
| 53 | − .min_len = sizeof(u16), | |
| 54 | − } | |
| 55 | − }, | |
| 56 | − [MLX5_IB_FLOW_TYPE_SNIFFER] = { | |
| 57 | − .type = UVERBS_ATTR_TYPE_PTR_IN, | |
| 58 | − UVERBS_ATTR_NO_DATA(), | |
| 59 | − }, | |
| 60 | − [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { | |
| 61 | − .type = UVERBS_ATTR_TYPE_PTR_IN, | |
| 62 | − UVERBS_ATTR_NO_DATA(), | |
| 63 | − }, | |
| 64 | − [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { | |
| 65 | − .type = UVERBS_ATTR_TYPE_PTR_IN, | |
| 66 | − UVERBS_ATTR_NO_DATA(), | |
| 67 | − }, | |
| 68 | −}; | |
| 69 | − | |
| 70 | −static int get_dests(struct uverbs_attr_bundle *attrs, | |
| 71 | − struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, | |
| 72 | − int *dest_type, struct ib_qp **qp, u32 *flags) | |
| 73 | −{ | |
| 74 | − bool dest_devx, dest_qp; | |
| 75 | − void *devx_obj; | |
| 76 | − int err; | |
| 77 | − | |
| 78 | − dest_devx = uverbs_attr_is_valid(attrs, | |
| 79 | − MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); | |
| 80 | − dest_qp = uverbs_attr_is_valid(attrs, | |
| 81 | − MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); | |
| 82 | − | |
| 83 | − *flags = 0; | |
| 84 | − err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, | |
| 85 | − MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | | |
| 86 | − MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); | |
| 87 | − if (err) | |
| 88 | − return err; | |
| 89 | − | |
| 90 | − /* Both flags are not allowed */ | |
| 91 | − if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && | |
| 92 | − *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) | |
| 93 | − return -EINVAL; | |
| 94 | − | |
| 95 | − if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { | |
| 96 | − if (dest_devx && (dest_qp || *flags)) | |
| 97 | − return -EINVAL; | |
| 98 | − else if (dest_qp && *flags) | |
| 99 | − return -EINVAL; | |
| 100 | − } | |
| 101 | − | |
| 102 | − /* Allow only DEVX object, drop as dest for FDB */ | |
| 103 | − if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx || | |
| 104 | − (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) | |
| 105 | − return -EINVAL; | |
| 106 | − | |
| 107 | − /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ | |
| 108 | − if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && | |
| 109 | − ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) | |
| 110 | − return -EINVAL; | |
| 111 | − | |
| 112 | − *qp = NULL; | |
| 113 | − if (dest_devx) { | |
| 114 | − devx_obj = | |
| 115 | − uverbs_attr_get_obj(attrs, | |
| 116 | − MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); | |
| 117 | − | |
| 118 | − /* Verify that the given DEVX object is a flow | |
| 119 | − * steering destination. | |
| 120 | − */ | |
| 121 | − if (!mlx5_ib_devx_is_flow_dest(devx_obj, dest_id, dest_type)) | |
| 122 | − return -EINVAL; | |
| 123 | − /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ | |
| 124 | − if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB || | |
| 125 | − fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && | |
| 126 | − *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) | |
| 127 | − return -EINVAL; | |
| 128 | − } else if (dest_qp) { | |
| 129 | − struct mlx5_ib_qp *mqp; | |
| 130 | − | |
| 131 | − *qp = uverbs_attr_get_obj(attrs, | |
| 132 | − MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); | |
| 133 | − if (IS_ERR(*qp)) | |
| 134 | − return PTR_ERR(*qp); | |
| 135 | − | |
| 136 | − if ((*qp)->qp_type != IB_QPT_RAW_PACKET) | |
| 137 | − return -EINVAL; | |
| 138 | − | |
| 139 | − mqp = to_mqp(*qp); | |
| 140 | − if (mqp->is_rss) | |
| 141 | − *dest_id = mqp->rss_qp.tirn; | |
| 142 | − else | |
| 143 | − *dest_id = mqp->raw_packet_qp.rq.tirn; | |
| 144 | − *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; | |
| 145 | − } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { | |
| 146 | − *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; | |
| 147 | − } | |
| 148 | − | |
| 149 | − if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && | |
| 150 | − fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) | |
| 151 | − return -EINVAL; | |
| 152 | − | |
| 153 | − return 0; | |
| 154 | −} | |
| 155 | − | |
| 156 | −#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 | |
| 157 | −static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( | |
| 158 | − struct uverbs_attr_bundle *attrs) | |
| 159 | −{ | |
| 160 | − struct mlx5_flow_context flow_context = {.flow_tag = | |
| 161 | − MLX5_FS_DEFAULT_FLOW_TAG}; | |
| 162 | − u32 *offset_attr, offset = 0, counter_id = 0; | |
| 163 | − int dest_id, dest_type, inlen, len, ret, i; | |
| 164 | − struct mlx5_ib_flow_handler *flow_handler; | |
| 165 | − struct mlx5_ib_flow_matcher *fs_matcher; | |
| 166 | − struct ib_uobject **arr_flow_actions; | |
| 167 | − struct ib_uflow_resources *uflow_res; | |
| 168 | − struct mlx5_flow_act flow_act = {}; | |
| 169 | − struct ib_qp *qp = NULL; | |
| 170 | − void *devx_obj, *cmd_in; | |
| 171 | − struct ib_uobject *uobj; | |
| 172 | − struct mlx5_ib_dev *dev; | |
| 173 | − u32 flags; | |
| 174 | − | |
| 175 | − if (!capable(CAP_NET_RAW)) | |
| 176 | − return -EPERM; | |
| 177 | − | |
| 178 | − fs_matcher = uverbs_attr_get_obj(attrs, | |
| 179 | − MLX5_IB_ATTR_CREATE_FLOW_MATCHER); | |
| 180 | − uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); | |
| 181 | − dev = mlx5_udata_to_mdev(&attrs->driver_udata); | |
| 182 | − | |
| 183 | − if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) | |
| 184 | − return -EINVAL; | |
| 185 | − | |
| 186 | − if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) | |
| 187 | − flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; | |
| 188 | − | |
| 189 | − if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) | |
| 190 | − flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; | |
| 191 | − | |
| 192 | − len = uverbs_attr_get_uobjs_arr(attrs, | |
| 193 | − MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); | |
| 194 | − if (len) { | |
| 195 | − devx_obj = arr_flow_actions[0]->object; | |
| 196 | − | |
| 197 | − if (uverbs_attr_is_valid(attrs, | |
| 198 | − MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { | |
| 199 | − | |
| 200 | − int num_offsets = uverbs_attr_ptr_get_array_size( | |
| 201 | − attrs, | |
| 202 | − MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, | |
| 203 | − sizeof(u32)); | |
| 204 | − | |
| 205 | − if (num_offsets != 1) | |
| 206 | − return -EINVAL; | |
| 207 | − | |
| 208 | − offset_attr = uverbs_attr_get_alloced_ptr( | |
| 209 | − attrs, | |
| 210 | − MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); | |
| 211 | − offset = *offset_attr; | |
| 212 | − } | |
| 213 | − | |
| 214 | − if (!mlx5_ib_devx_is_flow_counter(devx_obj, offset, | |
| 215 | − &counter_id)) | |
| 216 | − return -EINVAL; | |
| 217 | − | |
| 218 | − flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; | |
| 219 | − } | |
| 220 | − | |
| 221 | − cmd_in = uverbs_attr_get_alloced_ptr( | |
| 222 | − attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); | |
| 223 | − inlen = uverbs_attr_get_len(attrs, | |
| 224 | − MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); | |
| 225 | − | |
| 226 | − uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); | |
| 227 | − if (!uflow_res) | |
| 228 | − return -ENOMEM; | |
| 229 | − | |
| 230 | − len = uverbs_attr_get_uobjs_arr(attrs, | |
| 231 | − MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); | |
| 232 | − for (i = 0; i < len; i++) { | |
| 233 | − struct mlx5_ib_flow_action *maction = | |
| 234 | − to_mflow_act(arr_flow_actions[i]->object); | |
| 235 | − | |
| 236 | − ret = parse_flow_flow_action(maction, false, &flow_act); | |
| 237 | − if (ret) | |
| 238 | − goto err_out; | |
| 239 | − flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, | |
| 240 | − arr_flow_actions[i]->object); | |
| 241 | − } | |
| 242 | − | |
| 243 | − ret = uverbs_copy_from(&flow_context.flow_tag, attrs, | |
| 244 | − MLX5_IB_ATTR_CREATE_FLOW_TAG); | |
| 245 | − if (!ret) { | |
| 246 | − if (flow_context.flow_tag >= BIT(24)) { | |
| 247 | − ret = -EINVAL; | |
| 248 | − goto err_out; | |
| 249 | − } | |
| 250 | − flow_context.flags |= FLOW_CONTEXT_HAS_TAG; | |
| 251 | − } | |
| 252 | − | |
| 253 | − flow_handler = mlx5_ib_raw_fs_rule_add(dev, fs_matcher, | |
| 254 | − &flow_context, | |
| 255 | − &flow_act, | |
| 256 | − counter_id, | |
| 257 | − cmd_in, inlen, | |
| 258 | − dest_id, dest_type); | |
| 259 | − if (IS_ERR(flow_handler)) { | |
| 260 | − ret = PTR_ERR(flow_handler); | |
| 261 | − goto err_out; | |
| 262 | − } | |
| 263 | − | |
| 264 | − ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); | |
| 265 | − | |
| 266 | − return 0; | |
| 267 | −err_out: | |
| 268 | − ib_uverbs_flow_resources_free(uflow_res); | |
| 269 | − return ret; | |
| 270 | −} | |
| 271 | − | |
| 272 | −static int flow_matcher_cleanup(struct ib_uobject *uobject, | |
| 273 | − enum rdma_remove_reason why, | |
| 274 | − struct uverbs_attr_bundle *attrs) | |
| 275 | −{ | |
| 276 | − struct mlx5_ib_flow_matcher *obj = uobject->object; | |
| 277 | − int ret; | |
| 278 | − | |
| 279 | − ret = ib_destroy_usecnt(&obj->usecnt, why, uobject); | |
| 280 | − if (ret) | |
| 281 | − return ret; | |
| 282 | − | |
| 283 | − kfree(obj); | |
| 284 | − return 0; | |
| 285 | −} | |
| 286 | − | |
| 287 | −static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, | |
| 288 | − struct mlx5_ib_flow_matcher *obj) | |
| 289 | −{ | |
| 290 | − enum mlx5_ib_uapi_flow_table_type ft_type = | |
| 291 | − MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; | |
| 292 | − u32 flags; | |
| 293 | − int err; | |
| 294 | − | |
| 295 | − /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older | |
| 296 | − * users should switch to it. We leave this to not break userspace | |
| 297 | − */ | |
| 298 | − if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && | |
| 299 | − uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) | |
| 300 | − return -EINVAL; | |
| 301 | − | |
| 302 | − if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { | |
| 303 | − err = uverbs_get_const(&ft_type, attrs, | |
| 304 | − MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); | |
| 305 | − if (err) | |
| 306 | − return err; | |
| 307 | − | |
| 308 | − err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); | |
| 309 | − if (err) | |
| 310 | − return err; | |
| 311 | − | |
| 312 | − return 0; | |
| 313 | − } | |
| 314 | − | |
| 315 | − if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { | |
| 316 | − err = uverbs_get_flags32(&flags, attrs, | |
| 317 | − MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, | |
| 318 | − IB_FLOW_ATTR_FLAGS_EGRESS); | |
| 319 | − if (err) | |
| 320 | − return err; | |
| 321 | − | |
| 322 | − if (flags) { | |
| 323 | − mlx5_ib_ft_type_to_namespace( | |
| 324 | − MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, | |
| 325 | − &obj->ns_type); | |
| 326 | − return 0; | |
| 327 | − } | |
| 328 | − } | |
| 329 | − | |
| 330 | − obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; | |
| 331 | − | |
| 332 | − return 0; | |
| 333 | −} | |
| 334 | − | |
| 335 | −static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( | |
| 336 | − struct uverbs_attr_bundle *attrs) | |
| 337 | −{ | |
| 338 | − struct ib_uobject *uobj = uverbs_attr_get_uobject( | |
| 339 | − attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); | |
| 340 | − struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); | |
| 341 | − struct mlx5_ib_flow_matcher *obj; | |
| 342 | − int err; | |
| 343 | − | |
| 344 | − obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); | |
| 345 | − if (!obj) | |
| 346 | − return -ENOMEM; | |
| 347 | − | |
| 348 | − obj->mask_len = uverbs_attr_get_len( | |
| 349 | − attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); | |
| 350 | − err = uverbs_copy_from(&obj->matcher_mask, | |
| 351 | − attrs, | |
| 352 | − MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); | |
| 353 | − if (err) | |
| 354 | − goto end; | |
| 355 | − | |
| 356 | − obj->flow_type = uverbs_attr_get_enum_id( | |
| 357 | − attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); | |
| 358 | − | |
| 359 | − if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { | |
| 360 | − err = uverbs_copy_from(&obj->priority, | |
| 361 | − attrs, | |
| 362 | − MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); | |
| 363 | − if (err) | |
| 364 | − goto end; | |
| 365 | − } | |
| 366 | − | |
| 367 | − err = uverbs_copy_from(&obj->match_criteria_enable, | |
| 368 | − attrs, | |
| 369 | − MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); | |
| 370 | − if (err) | |
| 371 | − goto end; | |
| 372 | − | |
| 373 | − err = mlx5_ib_matcher_ns(attrs, obj); | |
| 374 | − if (err) | |
| 375 | − goto end; | |
| 376 | − | |
| 377 | − uobj->object = obj; | |
| 378 | − obj->mdev = dev->mdev; | |
| 379 | − atomic_set(&obj->usecnt, 0); | |
| 380 | − return 0; | |
| 381 | − | |
| 382 | −end: | |
| 383 | − kfree(obj); | |
| 384 | − return err; | |
| 385 | −} | |
| 386 | − | |
| 387 | −void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) | |
| 388 | −{ | |
| 389 | − switch (maction->flow_action_raw.sub_type) { | |
| 390 | − case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: | |
| 391 | − mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, | |
| 392 | − maction->flow_action_raw.modify_hdr); | |
| 393 | − break; | |
| 394 | − case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: | |
| 395 | − mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, | |
| 396 | − maction->flow_action_raw.pkt_reformat); | |
| 397 | − break; | |
| 398 | − case MLX5_IB_FLOW_ACTION_DECAP: | |
| 399 | − break; | |
| 400 | − default: | |
| 401 | − break; | |
| 402 | − } | |
| 403 | −} | |
| 404 | − | |
| 405 | −static struct ib_flow_action * | |
| 406 | −mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, | |
| 407 | − enum mlx5_ib_uapi_flow_table_type ft_type, | |
| 408 | − u8 num_actions, void *in) | |
| 409 | −{ | |
| 410 | − enum mlx5_flow_namespace_type namespace; | |
| 411 | − struct mlx5_ib_flow_action *maction; | |
| 412 | − int ret; | |
| 413 | − | |
| 414 | − ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); | |
| 415 | − if (ret) | |
| 416 | − return ERR_PTR(-EINVAL); | |
| 417 | − | |
| 418 | − maction = kzalloc(sizeof(*maction), GFP_KERNEL); | |
| 419 | − if (!maction) | |
| 420 | − return ERR_PTR(-ENOMEM); | |
| 421 | − | |
| 422 | − maction->flow_action_raw.modify_hdr = | |
| 423 | − mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); | |
| 424 | − | |
| 425 | − if (IS_ERR(maction->flow_action_raw.modify_hdr)) { | |
| 426 | − ret = PTR_ERR(maction->flow_action_raw.modify_hdr); | |
| 427 | − kfree(maction); | |
| 428 | − return ERR_PTR(ret); | |
| 429 | − } | |
| 430 | − maction->flow_action_raw.sub_type = | |
| 431 | − MLX5_IB_FLOW_ACTION_MODIFY_HEADER; | |
| 432 | − maction->flow_action_raw.dev = dev; | |
| 433 | − | |
| 434 | − return &maction->ib_action; | |
| 435 | −} | |
| 436 | − | |
| 437 | −static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) | |
| 438 | −{ | |
| 439 | − return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 440 | − max_modify_header_actions) || | |
| 441 | − MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, | |
| 442 | − max_modify_header_actions) || | |
| 443 | − MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, | |
| 444 | − max_modify_header_actions); | |
| 445 | −} | |
| 446 | − | |
| 447 | −static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( | |
| 448 | − struct uverbs_attr_bundle *attrs) | |
| 449 | −{ | |
| 450 | − struct ib_uobject *uobj = uverbs_attr_get_uobject( | |
| 451 | − attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); | |
| 452 | − struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); | |
| 453 | − enum mlx5_ib_uapi_flow_table_type ft_type; | |
| 454 | − struct ib_flow_action *action; | |
| 455 | − int num_actions; | |
| 456 | − void *in; | |
| 457 | − int ret; | |
| 458 | − | |
| 459 | − if (!mlx5_ib_modify_header_supported(mdev)) | |
| 460 | − return -EOPNOTSUPP; | |
| 461 | − | |
| 462 | − in = uverbs_attr_get_alloced_ptr(attrs, | |
| 463 | − MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); | |
| 464 | − | |
| 465 | − num_actions = uverbs_attr_ptr_get_array_size( | |
| 466 | − attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, | |
| 467 | − MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); | |
| 468 | − if (num_actions < 0) | |
| 469 | − return num_actions; | |
| 470 | − | |
| 471 | − ret = uverbs_get_const(&ft_type, attrs, | |
| 472 | − MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); | |
| 473 | − if (ret) | |
| 474 | − return ret; | |
| 475 | − action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); | |
| 476 | − if (IS_ERR(action)) | |
| 477 | − return PTR_ERR(action); | |
| 478 | − | |
| 479 | − uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, | |
| 480 | − IB_FLOW_ACTION_UNSPECIFIED); | |
| 481 | − | |
| 482 | − return 0; | |
| 483 | −} | |
| 484 | − | |
| 485 | −static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, | |
| 486 | − u8 packet_reformat_type, | |
| 487 | − u8 ft_type) | |
| 488 | −{ | |
| 489 | − switch (packet_reformat_type) { | |
| 490 | − case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: | |
| 491 | − if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) | |
| 492 | − return MLX5_CAP_FLOWTABLE(ibdev->mdev, | |
| 493 | − encap_general_header); | |
| 494 | − break; | |
| 495 | − case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: | |
| 496 | − if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) | |
| 497 | − return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, | |
| 498 | − reformat_l2_to_l3_tunnel); | |
| 499 | − break; | |
| 500 | − case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: | |
| 501 | − if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) | |
| 502 | − return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, | |
| 503 | − reformat_l3_tunnel_to_l2); | |
| 504 | − break; | |
| 505 | − case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: | |
| 506 | − if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) | |
| 507 | − return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); | |
| 508 | − break; | |
| 509 | − default: | |
| 510 | − break; | |
| 511 | − } | |
| 512 | − | |
| 513 | − return false; | |
| 514 | −} | |
| 515 | − | |
| 516 | −static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) | |
| 517 | −{ | |
| 518 | − switch (dv_prt) { | |
| 519 | − case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: | |
| 520 | − *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; | |
| 521 | − break; | |
| 522 | − case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: | |
| 523 | − *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; | |
| 524 | − break; | |
| 525 | − case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: | |
| 526 | − *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; | |
| 527 | − break; | |
| 528 | − default: | |
| 529 | − return -EINVAL; | |
| 530 | − } | |
| 531 | − | |
| 532 | − return 0; | |
| 533 | −} | |
| 534 | − | |
| 535 | −static int mlx5_ib_flow_action_create_packet_reformat_ctx( | |
| 536 | − struct mlx5_ib_dev *dev, | |
| 537 | − struct mlx5_ib_flow_action *maction, | |
| 538 | − u8 ft_type, u8 dv_prt, | |
| 539 | − void *in, size_t len) | |
| 540 | −{ | |
| 541 | − enum mlx5_flow_namespace_type namespace; | |
| 542 | − u8 prm_prt; | |
| 543 | − int ret; | |
| 544 | − | |
| 545 | − ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); | |
| 546 | − if (ret) | |
| 547 | − return ret; | |
| 548 | − | |
| 549 | − ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); | |
| 550 | − if (ret) | |
| 551 | − return ret; | |
| 552 | − | |
| 553 | − maction->flow_action_raw.pkt_reformat = | |
| 554 | − mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len, | |
| 555 | − in, namespace); | |
| 556 | − if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { | |
| 557 | − ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); | |
| 558 | − return ret; | |
| 559 | − } | |
| 560 | − | |
| 561 | − maction->flow_action_raw.sub_type = | |
| 562 | − MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; | |
| 563 | − maction->flow_action_raw.dev = dev; | |
| 564 | − | |
| 565 | − return 0; | |
| 566 | −} | |
| 567 | − | |
| 568 | −static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( | |
| 569 | − struct uverbs_attr_bundle *attrs) | |
| 570 | −{ | |
| 571 | − struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, | |
| 572 | − MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); | |
| 573 | − struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); | |
| 574 | − enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; | |
| 575 | − enum mlx5_ib_uapi_flow_table_type ft_type; | |
| 576 | − struct mlx5_ib_flow_action *maction; | |
| 577 | − int ret; | |
| 578 | − | |
| 579 | − ret = uverbs_get_const(&ft_type, attrs, | |
| 580 | − MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); | |
| 581 | − if (ret) | |
| 582 | − return ret; | |
| 583 | − | |
| 584 | − ret = uverbs_get_const(&dv_prt, attrs, | |
| 585 | − MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); | |
| 586 | − if (ret) | |
| 587 | − return ret; | |
| 588 | − | |
| 589 | − if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) | |
| 590 | − return -EOPNOTSUPP; | |
| 591 | − | |
| 592 | − maction = kzalloc(sizeof(*maction), GFP_KERNEL); | |
| 593 | − if (!maction) | |
| 594 | − return -ENOMEM; | |
| 595 | − | |
| 596 | − if (dv_prt == | |
| 597 | − MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { | |
| 598 | − maction->flow_action_raw.sub_type = | |
| 599 | − MLX5_IB_FLOW_ACTION_DECAP; | |
| 600 | − maction->flow_action_raw.dev = mdev; | |
| 601 | − } else { | |
| 602 | − void *in; | |
| 603 | − int len; | |
| 604 | − | |
| 605 | − in = uverbs_attr_get_alloced_ptr(attrs, | |
| 606 | − MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); | |
| 607 | − if (IS_ERR(in)) { | |
| 608 | − ret = PTR_ERR(in); | |
| 609 | − goto free_maction; | |
| 610 | − } | |
| 611 | − | |
| 612 | − len = uverbs_attr_get_len(attrs, | |
| 613 | − MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); | |
| 614 | − | |
| 615 | − ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, | |
| 616 | − maction, ft_type, dv_prt, in, len); | |
| 617 | − if (ret) | |
| 618 | − goto free_maction; | |
| 619 | − } | |
| 620 | − | |
| 621 | − uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, | |
| 622 | − IB_FLOW_ACTION_UNSPECIFIED); | |
| 623 | − return 0; | |
| 624 | − | |
| 625 | −free_maction: | |
| 626 | − kfree(maction); | |
| 627 | − return ret; | |
| 628 | −} | |
| 629 | − | |
| 630 | −DECLARE_UVERBS_NAMED_METHOD( | |
| 631 | − MLX5_IB_METHOD_CREATE_FLOW, | |
| 632 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, | |
| 633 | − UVERBS_OBJECT_FLOW, | |
| 634 | − UVERBS_ACCESS_NEW, | |
| 635 | − UA_MANDATORY), | |
| 636 | − UVERBS_ATTR_PTR_IN( | |
| 637 | − MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, | |
| 638 | − UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), | |
| 639 | − UA_MANDATORY, | |
| 640 | − UA_ALLOC_AND_COPY), | |
| 641 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, | |
| 642 | − MLX5_IB_OBJECT_FLOW_MATCHER, | |
| 643 | − UVERBS_ACCESS_READ, | |
| 644 | − UA_MANDATORY), | |
| 645 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, | |
| 646 | − UVERBS_OBJECT_QP, | |
| 647 | − UVERBS_ACCESS_READ), | |
| 648 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, | |
| 649 | − MLX5_IB_OBJECT_DEVX_OBJ, | |
| 650 | − UVERBS_ACCESS_READ), | |
| 651 | − UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, | |
| 652 | − UVERBS_OBJECT_FLOW_ACTION, | |
| 653 | − UVERBS_ACCESS_READ, 1, | |
| 654 | − MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, | |
| 655 | − UA_OPTIONAL), | |
| 656 | − UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, | |
| 657 | − UVERBS_ATTR_TYPE(u32), | |
| 658 | − UA_OPTIONAL), | |
| 659 | − UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, | |
| 660 | − MLX5_IB_OBJECT_DEVX_OBJ, | |
| 661 | − UVERBS_ACCESS_READ, 1, 1, | |
| 662 | − UA_OPTIONAL), | |
| 663 | − UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, | |
| 664 | − UVERBS_ATTR_MIN_SIZE(sizeof(u32)), | |
| 665 | − UA_OPTIONAL, | |
| 666 | − UA_ALLOC_AND_COPY), | |
| 667 | − UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, | |
| 668 | − enum mlx5_ib_create_flow_flags, | |
| 669 | − UA_OPTIONAL)); | |
| 670 | − | |
| 671 | −DECLARE_UVERBS_NAMED_METHOD_DESTROY( | |
| 672 | − MLX5_IB_METHOD_DESTROY_FLOW, | |
| 673 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, | |
| 674 | − UVERBS_OBJECT_FLOW, | |
| 675 | − UVERBS_ACCESS_DESTROY, | |
| 676 | − UA_MANDATORY)); | |
| 677 | − | |
| 678 | −ADD_UVERBS_METHODS(mlx5_ib_fs, | |
| 679 | − UVERBS_OBJECT_FLOW, | |
| 680 | − &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), | |
| 681 | − &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); | |
| 682 | − | |
| 683 | −DECLARE_UVERBS_NAMED_METHOD( | |
| 684 | − MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, | |
| 685 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, | |
| 686 | − UVERBS_OBJECT_FLOW_ACTION, | |
| 687 | − UVERBS_ACCESS_NEW, | |
| 688 | − UA_MANDATORY), | |
| 689 | − UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, | |
| 690 | − UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( | |
| 691 | − set_add_copy_action_in_auto)), | |
| 692 | − UA_MANDATORY, | |
| 693 | − UA_ALLOC_AND_COPY), | |
| 694 | − UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, | |
| 695 | − enum mlx5_ib_uapi_flow_table_type, | |
| 696 | − UA_MANDATORY)); | |
| 697 | − | |
| 698 | −DECLARE_UVERBS_NAMED_METHOD( | |
| 699 | − MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, | |
| 700 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, | |
| 701 | − UVERBS_OBJECT_FLOW_ACTION, | |
| 702 | − UVERBS_ACCESS_NEW, | |
| 703 | − UA_MANDATORY), | |
| 704 | − UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, | |
| 705 | − UVERBS_ATTR_MIN_SIZE(1), | |
| 706 | − UA_ALLOC_AND_COPY, | |
| 707 | − UA_OPTIONAL), | |
| 708 | − UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, | |
| 709 | − enum mlx5_ib_uapi_flow_action_packet_reformat_type, | |
| 710 | − UA_MANDATORY), | |
| 711 | − UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, | |
| 712 | − enum mlx5_ib_uapi_flow_table_type, | |
| 713 | − UA_MANDATORY)); | |
| 714 | − | |
| 715 | −ADD_UVERBS_METHODS( | |
| 716 | − mlx5_ib_flow_actions, | |
| 717 | − UVERBS_OBJECT_FLOW_ACTION, | |
| 718 | − &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), | |
| 719 | − &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); | |
| 720 | − | |
| 721 | −DECLARE_UVERBS_NAMED_METHOD( | |
| 722 | − MLX5_IB_METHOD_FLOW_MATCHER_CREATE, | |
| 723 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, | |
| 724 | − MLX5_IB_OBJECT_FLOW_MATCHER, | |
| 725 | − UVERBS_ACCESS_NEW, | |
| 726 | − UA_MANDATORY), | |
| 727 | − UVERBS_ATTR_PTR_IN( | |
| 728 | − MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, | |
| 729 | − UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), | |
| 730 | − UA_MANDATORY), | |
| 731 | − UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, | |
| 732 | − mlx5_ib_flow_type, | |
| 733 | − UA_MANDATORY), | |
| 734 | − UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, | |
| 735 | − UVERBS_ATTR_TYPE(u8), | |
| 736 | − UA_MANDATORY), | |
| 737 | − UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, | |
| 738 | − enum ib_flow_flags, | |
| 739 | − UA_OPTIONAL), | |
| 740 | − UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, | |
| 741 | − enum mlx5_ib_uapi_flow_table_type, | |
| 742 | − UA_OPTIONAL)); | |
| 743 | − | |
| 744 | −DECLARE_UVERBS_NAMED_METHOD_DESTROY( | |
| 745 | − MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, | |
| 746 | − UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, | |
| 747 | − MLX5_IB_OBJECT_FLOW_MATCHER, | |
| 748 | − UVERBS_ACCESS_DESTROY, | |
| 749 | − UA_MANDATORY)); | |
| 750 | − | |
| 751 | −DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, | |
| 752 | − UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), | |
| 753 | − &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), | |
| 754 | − &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); | |
| 755 | − | |
| 756 | −const struct uapi_definition mlx5_ib_flow_defs[] = { | |
| 757 | − UAPI_DEF_CHAIN_OBJ_TREE_NAMED( | |
| 758 | − MLX5_IB_OBJECT_FLOW_MATCHER), | |
| 759 | − UAPI_DEF_CHAIN_OBJ_TREE( | |
| 760 | − UVERBS_OBJECT_FLOW, | |
| 761 | − &mlx5_ib_fs), | |
| 762 | − UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, | |
| 763 | − &mlx5_ib_flow_actions), | |
| 764 | − {}, | |
| 765 | −}; | |
| @@ -0,0 +1,2516 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB | |
| 2 | +/* | |
| 3 | + * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#include <rdma/ib_user_verbs.h> | |
| 7 | +#include <rdma/ib_verbs.h> | |
| 8 | +#include <rdma/uverbs_types.h> | |
| 9 | +#include <rdma/uverbs_ioctl.h> | |
| 10 | +#include <rdma/uverbs_std_types.h> | |
| 11 | +#include <rdma/mlx5_user_ioctl_cmds.h> | |
| 12 | +#include <rdma/mlx5_user_ioctl_verbs.h> | |
| 13 | +#include <rdma/ib_umem.h> | |
| 14 | +#include <linux/mlx5/driver.h> | |
| 15 | +#include <linux/mlx5/fs.h> | |
| 16 | +#include <linux/mlx5/fs_helpers.h> | |
| 17 | +#include <linux/mlx5/accel.h> | |
| 18 | +#include <linux/mlx5/eswitch.h> | |
| 19 | +#include "mlx5_ib.h" | |
| 20 | +#include "counters.h" | |
| 21 | +#include "devx.h" | |
| 22 | +#include "fs.h" | |
| 23 | + | |
| 24 | +#define UVERBS_MODULE_NAME mlx5_ib | |
| 25 | +#include <rdma/uverbs_named_ioctl.h> | |
| 26 | + | |
| 27 | +enum { | |
| 28 | + MATCH_CRITERIA_ENABLE_OUTER_BIT, | |
| 29 | + MATCH_CRITERIA_ENABLE_MISC_BIT, | |
| 30 | + MATCH_CRITERIA_ENABLE_INNER_BIT, | |
| 31 | + MATCH_CRITERIA_ENABLE_MISC2_BIT | |
| 32 | +}; | |
| 33 | + | |
| 34 | +#define HEADER_IS_ZERO(match_criteria, headers) \ | |
| 35 | + !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ | |
| 36 | + 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ | |
| 37 | + | |
| 38 | +static u8 get_match_criteria_enable(u32 *match_criteria) | |
| 39 | +{ | |
| 40 | + u8 match_criteria_enable; | |
| 41 | + | |
| 42 | + match_criteria_enable = | |
| 43 | + (!HEADER_IS_ZERO(match_criteria, outer_headers)) << | |
| 44 | + MATCH_CRITERIA_ENABLE_OUTER_BIT; | |
| 45 | + match_criteria_enable |= | |
| 46 | + (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << | |
| 47 | + MATCH_CRITERIA_ENABLE_MISC_BIT; | |
| 48 | + match_criteria_enable |= | |
| 49 | + (!HEADER_IS_ZERO(match_criteria, inner_headers)) << | |
| 50 | + MATCH_CRITERIA_ENABLE_INNER_BIT; | |
| 51 | + match_criteria_enable |= | |
| 52 | + (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << | |
| 53 | + MATCH_CRITERIA_ENABLE_MISC2_BIT; | |
| 54 | + | |
| 55 | + return match_criteria_enable; | |
| 56 | +} | |
| 57 | + | |
| 58 | +static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) | |
| 59 | +{ | |
| 60 | + u8 entry_mask; | |
| 61 | + u8 entry_val; | |
| 62 | + int err = 0; | |
| 63 | + | |
| 64 | + if (!mask) | |
| 65 | + goto out; | |
| 66 | + | |
| 67 | + entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, | |
| 68 | + ip_protocol); | |
| 69 | + entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, | |
| 70 | + ip_protocol); | |
| 71 | + if (!entry_mask) { | |
| 72 | + MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); | |
| 73 | + MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); | |
| 74 | + goto out; | |
| 75 | + } | |
| 76 | + /* Don't override existing ip protocol */ | |
| 77 | + if (mask != entry_mask || val != entry_val) | |
| 78 | + err = -EINVAL; | |
| 79 | +out: | |
| 80 | + return err; | |
| 81 | +} | |
| 82 | + | |
| 83 | +static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, | |
| 84 | + bool inner) | |
| 85 | +{ | |
| 86 | + if (inner) { | |
| 87 | + MLX5_SET(fte_match_set_misc, | |
| 88 | + misc_c, inner_ipv6_flow_label, mask); | |
| 89 | + MLX5_SET(fte_match_set_misc, | |
| 90 | + misc_v, inner_ipv6_flow_label, val); | |
| 91 | + } else { | |
| 92 | + MLX5_SET(fte_match_set_misc, | |
| 93 | + misc_c, outer_ipv6_flow_label, mask); | |
| 94 | + MLX5_SET(fte_match_set_misc, | |
| 95 | + misc_v, outer_ipv6_flow_label, val); | |
| 96 | + } | |
| 97 | +} | |
| 98 | + | |
| 99 | +static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) | |
| 100 | +{ | |
| 101 | + MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); | |
| 102 | + MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); | |
| 103 | + MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); | |
| 104 | + MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); | |
| 105 | +} | |
| 106 | + | |
| 107 | +static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) | |
| 108 | +{ | |
| 109 | + if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && | |
| 110 | + !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) | |
| 111 | + return -EOPNOTSUPP; | |
| 112 | + | |
| 113 | + if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && | |
| 114 | + !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) | |
| 115 | + return -EOPNOTSUPP; | |
| 116 | + | |
| 117 | + if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && | |
| 118 | + !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) | |
| 119 | + return -EOPNOTSUPP; | |
| 120 | + | |
| 121 | + if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && | |
| 122 | + !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) | |
| 123 | + return -EOPNOTSUPP; | |
| 124 | + | |
| 125 | + return 0; | |
| 126 | +} | |
| 127 | + | |
| 128 | +#define LAST_ETH_FIELD vlan_tag | |
| 129 | +#define LAST_IB_FIELD sl | |
| 130 | +#define LAST_IPV4_FIELD tos | |
| 131 | +#define LAST_IPV6_FIELD traffic_class | |
| 132 | +#define LAST_TCP_UDP_FIELD src_port | |
| 133 | +#define LAST_TUNNEL_FIELD tunnel_id | |
| 134 | +#define LAST_FLOW_TAG_FIELD tag_id | |
| 135 | +#define LAST_DROP_FIELD size | |
| 136 | +#define LAST_COUNTERS_FIELD counters | |
| 137 | + | |
| 138 | +/* Field is the last supported field */ | |
| 139 | +#define FIELDS_NOT_SUPPORTED(filter, field)\ | |
| 140 | + memchr_inv((void *)&filter.field +\ | |
| 141 | + sizeof(filter.field), 0,\ | |
| 142 | + sizeof(filter) -\ | |
| 143 | + offsetof(typeof(filter), field) -\ | |
| 144 | + sizeof(filter.field)) | |
| 145 | + | |
| 146 | +int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, | |
| 147 | + bool is_egress, | |
| 148 | + struct mlx5_flow_act *action) | |
| 149 | +{ | |
| 150 | + | |
| 151 | + switch (maction->ib_action.type) { | |
| 152 | + case IB_FLOW_ACTION_ESP: | |
| 153 | + if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | | |
| 154 | + MLX5_FLOW_CONTEXT_ACTION_DECRYPT)) | |
| 155 | + return -EINVAL; | |
| 156 | + /* Currently only AES_GCM keymat is supported by the driver */ | |
| 157 | + action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx; | |
| 158 | + action->action |= is_egress ? | |
| 159 | + MLX5_FLOW_CONTEXT_ACTION_ENCRYPT : | |
| 160 | + MLX5_FLOW_CONTEXT_ACTION_DECRYPT; | |
| 161 | + return 0; | |
| 162 | + case IB_FLOW_ACTION_UNSPECIFIED: | |
| 163 | + if (maction->flow_action_raw.sub_type == | |
| 164 | + MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { | |
| 165 | + if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) | |
| 166 | + return -EINVAL; | |
| 167 | + action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; | |
| 168 | + action->modify_hdr = | |
| 169 | + maction->flow_action_raw.modify_hdr; | |
| 170 | + return 0; | |
| 171 | + } | |
| 172 | + if (maction->flow_action_raw.sub_type == | |
| 173 | + MLX5_IB_FLOW_ACTION_DECAP) { | |
| 174 | + if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) | |
| 175 | + return -EINVAL; | |
| 176 | + action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; | |
| 177 | + return 0; | |
| 178 | + } | |
| 179 | + if (maction->flow_action_raw.sub_type == | |
| 180 | + MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { | |
| 181 | + if (action->action & | |
| 182 | + MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) | |
| 183 | + return -EINVAL; | |
| 184 | + action->action |= | |
| 185 | + MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; | |
| 186 | + action->pkt_reformat = | |
| 187 | + maction->flow_action_raw.pkt_reformat; | |
| 188 | + return 0; | |
| 189 | + } | |
| 190 | + fallthrough; | |
| 191 | + default: | |
| 192 | + return -EOPNOTSUPP; | |
| 193 | + } | |
| 194 | +} | |
| 195 | + | |
| 196 | +static int parse_flow_attr(struct mlx5_core_dev *mdev, | |
| 197 | + struct mlx5_flow_spec *spec, | |
| 198 | + const union ib_flow_spec *ib_spec, | |
| 199 | + const struct ib_flow_attr *flow_attr, | |
| 200 | + struct mlx5_flow_act *action, u32 prev_type) | |
| 201 | +{ | |
| 202 | + struct mlx5_flow_context *flow_context = &spec->flow_context; | |
| 203 | + u32 *match_c = spec->match_criteria; | |
| 204 | + u32 *match_v = spec->match_value; | |
| 205 | + void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 206 | + misc_parameters); | |
| 207 | + void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 208 | + misc_parameters); | |
| 209 | + void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 210 | + misc_parameters_2); | |
| 211 | + void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 212 | + misc_parameters_2); | |
| 213 | + void *headers_c; | |
| 214 | + void *headers_v; | |
| 215 | + int match_ipv; | |
| 216 | + int ret; | |
| 217 | + | |
| 218 | + if (ib_spec->type & IB_FLOW_SPEC_INNER) { | |
| 219 | + headers_c = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 220 | + inner_headers); | |
| 221 | + headers_v = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 222 | + inner_headers); | |
| 223 | + match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 224 | + ft_field_support.inner_ip_version); | |
| 225 | + } else { | |
| 226 | + headers_c = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 227 | + outer_headers); | |
| 228 | + headers_v = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 229 | + outer_headers); | |
| 230 | + match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 231 | + ft_field_support.outer_ip_version); | |
| 232 | + } | |
| 233 | + | |
| 234 | + switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { | |
| 235 | + case IB_FLOW_SPEC_ETH: | |
| 236 | + if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) | |
| 237 | + return -EOPNOTSUPP; | |
| 238 | + | |
| 239 | + ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 240 | + dmac_47_16), | |
| 241 | + ib_spec->eth.mask.dst_mac); | |
| 242 | + ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 243 | + dmac_47_16), | |
| 244 | + ib_spec->eth.val.dst_mac); | |
| 245 | + | |
| 246 | + ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 247 | + smac_47_16), | |
| 248 | + ib_spec->eth.mask.src_mac); | |
| 249 | + ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 250 | + smac_47_16), | |
| 251 | + ib_spec->eth.val.src_mac); | |
| 252 | + | |
| 253 | + if (ib_spec->eth.mask.vlan_tag) { | |
| 254 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 255 | + cvlan_tag, 1); | |
| 256 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 257 | + cvlan_tag, 1); | |
| 258 | + | |
| 259 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 260 | + first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); | |
| 261 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 262 | + first_vid, ntohs(ib_spec->eth.val.vlan_tag)); | |
| 263 | + | |
| 264 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 265 | + first_cfi, | |
| 266 | + ntohs(ib_spec->eth.mask.vlan_tag) >> 12); | |
| 267 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 268 | + first_cfi, | |
| 269 | + ntohs(ib_spec->eth.val.vlan_tag) >> 12); | |
| 270 | + | |
| 271 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 272 | + first_prio, | |
| 273 | + ntohs(ib_spec->eth.mask.vlan_tag) >> 13); | |
| 274 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 275 | + first_prio, | |
| 276 | + ntohs(ib_spec->eth.val.vlan_tag) >> 13); | |
| 277 | + } | |
| 278 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 279 | + ethertype, ntohs(ib_spec->eth.mask.ether_type)); | |
| 280 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 281 | + ethertype, ntohs(ib_spec->eth.val.ether_type)); | |
| 282 | + break; | |
| 283 | + case IB_FLOW_SPEC_IPV4: | |
| 284 | + if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) | |
| 285 | + return -EOPNOTSUPP; | |
| 286 | + | |
| 287 | + if (match_ipv) { | |
| 288 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 289 | + ip_version, 0xf); | |
| 290 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 291 | + ip_version, MLX5_FS_IPV4_VERSION); | |
| 292 | + } else { | |
| 293 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 294 | + ethertype, 0xffff); | |
| 295 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 296 | + ethertype, ETH_P_IP); | |
| 297 | + } | |
| 298 | + | |
| 299 | + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 300 | + src_ipv4_src_ipv6.ipv4_layout.ipv4), | |
| 301 | + &ib_spec->ipv4.mask.src_ip, | |
| 302 | + sizeof(ib_spec->ipv4.mask.src_ip)); | |
| 303 | + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 304 | + src_ipv4_src_ipv6.ipv4_layout.ipv4), | |
| 305 | + &ib_spec->ipv4.val.src_ip, | |
| 306 | + sizeof(ib_spec->ipv4.val.src_ip)); | |
| 307 | + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 308 | + dst_ipv4_dst_ipv6.ipv4_layout.ipv4), | |
| 309 | + &ib_spec->ipv4.mask.dst_ip, | |
| 310 | + sizeof(ib_spec->ipv4.mask.dst_ip)); | |
| 311 | + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 312 | + dst_ipv4_dst_ipv6.ipv4_layout.ipv4), | |
| 313 | + &ib_spec->ipv4.val.dst_ip, | |
| 314 | + sizeof(ib_spec->ipv4.val.dst_ip)); | |
| 315 | + | |
| 316 | + set_tos(headers_c, headers_v, | |
| 317 | + ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); | |
| 318 | + | |
| 319 | + if (set_proto(headers_c, headers_v, | |
| 320 | + ib_spec->ipv4.mask.proto, | |
| 321 | + ib_spec->ipv4.val.proto)) | |
| 322 | + return -EINVAL; | |
| 323 | + break; | |
| 324 | + case IB_FLOW_SPEC_IPV6: | |
| 325 | + if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) | |
| 326 | + return -EOPNOTSUPP; | |
| 327 | + | |
| 328 | + if (match_ipv) { | |
| 329 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 330 | + ip_version, 0xf); | |
| 331 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 332 | + ip_version, MLX5_FS_IPV6_VERSION); | |
| 333 | + } else { | |
| 334 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 335 | + ethertype, 0xffff); | |
| 336 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 337 | + ethertype, ETH_P_IPV6); | |
| 338 | + } | |
| 339 | + | |
| 340 | + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 341 | + src_ipv4_src_ipv6.ipv6_layout.ipv6), | |
| 342 | + &ib_spec->ipv6.mask.src_ip, | |
| 343 | + sizeof(ib_spec->ipv6.mask.src_ip)); | |
| 344 | + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 345 | + src_ipv4_src_ipv6.ipv6_layout.ipv6), | |
| 346 | + &ib_spec->ipv6.val.src_ip, | |
| 347 | + sizeof(ib_spec->ipv6.val.src_ip)); | |
| 348 | + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 349 | + dst_ipv4_dst_ipv6.ipv6_layout.ipv6), | |
| 350 | + &ib_spec->ipv6.mask.dst_ip, | |
| 351 | + sizeof(ib_spec->ipv6.mask.dst_ip)); | |
| 352 | + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 353 | + dst_ipv4_dst_ipv6.ipv6_layout.ipv6), | |
| 354 | + &ib_spec->ipv6.val.dst_ip, | |
| 355 | + sizeof(ib_spec->ipv6.val.dst_ip)); | |
| 356 | + | |
| 357 | + set_tos(headers_c, headers_v, | |
| 358 | + ib_spec->ipv6.mask.traffic_class, | |
| 359 | + ib_spec->ipv6.val.traffic_class); | |
| 360 | + | |
| 361 | + if (set_proto(headers_c, headers_v, | |
| 362 | + ib_spec->ipv6.mask.next_hdr, | |
| 363 | + ib_spec->ipv6.val.next_hdr)) | |
| 364 | + return -EINVAL; | |
| 365 | + | |
| 366 | + set_flow_label(misc_params_c, misc_params_v, | |
| 367 | + ntohl(ib_spec->ipv6.mask.flow_label), | |
| 368 | + ntohl(ib_spec->ipv6.val.flow_label), | |
| 369 | + ib_spec->type & IB_FLOW_SPEC_INNER); | |
| 370 | + break; | |
| 371 | + case IB_FLOW_SPEC_ESP: | |
| 372 | + if (ib_spec->esp.mask.seq) | |
| 373 | + return -EOPNOTSUPP; | |
| 374 | + | |
| 375 | + MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, | |
| 376 | + ntohl(ib_spec->esp.mask.spi)); | |
| 377 | + MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi, | |
| 378 | + ntohl(ib_spec->esp.val.spi)); | |
| 379 | + break; | |
| 380 | + case IB_FLOW_SPEC_TCP: | |
| 381 | + if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, | |
| 382 | + LAST_TCP_UDP_FIELD)) | |
| 383 | + return -EOPNOTSUPP; | |
| 384 | + | |
| 385 | + if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) | |
| 386 | + return -EINVAL; | |
| 387 | + | |
| 388 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, | |
| 389 | + ntohs(ib_spec->tcp_udp.mask.src_port)); | |
| 390 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, | |
| 391 | + ntohs(ib_spec->tcp_udp.val.src_port)); | |
| 392 | + | |
| 393 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, | |
| 394 | + ntohs(ib_spec->tcp_udp.mask.dst_port)); | |
| 395 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, | |
| 396 | + ntohs(ib_spec->tcp_udp.val.dst_port)); | |
| 397 | + break; | |
| 398 | + case IB_FLOW_SPEC_UDP: | |
| 399 | + if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, | |
| 400 | + LAST_TCP_UDP_FIELD)) | |
| 401 | + return -EOPNOTSUPP; | |
| 402 | + | |
| 403 | + if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) | |
| 404 | + return -EINVAL; | |
| 405 | + | |
| 406 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, | |
| 407 | + ntohs(ib_spec->tcp_udp.mask.src_port)); | |
| 408 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, | |
| 409 | + ntohs(ib_spec->tcp_udp.val.src_port)); | |
| 410 | + | |
| 411 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, | |
| 412 | + ntohs(ib_spec->tcp_udp.mask.dst_port)); | |
| 413 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, | |
| 414 | + ntohs(ib_spec->tcp_udp.val.dst_port)); | |
| 415 | + break; | |
| 416 | + case IB_FLOW_SPEC_GRE: | |
| 417 | + if (ib_spec->gre.mask.c_ks_res0_ver) | |
| 418 | + return -EOPNOTSUPP; | |
| 419 | + | |
| 420 | + if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) | |
| 421 | + return -EINVAL; | |
| 422 | + | |
| 423 | + MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, | |
| 424 | + 0xff); | |
| 425 | + MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, | |
| 426 | + IPPROTO_GRE); | |
| 427 | + | |
| 428 | + MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, | |
| 429 | + ntohs(ib_spec->gre.mask.protocol)); | |
| 430 | + MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, | |
| 431 | + ntohs(ib_spec->gre.val.protocol)); | |
| 432 | + | |
| 433 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, | |
| 434 | + gre_key.nvgre.hi), | |
| 435 | + &ib_spec->gre.mask.key, | |
| 436 | + sizeof(ib_spec->gre.mask.key)); | |
| 437 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, | |
| 438 | + gre_key.nvgre.hi), | |
| 439 | + &ib_spec->gre.val.key, | |
| 440 | + sizeof(ib_spec->gre.val.key)); | |
| 441 | + break; | |
| 442 | + case IB_FLOW_SPEC_MPLS: | |
| 443 | + switch (prev_type) { | |
| 444 | + case IB_FLOW_SPEC_UDP: | |
| 445 | + if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 446 | + ft_field_support.outer_first_mpls_over_udp), | |
| 447 | + &ib_spec->mpls.mask.tag)) | |
| 448 | + return -EOPNOTSUPP; | |
| 449 | + | |
| 450 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, | |
| 451 | + outer_first_mpls_over_udp), | |
| 452 | + &ib_spec->mpls.val.tag, | |
| 453 | + sizeof(ib_spec->mpls.val.tag)); | |
| 454 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, | |
| 455 | + outer_first_mpls_over_udp), | |
| 456 | + &ib_spec->mpls.mask.tag, | |
| 457 | + sizeof(ib_spec->mpls.mask.tag)); | |
| 458 | + break; | |
| 459 | + case IB_FLOW_SPEC_GRE: | |
| 460 | + if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 461 | + ft_field_support.outer_first_mpls_over_gre), | |
| 462 | + &ib_spec->mpls.mask.tag)) | |
| 463 | + return -EOPNOTSUPP; | |
| 464 | + | |
| 465 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, | |
| 466 | + outer_first_mpls_over_gre), | |
| 467 | + &ib_spec->mpls.val.tag, | |
| 468 | + sizeof(ib_spec->mpls.val.tag)); | |
| 469 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, | |
| 470 | + outer_first_mpls_over_gre), | |
| 471 | + &ib_spec->mpls.mask.tag, | |
| 472 | + sizeof(ib_spec->mpls.mask.tag)); | |
| 473 | + break; | |
| 474 | + default: | |
| 475 | + if (ib_spec->type & IB_FLOW_SPEC_INNER) { | |
| 476 | + if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 477 | + ft_field_support.inner_first_mpls), | |
| 478 | + &ib_spec->mpls.mask.tag)) | |
| 479 | + return -EOPNOTSUPP; | |
| 480 | + | |
| 481 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, | |
| 482 | + inner_first_mpls), | |
| 483 | + &ib_spec->mpls.val.tag, | |
| 484 | + sizeof(ib_spec->mpls.val.tag)); | |
| 485 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, | |
| 486 | + inner_first_mpls), | |
| 487 | + &ib_spec->mpls.mask.tag, | |
| 488 | + sizeof(ib_spec->mpls.mask.tag)); | |
| 489 | + } else { | |
| 490 | + if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 491 | + ft_field_support.outer_first_mpls), | |
| 492 | + &ib_spec->mpls.mask.tag)) | |
| 493 | + return -EOPNOTSUPP; | |
| 494 | + | |
| 495 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, | |
| 496 | + outer_first_mpls), | |
| 497 | + &ib_spec->mpls.val.tag, | |
| 498 | + sizeof(ib_spec->mpls.val.tag)); | |
| 499 | + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, | |
| 500 | + outer_first_mpls), | |
| 501 | + &ib_spec->mpls.mask.tag, | |
| 502 | + sizeof(ib_spec->mpls.mask.tag)); | |
| 503 | + } | |
| 504 | + } | |
| 505 | + break; | |
| 506 | + case IB_FLOW_SPEC_VXLAN_TUNNEL: | |
| 507 | + if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, | |
| 508 | + LAST_TUNNEL_FIELD)) | |
| 509 | + return -EOPNOTSUPP; | |
| 510 | + | |
| 511 | + MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, | |
| 512 | + ntohl(ib_spec->tunnel.mask.tunnel_id)); | |
| 513 | + MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, | |
| 514 | + ntohl(ib_spec->tunnel.val.tunnel_id)); | |
| 515 | + break; | |
| 516 | + case IB_FLOW_SPEC_ACTION_TAG: | |
| 517 | + if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, | |
| 518 | + LAST_FLOW_TAG_FIELD)) | |
| 519 | + return -EOPNOTSUPP; | |
| 520 | + if (ib_spec->flow_tag.tag_id >= BIT(24)) | |
| 521 | + return -EINVAL; | |
| 522 | + | |
| 523 | + flow_context->flow_tag = ib_spec->flow_tag.tag_id; | |
| 524 | + flow_context->flags |= FLOW_CONTEXT_HAS_TAG; | |
| 525 | + break; | |
| 526 | + case IB_FLOW_SPEC_ACTION_DROP: | |
| 527 | + if (FIELDS_NOT_SUPPORTED(ib_spec->drop, | |
| 528 | + LAST_DROP_FIELD)) | |
| 529 | + return -EOPNOTSUPP; | |
| 530 | + action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; | |
| 531 | + break; | |
| 532 | + case IB_FLOW_SPEC_ACTION_HANDLE: | |
| 533 | + ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), | |
| 534 | + flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); | |
| 535 | + if (ret) | |
| 536 | + return ret; | |
| 537 | + break; | |
| 538 | + case IB_FLOW_SPEC_ACTION_COUNT: | |
| 539 | + if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, | |
| 540 | + LAST_COUNTERS_FIELD)) | |
| 541 | + return -EOPNOTSUPP; | |
| 542 | + | |
| 543 | + /* for now support only one counters spec per flow */ | |
| 544 | + if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) | |
| 545 | + return -EINVAL; | |
| 546 | + | |
| 547 | + action->counters = ib_spec->flow_count.counters; | |
| 548 | + action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; | |
| 549 | + break; | |
| 550 | + default: | |
| 551 | + return -EINVAL; | |
| 552 | + } | |
| 553 | + | |
| 554 | + return 0; | |
| 555 | +} | |
| 556 | + | |
| 557 | +/* If a flow could catch both multicast and unicast packets, | |
| 558 | + * it won't fall into the multicast flow steering table and this rule | |
| 559 | + * could steal other multicast packets. | |
| 560 | + */ | |
| 561 | +static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) | |
| 562 | +{ | |
| 563 | + union ib_flow_spec *flow_spec; | |
| 564 | + | |
| 565 | + if (ib_attr->type != IB_FLOW_ATTR_NORMAL || | |
| 566 | + ib_attr->num_of_specs < 1) | |
| 567 | + return false; | |
| 568 | + | |
| 569 | + flow_spec = (union ib_flow_spec *)(ib_attr + 1); | |
| 570 | + if (flow_spec->type == IB_FLOW_SPEC_IPV4) { | |
| 571 | + struct ib_flow_spec_ipv4 *ipv4_spec; | |
| 572 | + | |
| 573 | + ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; | |
| 574 | + if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) | |
| 575 | + return true; | |
| 576 | + | |
| 577 | + return false; | |
| 578 | + } | |
| 579 | + | |
| 580 | + if (flow_spec->type == IB_FLOW_SPEC_ETH) { | |
| 581 | + struct ib_flow_spec_eth *eth_spec; | |
| 582 | + | |
| 583 | + eth_spec = (struct ib_flow_spec_eth *)flow_spec; | |
| 584 | + return is_multicast_ether_addr(eth_spec->mask.dst_mac) && | |
| 585 | + is_multicast_ether_addr(eth_spec->val.dst_mac); | |
| 586 | + } | |
| 587 | + | |
| 588 | + return false; | |
| 589 | +} | |
| 590 | + | |
| 591 | +enum valid_spec { | |
| 592 | + VALID_SPEC_INVALID, | |
| 593 | + VALID_SPEC_VALID, | |
| 594 | + VALID_SPEC_NA, | |
| 595 | +}; | |
| 596 | + | |
| 597 | +static enum valid_spec | |
| 598 | +is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev, | |
| 599 | + const struct mlx5_flow_spec *spec, | |
| 600 | + const struct mlx5_flow_act *flow_act, | |
| 601 | + bool egress) | |
| 602 | +{ | |
| 603 | + const u32 *match_c = spec->match_criteria; | |
| 604 | + bool is_crypto = | |
| 605 | + (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | | |
| 606 | + MLX5_FLOW_CONTEXT_ACTION_DECRYPT)); | |
| 607 | + bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c); | |
| 608 | + bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP; | |
| 609 | + | |
| 610 | + /* | |
| 611 | + * Currently only crypto is supported in egress, when regular egress | |
| 612 | + * rules would be supported, always return VALID_SPEC_NA. | |
| 613 | + */ | |
| 614 | + if (!is_crypto) | |
| 615 | + return VALID_SPEC_NA; | |
| 616 | + | |
| 617 | + return is_crypto && is_ipsec && | |
| 618 | + (!egress || (!is_drop && | |
| 619 | + !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ? | |
| 620 | + VALID_SPEC_VALID : VALID_SPEC_INVALID; | |
| 621 | +} | |
| 622 | + | |
| 623 | +static bool is_valid_spec(struct mlx5_core_dev *mdev, | |
| 624 | + const struct mlx5_flow_spec *spec, | |
| 625 | + const struct mlx5_flow_act *flow_act, | |
| 626 | + bool egress) | |
| 627 | +{ | |
| 628 | + /* We curretly only support ipsec egress flow */ | |
| 629 | + return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID; | |
| 630 | +} | |
| 631 | + | |
| 632 | +static bool is_valid_ethertype(struct mlx5_core_dev *mdev, | |
| 633 | + const struct ib_flow_attr *flow_attr, | |
| 634 | + bool check_inner) | |
| 635 | +{ | |
| 636 | + union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); | |
| 637 | + int match_ipv = check_inner ? | |
| 638 | + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 639 | + ft_field_support.inner_ip_version) : | |
| 640 | + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 641 | + ft_field_support.outer_ip_version); | |
| 642 | + int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; | |
| 643 | + bool ipv4_spec_valid, ipv6_spec_valid; | |
| 644 | + unsigned int ip_spec_type = 0; | |
| 645 | + bool has_ethertype = false; | |
| 646 | + unsigned int spec_index; | |
| 647 | + bool mask_valid = true; | |
| 648 | + u16 eth_type = 0; | |
| 649 | + bool type_valid; | |
| 650 | + | |
| 651 | + /* Validate that ethertype is correct */ | |
| 652 | + for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { | |
| 653 | + if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && | |
| 654 | + ib_spec->eth.mask.ether_type) { | |
| 655 | + mask_valid = (ib_spec->eth.mask.ether_type == | |
| 656 | + htons(0xffff)); | |
| 657 | + has_ethertype = true; | |
| 658 | + eth_type = ntohs(ib_spec->eth.val.ether_type); | |
| 659 | + } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || | |
| 660 | + (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { | |
| 661 | + ip_spec_type = ib_spec->type; | |
| 662 | + } | |
| 663 | + ib_spec = (void *)ib_spec + ib_spec->size; | |
| 664 | + } | |
| 665 | + | |
| 666 | + type_valid = (!has_ethertype) || (!ip_spec_type); | |
| 667 | + if (!type_valid && mask_valid) { | |
| 668 | + ipv4_spec_valid = (eth_type == ETH_P_IP) && | |
| 669 | + (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); | |
| 670 | + ipv6_spec_valid = (eth_type == ETH_P_IPV6) && | |
| 671 | + (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); | |
| 672 | + | |
| 673 | + type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || | |
| 674 | + (((eth_type == ETH_P_MPLS_UC) || | |
| 675 | + (eth_type == ETH_P_MPLS_MC)) && match_ipv); | |
| 676 | + } | |
| 677 | + | |
| 678 | + return type_valid; | |
| 679 | +} | |
| 680 | + | |
| 681 | +static bool is_valid_attr(struct mlx5_core_dev *mdev, | |
| 682 | + const struct ib_flow_attr *flow_attr) | |
| 683 | +{ | |
| 684 | + return is_valid_ethertype(mdev, flow_attr, false) && | |
| 685 | + is_valid_ethertype(mdev, flow_attr, true); | |
| 686 | +} | |
| 687 | + | |
| 688 | +static void put_flow_table(struct mlx5_ib_dev *dev, | |
| 689 | + struct mlx5_ib_flow_prio *prio, bool ft_added) | |
| 690 | +{ | |
| 691 | + prio->refcount -= !!ft_added; | |
| 692 | + if (!prio->refcount) { | |
| 693 | + mlx5_destroy_flow_table(prio->flow_table); | |
| 694 | + prio->flow_table = NULL; | |
| 695 | + } | |
| 696 | +} | |
| 697 | + | |
| 698 | +static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) | |
| 699 | +{ | |
| 700 | + struct mlx5_ib_flow_handler *handler = container_of(flow_id, | |
| 701 | + struct mlx5_ib_flow_handler, | |
| 702 | + ibflow); | |
| 703 | + struct mlx5_ib_flow_handler *iter, *tmp; | |
| 704 | + struct mlx5_ib_dev *dev = handler->dev; | |
| 705 | + | |
| 706 | + mutex_lock(&dev->flow_db->lock); | |
| 707 | + | |
| 708 | + list_for_each_entry_safe(iter, tmp, &handler->list, list) { | |
| 709 | + mlx5_del_flow_rules(iter->rule); | |
| 710 | + put_flow_table(dev, iter->prio, true); | |
| 711 | + list_del(&iter->list); | |
| 712 | + kfree(iter); | |
| 713 | + } | |
| 714 | + | |
| 715 | + mlx5_del_flow_rules(handler->rule); | |
| 716 | + put_flow_table(dev, handler->prio, true); | |
| 717 | + mlx5_ib_counters_clear_description(handler->ibcounters); | |
| 718 | + mutex_unlock(&dev->flow_db->lock); | |
| 719 | + if (handler->flow_matcher) | |
| 720 | + atomic_dec(&handler->flow_matcher->usecnt); | |
| 721 | + kfree(handler); | |
| 722 | + | |
| 723 | + return 0; | |
| 724 | +} | |
| 725 | + | |
| 726 | +static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) | |
| 727 | +{ | |
| 728 | + priority *= 2; | |
| 729 | + if (!dont_trap) | |
| 730 | + priority++; | |
| 731 | + return priority; | |
| 732 | +} | |
| 733 | + | |
| 734 | +enum flow_table_type { | |
| 735 | + MLX5_IB_FT_RX, | |
| 736 | + MLX5_IB_FT_TX | |
| 737 | +}; | |
| 738 | + | |
| 739 | +#define MLX5_FS_MAX_TYPES 6 | |
| 740 | +#define MLX5_FS_MAX_ENTRIES BIT(16) | |
| 741 | + | |
| 742 | +static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns, | |
| 743 | + struct mlx5_ib_flow_prio *prio, | |
| 744 | + int priority, | |
| 745 | + int num_entries, int num_groups, | |
| 746 | + u32 flags) | |
| 747 | +{ | |
| 748 | + struct mlx5_flow_table_attr ft_attr = {}; | |
| 749 | + struct mlx5_flow_table *ft; | |
| 750 | + | |
| 751 | + ft_attr.prio = priority; | |
| 752 | + ft_attr.max_fte = num_entries; | |
| 753 | + ft_attr.flags = flags; | |
| 754 | + ft_attr.autogroup.max_num_groups = num_groups; | |
| 755 | + ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); | |
| 756 | + if (IS_ERR(ft)) | |
| 757 | + return ERR_CAST(ft); | |
| 758 | + | |
| 759 | + prio->flow_table = ft; | |
| 760 | + prio->refcount = 0; | |
| 761 | + return prio; | |
| 762 | +} | |
| 763 | + | |
| 764 | +static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, | |
| 765 | + struct ib_flow_attr *flow_attr, | |
| 766 | + enum flow_table_type ft_type) | |
| 767 | +{ | |
| 768 | + bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; | |
| 769 | + struct mlx5_flow_namespace *ns = NULL; | |
| 770 | + struct mlx5_ib_flow_prio *prio; | |
| 771 | + struct mlx5_flow_table *ft; | |
| 772 | + int max_table_size; | |
| 773 | + int num_entries; | |
| 774 | + int num_groups; | |
| 775 | + bool esw_encap; | |
| 776 | + u32 flags = 0; | |
| 777 | + int priority; | |
| 778 | + | |
| 779 | + max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 780 | + log_max_ft_size)); | |
| 781 | + esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != | |
| 782 | + DEVLINK_ESWITCH_ENCAP_MODE_NONE; | |
| 783 | + if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { | |
| 784 | + enum mlx5_flow_namespace_type fn_type; | |
| 785 | + | |
| 786 | + if (flow_is_multicast_only(flow_attr) && | |
| 787 | + !dont_trap) | |
| 788 | + priority = MLX5_IB_FLOW_MCAST_PRIO; | |
| 789 | + else | |
| 790 | + priority = ib_prio_to_core_prio(flow_attr->priority, | |
| 791 | + dont_trap); | |
| 792 | + if (ft_type == MLX5_IB_FT_RX) { | |
| 793 | + fn_type = MLX5_FLOW_NAMESPACE_BYPASS; | |
| 794 | + prio = &dev->flow_db->prios[priority]; | |
| 795 | + if (!dev->is_rep && !esw_encap && | |
| 796 | + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) | |
| 797 | + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; | |
| 798 | + if (!dev->is_rep && !esw_encap && | |
| 799 | + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 800 | + reformat_l3_tunnel_to_l2)) | |
| 801 | + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 802 | + } else { | |
| 803 | + max_table_size = | |
| 804 | + BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, | |
| 805 | + log_max_ft_size)); | |
| 806 | + fn_type = MLX5_FLOW_NAMESPACE_EGRESS; | |
| 807 | + prio = &dev->flow_db->egress_prios[priority]; | |
| 808 | + if (!dev->is_rep && !esw_encap && | |
| 809 | + MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) | |
| 810 | + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 811 | + } | |
| 812 | + ns = mlx5_get_flow_namespace(dev->mdev, fn_type); | |
| 813 | + num_entries = MLX5_FS_MAX_ENTRIES; | |
| 814 | + num_groups = MLX5_FS_MAX_TYPES; | |
| 815 | + } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || | |
| 816 | + flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { | |
| 817 | + ns = mlx5_get_flow_namespace(dev->mdev, | |
| 818 | + MLX5_FLOW_NAMESPACE_LEFTOVERS); | |
| 819 | + build_leftovers_ft_param(&priority, | |
| 820 | + &num_entries, | |
| 821 | + &num_groups); | |
| 822 | + prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; | |
| 823 | + } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { | |
| 824 | + if (!MLX5_CAP_FLOWTABLE(dev->mdev, | |
| 825 | + allow_sniffer_and_nic_rx_shared_tir)) | |
| 826 | + return ERR_PTR(-EOPNOTSUPP); | |
| 827 | + | |
| 828 | + ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ? | |
| 829 | + MLX5_FLOW_NAMESPACE_SNIFFER_RX : | |
| 830 | + MLX5_FLOW_NAMESPACE_SNIFFER_TX); | |
| 831 | + | |
| 832 | + prio = &dev->flow_db->sniffer[ft_type]; | |
| 833 | + priority = 0; | |
| 834 | + num_entries = 1; | |
| 835 | + num_groups = 1; | |
| 836 | + } | |
| 837 | + | |
| 838 | + if (!ns) | |
| 839 | + return ERR_PTR(-EOPNOTSUPP); | |
| 840 | + | |
| 841 | + max_table_size = min_t(int, num_entries, max_table_size); | |
| 842 | + | |
| 843 | + ft = prio->flow_table; | |
| 844 | + if (!ft) | |
| 845 | + return _get_prio(ns, prio, priority, max_table_size, num_groups, | |
| 846 | + flags); | |
| 847 | + | |
| 848 | + return prio; | |
| 849 | +} | |
| 850 | + | |
| 851 | +static void set_underlay_qp(struct mlx5_ib_dev *dev, | |
| 852 | + struct mlx5_flow_spec *spec, | |
| 853 | + u32 underlay_qpn) | |
| 854 | +{ | |
| 855 | + void *misc_params_c = MLX5_ADDR_OF(fte_match_param, | |
| 856 | + spec->match_criteria, | |
| 857 | + misc_parameters); | |
| 858 | + void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, | |
| 859 | + misc_parameters); | |
| 860 | + | |
| 861 | + if (underlay_qpn && | |
| 862 | + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 863 | + ft_field_support.bth_dst_qp)) { | |
| 864 | + MLX5_SET(fte_match_set_misc, | |
| 865 | + misc_params_v, bth_dst_qp, underlay_qpn); | |
| 866 | + MLX5_SET(fte_match_set_misc, | |
| 867 | + misc_params_c, bth_dst_qp, 0xffffff); | |
| 868 | + } | |
| 869 | +} | |
| 870 | + | |
| 871 | +static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, | |
| 872 | + struct mlx5_flow_spec *spec, | |
| 873 | + struct mlx5_eswitch_rep *rep) | |
| 874 | +{ | |
| 875 | + struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; | |
| 876 | + void *misc; | |
| 877 | + | |
| 878 | + if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { | |
| 879 | + misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, | |
| 880 | + misc_parameters_2); | |
| 881 | + | |
| 882 | + MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, | |
| 883 | + mlx5_eswitch_get_vport_metadata_for_match(esw, | |
| 884 | + rep->vport)); | |
| 885 | + misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, | |
| 886 | + misc_parameters_2); | |
| 887 | + | |
| 888 | + MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, | |
| 889 | + mlx5_eswitch_get_vport_metadata_mask()); | |
| 890 | + } else { | |
| 891 | + misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, | |
| 892 | + misc_parameters); | |
| 893 | + | |
| 894 | + MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); | |
| 895 | + | |
| 896 | + misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, | |
| 897 | + misc_parameters); | |
| 898 | + | |
| 899 | + MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); | |
| 900 | + } | |
| 901 | +} | |
| 902 | + | |
| 903 | +static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, | |
| 904 | + struct mlx5_ib_flow_prio *ft_prio, | |
| 905 | + const struct ib_flow_attr *flow_attr, | |
| 906 | + struct mlx5_flow_destination *dst, | |
| 907 | + u32 underlay_qpn, | |
| 908 | + struct mlx5_ib_create_flow *ucmd) | |
| 909 | +{ | |
| 910 | + struct mlx5_flow_table *ft = ft_prio->flow_table; | |
| 911 | + struct mlx5_ib_flow_handler *handler; | |
| 912 | + struct mlx5_flow_act flow_act = {}; | |
| 913 | + struct mlx5_flow_spec *spec; | |
| 914 | + struct mlx5_flow_destination dest_arr[2] = {}; | |
| 915 | + struct mlx5_flow_destination *rule_dst = dest_arr; | |
| 916 | + const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); | |
| 917 | + unsigned int spec_index; | |
| 918 | + u32 prev_type = 0; | |
| 919 | + int err = 0; | |
| 920 | + int dest_num = 0; | |
| 921 | + bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; | |
| 922 | + | |
| 923 | + if (!is_valid_attr(dev->mdev, flow_attr)) | |
| 924 | + return ERR_PTR(-EINVAL); | |
| 925 | + | |
| 926 | + if (dev->is_rep && is_egress) | |
| 927 | + return ERR_PTR(-EINVAL); | |
| 928 | + | |
| 929 | + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); | |
| 930 | + handler = kzalloc(sizeof(*handler), GFP_KERNEL); | |
| 931 | + if (!handler || !spec) { | |
| 932 | + err = -ENOMEM; | |
| 933 | + goto free; | |
| 934 | + } | |
| 935 | + | |
| 936 | + INIT_LIST_HEAD(&handler->list); | |
| 937 | + | |
| 938 | + for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { | |
| 939 | + err = parse_flow_attr(dev->mdev, spec, | |
| 940 | + ib_flow, flow_attr, &flow_act, | |
| 941 | + prev_type); | |
| 942 | + if (err < 0) | |
| 943 | + goto free; | |
| 944 | + | |
| 945 | + prev_type = ((union ib_flow_spec *)ib_flow)->type; | |
| 946 | + ib_flow += ((union ib_flow_spec *)ib_flow)->size; | |
| 947 | + } | |
| 948 | + | |
| 949 | + if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { | |
| 950 | + memcpy(&dest_arr[0], dst, sizeof(*dst)); | |
| 951 | + dest_num++; | |
| 952 | + } | |
| 953 | + | |
| 954 | + if (!flow_is_multicast_only(flow_attr)) | |
| 955 | + set_underlay_qp(dev, spec, underlay_qpn); | |
| 956 | + | |
| 957 | + if (dev->is_rep) { | |
| 958 | + struct mlx5_eswitch_rep *rep; | |
| 959 | + | |
| 960 | + rep = dev->port[flow_attr->port - 1].rep; | |
| 961 | + if (!rep) { | |
| 962 | + err = -EINVAL; | |
| 963 | + goto free; | |
| 964 | + } | |
| 965 | + | |
| 966 | + mlx5_ib_set_rule_source_port(dev, spec, rep); | |
| 967 | + } | |
| 968 | + | |
| 969 | + spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); | |
| 970 | + | |
| 971 | + if (is_egress && | |
| 972 | + !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) { | |
| 973 | + err = -EINVAL; | |
| 974 | + goto free; | |
| 975 | + } | |
| 976 | + | |
| 977 | + if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { | |
| 978 | + struct mlx5_ib_mcounters *mcounters; | |
| 979 | + | |
| 980 | + err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); | |
| 981 | + if (err) | |
| 982 | + goto free; | |
| 983 | + | |
| 984 | + mcounters = to_mcounters(flow_act.counters); | |
| 985 | + handler->ibcounters = flow_act.counters; | |
| 986 | + dest_arr[dest_num].type = | |
| 987 | + MLX5_FLOW_DESTINATION_TYPE_COUNTER; | |
| 988 | + dest_arr[dest_num].counter_id = | |
| 989 | + mlx5_fc_id(mcounters->hw_cntrs_hndl); | |
| 990 | + dest_num++; | |
| 991 | + } | |
| 992 | + | |
| 993 | + if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { | |
| 994 | + if (!dest_num) | |
| 995 | + rule_dst = NULL; | |
| 996 | + } else { | |
| 997 | + if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) | |
| 998 | + flow_act.action |= | |
| 999 | + MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; | |
| 1000 | + if (is_egress) | |
| 1001 | + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; | |
| 1002 | + else if (dest_num) | |
| 1003 | + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; | |
| 1004 | + } | |
| 1005 | + | |
| 1006 | + if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && | |
| 1007 | + (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || | |
| 1008 | + flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { | |
| 1009 | + mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", | |
| 1010 | + spec->flow_context.flow_tag, flow_attr->type); | |
| 1011 | + err = -EINVAL; | |
| 1012 | + goto free; | |
| 1013 | + } | |
| 1014 | + handler->rule = mlx5_add_flow_rules(ft, spec, | |
| 1015 | + &flow_act, | |
| 1016 | + rule_dst, dest_num); | |
| 1017 | + | |
| 1018 | + if (IS_ERR(handler->rule)) { | |
| 1019 | + err = PTR_ERR(handler->rule); | |
| 1020 | + goto free; | |
| 1021 | + } | |
| 1022 | + | |
| 1023 | + ft_prio->refcount++; | |
| 1024 | + handler->prio = ft_prio; | |
| 1025 | + handler->dev = dev; | |
| 1026 | + | |
| 1027 | + ft_prio->flow_table = ft; | |
| 1028 | +free: | |
| 1029 | + if (err && handler) { | |
| 1030 | + mlx5_ib_counters_clear_description(handler->ibcounters); | |
| 1031 | + kfree(handler); | |
| 1032 | + } | |
| 1033 | + kvfree(spec); | |
| 1034 | + return err ? ERR_PTR(err) : handler; | |
| 1035 | +} | |
| 1036 | + | |
| 1037 | +static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, | |
| 1038 | + struct mlx5_ib_flow_prio *ft_prio, | |
| 1039 | + const struct ib_flow_attr *flow_attr, | |
| 1040 | + struct mlx5_flow_destination *dst) | |
| 1041 | +{ | |
| 1042 | + return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); | |
| 1043 | +} | |
| 1044 | + | |
| 1045 | +enum { | |
| 1046 | + LEFTOVERS_MC, | |
| 1047 | + LEFTOVERS_UC, | |
| 1048 | +}; | |
| 1049 | + | |
| 1050 | +static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, | |
| 1051 | + struct mlx5_ib_flow_prio *ft_prio, | |
| 1052 | + struct ib_flow_attr *flow_attr, | |
| 1053 | + struct mlx5_flow_destination *dst) | |
| 1054 | +{ | |
| 1055 | + struct mlx5_ib_flow_handler *handler_ucast = NULL; | |
| 1056 | + struct mlx5_ib_flow_handler *handler = NULL; | |
| 1057 | + | |
| 1058 | + static struct { | |
| 1059 | + struct ib_flow_attr flow_attr; | |
| 1060 | + struct ib_flow_spec_eth eth_flow; | |
| 1061 | + } leftovers_specs[] = { | |
| 1062 | + [LEFTOVERS_MC] = { | |
| 1063 | + .flow_attr = { | |
| 1064 | + .num_of_specs = 1, | |
| 1065 | + .size = sizeof(leftovers_specs[0]) | |
| 1066 | + }, | |
| 1067 | + .eth_flow = { | |
| 1068 | + .type = IB_FLOW_SPEC_ETH, | |
| 1069 | + .size = sizeof(struct ib_flow_spec_eth), | |
| 1070 | + .mask = {.dst_mac = {0x1} }, | |
| 1071 | + .val = {.dst_mac = {0x1} } | |
| 1072 | + } | |
| 1073 | + }, | |
| 1074 | + [LEFTOVERS_UC] = { | |
| 1075 | + .flow_attr = { | |
| 1076 | + .num_of_specs = 1, | |
| 1077 | + .size = sizeof(leftovers_specs[0]) | |
| 1078 | + }, | |
| 1079 | + .eth_flow = { | |
| 1080 | + .type = IB_FLOW_SPEC_ETH, | |
| 1081 | + .size = sizeof(struct ib_flow_spec_eth), | |
| 1082 | + .mask = {.dst_mac = {0x1} }, | |
| 1083 | + .val = {.dst_mac = {} } | |
| 1084 | + } | |
| 1085 | + } | |
| 1086 | + }; | |
| 1087 | + | |
| 1088 | + handler = create_flow_rule(dev, ft_prio, | |
| 1089 | + &leftovers_specs[LEFTOVERS_MC].flow_attr, | |
| 1090 | + dst); | |
| 1091 | + if (!IS_ERR(handler) && | |
| 1092 | + flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { | |
| 1093 | + handler_ucast = create_flow_rule(dev, ft_prio, | |
| 1094 | + &leftovers_specs[LEFTOVERS_UC].flow_attr, | |
| 1095 | + dst); | |
| 1096 | + if (IS_ERR(handler_ucast)) { | |
| 1097 | + mlx5_del_flow_rules(handler->rule); | |
| 1098 | + ft_prio->refcount--; | |
| 1099 | + kfree(handler); | |
| 1100 | + handler = handler_ucast; | |
| 1101 | + } else { | |
| 1102 | + list_add(&handler_ucast->list, &handler->list); | |
| 1103 | + } | |
| 1104 | + } | |
| 1105 | + | |
| 1106 | + return handler; | |
| 1107 | +} | |
| 1108 | + | |
| 1109 | +static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, | |
| 1110 | + struct mlx5_ib_flow_prio *ft_rx, | |
| 1111 | + struct mlx5_ib_flow_prio *ft_tx, | |
| 1112 | + struct mlx5_flow_destination *dst) | |
| 1113 | +{ | |
| 1114 | + struct mlx5_ib_flow_handler *handler_rx; | |
| 1115 | + struct mlx5_ib_flow_handler *handler_tx; | |
| 1116 | + int err; | |
| 1117 | + static const struct ib_flow_attr flow_attr = { | |
| 1118 | + .num_of_specs = 0, | |
| 1119 | + .size = sizeof(flow_attr) | |
| 1120 | + }; | |
| 1121 | + | |
| 1122 | + handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); | |
| 1123 | + if (IS_ERR(handler_rx)) { | |
| 1124 | + err = PTR_ERR(handler_rx); | |
| 1125 | + goto err; | |
| 1126 | + } | |
| 1127 | + | |
| 1128 | + handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); | |
| 1129 | + if (IS_ERR(handler_tx)) { | |
| 1130 | + err = PTR_ERR(handler_tx); | |
| 1131 | + goto err_tx; | |
| 1132 | + } | |
| 1133 | + | |
| 1134 | + list_add(&handler_tx->list, &handler_rx->list); | |
| 1135 | + | |
| 1136 | + return handler_rx; | |
| 1137 | + | |
| 1138 | +err_tx: | |
| 1139 | + mlx5_del_flow_rules(handler_rx->rule); | |
| 1140 | + ft_rx->refcount--; | |
| 1141 | + kfree(handler_rx); | |
| 1142 | +err: | |
| 1143 | + return ERR_PTR(err); | |
| 1144 | +} | |
| 1145 | + | |
| 1146 | + | |
| 1147 | +static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, | |
| 1148 | + struct ib_flow_attr *flow_attr, | |
| 1149 | + int domain, | |
| 1150 | + struct ib_udata *udata) | |
| 1151 | +{ | |
| 1152 | + struct mlx5_ib_dev *dev = to_mdev(qp->device); | |
| 1153 | + struct mlx5_ib_qp *mqp = to_mqp(qp); | |
| 1154 | + struct mlx5_ib_flow_handler *handler = NULL; | |
| 1155 | + struct mlx5_flow_destination *dst = NULL; | |
| 1156 | + struct mlx5_ib_flow_prio *ft_prio_tx = NULL; | |
| 1157 | + struct mlx5_ib_flow_prio *ft_prio; | |
| 1158 | + bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; | |
| 1159 | + struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; | |
| 1160 | + size_t min_ucmd_sz, required_ucmd_sz; | |
| 1161 | + int err; | |
| 1162 | + int underlay_qpn; | |
| 1163 | + | |
| 1164 | + if (udata && udata->inlen) { | |
| 1165 | + min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) + | |
| 1166 | + sizeof(ucmd_hdr.reserved); | |
| 1167 | + if (udata->inlen < min_ucmd_sz) | |
| 1168 | + return ERR_PTR(-EOPNOTSUPP); | |
| 1169 | + | |
| 1170 | + err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); | |
| 1171 | + if (err) | |
| 1172 | + return ERR_PTR(err); | |
| 1173 | + | |
| 1174 | + /* currently supports only one counters data */ | |
| 1175 | + if (ucmd_hdr.ncounters_data > 1) | |
| 1176 | + return ERR_PTR(-EINVAL); | |
| 1177 | + | |
| 1178 | + required_ucmd_sz = min_ucmd_sz + | |
| 1179 | + sizeof(struct mlx5_ib_flow_counters_data) * | |
| 1180 | + ucmd_hdr.ncounters_data; | |
| 1181 | + if (udata->inlen > required_ucmd_sz && | |
| 1182 | + !ib_is_udata_cleared(udata, required_ucmd_sz, | |
| 1183 | + udata->inlen - required_ucmd_sz)) | |
| 1184 | + return ERR_PTR(-EOPNOTSUPP); | |
| 1185 | + | |
| 1186 | + ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); | |
| 1187 | + if (!ucmd) | |
| 1188 | + return ERR_PTR(-ENOMEM); | |
| 1189 | + | |
| 1190 | + err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); | |
| 1191 | + if (err) | |
| 1192 | + goto free_ucmd; | |
| 1193 | + } | |
| 1194 | + | |
| 1195 | + if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { | |
| 1196 | + err = -ENOMEM; | |
| 1197 | + goto free_ucmd; | |
| 1198 | + } | |
| 1199 | + | |
| 1200 | + if (domain != IB_FLOW_DOMAIN_USER || | |
| 1201 | + flow_attr->port > dev->num_ports || | |
| 1202 | + (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | | |
| 1203 | + IB_FLOW_ATTR_FLAGS_EGRESS))) { | |
| 1204 | + err = -EINVAL; | |
| 1205 | + goto free_ucmd; | |
| 1206 | + } | |
| 1207 | + | |
| 1208 | + if (is_egress && | |
| 1209 | + (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || | |
| 1210 | + flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { | |
| 1211 | + err = -EINVAL; | |
| 1212 | + goto free_ucmd; | |
| 1213 | + } | |
| 1214 | + | |
| 1215 | + dst = kzalloc(sizeof(*dst), GFP_KERNEL); | |
| 1216 | + if (!dst) { | |
| 1217 | + err = -ENOMEM; | |
| 1218 | + goto free_ucmd; | |
| 1219 | + } | |
| 1220 | + | |
| 1221 | + mutex_lock(&dev->flow_db->lock); | |
| 1222 | + | |
| 1223 | + ft_prio = get_flow_table(dev, flow_attr, | |
| 1224 | + is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); | |
| 1225 | + if (IS_ERR(ft_prio)) { | |
| 1226 | + err = PTR_ERR(ft_prio); | |
| 1227 | + goto unlock; | |
| 1228 | + } | |
| 1229 | + if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { | |
| 1230 | + ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); | |
| 1231 | + if (IS_ERR(ft_prio_tx)) { | |
| 1232 | + err = PTR_ERR(ft_prio_tx); | |
| 1233 | + ft_prio_tx = NULL; | |
| 1234 | + goto destroy_ft; | |
| 1235 | + } | |
| 1236 | + } | |
| 1237 | + | |
| 1238 | + if (is_egress) { | |
| 1239 | + dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; | |
| 1240 | + } else { | |
| 1241 | + dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; | |
| 1242 | + if (mqp->is_rss) | |
| 1243 | + dst->tir_num = mqp->rss_qp.tirn; | |
| 1244 | + else | |
| 1245 | + dst->tir_num = mqp->raw_packet_qp.rq.tirn; | |
| 1246 | + } | |
| 1247 | + | |
| 1248 | + if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { | |
| 1249 | + underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? | |
| 1250 | + mqp->underlay_qpn : | |
| 1251 | + 0; | |
| 1252 | + handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, | |
| 1253 | + underlay_qpn, ucmd); | |
| 1254 | + } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || | |
| 1255 | + flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { | |
| 1256 | + handler = create_leftovers_rule(dev, ft_prio, flow_attr, | |
| 1257 | + dst); | |
| 1258 | + } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { | |
| 1259 | + handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); | |
| 1260 | + } else { | |
| 1261 | + err = -EINVAL; | |
| 1262 | + goto destroy_ft; | |
| 1263 | + } | |
| 1264 | + | |
| 1265 | + if (IS_ERR(handler)) { | |
| 1266 | + err = PTR_ERR(handler); | |
| 1267 | + handler = NULL; | |
| 1268 | + goto destroy_ft; | |
| 1269 | + } | |
| 1270 | + | |
| 1271 | + mutex_unlock(&dev->flow_db->lock); | |
| 1272 | + kfree(dst); | |
| 1273 | + kfree(ucmd); | |
| 1274 | + | |
| 1275 | + return &handler->ibflow; | |
| 1276 | + | |
| 1277 | +destroy_ft: | |
| 1278 | + put_flow_table(dev, ft_prio, false); | |
| 1279 | + if (ft_prio_tx) | |
| 1280 | + put_flow_table(dev, ft_prio_tx, false); | |
| 1281 | +unlock: | |
| 1282 | + mutex_unlock(&dev->flow_db->lock); | |
| 1283 | + kfree(dst); | |
| 1284 | +free_ucmd: | |
| 1285 | + kfree(ucmd); | |
| 1286 | + return ERR_PTR(err); | |
| 1287 | +} | |
| 1288 | + | |
| 1289 | +static struct mlx5_ib_flow_prio * | |
| 1290 | +_get_flow_table(struct mlx5_ib_dev *dev, | |
| 1291 | + struct mlx5_ib_flow_matcher *fs_matcher, | |
| 1292 | + bool mcast) | |
| 1293 | +{ | |
| 1294 | + struct mlx5_flow_namespace *ns = NULL; | |
| 1295 | + struct mlx5_ib_flow_prio *prio = NULL; | |
| 1296 | + int max_table_size = 0; | |
| 1297 | + bool esw_encap; | |
| 1298 | + u32 flags = 0; | |
| 1299 | + int priority; | |
| 1300 | + | |
| 1301 | + if (mcast) | |
| 1302 | + priority = MLX5_IB_FLOW_MCAST_PRIO; | |
| 1303 | + else | |
| 1304 | + priority = ib_prio_to_core_prio(fs_matcher->priority, false); | |
| 1305 | + | |
| 1306 | + esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != | |
| 1307 | + DEVLINK_ESWITCH_ENCAP_MODE_NONE; | |
| 1308 | + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { | |
| 1309 | + max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 1310 | + log_max_ft_size)); | |
| 1311 | + if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) | |
| 1312 | + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; | |
| 1313 | + if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 1314 | + reformat_l3_tunnel_to_l2) && | |
| 1315 | + !esw_encap) | |
| 1316 | + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 1317 | + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { | |
| 1318 | + max_table_size = BIT( | |
| 1319 | + MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); | |
| 1320 | + if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) | |
| 1321 | + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 1322 | + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { | |
| 1323 | + max_table_size = BIT( | |
| 1324 | + MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); | |
| 1325 | + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) | |
| 1326 | + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; | |
| 1327 | + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && | |
| 1328 | + esw_encap) | |
| 1329 | + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 1330 | + priority = FDB_BYPASS_PATH; | |
| 1331 | + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) { | |
| 1332 | + max_table_size = | |
| 1333 | + BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, | |
| 1334 | + log_max_ft_size)); | |
| 1335 | + priority = fs_matcher->priority; | |
| 1336 | + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { | |
| 1337 | + max_table_size = | |
| 1338 | + BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, | |
| 1339 | + log_max_ft_size)); | |
| 1340 | + priority = fs_matcher->priority; | |
| 1341 | + } | |
| 1342 | + | |
| 1343 | + max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); | |
| 1344 | + | |
| 1345 | + ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type); | |
| 1346 | + if (!ns) | |
| 1347 | + return ERR_PTR(-EOPNOTSUPP); | |
| 1348 | + | |
| 1349 | + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) | |
| 1350 | + prio = &dev->flow_db->prios[priority]; | |
| 1351 | + else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) | |
| 1352 | + prio = &dev->flow_db->egress_prios[priority]; | |
| 1353 | + else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) | |
| 1354 | + prio = &dev->flow_db->fdb; | |
| 1355 | + else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) | |
| 1356 | + prio = &dev->flow_db->rdma_rx[priority]; | |
| 1357 | + else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) | |
| 1358 | + prio = &dev->flow_db->rdma_tx[priority]; | |
| 1359 | + | |
| 1360 | + if (!prio) | |
| 1361 | + return ERR_PTR(-EINVAL); | |
| 1362 | + | |
| 1363 | + if (prio->flow_table) | |
| 1364 | + return prio; | |
| 1365 | + | |
| 1366 | + return _get_prio(ns, prio, priority, max_table_size, | |
| 1367 | + MLX5_FS_MAX_TYPES, flags); | |
| 1368 | +} | |
| 1369 | + | |
| 1370 | +static struct mlx5_ib_flow_handler * | |
| 1371 | +_create_raw_flow_rule(struct mlx5_ib_dev *dev, | |
| 1372 | + struct mlx5_ib_flow_prio *ft_prio, | |
| 1373 | + struct mlx5_flow_destination *dst, | |
| 1374 | + struct mlx5_ib_flow_matcher *fs_matcher, | |
| 1375 | + struct mlx5_flow_context *flow_context, | |
| 1376 | + struct mlx5_flow_act *flow_act, | |
| 1377 | + void *cmd_in, int inlen, | |
| 1378 | + int dst_num) | |
| 1379 | +{ | |
| 1380 | + struct mlx5_ib_flow_handler *handler; | |
| 1381 | + struct mlx5_flow_spec *spec; | |
| 1382 | + struct mlx5_flow_table *ft = ft_prio->flow_table; | |
| 1383 | + int err = 0; | |
| 1384 | + | |
| 1385 | + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); | |
| 1386 | + handler = kzalloc(sizeof(*handler), GFP_KERNEL); | |
| 1387 | + if (!handler || !spec) { | |
| 1388 | + err = -ENOMEM; | |
| 1389 | + goto free; | |
| 1390 | + } | |
| 1391 | + | |
| 1392 | + INIT_LIST_HEAD(&handler->list); | |
| 1393 | + | |
| 1394 | + memcpy(spec->match_value, cmd_in, inlen); | |
| 1395 | + memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, | |
| 1396 | + fs_matcher->mask_len); | |
| 1397 | + spec->match_criteria_enable = fs_matcher->match_criteria_enable; | |
| 1398 | + spec->flow_context = *flow_context; | |
| 1399 | + | |
| 1400 | + handler->rule = mlx5_add_flow_rules(ft, spec, | |
| 1401 | + flow_act, dst, dst_num); | |
| 1402 | + | |
| 1403 | + if (IS_ERR(handler->rule)) { | |
| 1404 | + err = PTR_ERR(handler->rule); | |
| 1405 | + goto free; | |
| 1406 | + } | |
| 1407 | + | |
| 1408 | + ft_prio->refcount++; | |
| 1409 | + handler->prio = ft_prio; | |
| 1410 | + handler->dev = dev; | |
| 1411 | + ft_prio->flow_table = ft; | |
| 1412 | + | |
| 1413 | +free: | |
| 1414 | + if (err) | |
| 1415 | + kfree(handler); | |
| 1416 | + kvfree(spec); | |
| 1417 | + return err ? ERR_PTR(err) : handler; | |
| 1418 | +} | |
| 1419 | + | |
| 1420 | +static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, | |
| 1421 | + void *match_v) | |
| 1422 | +{ | |
| 1423 | + void *match_c; | |
| 1424 | + void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; | |
| 1425 | + void *dmac, *dmac_mask; | |
| 1426 | + void *ipv4, *ipv4_mask; | |
| 1427 | + | |
| 1428 | + if (!(fs_matcher->match_criteria_enable & | |
| 1429 | + (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) | |
| 1430 | + return false; | |
| 1431 | + | |
| 1432 | + match_c = fs_matcher->matcher_mask.match_params; | |
| 1433 | + match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 1434 | + outer_headers); | |
| 1435 | + match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 1436 | + outer_headers); | |
| 1437 | + | |
| 1438 | + dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, | |
| 1439 | + dmac_47_16); | |
| 1440 | + dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, | |
| 1441 | + dmac_47_16); | |
| 1442 | + | |
| 1443 | + if (is_multicast_ether_addr(dmac) && | |
| 1444 | + is_multicast_ether_addr(dmac_mask)) | |
| 1445 | + return true; | |
| 1446 | + | |
| 1447 | + ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, | |
| 1448 | + dst_ipv4_dst_ipv6.ipv4_layout.ipv4); | |
| 1449 | + | |
| 1450 | + ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, | |
| 1451 | + dst_ipv4_dst_ipv6.ipv4_layout.ipv4); | |
| 1452 | + | |
| 1453 | + if (ipv4_is_multicast(*(__be32 *)(ipv4)) && | |
| 1454 | + ipv4_is_multicast(*(__be32 *)(ipv4_mask))) | |
| 1455 | + return true; | |
| 1456 | + | |
| 1457 | + return false; | |
| 1458 | +} | |
| 1459 | + | |
| 1460 | +static struct mlx5_ib_flow_handler *raw_fs_rule_add( | |
| 1461 | + struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, | |
| 1462 | + struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, | |
| 1463 | + u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type) | |
| 1464 | +{ | |
| 1465 | + struct mlx5_flow_destination *dst; | |
| 1466 | + struct mlx5_ib_flow_prio *ft_prio; | |
| 1467 | + struct mlx5_ib_flow_handler *handler; | |
| 1468 | + int dst_num = 0; | |
| 1469 | + bool mcast; | |
| 1470 | + int err; | |
| 1471 | + | |
| 1472 | + if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) | |
| 1473 | + return ERR_PTR(-EOPNOTSUPP); | |
| 1474 | + | |
| 1475 | + if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) | |
| 1476 | + return ERR_PTR(-ENOMEM); | |
| 1477 | + | |
| 1478 | + dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); | |
| 1479 | + if (!dst) | |
| 1480 | + return ERR_PTR(-ENOMEM); | |
| 1481 | + | |
| 1482 | + mcast = raw_fs_is_multicast(fs_matcher, cmd_in); | |
| 1483 | + mutex_lock(&dev->flow_db->lock); | |
| 1484 | + | |
| 1485 | + ft_prio = _get_flow_table(dev, fs_matcher, mcast); | |
| 1486 | + if (IS_ERR(ft_prio)) { | |
| 1487 | + err = PTR_ERR(ft_prio); | |
| 1488 | + goto unlock; | |
| 1489 | + } | |
| 1490 | + | |
| 1491 | + if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) { | |
| 1492 | + dst[dst_num].type = dest_type; | |
| 1493 | + dst[dst_num++].tir_num = dest_id; | |
| 1494 | + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; | |
| 1495 | + } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) { | |
| 1496 | + dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; | |
| 1497 | + dst[dst_num++].ft_num = dest_id; | |
| 1498 | + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; | |
| 1499 | + } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) { | |
| 1500 | + dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; | |
| 1501 | + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; | |
| 1502 | + } | |
| 1503 | + | |
| 1504 | + | |
| 1505 | + if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { | |
| 1506 | + dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; | |
| 1507 | + dst[dst_num].counter_id = counter_id; | |
| 1508 | + dst_num++; | |
| 1509 | + } | |
| 1510 | + | |
| 1511 | + handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher, | |
| 1512 | + flow_context, flow_act, | |
| 1513 | + cmd_in, inlen, dst_num); | |
| 1514 | + | |
| 1515 | + if (IS_ERR(handler)) { | |
| 1516 | + err = PTR_ERR(handler); | |
| 1517 | + goto destroy_ft; | |
| 1518 | + } | |
| 1519 | + | |
| 1520 | + mutex_unlock(&dev->flow_db->lock); | |
| 1521 | + atomic_inc(&fs_matcher->usecnt); | |
| 1522 | + handler->flow_matcher = fs_matcher; | |
| 1523 | + | |
| 1524 | + kfree(dst); | |
| 1525 | + | |
| 1526 | + return handler; | |
| 1527 | + | |
| 1528 | +destroy_ft: | |
| 1529 | + put_flow_table(dev, ft_prio, false); | |
| 1530 | +unlock: | |
| 1531 | + mutex_unlock(&dev->flow_db->lock); | |
| 1532 | + kfree(dst); | |
| 1533 | + | |
| 1534 | + return ERR_PTR(err); | |
| 1535 | +} | |
| 1536 | + | |
| 1537 | +static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags) | |
| 1538 | +{ | |
| 1539 | + u32 flags = 0; | |
| 1540 | + | |
| 1541 | + if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA) | |
| 1542 | + flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA; | |
| 1543 | + | |
| 1544 | + return flags; | |
| 1545 | +} | |
| 1546 | + | |
| 1547 | +#define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED \ | |
| 1548 | + MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA | |
| 1549 | +static struct ib_flow_action * | |
| 1550 | +mlx5_ib_create_flow_action_esp(struct ib_device *device, | |
| 1551 | + const struct ib_flow_action_attrs_esp *attr, | |
| 1552 | + struct uverbs_attr_bundle *attrs) | |
| 1553 | +{ | |
| 1554 | + struct mlx5_ib_dev *mdev = to_mdev(device); | |
| 1555 | + struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm; | |
| 1556 | + struct mlx5_accel_esp_xfrm_attrs accel_attrs = {}; | |
| 1557 | + struct mlx5_ib_flow_action *action; | |
| 1558 | + u64 action_flags; | |
| 1559 | + u64 flags; | |
| 1560 | + int err = 0; | |
| 1561 | + | |
| 1562 | + err = uverbs_get_flags64( | |
| 1563 | + &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS, | |
| 1564 | + ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1)); | |
| 1565 | + if (err) | |
| 1566 | + return ERR_PTR(err); | |
| 1567 | + | |
| 1568 | + flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags); | |
| 1569 | + | |
| 1570 | + /* We current only support a subset of the standard features. Only a | |
| 1571 | + * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn | |
| 1572 | + * (with overlap). Full offload mode isn't supported. | |
| 1573 | + */ | |
| 1574 | + if (!attr->keymat || attr->replay || attr->encap || | |
| 1575 | + attr->spi || attr->seq || attr->tfc_pad || | |
| 1576 | + attr->hard_limit_pkts || | |
| 1577 | + (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | | |
| 1578 | + IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT))) | |
| 1579 | + return ERR_PTR(-EOPNOTSUPP); | |
| 1580 | + | |
| 1581 | + if (attr->keymat->protocol != | |
| 1582 | + IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM) | |
| 1583 | + return ERR_PTR(-EOPNOTSUPP); | |
| 1584 | + | |
| 1585 | + aes_gcm = &attr->keymat->keymat.aes_gcm; | |
| 1586 | + | |
| 1587 | + if (aes_gcm->icv_len != 16 || | |
| 1588 | + aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ) | |
| 1589 | + return ERR_PTR(-EOPNOTSUPP); | |
| 1590 | + | |
| 1591 | + action = kmalloc(sizeof(*action), GFP_KERNEL); | |
| 1592 | + if (!action) | |
| 1593 | + return ERR_PTR(-ENOMEM); | |
| 1594 | + | |
| 1595 | + action->esp_aes_gcm.ib_flags = attr->flags; | |
| 1596 | + memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key, | |
| 1597 | + sizeof(accel_attrs.keymat.aes_gcm.aes_key)); | |
| 1598 | + accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8; | |
| 1599 | + memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt, | |
| 1600 | + sizeof(accel_attrs.keymat.aes_gcm.salt)); | |
| 1601 | + memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv, | |
| 1602 | + sizeof(accel_attrs.keymat.aes_gcm.seq_iv)); | |
| 1603 | + accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8; | |
| 1604 | + accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ; | |
| 1605 | + accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM; | |
| 1606 | + | |
| 1607 | + accel_attrs.esn = attr->esn; | |
| 1608 | + if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) | |
| 1609 | + accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; | |
| 1610 | + if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) | |
| 1611 | + accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; | |
| 1612 | + | |
| 1613 | + if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT) | |
| 1614 | + accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT; | |
| 1615 | + | |
| 1616 | + action->esp_aes_gcm.ctx = | |
| 1617 | + mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags); | |
| 1618 | + if (IS_ERR(action->esp_aes_gcm.ctx)) { | |
| 1619 | + err = PTR_ERR(action->esp_aes_gcm.ctx); | |
| 1620 | + goto err_parse; | |
| 1621 | + } | |
| 1622 | + | |
| 1623 | + action->esp_aes_gcm.ib_flags = attr->flags; | |
| 1624 | + | |
| 1625 | + return &action->ib_action; | |
| 1626 | + | |
| 1627 | +err_parse: | |
| 1628 | + kfree(action); | |
| 1629 | + return ERR_PTR(err); | |
| 1630 | +} | |
| 1631 | + | |
| 1632 | +static int | |
| 1633 | +mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action, | |
| 1634 | + const struct ib_flow_action_attrs_esp *attr, | |
| 1635 | + struct uverbs_attr_bundle *attrs) | |
| 1636 | +{ | |
| 1637 | + struct mlx5_ib_flow_action *maction = to_mflow_act(action); | |
| 1638 | + struct mlx5_accel_esp_xfrm_attrs accel_attrs; | |
| 1639 | + int err = 0; | |
| 1640 | + | |
| 1641 | + if (attr->keymat || attr->replay || attr->encap || | |
| 1642 | + attr->spi || attr->seq || attr->tfc_pad || | |
| 1643 | + attr->hard_limit_pkts || | |
| 1644 | + (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | | |
| 1645 | + IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS | | |
| 1646 | + IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))) | |
| 1647 | + return -EOPNOTSUPP; | |
| 1648 | + | |
| 1649 | + /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can | |
| 1650 | + * be modified. | |
| 1651 | + */ | |
| 1652 | + if (!(maction->esp_aes_gcm.ib_flags & | |
| 1653 | + IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) && | |
| 1654 | + attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | | |
| 1655 | + IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)) | |
| 1656 | + return -EINVAL; | |
| 1657 | + | |
| 1658 | + memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs, | |
| 1659 | + sizeof(accel_attrs)); | |
| 1660 | + | |
| 1661 | + accel_attrs.esn = attr->esn; | |
| 1662 | + if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) | |
| 1663 | + accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; | |
| 1664 | + else | |
| 1665 | + accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; | |
| 1666 | + | |
| 1667 | + err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx, | |
| 1668 | + &accel_attrs); | |
| 1669 | + if (err) | |
| 1670 | + return err; | |
| 1671 | + | |
| 1672 | + maction->esp_aes_gcm.ib_flags &= | |
| 1673 | + ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; | |
| 1674 | + maction->esp_aes_gcm.ib_flags |= | |
| 1675 | + attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; | |
| 1676 | + | |
| 1677 | + return 0; | |
| 1678 | +} | |
| 1679 | + | |
| 1680 | +static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) | |
| 1681 | +{ | |
| 1682 | + switch (maction->flow_action_raw.sub_type) { | |
| 1683 | + case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: | |
| 1684 | + mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, | |
| 1685 | + maction->flow_action_raw.modify_hdr); | |
| 1686 | + break; | |
| 1687 | + case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: | |
| 1688 | + mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, | |
| 1689 | + maction->flow_action_raw.pkt_reformat); | |
| 1690 | + break; | |
| 1691 | + case MLX5_IB_FLOW_ACTION_DECAP: | |
| 1692 | + break; | |
| 1693 | + default: | |
| 1694 | + break; | |
| 1695 | + } | |
| 1696 | +} | |
| 1697 | + | |
| 1698 | +static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) | |
| 1699 | +{ | |
| 1700 | + struct mlx5_ib_flow_action *maction = to_mflow_act(action); | |
| 1701 | + | |
| 1702 | + switch (action->type) { | |
| 1703 | + case IB_FLOW_ACTION_ESP: | |
| 1704 | + /* | |
| 1705 | + * We only support aes_gcm by now, so we implicitly know this is | |
| 1706 | + * the underline crypto. | |
| 1707 | + */ | |
| 1708 | + mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx); | |
| 1709 | + break; | |
| 1710 | + case IB_FLOW_ACTION_UNSPECIFIED: | |
| 1711 | + destroy_flow_action_raw(maction); | |
| 1712 | + break; | |
| 1713 | + default: | |
| 1714 | + WARN_ON(true); | |
| 1715 | + break; | |
| 1716 | + } | |
| 1717 | + | |
| 1718 | + kfree(maction); | |
| 1719 | + return 0; | |
| 1720 | +} | |
| 1721 | + | |
| 1722 | +static int | |
| 1723 | +mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, | |
| 1724 | + enum mlx5_flow_namespace_type *namespace) | |
| 1725 | +{ | |
| 1726 | + switch (table_type) { | |
| 1727 | + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: | |
| 1728 | + *namespace = MLX5_FLOW_NAMESPACE_BYPASS; | |
| 1729 | + break; | |
| 1730 | + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: | |
| 1731 | + *namespace = MLX5_FLOW_NAMESPACE_EGRESS; | |
| 1732 | + break; | |
| 1733 | + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: | |
| 1734 | + *namespace = MLX5_FLOW_NAMESPACE_FDB; | |
| 1735 | + break; | |
| 1736 | + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: | |
| 1737 | + *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; | |
| 1738 | + break; | |
| 1739 | + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: | |
| 1740 | + *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; | |
| 1741 | + break; | |
| 1742 | + default: | |
| 1743 | + return -EINVAL; | |
| 1744 | + } | |
| 1745 | + | |
| 1746 | + return 0; | |
| 1747 | +} | |
| 1748 | + | |
| 1749 | +static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { | |
| 1750 | + [MLX5_IB_FLOW_TYPE_NORMAL] = { | |
| 1751 | + .type = UVERBS_ATTR_TYPE_PTR_IN, | |
| 1752 | + .u.ptr = { | |
| 1753 | + .len = sizeof(u16), /* data is priority */ | |
| 1754 | + .min_len = sizeof(u16), | |
| 1755 | + } | |
| 1756 | + }, | |
| 1757 | + [MLX5_IB_FLOW_TYPE_SNIFFER] = { | |
| 1758 | + .type = UVERBS_ATTR_TYPE_PTR_IN, | |
| 1759 | + UVERBS_ATTR_NO_DATA(), | |
| 1760 | + }, | |
| 1761 | + [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { | |
| 1762 | + .type = UVERBS_ATTR_TYPE_PTR_IN, | |
| 1763 | + UVERBS_ATTR_NO_DATA(), | |
| 1764 | + }, | |
| 1765 | + [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { | |
| 1766 | + .type = UVERBS_ATTR_TYPE_PTR_IN, | |
| 1767 | + UVERBS_ATTR_NO_DATA(), | |
| 1768 | + }, | |
| 1769 | +}; | |
| 1770 | + | |
| 1771 | +static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) | |
| 1772 | +{ | |
| 1773 | + struct devx_obj *devx_obj = obj; | |
| 1774 | + u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); | |
| 1775 | + | |
| 1776 | + switch (opcode) { | |
| 1777 | + case MLX5_CMD_OP_DESTROY_TIR: | |
| 1778 | + *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; | |
| 1779 | + *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, | |
| 1780 | + obj_id); | |
| 1781 | + return true; | |
| 1782 | + | |
| 1783 | + case MLX5_CMD_OP_DESTROY_FLOW_TABLE: | |
| 1784 | + *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; | |
| 1785 | + *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, | |
| 1786 | + table_id); | |
| 1787 | + return true; | |
| 1788 | + default: | |
| 1789 | + return false; | |
| 1790 | + } | |
| 1791 | +} | |
| 1792 | + | |
| 1793 | +static int get_dests(struct uverbs_attr_bundle *attrs, | |
| 1794 | + struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, | |
| 1795 | + int *dest_type, struct ib_qp **qp, u32 *flags) | |
| 1796 | +{ | |
| 1797 | + bool dest_devx, dest_qp; | |
| 1798 | + void *devx_obj; | |
| 1799 | + int err; | |
| 1800 | + | |
| 1801 | + dest_devx = uverbs_attr_is_valid(attrs, | |
| 1802 | + MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); | |
| 1803 | + dest_qp = uverbs_attr_is_valid(attrs, | |
| 1804 | + MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); | |
| 1805 | + | |
| 1806 | + *flags = 0; | |
| 1807 | + err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, | |
| 1808 | + MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | | |
| 1809 | + MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); | |
| 1810 | + if (err) | |
| 1811 | + return err; | |
| 1812 | + | |
| 1813 | + /* Both flags are not allowed */ | |
| 1814 | + if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && | |
| 1815 | + *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) | |
| 1816 | + return -EINVAL; | |
| 1817 | + | |
| 1818 | + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { | |
| 1819 | + if (dest_devx && (dest_qp || *flags)) | |
| 1820 | + return -EINVAL; | |
| 1821 | + else if (dest_qp && *flags) | |
| 1822 | + return -EINVAL; | |
| 1823 | + } | |
| 1824 | + | |
| 1825 | + /* Allow only DEVX object, drop as dest for FDB */ | |
| 1826 | + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx || | |
| 1827 | + (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) | |
| 1828 | + return -EINVAL; | |
| 1829 | + | |
| 1830 | + /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ | |
| 1831 | + if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && | |
| 1832 | + ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) | |
| 1833 | + return -EINVAL; | |
| 1834 | + | |
| 1835 | + *qp = NULL; | |
| 1836 | + if (dest_devx) { | |
| 1837 | + devx_obj = | |
| 1838 | + uverbs_attr_get_obj(attrs, | |
| 1839 | + MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); | |
| 1840 | + | |
| 1841 | + /* Verify that the given DEVX object is a flow | |
| 1842 | + * steering destination. | |
| 1843 | + */ | |
| 1844 | + if (!is_flow_dest(devx_obj, dest_id, dest_type)) | |
| 1845 | + return -EINVAL; | |
| 1846 | + /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ | |
| 1847 | + if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB || | |
| 1848 | + fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && | |
| 1849 | + *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) | |
| 1850 | + return -EINVAL; | |
| 1851 | + } else if (dest_qp) { | |
| 1852 | + struct mlx5_ib_qp *mqp; | |
| 1853 | + | |
| 1854 | + *qp = uverbs_attr_get_obj(attrs, | |
| 1855 | + MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); | |
| 1856 | + if (IS_ERR(*qp)) | |
| 1857 | + return PTR_ERR(*qp); | |
| 1858 | + | |
| 1859 | + if ((*qp)->qp_type != IB_QPT_RAW_PACKET) | |
| 1860 | + return -EINVAL; | |
| 1861 | + | |
| 1862 | + mqp = to_mqp(*qp); | |
| 1863 | + if (mqp->is_rss) | |
| 1864 | + *dest_id = mqp->rss_qp.tirn; | |
| 1865 | + else | |
| 1866 | + *dest_id = mqp->raw_packet_qp.rq.tirn; | |
| 1867 | + *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; | |
| 1868 | + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || | |
| 1869 | + fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { | |
| 1870 | + *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; | |
| 1871 | + } | |
| 1872 | + | |
| 1873 | + if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && | |
| 1874 | + (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || | |
| 1875 | + fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)) | |
| 1876 | + return -EINVAL; | |
| 1877 | + | |
| 1878 | + return 0; | |
| 1879 | +} | |
| 1880 | + | |
| 1881 | +static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id) | |
| 1882 | +{ | |
| 1883 | + struct devx_obj *devx_obj = obj; | |
| 1884 | + u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); | |
| 1885 | + | |
| 1886 | + if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { | |
| 1887 | + | |
| 1888 | + if (offset && offset >= devx_obj->flow_counter_bulk_size) | |
| 1889 | + return false; | |
| 1890 | + | |
| 1891 | + *counter_id = MLX5_GET(dealloc_flow_counter_in, | |
| 1892 | + devx_obj->dinbox, | |
| 1893 | + flow_counter_id); | |
| 1894 | + *counter_id += offset; | |
| 1895 | + return true; | |
| 1896 | + } | |
| 1897 | + | |
| 1898 | + return false; | |
| 1899 | +} | |
| 1900 | + | |
| 1901 | +#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 | |
| 1902 | +static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( | |
| 1903 | + struct uverbs_attr_bundle *attrs) | |
| 1904 | +{ | |
| 1905 | + struct mlx5_flow_context flow_context = {.flow_tag = | |
| 1906 | + MLX5_FS_DEFAULT_FLOW_TAG}; | |
| 1907 | + u32 *offset_attr, offset = 0, counter_id = 0; | |
| 1908 | + int dest_id, dest_type = -1, inlen, len, ret, i; | |
| 1909 | + struct mlx5_ib_flow_handler *flow_handler; | |
| 1910 | + struct mlx5_ib_flow_matcher *fs_matcher; | |
| 1911 | + struct ib_uobject **arr_flow_actions; | |
| 1912 | + struct ib_uflow_resources *uflow_res; | |
| 1913 | + struct mlx5_flow_act flow_act = {}; | |
| 1914 | + struct ib_qp *qp = NULL; | |
| 1915 | + void *devx_obj, *cmd_in; | |
| 1916 | + struct ib_uobject *uobj; | |
| 1917 | + struct mlx5_ib_dev *dev; | |
| 1918 | + u32 flags; | |
| 1919 | + | |
| 1920 | + if (!capable(CAP_NET_RAW)) | |
| 1921 | + return -EPERM; | |
| 1922 | + | |
| 1923 | + fs_matcher = uverbs_attr_get_obj(attrs, | |
| 1924 | + MLX5_IB_ATTR_CREATE_FLOW_MATCHER); | |
| 1925 | + uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); | |
| 1926 | + dev = mlx5_udata_to_mdev(&attrs->driver_udata); | |
| 1927 | + | |
| 1928 | + if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) | |
| 1929 | + return -EINVAL; | |
| 1930 | + | |
| 1931 | + if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) | |
| 1932 | + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; | |
| 1933 | + | |
| 1934 | + if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) | |
| 1935 | + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; | |
| 1936 | + | |
| 1937 | + len = uverbs_attr_get_uobjs_arr(attrs, | |
| 1938 | + MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); | |
| 1939 | + if (len) { | |
| 1940 | + devx_obj = arr_flow_actions[0]->object; | |
| 1941 | + | |
| 1942 | + if (uverbs_attr_is_valid(attrs, | |
| 1943 | + MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { | |
| 1944 | + | |
| 1945 | + int num_offsets = uverbs_attr_ptr_get_array_size( | |
| 1946 | + attrs, | |
| 1947 | + MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, | |
| 1948 | + sizeof(u32)); | |
| 1949 | + | |
| 1950 | + if (num_offsets != 1) | |
| 1951 | + return -EINVAL; | |
| 1952 | + | |
| 1953 | + offset_attr = uverbs_attr_get_alloced_ptr( | |
| 1954 | + attrs, | |
| 1955 | + MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); | |
| 1956 | + offset = *offset_attr; | |
| 1957 | + } | |
| 1958 | + | |
| 1959 | + if (!is_flow_counter(devx_obj, offset, &counter_id)) | |
| 1960 | + return -EINVAL; | |
| 1961 | + | |
| 1962 | + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; | |
| 1963 | + } | |
| 1964 | + | |
| 1965 | + cmd_in = uverbs_attr_get_alloced_ptr( | |
| 1966 | + attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); | |
| 1967 | + inlen = uverbs_attr_get_len(attrs, | |
| 1968 | + MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); | |
| 1969 | + | |
| 1970 | + uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); | |
| 1971 | + if (!uflow_res) | |
| 1972 | + return -ENOMEM; | |
| 1973 | + | |
| 1974 | + len = uverbs_attr_get_uobjs_arr(attrs, | |
| 1975 | + MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); | |
| 1976 | + for (i = 0; i < len; i++) { | |
| 1977 | + struct mlx5_ib_flow_action *maction = | |
| 1978 | + to_mflow_act(arr_flow_actions[i]->object); | |
| 1979 | + | |
| 1980 | + ret = parse_flow_flow_action(maction, false, &flow_act); | |
| 1981 | + if (ret) | |
| 1982 | + goto err_out; | |
| 1983 | + flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, | |
| 1984 | + arr_flow_actions[i]->object); | |
| 1985 | + } | |
| 1986 | + | |
| 1987 | + ret = uverbs_copy_from(&flow_context.flow_tag, attrs, | |
| 1988 | + MLX5_IB_ATTR_CREATE_FLOW_TAG); | |
| 1989 | + if (!ret) { | |
| 1990 | + if (flow_context.flow_tag >= BIT(24)) { | |
| 1991 | + ret = -EINVAL; | |
| 1992 | + goto err_out; | |
| 1993 | + } | |
| 1994 | + flow_context.flags |= FLOW_CONTEXT_HAS_TAG; | |
| 1995 | + } | |
| 1996 | + | |
| 1997 | + flow_handler = | |
| 1998 | + raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, | |
| 1999 | + counter_id, cmd_in, inlen, dest_id, dest_type); | |
| 2000 | + if (IS_ERR(flow_handler)) { | |
| 2001 | + ret = PTR_ERR(flow_handler); | |
| 2002 | + goto err_out; | |
| 2003 | + } | |
| 2004 | + | |
| 2005 | + ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); | |
| 2006 | + | |
| 2007 | + return 0; | |
| 2008 | +err_out: | |
| 2009 | + ib_uverbs_flow_resources_free(uflow_res); | |
| 2010 | + return ret; | |
| 2011 | +} | |
| 2012 | + | |
| 2013 | +static int flow_matcher_cleanup(struct ib_uobject *uobject, | |
| 2014 | + enum rdma_remove_reason why, | |
| 2015 | + struct uverbs_attr_bundle *attrs) | |
| 2016 | +{ | |
| 2017 | + struct mlx5_ib_flow_matcher *obj = uobject->object; | |
| 2018 | + int ret; | |
| 2019 | + | |
| 2020 | + ret = ib_destroy_usecnt(&obj->usecnt, why, uobject); | |
| 2021 | + if (ret) | |
| 2022 | + return ret; | |
| 2023 | + | |
| 2024 | + kfree(obj); | |
| 2025 | + return 0; | |
| 2026 | +} | |
| 2027 | + | |
| 2028 | +static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, | |
| 2029 | + struct mlx5_ib_flow_matcher *obj) | |
| 2030 | +{ | |
| 2031 | + enum mlx5_ib_uapi_flow_table_type ft_type = | |
| 2032 | + MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; | |
| 2033 | + u32 flags; | |
| 2034 | + int err; | |
| 2035 | + | |
| 2036 | + /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older | |
| 2037 | + * users should switch to it. We leave this to not break userspace | |
| 2038 | + */ | |
| 2039 | + if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && | |
| 2040 | + uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) | |
| 2041 | + return -EINVAL; | |
| 2042 | + | |
| 2043 | + if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { | |
| 2044 | + err = uverbs_get_const(&ft_type, attrs, | |
| 2045 | + MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); | |
| 2046 | + if (err) | |
| 2047 | + return err; | |
| 2048 | + | |
| 2049 | + err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); | |
| 2050 | + if (err) | |
| 2051 | + return err; | |
| 2052 | + | |
| 2053 | + return 0; | |
| 2054 | + } | |
| 2055 | + | |
| 2056 | + if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { | |
| 2057 | + err = uverbs_get_flags32(&flags, attrs, | |
| 2058 | + MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, | |
| 2059 | + IB_FLOW_ATTR_FLAGS_EGRESS); | |
| 2060 | + if (err) | |
| 2061 | + return err; | |
| 2062 | + | |
| 2063 | + if (flags) { | |
| 2064 | + mlx5_ib_ft_type_to_namespace( | |
| 2065 | + MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, | |
| 2066 | + &obj->ns_type); | |
| 2067 | + return 0; | |
| 2068 | + } | |
| 2069 | + } | |
| 2070 | + | |
| 2071 | + obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; | |
| 2072 | + | |
| 2073 | + return 0; | |
| 2074 | +} | |
| 2075 | + | |
| 2076 | +static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( | |
| 2077 | + struct uverbs_attr_bundle *attrs) | |
| 2078 | +{ | |
| 2079 | + struct ib_uobject *uobj = uverbs_attr_get_uobject( | |
| 2080 | + attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); | |
| 2081 | + struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); | |
| 2082 | + struct mlx5_ib_flow_matcher *obj; | |
| 2083 | + int err; | |
| 2084 | + | |
| 2085 | + obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); | |
| 2086 | + if (!obj) | |
| 2087 | + return -ENOMEM; | |
| 2088 | + | |
| 2089 | + obj->mask_len = uverbs_attr_get_len( | |
| 2090 | + attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); | |
| 2091 | + err = uverbs_copy_from(&obj->matcher_mask, | |
| 2092 | + attrs, | |
| 2093 | + MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); | |
| 2094 | + if (err) | |
| 2095 | + goto end; | |
| 2096 | + | |
| 2097 | + obj->flow_type = uverbs_attr_get_enum_id( | |
| 2098 | + attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); | |
| 2099 | + | |
| 2100 | + if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { | |
| 2101 | + err = uverbs_copy_from(&obj->priority, | |
| 2102 | + attrs, | |
| 2103 | + MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); | |
| 2104 | + if (err) | |
| 2105 | + goto end; | |
| 2106 | + } | |
| 2107 | + | |
| 2108 | + err = uverbs_copy_from(&obj->match_criteria_enable, | |
| 2109 | + attrs, | |
| 2110 | + MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); | |
| 2111 | + if (err) | |
| 2112 | + goto end; | |
| 2113 | + | |
| 2114 | + err = mlx5_ib_matcher_ns(attrs, obj); | |
| 2115 | + if (err) | |
| 2116 | + goto end; | |
| 2117 | + | |
| 2118 | + uobj->object = obj; | |
| 2119 | + obj->mdev = dev->mdev; | |
| 2120 | + atomic_set(&obj->usecnt, 0); | |
| 2121 | + return 0; | |
| 2122 | + | |
| 2123 | +end: | |
| 2124 | + kfree(obj); | |
| 2125 | + return err; | |
| 2126 | +} | |
| 2127 | + | |
| 2128 | +static struct ib_flow_action * | |
| 2129 | +mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, | |
| 2130 | + enum mlx5_ib_uapi_flow_table_type ft_type, | |
| 2131 | + u8 num_actions, void *in) | |
| 2132 | +{ | |
| 2133 | + enum mlx5_flow_namespace_type namespace; | |
| 2134 | + struct mlx5_ib_flow_action *maction; | |
| 2135 | + int ret; | |
| 2136 | + | |
| 2137 | + ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); | |
| 2138 | + if (ret) | |
| 2139 | + return ERR_PTR(-EINVAL); | |
| 2140 | + | |
| 2141 | + maction = kzalloc(sizeof(*maction), GFP_KERNEL); | |
| 2142 | + if (!maction) | |
| 2143 | + return ERR_PTR(-ENOMEM); | |
| 2144 | + | |
| 2145 | + maction->flow_action_raw.modify_hdr = | |
| 2146 | + mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); | |
| 2147 | + | |
| 2148 | + if (IS_ERR(maction->flow_action_raw.modify_hdr)) { | |
| 2149 | + ret = PTR_ERR(maction->flow_action_raw.modify_hdr); | |
| 2150 | + kfree(maction); | |
| 2151 | + return ERR_PTR(ret); | |
| 2152 | + } | |
| 2153 | + maction->flow_action_raw.sub_type = | |
| 2154 | + MLX5_IB_FLOW_ACTION_MODIFY_HEADER; | |
| 2155 | + maction->flow_action_raw.dev = dev; | |
| 2156 | + | |
| 2157 | + return &maction->ib_action; | |
| 2158 | +} | |
| 2159 | + | |
| 2160 | +static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) | |
| 2161 | +{ | |
| 2162 | + return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 2163 | + max_modify_header_actions) || | |
| 2164 | + MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, | |
| 2165 | + max_modify_header_actions) || | |
| 2166 | + MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, | |
| 2167 | + max_modify_header_actions); | |
| 2168 | +} | |
| 2169 | + | |
| 2170 | +static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( | |
| 2171 | + struct uverbs_attr_bundle *attrs) | |
| 2172 | +{ | |
| 2173 | + struct ib_uobject *uobj = uverbs_attr_get_uobject( | |
| 2174 | + attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); | |
| 2175 | + struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); | |
| 2176 | + enum mlx5_ib_uapi_flow_table_type ft_type; | |
| 2177 | + struct ib_flow_action *action; | |
| 2178 | + int num_actions; | |
| 2179 | + void *in; | |
| 2180 | + int ret; | |
| 2181 | + | |
| 2182 | + if (!mlx5_ib_modify_header_supported(mdev)) | |
| 2183 | + return -EOPNOTSUPP; | |
| 2184 | + | |
| 2185 | + in = uverbs_attr_get_alloced_ptr(attrs, | |
| 2186 | + MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); | |
| 2187 | + | |
| 2188 | + num_actions = uverbs_attr_ptr_get_array_size( | |
| 2189 | + attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, | |
| 2190 | + MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); | |
| 2191 | + if (num_actions < 0) | |
| 2192 | + return num_actions; | |
| 2193 | + | |
| 2194 | + ret = uverbs_get_const(&ft_type, attrs, | |
| 2195 | + MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); | |
| 2196 | + if (ret) | |
| 2197 | + return ret; | |
| 2198 | + action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); | |
| 2199 | + if (IS_ERR(action)) | |
| 2200 | + return PTR_ERR(action); | |
| 2201 | + | |
| 2202 | + uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, | |
| 2203 | + IB_FLOW_ACTION_UNSPECIFIED); | |
| 2204 | + | |
| 2205 | + return 0; | |
| 2206 | +} | |
| 2207 | + | |
| 2208 | +static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, | |
| 2209 | + u8 packet_reformat_type, | |
| 2210 | + u8 ft_type) | |
| 2211 | +{ | |
| 2212 | + switch (packet_reformat_type) { | |
| 2213 | + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: | |
| 2214 | + if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) | |
| 2215 | + return MLX5_CAP_FLOWTABLE(ibdev->mdev, | |
| 2216 | + encap_general_header); | |
| 2217 | + break; | |
| 2218 | + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: | |
| 2219 | + if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) | |
| 2220 | + return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, | |
| 2221 | + reformat_l2_to_l3_tunnel); | |
| 2222 | + break; | |
| 2223 | + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: | |
| 2224 | + if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) | |
| 2225 | + return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, | |
| 2226 | + reformat_l3_tunnel_to_l2); | |
| 2227 | + break; | |
| 2228 | + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: | |
| 2229 | + if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) | |
| 2230 | + return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); | |
| 2231 | + break; | |
| 2232 | + default: | |
| 2233 | + break; | |
| 2234 | + } | |
| 2235 | + | |
| 2236 | + return false; | |
| 2237 | +} | |
| 2238 | + | |
| 2239 | +static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) | |
| 2240 | +{ | |
| 2241 | + switch (dv_prt) { | |
| 2242 | + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: | |
| 2243 | + *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; | |
| 2244 | + break; | |
| 2245 | + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: | |
| 2246 | + *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; | |
| 2247 | + break; | |
| 2248 | + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: | |
| 2249 | + *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; | |
| 2250 | + break; | |
| 2251 | + default: | |
| 2252 | + return -EINVAL; | |
| 2253 | + } | |
| 2254 | + | |
| 2255 | + return 0; | |
| 2256 | +} | |
| 2257 | + | |
| 2258 | +static int mlx5_ib_flow_action_create_packet_reformat_ctx( | |
| 2259 | + struct mlx5_ib_dev *dev, | |
| 2260 | + struct mlx5_ib_flow_action *maction, | |
| 2261 | + u8 ft_type, u8 dv_prt, | |
| 2262 | + void *in, size_t len) | |
| 2263 | +{ | |
| 2264 | + enum mlx5_flow_namespace_type namespace; | |
| 2265 | + u8 prm_prt; | |
| 2266 | + int ret; | |
| 2267 | + | |
| 2268 | + ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); | |
| 2269 | + if (ret) | |
| 2270 | + return ret; | |
| 2271 | + | |
| 2272 | + ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); | |
| 2273 | + if (ret) | |
| 2274 | + return ret; | |
| 2275 | + | |
| 2276 | + maction->flow_action_raw.pkt_reformat = | |
| 2277 | + mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len, | |
| 2278 | + in, namespace); | |
| 2279 | + if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { | |
| 2280 | + ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); | |
| 2281 | + return ret; | |
| 2282 | + } | |
| 2283 | + | |
| 2284 | + maction->flow_action_raw.sub_type = | |
| 2285 | + MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; | |
| 2286 | + maction->flow_action_raw.dev = dev; | |
| 2287 | + | |
| 2288 | + return 0; | |
| 2289 | +} | |
| 2290 | + | |
| 2291 | +static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( | |
| 2292 | + struct uverbs_attr_bundle *attrs) | |
| 2293 | +{ | |
| 2294 | + struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, | |
| 2295 | + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); | |
| 2296 | + struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); | |
| 2297 | + enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; | |
| 2298 | + enum mlx5_ib_uapi_flow_table_type ft_type; | |
| 2299 | + struct mlx5_ib_flow_action *maction; | |
| 2300 | + int ret; | |
| 2301 | + | |
| 2302 | + ret = uverbs_get_const(&ft_type, attrs, | |
| 2303 | + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); | |
| 2304 | + if (ret) | |
| 2305 | + return ret; | |
| 2306 | + | |
| 2307 | + ret = uverbs_get_const(&dv_prt, attrs, | |
| 2308 | + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); | |
| 2309 | + if (ret) | |
| 2310 | + return ret; | |
| 2311 | + | |
| 2312 | + if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) | |
| 2313 | + return -EOPNOTSUPP; | |
| 2314 | + | |
| 2315 | + maction = kzalloc(sizeof(*maction), GFP_KERNEL); | |
| 2316 | + if (!maction) | |
| 2317 | + return -ENOMEM; | |
| 2318 | + | |
| 2319 | + if (dv_prt == | |
| 2320 | + MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { | |
| 2321 | + maction->flow_action_raw.sub_type = | |
| 2322 | + MLX5_IB_FLOW_ACTION_DECAP; | |
| 2323 | + maction->flow_action_raw.dev = mdev; | |
| 2324 | + } else { | |
| 2325 | + void *in; | |
| 2326 | + int len; | |
| 2327 | + | |
| 2328 | + in = uverbs_attr_get_alloced_ptr(attrs, | |
| 2329 | + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); | |
| 2330 | + if (IS_ERR(in)) { | |
| 2331 | + ret = PTR_ERR(in); | |
| 2332 | + goto free_maction; | |
| 2333 | + } | |
| 2334 | + | |
| 2335 | + len = uverbs_attr_get_len(attrs, | |
| 2336 | + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); | |
| 2337 | + | |
| 2338 | + ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, | |
| 2339 | + maction, ft_type, dv_prt, in, len); | |
| 2340 | + if (ret) | |
| 2341 | + goto free_maction; | |
| 2342 | + } | |
| 2343 | + | |
| 2344 | + uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, | |
| 2345 | + IB_FLOW_ACTION_UNSPECIFIED); | |
| 2346 | + return 0; | |
| 2347 | + | |
| 2348 | +free_maction: | |
| 2349 | + kfree(maction); | |
| 2350 | + return ret; | |
| 2351 | +} | |
| 2352 | + | |
| 2353 | +DECLARE_UVERBS_NAMED_METHOD( | |
| 2354 | + MLX5_IB_METHOD_CREATE_FLOW, | |
| 2355 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, | |
| 2356 | + UVERBS_OBJECT_FLOW, | |
| 2357 | + UVERBS_ACCESS_NEW, | |
| 2358 | + UA_MANDATORY), | |
| 2359 | + UVERBS_ATTR_PTR_IN( | |
| 2360 | + MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, | |
| 2361 | + UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), | |
| 2362 | + UA_MANDATORY, | |
| 2363 | + UA_ALLOC_AND_COPY), | |
| 2364 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, | |
| 2365 | + MLX5_IB_OBJECT_FLOW_MATCHER, | |
| 2366 | + UVERBS_ACCESS_READ, | |
| 2367 | + UA_MANDATORY), | |
| 2368 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, | |
| 2369 | + UVERBS_OBJECT_QP, | |
| 2370 | + UVERBS_ACCESS_READ), | |
| 2371 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, | |
| 2372 | + MLX5_IB_OBJECT_DEVX_OBJ, | |
| 2373 | + UVERBS_ACCESS_READ), | |
| 2374 | + UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, | |
| 2375 | + UVERBS_OBJECT_FLOW_ACTION, | |
| 2376 | + UVERBS_ACCESS_READ, 1, | |
| 2377 | + MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, | |
| 2378 | + UA_OPTIONAL), | |
| 2379 | + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, | |
| 2380 | + UVERBS_ATTR_TYPE(u32), | |
| 2381 | + UA_OPTIONAL), | |
| 2382 | + UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, | |
| 2383 | + MLX5_IB_OBJECT_DEVX_OBJ, | |
| 2384 | + UVERBS_ACCESS_READ, 1, 1, | |
| 2385 | + UA_OPTIONAL), | |
| 2386 | + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, | |
| 2387 | + UVERBS_ATTR_MIN_SIZE(sizeof(u32)), | |
| 2388 | + UA_OPTIONAL, | |
| 2389 | + UA_ALLOC_AND_COPY), | |
| 2390 | + UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, | |
| 2391 | + enum mlx5_ib_create_flow_flags, | |
| 2392 | + UA_OPTIONAL)); | |
| 2393 | + | |
| 2394 | +DECLARE_UVERBS_NAMED_METHOD_DESTROY( | |
| 2395 | + MLX5_IB_METHOD_DESTROY_FLOW, | |
| 2396 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, | |
| 2397 | + UVERBS_OBJECT_FLOW, | |
| 2398 | + UVERBS_ACCESS_DESTROY, | |
| 2399 | + UA_MANDATORY)); | |
| 2400 | + | |
| 2401 | +ADD_UVERBS_METHODS(mlx5_ib_fs, | |
| 2402 | + UVERBS_OBJECT_FLOW, | |
| 2403 | + &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), | |
| 2404 | + &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); | |
| 2405 | + | |
| 2406 | +DECLARE_UVERBS_NAMED_METHOD( | |
| 2407 | + MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, | |
| 2408 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, | |
| 2409 | + UVERBS_OBJECT_FLOW_ACTION, | |
| 2410 | + UVERBS_ACCESS_NEW, | |
| 2411 | + UA_MANDATORY), | |
| 2412 | + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, | |
| 2413 | + UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( | |
| 2414 | + set_add_copy_action_in_auto)), | |
| 2415 | + UA_MANDATORY, | |
| 2416 | + UA_ALLOC_AND_COPY), | |
| 2417 | + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, | |
| 2418 | + enum mlx5_ib_uapi_flow_table_type, | |
| 2419 | + UA_MANDATORY)); | |
| 2420 | + | |
| 2421 | +DECLARE_UVERBS_NAMED_METHOD( | |
| 2422 | + MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, | |
| 2423 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, | |
| 2424 | + UVERBS_OBJECT_FLOW_ACTION, | |
| 2425 | + UVERBS_ACCESS_NEW, | |
| 2426 | + UA_MANDATORY), | |
| 2427 | + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, | |
| 2428 | + UVERBS_ATTR_MIN_SIZE(1), | |
| 2429 | + UA_ALLOC_AND_COPY, | |
| 2430 | + UA_OPTIONAL), | |
| 2431 | + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, | |
| 2432 | + enum mlx5_ib_uapi_flow_action_packet_reformat_type, | |
| 2433 | + UA_MANDATORY), | |
| 2434 | + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, | |
| 2435 | + enum mlx5_ib_uapi_flow_table_type, | |
| 2436 | + UA_MANDATORY)); | |
| 2437 | + | |
| 2438 | +ADD_UVERBS_METHODS( | |
| 2439 | + mlx5_ib_flow_actions, | |
| 2440 | + UVERBS_OBJECT_FLOW_ACTION, | |
| 2441 | + &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), | |
| 2442 | + &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); | |
| 2443 | + | |
| 2444 | +DECLARE_UVERBS_NAMED_METHOD( | |
| 2445 | + MLX5_IB_METHOD_FLOW_MATCHER_CREATE, | |
| 2446 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, | |
| 2447 | + MLX5_IB_OBJECT_FLOW_MATCHER, | |
| 2448 | + UVERBS_ACCESS_NEW, | |
| 2449 | + UA_MANDATORY), | |
| 2450 | + UVERBS_ATTR_PTR_IN( | |
| 2451 | + MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, | |
| 2452 | + UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), | |
| 2453 | + UA_MANDATORY), | |
| 2454 | + UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, | |
| 2455 | + mlx5_ib_flow_type, | |
| 2456 | + UA_MANDATORY), | |
| 2457 | + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, | |
| 2458 | + UVERBS_ATTR_TYPE(u8), | |
| 2459 | + UA_MANDATORY), | |
| 2460 | + UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, | |
| 2461 | + enum ib_flow_flags, | |
| 2462 | + UA_OPTIONAL), | |
| 2463 | + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, | |
| 2464 | + enum mlx5_ib_uapi_flow_table_type, | |
| 2465 | + UA_OPTIONAL)); | |
| 2466 | + | |
| 2467 | +DECLARE_UVERBS_NAMED_METHOD_DESTROY( | |
| 2468 | + MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, | |
| 2469 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, | |
| 2470 | + MLX5_IB_OBJECT_FLOW_MATCHER, | |
| 2471 | + UVERBS_ACCESS_DESTROY, | |
| 2472 | + UA_MANDATORY)); | |
| 2473 | + | |
| 2474 | +DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, | |
| 2475 | + UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), | |
| 2476 | + &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), | |
| 2477 | + &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); | |
| 2478 | + | |
| 2479 | +const struct uapi_definition mlx5_ib_flow_defs[] = { | |
| 2480 | + UAPI_DEF_CHAIN_OBJ_TREE_NAMED( | |
| 2481 | + MLX5_IB_OBJECT_FLOW_MATCHER), | |
| 2482 | + UAPI_DEF_CHAIN_OBJ_TREE( | |
| 2483 | + UVERBS_OBJECT_FLOW, | |
| 2484 | + &mlx5_ib_fs), | |
| 2485 | + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, | |
| 2486 | + &mlx5_ib_flow_actions), | |
| 2487 | + {}, | |
| 2488 | +}; | |
| 2489 | + | |
| 2490 | +static const struct ib_device_ops flow_ops = { | |
| 2491 | + .create_flow = mlx5_ib_create_flow, | |
| 2492 | + .destroy_flow = mlx5_ib_destroy_flow, | |
| 2493 | + .destroy_flow_action = mlx5_ib_destroy_flow_action, | |
| 2494 | +}; | |
| 2495 | + | |
| 2496 | +static const struct ib_device_ops flow_ipsec_ops = { | |
| 2497 | + .create_flow_action_esp = mlx5_ib_create_flow_action_esp, | |
| 2498 | + .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp, | |
| 2499 | +}; | |
| 2500 | + | |
| 2501 | +int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) | |
| 2502 | +{ | |
| 2503 | + dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); | |
| 2504 | + | |
| 2505 | + if (!dev->flow_db) | |
| 2506 | + return -ENOMEM; | |
| 2507 | + | |
| 2508 | + mutex_init(&dev->flow_db->lock); | |
| 2509 | + | |
| 2510 | + ib_set_device_ops(&dev->ib_dev, &flow_ops); | |
| 2511 | + if (mlx5_accel_ipsec_device_caps(dev->mdev) & | |
| 2512 | + MLX5_ACCEL_IPSEC_CAP_DEVICE) | |
| 2513 | + ib_set_device_ops(&dev->ib_dev, &flow_ipsec_ops); | |
| 2514 | + | |
| 2515 | + return 0; | |
| 2516 | +} | |
| @@ -0,0 +1,29 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 2 | +/* | |
| 3 | + * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#ifndef _MLX5_IB_FS_H | |
| 7 | +#define _MLX5_IB_FS_H | |
| 8 | + | |
| 9 | +#include "mlx5_ib.h" | |
| 10 | + | |
| 11 | +#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) | |
| 12 | +int mlx5_ib_fs_init(struct mlx5_ib_dev *dev); | |
| 13 | +#else | |
| 14 | +static inline int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) | |
| 15 | +{ | |
| 16 | + dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); | |
| 17 | + | |
| 18 | + if (!dev->flow_db) | |
| 19 | + return -ENOMEM; | |
| 20 | + | |
| 21 | + mutex_init(&dev->flow_db->lock); | |
| 22 | + return 0; | |
| 23 | +} | |
| 24 | +#endif | |
| 25 | +static inline void mlx5_ib_fs_cleanup(struct mlx5_ib_dev *dev) | |
| 26 | +{ | |
| 27 | + kfree(dev->flow_db); | |
| 28 | +} | |
| 29 | +#endif /* _MLX5_IB_FS_H */ | |
| @@ -1,33 +1,6 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB | |
| 1 | 2 | /* |
| 2 | − * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. | |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 3 | + * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | 6 | #include <linux/debugfs.h> |
| @@ -59,10 +32,13 @@ · 25 unmodified lines | ||
| 59 | 32 | #include "mlx5_ib.h" |
| 60 | 33 | #include "ib_rep.h" |
| 61 | 34 | #include "cmd.h" |
| 35 | +#include "devx.h" | |
| 36 | +#include "fs.h" | |
| 62 | 37 | #include "srq.h" |
| 63 | 38 | #include "qp.h" |
| 64 | 39 | #include "wr.h" |
| 65 | −#include <linux/mlx5/fs_helpers.h> | |
| 40 | +#include "restrack.h" | |
| 41 | +#include "counters.h" | |
| 66 | 42 | #include <linux/mlx5/accel.h> |
| 67 | 43 | #include <rdma/uverbs_std_types.h> |
| 68 | 44 | #include <rdma/mlx5_user_ioctl_verbs.h> |
| @@ -311,9 +287,6 @@ struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev, · 242 unmodified lines | ||
| 311 | 287 | *native_port_num = 1; |
| 312 | 288 | |
| 313 | 289 | port = &ibdev->port[ib_port_num - 1]; |
| 314 | − if (!port) | |
| 315 | − return NULL; | |
| 316 | − | |
| 317 | 290 | spin_lock(&port->mp.mpi_lock); |
| 318 | 291 | mpi = ibdev->port[ib_port_num - 1].mp.mpi; |
| 319 | 292 | if (mpi && !mpi->unaffiliate) { |
| @@ -1765,6 +1738,92 @@ static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn, · 1445 unmodified lines | ||
| 1765 | 1738 | mlx5_ib_disable_lb(dev, true, false); |
| 1766 | 1739 | } |
| 1767 | 1740 | |
| 1741 | +static int set_ucontext_resp(struct ib_ucontext *uctx, | |
| 1742 | + struct mlx5_ib_alloc_ucontext_resp *resp) | |
| 1743 | +{ | |
| 1744 | + struct ib_device *ibdev = uctx->device; | |
| 1745 | + struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 1746 | + struct mlx5_ib_ucontext *context = to_mucontext(uctx); | |
| 1747 | + struct mlx5_bfreg_info *bfregi = &context->bfregi; | |
| 1748 | + int err; | |
| 1749 | + | |
| 1750 | + if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) { | |
| 1751 | + err = mlx5_cmd_dump_fill_mkey(dev->mdev, | |
| 1752 | + &resp->dump_fill_mkey); | |
| 1753 | + if (err) | |
| 1754 | + return err; | |
| 1755 | + resp->comp_mask |= | |
| 1756 | + MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY; | |
| 1757 | + } | |
| 1758 | + | |
| 1759 | + resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp); | |
| 1760 | + if (dev->wc_support) | |
| 1761 | + resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, | |
| 1762 | + log_bf_reg_size); | |
| 1763 | + resp->cache_line_size = cache_line_size(); | |
| 1764 | + resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq); | |
| 1765 | + resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq); | |
| 1766 | + resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz); | |
| 1767 | + resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz); | |
| 1768 | + resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); | |
| 1769 | + resp->cqe_version = context->cqe_version; | |
| 1770 | + resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ? | |
| 1771 | + MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT; | |
| 1772 | + resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? | |
| 1773 | + MLX5_CAP_GEN(dev->mdev, | |
| 1774 | + num_of_uars_per_page) : 1; | |
| 1775 | + | |
| 1776 | + if (mlx5_accel_ipsec_device_caps(dev->mdev) & | |
| 1777 | + MLX5_ACCEL_IPSEC_CAP_DEVICE) { | |
| 1778 | + if (mlx5_get_flow_namespace(dev->mdev, | |
| 1779 | + MLX5_FLOW_NAMESPACE_EGRESS)) | |
| 1780 | + resp->flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM; | |
| 1781 | + if (mlx5_accel_ipsec_device_caps(dev->mdev) & | |
| 1782 | + MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA) | |
| 1783 | + resp->flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA; | |
| 1784 | + if (MLX5_CAP_FLOWTABLE(dev->mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi)) | |
| 1785 | + resp->flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING; | |
| 1786 | + if (mlx5_accel_ipsec_device_caps(dev->mdev) & | |
| 1787 | + MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN) | |
| 1788 | + resp->flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN; | |
| 1789 | + /* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */ | |
| 1790 | + } | |
| 1791 | + | |
| 1792 | + resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 : | |
| 1793 | + bfregi->total_num_bfregs - bfregi->num_dyn_bfregs; | |
| 1794 | + resp->num_ports = dev->num_ports; | |
| 1795 | + resp->cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE | | |
| 1796 | + MLX5_USER_CMDS_SUPP_UHW_CREATE_AH; | |
| 1797 | + | |
| 1798 | + if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) { | |
| 1799 | + mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline); | |
| 1800 | + resp->eth_min_inline++; | |
| 1801 | + } | |
| 1802 | + | |
| 1803 | + if (dev->mdev->clock_info) | |
| 1804 | + resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1); | |
| 1805 | + | |
| 1806 | + /* | |
| 1807 | + * We don't want to expose information from the PCI bar that is located | |
| 1808 | + * after 4096 bytes, so if the arch only supports larger pages, let's | |
| 1809 | + * pretend we don't support reading the HCA's core clock. This is also | |
| 1810 | + * forced by mmap function. | |
| 1811 | + */ | |
| 1812 | + if (PAGE_SIZE <= 4096) { | |
| 1813 | + resp->comp_mask |= | |
| 1814 | + MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET; | |
| 1815 | + resp->hca_core_clock_offset = | |
| 1816 | + offsetof(struct mlx5_init_seg, | |
| 1817 | + internal_timer_h) % PAGE_SIZE; | |
| 1818 | + } | |
| 1819 | + | |
| 1820 | + if (MLX5_CAP_GEN(dev->mdev, ece_support)) | |
| 1821 | + resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE; | |
| 1822 | + | |
| 1823 | + resp->num_dyn_bfregs = bfregi->num_dyn_bfregs; | |
| 1824 | + return 0; | |
| 1825 | +} | |
| 1826 | + | |
| 1768 | 1827 | static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, |
| 1769 | 1828 | struct ib_udata *udata) |
| 1770 | 1829 | { |
| @@ -1772,14 +1831,12 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, · 1 unmodified line | ||
| 1772 | 1831 | struct mlx5_ib_dev *dev = to_mdev(ibdev); |
| 1773 | 1832 | struct mlx5_ib_alloc_ucontext_req_v2 req = {}; |
| 1774 | 1833 | struct mlx5_ib_alloc_ucontext_resp resp = {}; |
| 1775 | − struct mlx5_core_dev *mdev = dev->mdev; | |
| 1776 | 1834 | struct mlx5_ib_ucontext *context = to_mucontext(uctx); |
| 1777 | 1835 | struct mlx5_bfreg_info *bfregi; |
| 1778 | 1836 | int ver; |
| 1779 | 1837 | int err; |
| 1780 | 1838 | size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2, |
| 1781 | 1839 | max_cqe_version); |
| 1782 | − u32 dump_fill_mkey; | |
| 1783 | 1840 | bool lib_uar_4k; |
| 1784 | 1841 | bool lib_uar_dyn; |
| 1785 | 1842 | |
| @@ -1808,37 +1865,6 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, · 22 unmodified lines | ||
| 1808 | 1865 | if (req.num_low_latency_bfregs > req.total_num_bfregs - 1) |
| 1809 | 1866 | return -EINVAL; |
| 1810 | 1867 | |
| 1811 | − resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp); | |
| 1812 | − if (dev->wc_support) | |
| 1813 | − resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size); | |
| 1814 | − resp.cache_line_size = cache_line_size(); | |
| 1815 | − resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq); | |
| 1816 | − resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq); | |
| 1817 | − resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz); | |
| 1818 | − resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz); | |
| 1819 | − resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); | |
| 1820 | − resp.cqe_version = min_t(__u8, | |
| 1821 | − (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version), | |
| 1822 | − req.max_cqe_version); | |
| 1823 | − resp.log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ? | |
| 1824 | − MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT; | |
| 1825 | − resp.num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? | |
| 1826 | − MLX5_CAP_GEN(dev->mdev, num_of_uars_per_page) : 1; | |
| 1827 | − resp.response_length = min(offsetof(typeof(resp), response_length) + | |
| 1828 | − sizeof(resp.response_length), udata->outlen); | |
| 1829 | − | |
| 1830 | − if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) { | |
| 1831 | − if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_EGRESS)) | |
| 1832 | − resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM; | |
| 1833 | − if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA) | |
| 1834 | − resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA; | |
| 1835 | − if (MLX5_CAP_FLOWTABLE(dev->mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi)) | |
| 1836 | − resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING; | |
| 1837 | − if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN) | |
| 1838 | − resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN; | |
| 1839 | − /* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */ | |
| 1840 | − } | |
| 1841 | − | |
| 1842 | 1868 | lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR; |
| 1843 | 1869 | lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR; |
| 1844 | 1870 | bfregi = &context->bfregi; |
| @@ -1887,87 +1913,24 @@ uar_done: · 42 unmodified lines | ||
| 1887 | 1913 | if (err) |
| 1888 | 1914 | goto out_devx; |
| 1889 | 1915 | |
| 1890 | − if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) { | |
| 1891 | − err = mlx5_cmd_dump_fill_mkey(dev->mdev, &dump_fill_mkey); | |
| 1892 | − if (err) | |
| 1893 | − goto out_mdev; | |
| 1894 | − } | |
| 1895 | − | |
| 1896 | 1916 | INIT_LIST_HEAD(&context->db_page_list); |
| 1897 | 1917 | mutex_init(&context->db_page_mutex); |
| 1898 | 1918 | |
| 1899 | − resp.tot_bfregs = lib_uar_dyn ? 0 : req.total_num_bfregs; | |
| 1900 | − resp.num_ports = dev->num_ports; | |
| 1901 | − | |
| 1902 | − if (offsetofend(typeof(resp), cqe_version) <= udata->outlen) | |
| 1903 | − resp.response_length += sizeof(resp.cqe_version); | |
| 1904 | − | |
| 1905 | − if (offsetofend(typeof(resp), cmds_supp_uhw) <= udata->outlen) { | |
| 1906 | − resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE | | |
| 1907 | − MLX5_USER_CMDS_SUPP_UHW_CREATE_AH; | |
| 1908 | − resp.response_length += sizeof(resp.cmds_supp_uhw); | |
| 1909 | − } | |
| 1910 | − | |
| 1911 | − if (offsetofend(typeof(resp), eth_min_inline) <= udata->outlen) { | |
| 1912 | − if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) { | |
| 1913 | − mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline); | |
| 1914 | − resp.eth_min_inline++; | |
| 1915 | − } | |
| 1916 | − resp.response_length += sizeof(resp.eth_min_inline); | |
| 1917 | − } | |
| 1918 | − | |
| 1919 | − if (offsetofend(typeof(resp), clock_info_versions) <= udata->outlen) { | |
| 1920 | − if (mdev->clock_info) | |
| 1921 | − resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1); | |
| 1922 | − resp.response_length += sizeof(resp.clock_info_versions); | |
| 1923 | − } | |
| 1924 | − | |
| 1925 | − /* | |
| 1926 | − * We don't want to expose information from the PCI bar that is located | |
| 1927 | − * after 4096 bytes, so if the arch only supports larger pages, let's | |
| 1928 | − * pretend we don't support reading the HCA's core clock. This is also | |
| 1929 | − * forced by mmap function. | |
| 1930 | − */ | |
| 1931 | − if (offsetofend(typeof(resp), hca_core_clock_offset) <= udata->outlen) { | |
| 1932 | − if (PAGE_SIZE <= 4096) { | |
| 1933 | − resp.comp_mask |= | |
| 1934 | − MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET; | |
| 1935 | − resp.hca_core_clock_offset = | |
| 1936 | − offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE; | |
| 1937 | − } | |
| 1938 | − resp.response_length += sizeof(resp.hca_core_clock_offset); | |
| 1939 | − } | |
| 1940 | − | |
| 1941 | − if (offsetofend(typeof(resp), log_uar_size) <= udata->outlen) | |
| 1942 | − resp.response_length += sizeof(resp.log_uar_size); | |
| 1943 | − | |
| 1944 | − if (offsetofend(typeof(resp), num_uars_per_page) <= udata->outlen) | |
| 1945 | − resp.response_length += sizeof(resp.num_uars_per_page); | |
| 1946 | − | |
| 1947 | − if (offsetofend(typeof(resp), num_dyn_bfregs) <= udata->outlen) { | |
| 1948 | − resp.num_dyn_bfregs = bfregi->num_dyn_bfregs; | |
| 1949 | − resp.response_length += sizeof(resp.num_dyn_bfregs); | |
| 1950 | − } | |
| 1951 | − | |
| 1952 | − if (offsetofend(typeof(resp), dump_fill_mkey) <= udata->outlen) { | |
| 1953 | − if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) { | |
| 1954 | − resp.dump_fill_mkey = dump_fill_mkey; | |
| 1955 | − resp.comp_mask |= | |
| 1956 | − MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY; | |
| 1957 | − } | |
| 1958 | − resp.response_length += sizeof(resp.dump_fill_mkey); | |
| 1959 | − } | |
| 1919 | + context->cqe_version = min_t(__u8, | |
| 1920 | + (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version), | |
| 1921 | + req.max_cqe_version); | |
| 1960 | 1922 | |
| 1961 | − if (MLX5_CAP_GEN(dev->mdev, ece_support)) | |
| 1962 | − resp.comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE; | |
| 1923 | + err = set_ucontext_resp(uctx, &resp); | |
| 1924 | + if (err) | |
| 1925 | + goto out_mdev; | |
| 1963 | 1926 | |
| 1927 | + resp.response_length = min(udata->outlen, sizeof(resp)); | |
| 1964 | 1928 | err = ib_copy_to_udata(udata, &resp, resp.response_length); |
| 1965 | 1929 | if (err) |
| 1966 | 1930 | goto out_mdev; |
| 1967 | 1931 | |
| 1968 | 1932 | bfregi->ver = ver; |
| 1969 | 1933 | bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs; |
| 1970 | − context->cqe_version = resp.cqe_version; | |
| 1971 | 1934 | context->lib_caps = req.lib_caps; |
| 1972 | 1935 | print_lib_caps(dev, context->lib_caps); |
| 1973 | 1936 | |
| @@ -2000,6 +1963,29 @@ out_ctx: · 26 unmodified lines | ||
| 2000 | 1963 | return err; |
| 2001 | 1964 | } |
| 2002 | 1965 | |
| 1966 | +static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext, | |
| 1967 | + struct uverbs_attr_bundle *attrs) | |
| 1968 | +{ | |
| 1969 | + struct mlx5_ib_alloc_ucontext_resp uctx_resp = {}; | |
| 1970 | + int ret; | |
| 1971 | + | |
| 1972 | + ret = set_ucontext_resp(ibcontext, &uctx_resp); | |
| 1973 | + if (ret) | |
| 1974 | + return ret; | |
| 1975 | + | |
| 1976 | + uctx_resp.response_length = | |
| 1977 | + min_t(size_t, | |
| 1978 | + uverbs_attr_get_len(attrs, | |
| 1979 | + MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX), | |
| 1980 | + sizeof(uctx_resp)); | |
| 1981 | + | |
| 1982 | + ret = uverbs_copy_to_struct_or_zero(attrs, | |
| 1983 | + MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX, | |
| 1984 | + &uctx_resp, | |
| 1985 | + sizeof(uctx_resp)); | |
| 1986 | + return ret; | |
| 1987 | +} | |
| 1988 | + | |
| 2003 | 1989 | static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) |
| 2004 | 1990 | { |
| 2005 | 1991 | struct mlx5_ib_ucontext *context = to_mucontext(ibcontext); |
| @@ -2591,3205 +2577,786 @@ static void mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) · 585 unmodified lines | ||
| 2591 | 2577 | mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid); |
| 2592 | 2578 | } |
| 2593 | 2579 | |
| 2594 | −enum { | |
| 2595 | − MATCH_CRITERIA_ENABLE_OUTER_BIT, | |
| 2596 | − MATCH_CRITERIA_ENABLE_MISC_BIT, | |
| 2597 | − MATCH_CRITERIA_ENABLE_INNER_BIT, | |
| 2598 | − MATCH_CRITERIA_ENABLE_MISC2_BIT | |
| 2599 | −}; | |
| 2600 | − | |
| 2601 | −#define HEADER_IS_ZERO(match_criteria, headers) \ | |
| 2602 | − !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ | |
| 2603 | − 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ | |
| 2604 | − | |
| 2605 | −static u8 get_match_criteria_enable(u32 *match_criteria) | |
| 2580 | +static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) | |
| 2606 | 2581 | { |
| 2607 | − u8 match_criteria_enable; | |
| 2608 | − | |
| 2609 | − match_criteria_enable = | |
| 2610 | − (!HEADER_IS_ZERO(match_criteria, outer_headers)) << | |
| 2611 | − MATCH_CRITERIA_ENABLE_OUTER_BIT; | |
| 2612 | − match_criteria_enable |= | |
| 2613 | − (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << | |
| 2614 | − MATCH_CRITERIA_ENABLE_MISC_BIT; | |
| 2615 | − match_criteria_enable |= | |
| 2616 | − (!HEADER_IS_ZERO(match_criteria, inner_headers)) << | |
| 2617 | − MATCH_CRITERIA_ENABLE_INNER_BIT; | |
| 2618 | − match_criteria_enable |= | |
| 2619 | − (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << | |
| 2620 | − MATCH_CRITERIA_ENABLE_MISC2_BIT; | |
| 2582 | + struct mlx5_ib_dev *dev = to_mdev(ibqp->device); | |
| 2583 | + struct mlx5_ib_qp *mqp = to_mqp(ibqp); | |
| 2584 | + int err; | |
| 2585 | + u16 uid; | |
| 2621 | 2586 | |
| 2622 | − return match_criteria_enable; | |
| 2623 | −} | |
| 2587 | + uid = ibqp->pd ? | |
| 2588 | + to_mpd(ibqp->pd)->uid : 0; | |
| 2624 | 2589 | |
| 2625 | −static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) | |
| 2626 | −{ | |
| 2627 | − u8 entry_mask; | |
| 2628 | − u8 entry_val; | |
| 2629 | − int err = 0; | |
| 2590 | + if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) { | |
| 2591 | + mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n"); | |
| 2592 | + return -EOPNOTSUPP; | |
| 2593 | + } | |
| 2630 | 2594 | |
| 2631 | − if (!mask) | |
| 2632 | − goto out; | |
| 2595 | + err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid); | |
| 2596 | + if (err) | |
| 2597 | + mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n", | |
| 2598 | + ibqp->qp_num, gid->raw); | |
| 2633 | 2599 | |
| 2634 | − entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, | |
| 2635 | − ip_protocol); | |
| 2636 | − entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, | |
| 2637 | − ip_protocol); | |
| 2638 | − if (!entry_mask) { | |
| 2639 | − MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); | |
| 2640 | − MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); | |
| 2641 | − goto out; | |
| 2642 | − } | |
| 2643 | − /* Don't override existing ip protocol */ | |
| 2644 | − if (mask != entry_mask || val != entry_val) | |
| 2645 | − err = -EINVAL; | |
| 2646 | −out: | |
| 2647 | 2600 | return err; |
| 2648 | 2601 | } |
| 2649 | 2602 | |
| 2650 | −static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, | |
| 2651 | − bool inner) | |
| 2603 | +static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) | |
| 2652 | 2604 | { |
| 2653 | − if (inner) { | |
| 2654 | − MLX5_SET(fte_match_set_misc, | |
| 2655 | − misc_c, inner_ipv6_flow_label, mask); | |
| 2656 | − MLX5_SET(fte_match_set_misc, | |
| 2657 | − misc_v, inner_ipv6_flow_label, val); | |
| 2658 | − } else { | |
| 2659 | − MLX5_SET(fte_match_set_misc, | |
| 2660 | − misc_c, outer_ipv6_flow_label, mask); | |
| 2661 | − MLX5_SET(fte_match_set_misc, | |
| 2662 | − misc_v, outer_ipv6_flow_label, val); | |
| 2663 | − } | |
| 2664 | −} | |
| 2605 | + struct mlx5_ib_dev *dev = to_mdev(ibqp->device); | |
| 2606 | + int err; | |
| 2607 | + u16 uid; | |
| 2665 | 2608 | |
| 2666 | −static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) | |
| 2667 | −{ | |
| 2668 | − MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); | |
| 2669 | − MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); | |
| 2670 | − MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); | |
| 2671 | − MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); | |
| 2609 | + uid = ibqp->pd ? | |
| 2610 | + to_mpd(ibqp->pd)->uid : 0; | |
| 2611 | + err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid); | |
| 2612 | + if (err) | |
| 2613 | + mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n", | |
| 2614 | + ibqp->qp_num, gid->raw); | |
| 2615 | + | |
| 2616 | + return err; | |
| 2672 | 2617 | } |
| 2673 | 2618 | |
| 2674 | −static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) | |
| 2619 | +static int init_node_data(struct mlx5_ib_dev *dev) | |
| 2675 | 2620 | { |
| 2676 | − if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && | |
| 2677 | − !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) | |
| 2678 | − return -EOPNOTSUPP; | |
| 2679 | − | |
| 2680 | − if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && | |
| 2681 | − !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) | |
| 2682 | − return -EOPNOTSUPP; | |
| 2683 | − | |
| 2684 | − if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && | |
| 2685 | − !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) | |
| 2686 | − return -EOPNOTSUPP; | |
| 2621 | + int err; | |
| 2687 | 2622 | |
| 2688 | − if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && | |
| 2689 | − !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) | |
| 2690 | − return -EOPNOTSUPP; | |
| 2623 | + err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc); | |
| 2624 | + if (err) | |
| 2625 | + return err; | |
| 2691 | 2626 | |
| 2692 | − return 0; | |
| 2693 | −} | |
| 2627 | + dev->mdev->rev_id = dev->mdev->pdev->revision; | |
| 2694 | 2628 | |
| 2695 | −#define LAST_ETH_FIELD vlan_tag | |
| 2696 | −#define LAST_IB_FIELD sl | |
| 2697 | −#define LAST_IPV4_FIELD tos | |
| 2698 | −#define LAST_IPV6_FIELD traffic_class | |
| 2699 | −#define LAST_TCP_UDP_FIELD src_port | |
| 2700 | −#define LAST_TUNNEL_FIELD tunnel_id | |
| 2701 | −#define LAST_FLOW_TAG_FIELD tag_id | |
| 2702 | −#define LAST_DROP_FIELD size | |
| 2703 | −#define LAST_COUNTERS_FIELD counters | |
| 2704 | − | |
| 2705 | −/* Field is the last supported field */ | |
| 2706 | −#define FIELDS_NOT_SUPPORTED(filter, field)\ | |
| 2707 | − memchr_inv((void *)&filter.field +\ | |
| 2708 | − sizeof(filter.field), 0,\ | |
| 2709 | − sizeof(filter) -\ | |
| 2710 | − offsetof(typeof(filter), field) -\ | |
| 2711 | − sizeof(filter.field)) | |
| 2712 | − | |
| 2713 | −int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, | |
| 2714 | − bool is_egress, | |
| 2715 | − struct mlx5_flow_act *action) | |
| 2716 | −{ | |
| 2717 | − | |
| 2718 | − switch (maction->ib_action.type) { | |
| 2719 | − case IB_FLOW_ACTION_ESP: | |
| 2720 | − if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | | |
| 2721 | − MLX5_FLOW_CONTEXT_ACTION_DECRYPT)) | |
| 2722 | − return -EINVAL; | |
| 2723 | − /* Currently only AES_GCM keymat is supported by the driver */ | |
| 2724 | − action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx; | |
| 2725 | − action->action |= is_egress ? | |
| 2726 | − MLX5_FLOW_CONTEXT_ACTION_ENCRYPT : | |
| 2727 | − MLX5_FLOW_CONTEXT_ACTION_DECRYPT; | |
| 2728 | − return 0; | |
| 2729 | − case IB_FLOW_ACTION_UNSPECIFIED: | |
| 2730 | − if (maction->flow_action_raw.sub_type == | |
| 2731 | − MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { | |
| 2732 | − if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) | |
| 2733 | − return -EINVAL; | |
| 2734 | − action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; | |
| 2735 | − action->modify_hdr = | |
| 2736 | − maction->flow_action_raw.modify_hdr; | |
| 2737 | − return 0; | |
| 2738 | − } | |
| 2739 | − if (maction->flow_action_raw.sub_type == | |
| 2740 | − MLX5_IB_FLOW_ACTION_DECAP) { | |
| 2741 | − if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) | |
| 2742 | − return -EINVAL; | |
| 2743 | − action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; | |
| 2744 | − return 0; | |
| 2745 | − } | |
| 2746 | − if (maction->flow_action_raw.sub_type == | |
| 2747 | − MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { | |
| 2748 | − if (action->action & | |
| 2749 | − MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) | |
| 2750 | − return -EINVAL; | |
| 2751 | − action->action |= | |
| 2752 | − MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; | |
| 2753 | − action->pkt_reformat = | |
| 2754 | − maction->flow_action_raw.pkt_reformat; | |
| 2755 | − return 0; | |
| 2756 | − } | |
| 2757 | − /* fall through */ | |
| 2758 | − default: | |
| 2759 | − return -EOPNOTSUPP; | |
| 2760 | − } | |
| 2629 | + return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid); | |
| 2761 | 2630 | } |
| 2762 | 2631 | |
| 2763 | −static int parse_flow_attr(struct mlx5_core_dev *mdev, | |
| 2764 | − struct mlx5_flow_spec *spec, | |
| 2765 | − const union ib_flow_spec *ib_spec, | |
| 2766 | − const struct ib_flow_attr *flow_attr, | |
| 2767 | − struct mlx5_flow_act *action, u32 prev_type) | |
| 2768 | −{ | |
| 2769 | − struct mlx5_flow_context *flow_context = &spec->flow_context; | |
| 2770 | − u32 *match_c = spec->match_criteria; | |
| 2771 | − u32 *match_v = spec->match_value; | |
| 2772 | − void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 2773 | − misc_parameters); | |
| 2774 | − void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 2775 | − misc_parameters); | |
| 2776 | − void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 2777 | − misc_parameters_2); | |
| 2778 | − void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 2779 | − misc_parameters_2); | |
| 2780 | − void *headers_c; | |
| 2781 | − void *headers_v; | |
| 2782 | − int match_ipv; | |
| 2783 | − int ret; | |
| 2632 | +static ssize_t fw_pages_show(struct device *device, | |
| 2633 | + struct device_attribute *attr, char *buf) | |
| 2634 | +{ | |
| 2635 | + struct mlx5_ib_dev *dev = | |
| 2636 | + rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 2784 | 2637 | |
| 2785 | − if (ib_spec->type & IB_FLOW_SPEC_INNER) { | |
| 2786 | − headers_c = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 2787 | − inner_headers); | |
| 2788 | − headers_v = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 2789 | − inner_headers); | |
| 2790 | − match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 2791 | − ft_field_support.inner_ip_version); | |
| 2792 | − } else { | |
| 2793 | − headers_c = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 2794 | − outer_headers); | |
| 2795 | − headers_v = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 2796 | − outer_headers); | |
| 2797 | − match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 2798 | − ft_field_support.outer_ip_version); | |
| 2799 | − } | |
| 2638 | + return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages); | |
| 2639 | +} | |
| 2640 | +static DEVICE_ATTR_RO(fw_pages); | |
| 2800 | 2641 | |
| 2801 | − switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { | |
| 2802 | − case IB_FLOW_SPEC_ETH: | |
| 2803 | − if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) | |
| 2804 | − return -EOPNOTSUPP; | |
| 2642 | +static ssize_t reg_pages_show(struct device *device, | |
| 2643 | + struct device_attribute *attr, char *buf) | |
| 2644 | +{ | |
| 2645 | + struct mlx5_ib_dev *dev = | |
| 2646 | + rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 2805 | 2647 | |
| 2806 | − ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 2807 | − dmac_47_16), | |
| 2808 | − ib_spec->eth.mask.dst_mac); | |
| 2809 | − ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 2810 | − dmac_47_16), | |
| 2811 | − ib_spec->eth.val.dst_mac); | |
| 2812 | − | |
| 2813 | − ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 2814 | − smac_47_16), | |
| 2815 | − ib_spec->eth.mask.src_mac); | |
| 2816 | − ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 2817 | − smac_47_16), | |
| 2818 | − ib_spec->eth.val.src_mac); | |
| 2819 | − | |
| 2820 | − if (ib_spec->eth.mask.vlan_tag) { | |
| 2821 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2822 | − cvlan_tag, 1); | |
| 2823 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2824 | − cvlan_tag, 1); | |
| 2825 | − | |
| 2826 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2827 | − first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); | |
| 2828 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2829 | − first_vid, ntohs(ib_spec->eth.val.vlan_tag)); | |
| 2830 | − | |
| 2831 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2832 | − first_cfi, | |
| 2833 | − ntohs(ib_spec->eth.mask.vlan_tag) >> 12); | |
| 2834 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2835 | − first_cfi, | |
| 2836 | − ntohs(ib_spec->eth.val.vlan_tag) >> 12); | |
| 2837 | − | |
| 2838 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2839 | − first_prio, | |
| 2840 | − ntohs(ib_spec->eth.mask.vlan_tag) >> 13); | |
| 2841 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2842 | − first_prio, | |
| 2843 | − ntohs(ib_spec->eth.val.vlan_tag) >> 13); | |
| 2844 | − } | |
| 2845 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2846 | − ethertype, ntohs(ib_spec->eth.mask.ether_type)); | |
| 2847 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2848 | − ethertype, ntohs(ib_spec->eth.val.ether_type)); | |
| 2849 | − break; | |
| 2850 | − case IB_FLOW_SPEC_IPV4: | |
| 2851 | − if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) | |
| 2852 | − return -EOPNOTSUPP; | |
| 2648 | + return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages)); | |
| 2649 | +} | |
| 2650 | +static DEVICE_ATTR_RO(reg_pages); | |
| 2853 | 2651 | |
| 2854 | − if (match_ipv) { | |
| 2855 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2856 | − ip_version, 0xf); | |
| 2857 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2858 | − ip_version, MLX5_FS_IPV4_VERSION); | |
| 2859 | − } else { | |
| 2860 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2861 | − ethertype, 0xffff); | |
| 2862 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2863 | − ethertype, ETH_P_IP); | |
| 2864 | − } | |
| 2652 | +static ssize_t hca_type_show(struct device *device, | |
| 2653 | + struct device_attribute *attr, char *buf) | |
| 2654 | +{ | |
| 2655 | + struct mlx5_ib_dev *dev = | |
| 2656 | + rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 2865 | 2657 | |
| 2866 | − memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 2867 | − src_ipv4_src_ipv6.ipv4_layout.ipv4), | |
| 2868 | − &ib_spec->ipv4.mask.src_ip, | |
| 2869 | − sizeof(ib_spec->ipv4.mask.src_ip)); | |
| 2870 | − memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 2871 | − src_ipv4_src_ipv6.ipv4_layout.ipv4), | |
| 2872 | − &ib_spec->ipv4.val.src_ip, | |
| 2873 | − sizeof(ib_spec->ipv4.val.src_ip)); | |
| 2874 | − memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 2875 | − dst_ipv4_dst_ipv6.ipv4_layout.ipv4), | |
| 2876 | − &ib_spec->ipv4.mask.dst_ip, | |
| 2877 | − sizeof(ib_spec->ipv4.mask.dst_ip)); | |
| 2878 | − memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 2879 | − dst_ipv4_dst_ipv6.ipv4_layout.ipv4), | |
| 2880 | − &ib_spec->ipv4.val.dst_ip, | |
| 2881 | − sizeof(ib_spec->ipv4.val.dst_ip)); | |
| 2882 | − | |
| 2883 | − set_tos(headers_c, headers_v, | |
| 2884 | − ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); | |
| 2885 | − | |
| 2886 | − if (set_proto(headers_c, headers_v, | |
| 2887 | − ib_spec->ipv4.mask.proto, | |
| 2888 | − ib_spec->ipv4.val.proto)) | |
| 2889 | − return -EINVAL; | |
| 2890 | − break; | |
| 2891 | − case IB_FLOW_SPEC_IPV6: | |
| 2892 | − if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) | |
| 2893 | − return -EOPNOTSUPP; | |
| 2658 | + return sprintf(buf, "MT%d\n", dev->mdev->pdev->device); | |
| 2659 | +} | |
| 2660 | +static DEVICE_ATTR_RO(hca_type); | |
| 2894 | 2661 | |
| 2895 | − if (match_ipv) { | |
| 2896 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2897 | − ip_version, 0xf); | |
| 2898 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2899 | − ip_version, MLX5_FS_IPV6_VERSION); | |
| 2900 | − } else { | |
| 2901 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, | |
| 2902 | − ethertype, 0xffff); | |
| 2903 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, | |
| 2904 | − ethertype, ETH_P_IPV6); | |
| 2905 | − } | |
| 2662 | +static ssize_t hw_rev_show(struct device *device, | |
| 2663 | + struct device_attribute *attr, char *buf) | |
| 2664 | +{ | |
| 2665 | + struct mlx5_ib_dev *dev = | |
| 2666 | + rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 2906 | 2667 | |
| 2907 | − memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 2908 | − src_ipv4_src_ipv6.ipv6_layout.ipv6), | |
| 2909 | − &ib_spec->ipv6.mask.src_ip, | |
| 2910 | − sizeof(ib_spec->ipv6.mask.src_ip)); | |
| 2911 | − memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 2912 | − src_ipv4_src_ipv6.ipv6_layout.ipv6), | |
| 2913 | − &ib_spec->ipv6.val.src_ip, | |
| 2914 | − sizeof(ib_spec->ipv6.val.src_ip)); | |
| 2915 | − memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, | |
| 2916 | − dst_ipv4_dst_ipv6.ipv6_layout.ipv6), | |
| 2917 | − &ib_spec->ipv6.mask.dst_ip, | |
| 2918 | − sizeof(ib_spec->ipv6.mask.dst_ip)); | |
| 2919 | − memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, | |
| 2920 | − dst_ipv4_dst_ipv6.ipv6_layout.ipv6), | |
| 2921 | − &ib_spec->ipv6.val.dst_ip, | |
| 2922 | − sizeof(ib_spec->ipv6.val.dst_ip)); | |
| 2923 | − | |
| 2924 | − set_tos(headers_c, headers_v, | |
| 2925 | − ib_spec->ipv6.mask.traffic_class, | |
| 2926 | − ib_spec->ipv6.val.traffic_class); | |
| 2927 | − | |
| 2928 | − if (set_proto(headers_c, headers_v, | |
| 2929 | − ib_spec->ipv6.mask.next_hdr, | |
| 2930 | − ib_spec->ipv6.val.next_hdr)) | |
| 2931 | − return -EINVAL; | |
| 2668 | + return sprintf(buf, "%x\n", dev->mdev->rev_id); | |
| 2669 | +} | |
| 2670 | +static DEVICE_ATTR_RO(hw_rev); | |
| 2932 | 2671 | |
| 2933 | − set_flow_label(misc_params_c, misc_params_v, | |
| 2934 | − ntohl(ib_spec->ipv6.mask.flow_label), | |
| 2935 | − ntohl(ib_spec->ipv6.val.flow_label), | |
| 2936 | − ib_spec->type & IB_FLOW_SPEC_INNER); | |
| 2937 | − break; | |
| 2938 | − case IB_FLOW_SPEC_ESP: | |
| 2939 | − if (ib_spec->esp.mask.seq) | |
| 2940 | − return -EOPNOTSUPP; | |
| 2672 | +static ssize_t board_id_show(struct device *device, | |
| 2673 | + struct device_attribute *attr, char *buf) | |
| 2674 | +{ | |
| 2675 | + struct mlx5_ib_dev *dev = | |
| 2676 | + rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 2941 | 2677 | |
| 2942 | − MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, | |
| 2943 | − ntohl(ib_spec->esp.mask.spi)); | |
| 2944 | − MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi, | |
| 2945 | − ntohl(ib_spec->esp.val.spi)); | |
| 2946 | − break; | |
| 2947 | − case IB_FLOW_SPEC_TCP: | |
| 2948 | − if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, | |
| 2949 | − LAST_TCP_UDP_FIELD)) | |
| 2950 | − return -EOPNOTSUPP; | |
| 2678 | + return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN, | |
| 2679 | + dev->mdev->board_id); | |
| 2680 | +} | |
| 2681 | +static DEVICE_ATTR_RO(board_id); | |
| 2951 | 2682 | |
| 2952 | − if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) | |
| 2953 | − return -EINVAL; | |
| 2683 | +static struct attribute *mlx5_class_attributes[] = { | |
| 2684 | + &dev_attr_hw_rev.attr, | |
| 2685 | + &dev_attr_hca_type.attr, | |
| 2686 | + &dev_attr_board_id.attr, | |
| 2687 | + &dev_attr_fw_pages.attr, | |
| 2688 | + &dev_attr_reg_pages.attr, | |
| 2689 | + NULL, | |
| 2690 | +}; | |
| 2954 | 2691 | |
| 2955 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, | |
| 2956 | − ntohs(ib_spec->tcp_udp.mask.src_port)); | |
| 2957 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, | |
| 2958 | − ntohs(ib_spec->tcp_udp.val.src_port)); | |
| 2692 | +static const struct attribute_group mlx5_attr_group = { | |
| 2693 | + .attrs = mlx5_class_attributes, | |
| 2694 | +}; | |
| 2959 | 2695 | |
| 2960 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, | |
| 2961 | − ntohs(ib_spec->tcp_udp.mask.dst_port)); | |
| 2962 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, | |
| 2963 | − ntohs(ib_spec->tcp_udp.val.dst_port)); | |
| 2964 | − break; | |
| 2965 | − case IB_FLOW_SPEC_UDP: | |
| 2966 | − if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, | |
| 2967 | − LAST_TCP_UDP_FIELD)) | |
| 2968 | − return -EOPNOTSUPP; | |
| 2696 | +static void pkey_change_handler(struct work_struct *work) | |
| 2697 | +{ | |
| 2698 | + struct mlx5_ib_port_resources *ports = | |
| 2699 | + container_of(work, struct mlx5_ib_port_resources, | |
| 2700 | + pkey_change_work); | |
| 2969 | 2701 | |
| 2970 | − if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) | |
| 2971 | − return -EINVAL; | |
| 2702 | + mutex_lock(&ports->devr->mutex); | |
| 2703 | + mlx5_ib_gsi_pkey_change(ports->gsi); | |
| 2704 | + mutex_unlock(&ports->devr->mutex); | |
| 2705 | +} | |
| 2972 | 2706 | |
| 2973 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, | |
| 2974 | − ntohs(ib_spec->tcp_udp.mask.src_port)); | |
| 2975 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, | |
| 2976 | − ntohs(ib_spec->tcp_udp.val.src_port)); | |
| 2707 | +static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev) | |
| 2708 | +{ | |
| 2709 | + struct mlx5_ib_qp *mqp; | |
| 2710 | + struct mlx5_ib_cq *send_mcq, *recv_mcq; | |
| 2711 | + struct mlx5_core_cq *mcq; | |
| 2712 | + struct list_head cq_armed_list; | |
| 2713 | + unsigned long flags_qp; | |
| 2714 | + unsigned long flags_cq; | |
| 2715 | + unsigned long flags; | |
| 2977 | 2716 | |
| 2978 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, | |
| 2979 | − ntohs(ib_spec->tcp_udp.mask.dst_port)); | |
| 2980 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, | |
| 2981 | − ntohs(ib_spec->tcp_udp.val.dst_port)); | |
| 2982 | − break; | |
| 2983 | − case IB_FLOW_SPEC_GRE: | |
| 2984 | − if (ib_spec->gre.mask.c_ks_res0_ver) | |
| 2985 | − return -EOPNOTSUPP; | |
| 2717 | + INIT_LIST_HEAD(&cq_armed_list); | |
| 2986 | 2718 | |
| 2987 | − if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) | |
| 2988 | − return -EINVAL; | |
| 2989 | − | |
| 2990 | − MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, | |
| 2991 | − 0xff); | |
| 2992 | − MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, | |
| 2993 | − IPPROTO_GRE); | |
| 2994 | − | |
| 2995 | − MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, | |
| 2996 | − ntohs(ib_spec->gre.mask.protocol)); | |
| 2997 | − MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, | |
| 2998 | − ntohs(ib_spec->gre.val.protocol)); | |
| 2999 | − | |
| 3000 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, | |
| 3001 | − gre_key.nvgre.hi), | |
| 3002 | − &ib_spec->gre.mask.key, | |
| 3003 | − sizeof(ib_spec->gre.mask.key)); | |
| 3004 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, | |
| 3005 | − gre_key.nvgre.hi), | |
| 3006 | − &ib_spec->gre.val.key, | |
| 3007 | − sizeof(ib_spec->gre.val.key)); | |
| 3008 | − break; | |
| 3009 | − case IB_FLOW_SPEC_MPLS: | |
| 3010 | − switch (prev_type) { | |
| 3011 | − case IB_FLOW_SPEC_UDP: | |
| 3012 | − if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 3013 | − ft_field_support.outer_first_mpls_over_udp), | |
| 3014 | − &ib_spec->mpls.mask.tag)) | |
| 3015 | − return -EOPNOTSUPP; | |
| 3016 | − | |
| 3017 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, | |
| 3018 | − outer_first_mpls_over_udp), | |
| 3019 | − &ib_spec->mpls.val.tag, | |
| 3020 | − sizeof(ib_spec->mpls.val.tag)); | |
| 3021 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, | |
| 3022 | − outer_first_mpls_over_udp), | |
| 3023 | − &ib_spec->mpls.mask.tag, | |
| 3024 | − sizeof(ib_spec->mpls.mask.tag)); | |
| 3025 | − break; | |
| 3026 | − case IB_FLOW_SPEC_GRE: | |
| 3027 | − if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 3028 | − ft_field_support.outer_first_mpls_over_gre), | |
| 3029 | − &ib_spec->mpls.mask.tag)) | |
| 3030 | − return -EOPNOTSUPP; | |
| 3031 | − | |
| 3032 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, | |
| 3033 | − outer_first_mpls_over_gre), | |
| 3034 | − &ib_spec->mpls.val.tag, | |
| 3035 | − sizeof(ib_spec->mpls.val.tag)); | |
| 3036 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, | |
| 3037 | − outer_first_mpls_over_gre), | |
| 3038 | − &ib_spec->mpls.mask.tag, | |
| 3039 | − sizeof(ib_spec->mpls.mask.tag)); | |
| 3040 | − break; | |
| 3041 | − default: | |
| 3042 | − if (ib_spec->type & IB_FLOW_SPEC_INNER) { | |
| 3043 | − if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 3044 | − ft_field_support.inner_first_mpls), | |
| 3045 | − &ib_spec->mpls.mask.tag)) | |
| 3046 | − return -EOPNOTSUPP; | |
| 3047 | − | |
| 3048 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, | |
| 3049 | − inner_first_mpls), | |
| 3050 | − &ib_spec->mpls.val.tag, | |
| 3051 | − sizeof(ib_spec->mpls.val.tag)); | |
| 3052 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, | |
| 3053 | − inner_first_mpls), | |
| 3054 | − &ib_spec->mpls.mask.tag, | |
| 3055 | − sizeof(ib_spec->mpls.mask.tag)); | |
| 3056 | − } else { | |
| 3057 | − if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 3058 | − ft_field_support.outer_first_mpls), | |
| 3059 | − &ib_spec->mpls.mask.tag)) | |
| 3060 | − return -EOPNOTSUPP; | |
| 3061 | − | |
| 3062 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, | |
| 3063 | − outer_first_mpls), | |
| 3064 | − &ib_spec->mpls.val.tag, | |
| 3065 | − sizeof(ib_spec->mpls.val.tag)); | |
| 3066 | − memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, | |
| 3067 | − outer_first_mpls), | |
| 3068 | − &ib_spec->mpls.mask.tag, | |
| 3069 | − sizeof(ib_spec->mpls.mask.tag)); | |
| 2719 | + /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/ | |
| 2720 | + spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags); | |
| 2721 | + list_for_each_entry(mqp, &ibdev->qp_list, qps_list) { | |
| 2722 | + spin_lock_irqsave(&mqp->sq.lock, flags_qp); | |
| 2723 | + if (mqp->sq.tail != mqp->sq.head) { | |
| 2724 | + send_mcq = to_mcq(mqp->ibqp.send_cq); | |
| 2725 | + spin_lock_irqsave(&send_mcq->lock, flags_cq); | |
| 2726 | + if (send_mcq->mcq.comp && | |
| 2727 | + mqp->ibqp.send_cq->comp_handler) { | |
| 2728 | + if (!send_mcq->mcq.reset_notify_added) { | |
| 2729 | + send_mcq->mcq.reset_notify_added = 1; | |
| 2730 | + list_add_tail(&send_mcq->mcq.reset_notify, | |
| 2731 | + &cq_armed_list); | |
| 2732 | + } | |
| 3070 | 2733 | } |
| 2734 | + spin_unlock_irqrestore(&send_mcq->lock, flags_cq); | |
| 3071 | 2735 | } |
| 3072 | − break; | |
| 3073 | − case IB_FLOW_SPEC_VXLAN_TUNNEL: | |
| 3074 | − if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, | |
| 3075 | − LAST_TUNNEL_FIELD)) | |
| 3076 | − return -EOPNOTSUPP; | |
| 3077 | − | |
| 3078 | − MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, | |
| 3079 | − ntohl(ib_spec->tunnel.mask.tunnel_id)); | |
| 3080 | − MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, | |
| 3081 | − ntohl(ib_spec->tunnel.val.tunnel_id)); | |
| 3082 | − break; | |
| 3083 | − case IB_FLOW_SPEC_ACTION_TAG: | |
| 3084 | − if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, | |
| 3085 | − LAST_FLOW_TAG_FIELD)) | |
| 3086 | − return -EOPNOTSUPP; | |
| 3087 | − if (ib_spec->flow_tag.tag_id >= BIT(24)) | |
| 3088 | − return -EINVAL; | |
| 3089 | − | |
| 3090 | − flow_context->flow_tag = ib_spec->flow_tag.tag_id; | |
| 3091 | − flow_context->flags |= FLOW_CONTEXT_HAS_TAG; | |
| 3092 | − break; | |
| 3093 | − case IB_FLOW_SPEC_ACTION_DROP: | |
| 3094 | − if (FIELDS_NOT_SUPPORTED(ib_spec->drop, | |
| 3095 | − LAST_DROP_FIELD)) | |
| 3096 | − return -EOPNOTSUPP; | |
| 3097 | − action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; | |
| 3098 | − break; | |
| 3099 | − case IB_FLOW_SPEC_ACTION_HANDLE: | |
| 3100 | − ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), | |
| 3101 | − flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); | |
| 3102 | − if (ret) | |
| 3103 | − return ret; | |
| 3104 | − break; | |
| 3105 | − case IB_FLOW_SPEC_ACTION_COUNT: | |
| 3106 | − if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, | |
| 3107 | − LAST_COUNTERS_FIELD)) | |
| 3108 | − return -EOPNOTSUPP; | |
| 3109 | − | |
| 3110 | − /* for now support only one counters spec per flow */ | |
| 3111 | − if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) | |
| 3112 | − return -EINVAL; | |
| 3113 | − | |
| 3114 | − action->counters = ib_spec->flow_count.counters; | |
| 3115 | − action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; | |
| 3116 | − break; | |
| 3117 | − default: | |
| 3118 | − return -EINVAL; | |
| 3119 | − } | |
| 3120 | − | |
| 3121 | − return 0; | |
| 3122 | −} | |
| 3123 | − | |
| 3124 | −/* If a flow could catch both multicast and unicast packets, | |
| 3125 | − * it won't fall into the multicast flow steering table and this rule | |
| 3126 | − * could steal other multicast packets. | |
| 3127 | − */ | |
| 3128 | −static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) | |
| 3129 | −{ | |
| 3130 | − union ib_flow_spec *flow_spec; | |
| 3131 | − | |
| 3132 | − if (ib_attr->type != IB_FLOW_ATTR_NORMAL || | |
| 3133 | − ib_attr->num_of_specs < 1) | |
| 3134 | − return false; | |
| 3135 | − | |
| 3136 | − flow_spec = (union ib_flow_spec *)(ib_attr + 1); | |
| 3137 | − if (flow_spec->type == IB_FLOW_SPEC_IPV4) { | |
| 3138 | − struct ib_flow_spec_ipv4 *ipv4_spec; | |
| 3139 | − | |
| 3140 | − ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; | |
| 3141 | − if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) | |
| 3142 | − return true; | |
| 3143 | − | |
| 3144 | − return false; | |
| 2736 | + spin_unlock_irqrestore(&mqp->sq.lock, flags_qp); | |
| 2737 | + spin_lock_irqsave(&mqp->rq.lock, flags_qp); | |
| 2738 | + /* no handling is needed for SRQ */ | |
| 2739 | + if (!mqp->ibqp.srq) { | |
| 2740 | + if (mqp->rq.tail != mqp->rq.head) { | |
| 2741 | + recv_mcq = to_mcq(mqp->ibqp.recv_cq); | |
| 2742 | + spin_lock_irqsave(&recv_mcq->lock, flags_cq); | |
| 2743 | + if (recv_mcq->mcq.comp && | |
| 2744 | + mqp->ibqp.recv_cq->comp_handler) { | |
| 2745 | + if (!recv_mcq->mcq.reset_notify_added) { | |
| 2746 | + recv_mcq->mcq.reset_notify_added = 1; | |
| 2747 | + list_add_tail(&recv_mcq->mcq.reset_notify, | |
| 2748 | + &cq_armed_list); | |
| 2749 | + } | |
| 2750 | + } | |
| 2751 | + spin_unlock_irqrestore(&recv_mcq->lock, | |
| 2752 | + flags_cq); | |
| 2753 | + } | |
| 2754 | + } | |
| 2755 | + spin_unlock_irqrestore(&mqp->rq.lock, flags_qp); | |
| 3145 | 2756 | } |
| 3146 | − | |
| 3147 | − if (flow_spec->type == IB_FLOW_SPEC_ETH) { | |
| 3148 | − struct ib_flow_spec_eth *eth_spec; | |
| 3149 | − | |
| 3150 | − eth_spec = (struct ib_flow_spec_eth *)flow_spec; | |
| 3151 | − return is_multicast_ether_addr(eth_spec->mask.dst_mac) && | |
| 3152 | − is_multicast_ether_addr(eth_spec->val.dst_mac); | |
| 2757 | + /*At that point all inflight post send were put to be executed as of we | |
| 2758 | + * lock/unlock above locks Now need to arm all involved CQs. | |
| 2759 | + */ | |
| 2760 | + list_for_each_entry(mcq, &cq_armed_list, reset_notify) { | |
| 2761 | + mcq->comp(mcq, NULL); | |
| 3153 | 2762 | } |
| 3154 | − | |
| 3155 | − return false; | |
| 2763 | + spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags); | |
| 3156 | 2764 | } |
| 3157 | 2765 | |
| 3158 | −enum valid_spec { | |
| 3159 | − VALID_SPEC_INVALID, | |
| 3160 | − VALID_SPEC_VALID, | |
| 3161 | − VALID_SPEC_NA, | |
| 3162 | −}; | |
| 3163 | − | |
| 3164 | −static enum valid_spec | |
| 3165 | −is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev, | |
| 3166 | − const struct mlx5_flow_spec *spec, | |
| 3167 | − const struct mlx5_flow_act *flow_act, | |
| 3168 | − bool egress) | |
| 2766 | +static void delay_drop_handler(struct work_struct *work) | |
| 3169 | 2767 | { |
| 3170 | − const u32 *match_c = spec->match_criteria; | |
| 3171 | − bool is_crypto = | |
| 3172 | − (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | | |
| 3173 | − MLX5_FLOW_CONTEXT_ACTION_DECRYPT)); | |
| 3174 | − bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c); | |
| 3175 | − bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP; | |
| 3176 | − | |
| 3177 | − /* | |
| 3178 | − * Currently only crypto is supported in egress, when regular egress | |
| 3179 | − * rules would be supported, always return VALID_SPEC_NA. | |
| 3180 | − */ | |
| 3181 | − if (!is_crypto) | |
| 3182 | − return VALID_SPEC_NA; | |
| 3183 | − | |
| 3184 | − return is_crypto && is_ipsec && | |
| 3185 | − (!egress || (!is_drop && | |
| 3186 | − !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ? | |
| 3187 | − VALID_SPEC_VALID : VALID_SPEC_INVALID; | |
| 3188 | −} | |
| 3189 | − | |
| 3190 | −static bool is_valid_spec(struct mlx5_core_dev *mdev, | |
| 3191 | − const struct mlx5_flow_spec *spec, | |
| 3192 | − const struct mlx5_flow_act *flow_act, | |
| 3193 | − bool egress) | |
| 3194 | −{ | |
| 3195 | − /* We curretly only support ipsec egress flow */ | |
| 3196 | − return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID; | |
| 3197 | −} | |
| 3198 | − | |
| 3199 | −static bool is_valid_ethertype(struct mlx5_core_dev *mdev, | |
| 3200 | − const struct ib_flow_attr *flow_attr, | |
| 3201 | − bool check_inner) | |
| 3202 | −{ | |
| 3203 | − union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); | |
| 3204 | − int match_ipv = check_inner ? | |
| 3205 | − MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 3206 | − ft_field_support.inner_ip_version) : | |
| 3207 | − MLX5_CAP_FLOWTABLE_NIC_RX(mdev, | |
| 3208 | − ft_field_support.outer_ip_version); | |
| 3209 | − int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; | |
| 3210 | − bool ipv4_spec_valid, ipv6_spec_valid; | |
| 3211 | − unsigned int ip_spec_type = 0; | |
| 3212 | − bool has_ethertype = false; | |
| 3213 | − unsigned int spec_index; | |
| 3214 | − bool mask_valid = true; | |
| 3215 | − u16 eth_type = 0; | |
| 3216 | − bool type_valid; | |
| 3217 | − | |
| 3218 | − /* Validate that ethertype is correct */ | |
| 3219 | − for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { | |
| 3220 | − if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && | |
| 3221 | − ib_spec->eth.mask.ether_type) { | |
| 3222 | − mask_valid = (ib_spec->eth.mask.ether_type == | |
| 3223 | − htons(0xffff)); | |
| 3224 | − has_ethertype = true; | |
| 3225 | − eth_type = ntohs(ib_spec->eth.val.ether_type); | |
| 3226 | − } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || | |
| 3227 | − (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { | |
| 3228 | − ip_spec_type = ib_spec->type; | |
| 3229 | − } | |
| 3230 | − ib_spec = (void *)ib_spec + ib_spec->size; | |
| 3231 | − } | |
| 2768 | + int err; | |
| 2769 | + struct mlx5_ib_delay_drop *delay_drop = | |
| 2770 | + container_of(work, struct mlx5_ib_delay_drop, | |
| 2771 | + delay_drop_work); | |
| 3232 | 2772 | |
| 3233 | − type_valid = (!has_ethertype) || (!ip_spec_type); | |
| 3234 | − if (!type_valid && mask_valid) { | |
| 3235 | − ipv4_spec_valid = (eth_type == ETH_P_IP) && | |
| 3236 | − (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); | |
| 3237 | − ipv6_spec_valid = (eth_type == ETH_P_IPV6) && | |
| 3238 | − (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); | |
| 2773 | + atomic_inc(&delay_drop->events_cnt); | |
| 3239 | 2774 | |
| 3240 | − type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || | |
| 3241 | − (((eth_type == ETH_P_MPLS_UC) || | |
| 3242 | − (eth_type == ETH_P_MPLS_MC)) && match_ipv); | |
| 2775 | + mutex_lock(&delay_drop->lock); | |
| 2776 | + err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout); | |
| 2777 | + if (err) { | |
| 2778 | + mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n", | |
| 2779 | + delay_drop->timeout); | |
| 2780 | + delay_drop->activate = false; | |
| 3243 | 2781 | } |
| 3244 | − | |
| 3245 | − return type_valid; | |
| 2782 | + mutex_unlock(&delay_drop->lock); | |
| 3246 | 2783 | } |
| 3247 | 2784 | |
| 3248 | −static bool is_valid_attr(struct mlx5_core_dev *mdev, | |
| 3249 | − const struct ib_flow_attr *flow_attr) | |
| 2785 | +static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe, | |
| 2786 | + struct ib_event *ibev) | |
| 3250 | 2787 | { |
| 3251 | − return is_valid_ethertype(mdev, flow_attr, false) && | |
| 3252 | − is_valid_ethertype(mdev, flow_attr, true); | |
| 3253 | −} | |
| 2788 | + u8 port = (eqe->data.port.port >> 4) & 0xf; | |
| 3254 | 2789 | |
| 3255 | −static void put_flow_table(struct mlx5_ib_dev *dev, | |
| 3256 | − struct mlx5_ib_flow_prio *prio, bool ft_added) | |
| 3257 | −{ | |
| 3258 | − prio->refcount -= !!ft_added; | |
| 3259 | − if (!prio->refcount) { | |
| 3260 | − mlx5_destroy_flow_table(prio->flow_table); | |
| 3261 | − prio->flow_table = NULL; | |
| 2790 | + switch (eqe->sub_type) { | |
| 2791 | + case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT: | |
| 2792 | + if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) == | |
| 2793 | + IB_LINK_LAYER_ETHERNET) | |
| 2794 | + schedule_work(&ibdev->delay_drop.delay_drop_work); | |
| 2795 | + break; | |
| 2796 | + default: /* do nothing */ | |
| 2797 | + return; | |
| 3262 | 2798 | } |
| 3263 | 2799 | } |
| 3264 | 2800 | |
| 3265 | −static void counters_clear_description(struct ib_counters *counters) | |
| 2801 | +static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe, | |
| 2802 | + struct ib_event *ibev) | |
| 3266 | 2803 | { |
| 3267 | − struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); | |
| 2804 | + u8 port = (eqe->data.port.port >> 4) & 0xf; | |
| 3268 | 2805 | |
| 3269 | − mutex_lock(&mcounters->mcntrs_mutex); | |
| 3270 | − kfree(mcounters->counters_data); | |
| 3271 | − mcounters->counters_data = NULL; | |
| 3272 | − mcounters->cntrs_max_index = 0; | |
| 3273 | − mutex_unlock(&mcounters->mcntrs_mutex); | |
| 3274 | −} | |
| 2806 | + ibev->element.port_num = port; | |
| 3275 | 2807 | |
| 3276 | −static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) | |
| 3277 | −{ | |
| 3278 | − struct mlx5_ib_flow_handler *handler = container_of(flow_id, | |
| 3279 | − struct mlx5_ib_flow_handler, | |
| 3280 | − ibflow); | |
| 3281 | − struct mlx5_ib_flow_handler *iter, *tmp; | |
| 3282 | − struct mlx5_ib_dev *dev = handler->dev; | |
| 2808 | + switch (eqe->sub_type) { | |
| 2809 | + case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: | |
| 2810 | + case MLX5_PORT_CHANGE_SUBTYPE_DOWN: | |
| 2811 | + case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED: | |
| 2812 | + /* In RoCE, port up/down events are handled in | |
| 2813 | + * mlx5_netdev_event(). | |
| 2814 | + */ | |
| 2815 | + if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) == | |
| 2816 | + IB_LINK_LAYER_ETHERNET) | |
| 2817 | + return -EINVAL; | |
| 2818 | + | |
| 2819 | + ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ? | |
| 2820 | + IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; | |
| 2821 | + break; | |
| 3283 | 2822 | |
| 3284 | − mutex_lock(&dev->flow_db->lock); | |
| 2823 | + case MLX5_PORT_CHANGE_SUBTYPE_LID: | |
| 2824 | + ibev->event = IB_EVENT_LID_CHANGE; | |
| 2825 | + break; | |
| 3285 | 2826 | |
| 3286 | − list_for_each_entry_safe(iter, tmp, &handler->list, list) { | |
| 3287 | − mlx5_del_flow_rules(iter->rule); | |
| 3288 | − put_flow_table(dev, iter->prio, true); | |
| 3289 | − list_del(&iter->list); | |
| 3290 | − kfree(iter); | |
| 3291 | − } | |
| 2827 | + case MLX5_PORT_CHANGE_SUBTYPE_PKEY: | |
| 2828 | + ibev->event = IB_EVENT_PKEY_CHANGE; | |
| 2829 | + schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work); | |
| 2830 | + break; | |
| 3292 | 2831 | |
| 3293 | − mlx5_del_flow_rules(handler->rule); | |
| 3294 | − put_flow_table(dev, handler->prio, true); | |
| 3295 | − if (handler->ibcounters && | |
| 3296 | − atomic_read(&handler->ibcounters->usecnt) == 1) | |
| 3297 | − counters_clear_description(handler->ibcounters); | |
| 2832 | + case MLX5_PORT_CHANGE_SUBTYPE_GUID: | |
| 2833 | + ibev->event = IB_EVENT_GID_CHANGE; | |
| 2834 | + break; | |
| 3298 | 2835 | |
| 3299 | − mutex_unlock(&dev->flow_db->lock); | |
| 3300 | − if (handler->flow_matcher) | |
| 3301 | − atomic_dec(&handler->flow_matcher->usecnt); | |
| 3302 | − kfree(handler); | |
| 2836 | + case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG: | |
| 2837 | + ibev->event = IB_EVENT_CLIENT_REREGISTER; | |
| 2838 | + break; | |
| 2839 | + default: | |
| 2840 | + return -EINVAL; | |
| 2841 | + } | |
| 3303 | 2842 | |
| 3304 | 2843 | return 0; |
| 3305 | 2844 | } |
| 3306 | 2845 | |
| 3307 | −static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) | |
| 3308 | −{ | |
| 3309 | − priority *= 2; | |
| 3310 | − if (!dont_trap) | |
| 3311 | − priority++; | |
| 3312 | − return priority; | |
| 3313 | −} | |
| 3314 | − | |
| 3315 | −enum flow_table_type { | |
| 3316 | − MLX5_IB_FT_RX, | |
| 3317 | − MLX5_IB_FT_TX | |
| 3318 | −}; | |
| 3319 | − | |
| 3320 | −#define MLX5_FS_MAX_TYPES 6 | |
| 3321 | −#define MLX5_FS_MAX_ENTRIES BIT(16) | |
| 3322 | − | |
| 3323 | −static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns, | |
| 3324 | − struct mlx5_ib_flow_prio *prio, | |
| 3325 | − int priority, | |
| 3326 | − int num_entries, int num_groups, | |
| 3327 | − u32 flags) | |
| 2846 | +static void mlx5_ib_handle_event(struct work_struct *_work) | |
| 3328 | 2847 | { |
| 3329 | − struct mlx5_flow_table_attr ft_attr = {}; | |
| 3330 | − struct mlx5_flow_table *ft; | |
| 3331 | − | |
| 3332 | − ft_attr.prio = priority; | |
| 3333 | − ft_attr.max_fte = num_entries; | |
| 3334 | − ft_attr.flags = flags; | |
| 3335 | − ft_attr.autogroup.max_num_groups = num_groups; | |
| 3336 | − ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); | |
| 3337 | − if (IS_ERR(ft)) | |
| 3338 | − return ERR_CAST(ft); | |
| 3339 | − | |
| 3340 | − prio->flow_table = ft; | |
| 3341 | − prio->refcount = 0; | |
| 3342 | − return prio; | |
| 3343 | −} | |
| 2848 | + struct mlx5_ib_event_work *work = | |
| 2849 | + container_of(_work, struct mlx5_ib_event_work, work); | |
| 2850 | + struct mlx5_ib_dev *ibdev; | |
| 2851 | + struct ib_event ibev; | |
| 2852 | + bool fatal = false; | |
| 3344 | 2853 | |
| 3345 | −static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, | |
| 3346 | − struct ib_flow_attr *flow_attr, | |
| 3347 | − enum flow_table_type ft_type) | |
| 3348 | −{ | |
| 3349 | − bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; | |
| 3350 | − struct mlx5_flow_namespace *ns = NULL; | |
| 3351 | − struct mlx5_ib_flow_prio *prio; | |
| 3352 | − struct mlx5_flow_table *ft; | |
| 3353 | − int max_table_size; | |
| 3354 | − int num_entries; | |
| 3355 | − int num_groups; | |
| 3356 | − bool esw_encap; | |
| 3357 | − u32 flags = 0; | |
| 3358 | − int priority; | |
| 3359 | − | |
| 3360 | − max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 3361 | − log_max_ft_size)); | |
| 3362 | − esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != | |
| 3363 | − DEVLINK_ESWITCH_ENCAP_MODE_NONE; | |
| 3364 | − if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { | |
| 3365 | − enum mlx5_flow_namespace_type fn_type; | |
| 3366 | − | |
| 3367 | − if (flow_is_multicast_only(flow_attr) && | |
| 3368 | − !dont_trap) | |
| 3369 | − priority = MLX5_IB_FLOW_MCAST_PRIO; | |
| 3370 | − else | |
| 3371 | − priority = ib_prio_to_core_prio(flow_attr->priority, | |
| 3372 | − dont_trap); | |
| 3373 | − if (ft_type == MLX5_IB_FT_RX) { | |
| 3374 | − fn_type = MLX5_FLOW_NAMESPACE_BYPASS; | |
| 3375 | − prio = &dev->flow_db->prios[priority]; | |
| 3376 | − if (!dev->is_rep && !esw_encap && | |
| 3377 | − MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) | |
| 3378 | − flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; | |
| 3379 | − if (!dev->is_rep && !esw_encap && | |
| 3380 | − MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 3381 | − reformat_l3_tunnel_to_l2)) | |
| 3382 | − flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 3383 | − } else { | |
| 3384 | − max_table_size = | |
| 3385 | − BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, | |
| 3386 | − log_max_ft_size)); | |
| 3387 | − fn_type = MLX5_FLOW_NAMESPACE_EGRESS; | |
| 3388 | − prio = &dev->flow_db->egress_prios[priority]; | |
| 3389 | − if (!dev->is_rep && !esw_encap && | |
| 3390 | − MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) | |
| 3391 | − flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 3392 | − } | |
| 3393 | − ns = mlx5_get_flow_namespace(dev->mdev, fn_type); | |
| 3394 | − num_entries = MLX5_FS_MAX_ENTRIES; | |
| 3395 | − num_groups = MLX5_FS_MAX_TYPES; | |
| 3396 | − } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || | |
| 3397 | − flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { | |
| 3398 | − ns = mlx5_get_flow_namespace(dev->mdev, | |
| 3399 | − MLX5_FLOW_NAMESPACE_LEFTOVERS); | |
| 3400 | − build_leftovers_ft_param(&priority, | |
| 3401 | − &num_entries, | |
| 3402 | − &num_groups); | |
| 3403 | − prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; | |
| 3404 | − } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { | |
| 3405 | − if (!MLX5_CAP_FLOWTABLE(dev->mdev, | |
| 3406 | − allow_sniffer_and_nic_rx_shared_tir)) | |
| 3407 | − return ERR_PTR(-ENOTSUPP); | |
| 3408 | − | |
| 3409 | − ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ? | |
| 3410 | − MLX5_FLOW_NAMESPACE_SNIFFER_RX : | |
| 3411 | − MLX5_FLOW_NAMESPACE_SNIFFER_TX); | |
| 3412 | − | |
| 3413 | − prio = &dev->flow_db->sniffer[ft_type]; | |
| 3414 | − priority = 0; | |
| 3415 | − num_entries = 1; | |
| 3416 | − num_groups = 1; | |
| 2854 | + if (work->is_slave) { | |
| 2855 | + ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi); | |
| 2856 | + if (!ibdev) | |
| 2857 | + goto out; | |
| 2858 | + } else { | |
| 2859 | + ibdev = work->dev; | |
| 3417 | 2860 | } |
| 3418 | 2861 | |
| 3419 | − if (!ns) | |
| 3420 | − return ERR_PTR(-ENOTSUPP); | |
| 3421 | − | |
| 3422 | − max_table_size = min_t(int, num_entries, max_table_size); | |
| 3423 | − | |
| 3424 | − ft = prio->flow_table; | |
| 3425 | − if (!ft) | |
| 3426 | − return _get_prio(ns, prio, priority, max_table_size, num_groups, | |
| 3427 | − flags); | |
| 3428 | − | |
| 3429 | − return prio; | |
| 3430 | −} | |
| 2862 | + switch (work->event) { | |
| 2863 | + case MLX5_DEV_EVENT_SYS_ERROR: | |
| 2864 | + ibev.event = IB_EVENT_DEVICE_FATAL; | |
| 2865 | + mlx5_ib_handle_internal_error(ibdev); | |
| 2866 | + ibev.element.port_num = (u8)(unsigned long)work->param; | |
| 2867 | + fatal = true; | |
| 2868 | + break; | |
| 2869 | + case MLX5_EVENT_TYPE_PORT_CHANGE: | |
| 2870 | + if (handle_port_change(ibdev, work->param, &ibev)) | |
| 2871 | + goto out; | |
| 2872 | + break; | |
| 2873 | + case MLX5_EVENT_TYPE_GENERAL_EVENT: | |
| 2874 | + handle_general_event(ibdev, work->param, &ibev); | |
| 2875 | + /* fall through */ | |
| 2876 | + default: | |
| 2877 | + goto out; | |
| 2878 | + } | |
| 3431 | 2879 | |
| 3432 | −static void set_underlay_qp(struct mlx5_ib_dev *dev, | |
| 3433 | − struct mlx5_flow_spec *spec, | |
| 3434 | − u32 underlay_qpn) | |
| 3435 | −{ | |
| 3436 | − void *misc_params_c = MLX5_ADDR_OF(fte_match_param, | |
| 3437 | − spec->match_criteria, | |
| 3438 | − misc_parameters); | |
| 3439 | − void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, | |
| 3440 | − misc_parameters); | |
| 2880 | + ibev.device = &ibdev->ib_dev; | |
| 3441 | 2881 | |
| 3442 | − if (underlay_qpn && | |
| 3443 | − MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 3444 | − ft_field_support.bth_dst_qp)) { | |
| 3445 | − MLX5_SET(fte_match_set_misc, | |
| 3446 | − misc_params_v, bth_dst_qp, underlay_qpn); | |
| 3447 | − MLX5_SET(fte_match_set_misc, | |
| 3448 | − misc_params_c, bth_dst_qp, 0xffffff); | |
| 2882 | + if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) { | |
| 2883 | + mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num); | |
| 2884 | + goto out; | |
| 3449 | 2885 | } |
| 3450 | −} | |
| 3451 | 2886 | |
| 3452 | −static int read_flow_counters(struct ib_device *ibdev, | |
| 3453 | − struct mlx5_read_counters_attr *read_attr) | |
| 3454 | −{ | |
| 3455 | − struct mlx5_fc *fc = read_attr->hw_cntrs_hndl; | |
| 3456 | − struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 2887 | + if (ibdev->ib_active) | |
| 2888 | + ib_dispatch_event(&ibev); | |
| 3457 | 2889 | |
| 3458 | − return mlx5_fc_query(dev->mdev, fc, | |
| 3459 | − &read_attr->out[IB_COUNTER_PACKETS], | |
| 3460 | − &read_attr->out[IB_COUNTER_BYTES]); | |
| 2890 | + if (fatal) | |
| 2891 | + ibdev->ib_active = false; | |
| 2892 | +out: | |
| 2893 | + kfree(work); | |
| 3461 | 2894 | } |
| 3462 | 2895 | |
| 3463 | −/* flow counters currently expose two counters packets and bytes */ | |
| 3464 | −#define FLOW_COUNTERS_NUM 2 | |
| 3465 | −static int counters_set_description(struct ib_counters *counters, | |
| 3466 | − enum mlx5_ib_counters_type counters_type, | |
| 3467 | − struct mlx5_ib_flow_counters_desc *desc_data, | |
| 3468 | − u32 ncounters) | |
| 2896 | +static int mlx5_ib_event(struct notifier_block *nb, | |
| 2897 | + unsigned long event, void *param) | |
| 3469 | 2898 | { |
| 3470 | − struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); | |
| 3471 | − u32 cntrs_max_index = 0; | |
| 3472 | − int i; | |
| 3473 | − | |
| 3474 | − if (counters_type != MLX5_IB_COUNTERS_FLOW) | |
| 3475 | − return -EINVAL; | |
| 2899 | + struct mlx5_ib_event_work *work; | |
| 3476 | 2900 | |
| 3477 | − /* init the fields for the object */ | |
| 3478 | − mcounters->type = counters_type; | |
| 3479 | − mcounters->read_counters = read_flow_counters; | |
| 3480 | − mcounters->counters_num = FLOW_COUNTERS_NUM; | |
| 3481 | − mcounters->ncounters = ncounters; | |
| 3482 | − /* each counter entry have both description and index pair */ | |
| 3483 | − for (i = 0; i < ncounters; i++) { | |
| 3484 | − if (desc_data[i].description > IB_COUNTER_BYTES) | |
| 3485 | − return -EINVAL; | |
| 2901 | + work = kmalloc(sizeof(*work), GFP_ATOMIC); | |
| 2902 | + if (!work) | |
| 2903 | + return NOTIFY_DONE; | |
| 3486 | 2904 | |
| 3487 | − if (cntrs_max_index <= desc_data[i].index) | |
| 3488 | − cntrs_max_index = desc_data[i].index + 1; | |
| 3489 | − } | |
| 2905 | + INIT_WORK(&work->work, mlx5_ib_handle_event); | |
| 2906 | + work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events); | |
| 2907 | + work->is_slave = false; | |
| 2908 | + work->param = param; | |
| 2909 | + work->event = event; | |
| 3490 | 2910 | |
| 3491 | − mutex_lock(&mcounters->mcntrs_mutex); | |
| 3492 | − mcounters->counters_data = desc_data; | |
| 3493 | − mcounters->cntrs_max_index = cntrs_max_index; | |
| 3494 | − mutex_unlock(&mcounters->mcntrs_mutex); | |
| 2911 | + queue_work(mlx5_ib_event_wq, &work->work); | |
| 3495 | 2912 | |
| 3496 | − return 0; | |
| 2913 | + return NOTIFY_OK; | |
| 3497 | 2914 | } |
| 3498 | 2915 | |
| 3499 | −#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2)) | |
| 3500 | −static int flow_counters_set_data(struct ib_counters *ibcounters, | |
| 3501 | − struct mlx5_ib_create_flow *ucmd) | |
| 2916 | +static int mlx5_ib_event_slave_port(struct notifier_block *nb, | |
| 2917 | + unsigned long event, void *param) | |
| 3502 | 2918 | { |
| 3503 | − struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters); | |
| 3504 | − struct mlx5_ib_flow_counters_data *cntrs_data = NULL; | |
| 3505 | − struct mlx5_ib_flow_counters_desc *desc_data = NULL; | |
| 3506 | − bool hw_hndl = false; | |
| 3507 | − int ret = 0; | |
| 3508 | − | |
| 3509 | − if (ucmd && ucmd->ncounters_data != 0) { | |
| 3510 | − cntrs_data = ucmd->data; | |
| 3511 | − if (cntrs_data->ncounters > MAX_COUNTERS_NUM) | |
| 3512 | − return -EINVAL; | |
| 3513 | − | |
| 3514 | − desc_data = kcalloc(cntrs_data->ncounters, | |
| 3515 | − sizeof(*desc_data), | |
| 3516 | − GFP_KERNEL); | |
| 3517 | − if (!desc_data) | |
| 3518 | − return -ENOMEM; | |
| 3519 | − | |
| 3520 | − if (copy_from_user(desc_data, | |
| 3521 | − u64_to_user_ptr(cntrs_data->counters_data), | |
| 3522 | − sizeof(*desc_data) * cntrs_data->ncounters)) { | |
| 3523 | − ret = -EFAULT; | |
| 3524 | − goto free; | |
| 3525 | − } | |
| 3526 | − } | |
| 3527 | − | |
| 3528 | − if (!mcounters->hw_cntrs_hndl) { | |
| 3529 | − mcounters->hw_cntrs_hndl = mlx5_fc_create( | |
| 3530 | − to_mdev(ibcounters->device)->mdev, false); | |
| 3531 | − if (IS_ERR(mcounters->hw_cntrs_hndl)) { | |
| 3532 | − ret = PTR_ERR(mcounters->hw_cntrs_hndl); | |
| 3533 | − goto free; | |
| 3534 | − } | |
| 3535 | − hw_hndl = true; | |
| 3536 | − } | |
| 3537 | − | |
| 3538 | − if (desc_data) { | |
| 3539 | − /* counters already bound to at least one flow */ | |
| 3540 | − if (mcounters->cntrs_max_index) { | |
| 3541 | − ret = -EINVAL; | |
| 3542 | − goto free_hndl; | |
| 3543 | − } | |
| 3544 | − | |
| 3545 | − ret = counters_set_description(ibcounters, | |
| 3546 | − MLX5_IB_COUNTERS_FLOW, | |
| 3547 | − desc_data, | |
| 3548 | − cntrs_data->ncounters); | |
| 3549 | − if (ret) | |
| 3550 | − goto free_hndl; | |
| 3551 | − | |
| 3552 | − } else if (!mcounters->cntrs_max_index) { | |
| 3553 | − /* counters not bound yet, must have udata passed */ | |
| 3554 | − ret = -EINVAL; | |
| 3555 | − goto free_hndl; | |
| 3556 | − } | |
| 3557 | − | |
| 3558 | − return 0; | |
| 3559 | − | |
| 3560 | −free_hndl: | |
| 3561 | − if (hw_hndl) { | |
| 3562 | − mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev, | |
| 3563 | − mcounters->hw_cntrs_hndl); | |
| 3564 | − mcounters->hw_cntrs_hndl = NULL; | |
| 3565 | − } | |
| 3566 | −free: | |
| 3567 | − kfree(desc_data); | |
| 3568 | − return ret; | |
| 3569 | −} | |
| 3570 | − | |
| 3571 | −static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, | |
| 3572 | − struct mlx5_flow_spec *spec, | |
| 3573 | − struct mlx5_eswitch_rep *rep) | |
| 3574 | −{ | |
| 3575 | − struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; | |
| 3576 | − void *misc; | |
| 3577 | − | |
| 3578 | − if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { | |
| 3579 | − misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, | |
| 3580 | − misc_parameters_2); | |
| 3581 | − | |
| 3582 | − MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, | |
| 3583 | − mlx5_eswitch_get_vport_metadata_for_match(esw, | |
| 3584 | − rep->vport)); | |
| 3585 | − misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, | |
| 3586 | − misc_parameters_2); | |
| 3587 | − | |
| 3588 | − MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, | |
| 3589 | − mlx5_eswitch_get_vport_metadata_mask()); | |
| 3590 | − } else { | |
| 3591 | − misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, | |
| 3592 | − misc_parameters); | |
| 3593 | − | |
| 3594 | − MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); | |
| 3595 | − | |
| 3596 | − misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, | |
| 3597 | − misc_parameters); | |
| 3598 | − | |
| 3599 | − MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); | |
| 3600 | − } | |
| 3601 | −} | |
| 3602 | − | |
| 3603 | −static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, | |
| 3604 | − struct mlx5_ib_flow_prio *ft_prio, | |
| 3605 | − const struct ib_flow_attr *flow_attr, | |
| 3606 | − struct mlx5_flow_destination *dst, | |
| 3607 | − u32 underlay_qpn, | |
| 3608 | − struct mlx5_ib_create_flow *ucmd) | |
| 3609 | −{ | |
| 3610 | − struct mlx5_flow_table *ft = ft_prio->flow_table; | |
| 3611 | − struct mlx5_ib_flow_handler *handler; | |
| 3612 | − struct mlx5_flow_act flow_act = {}; | |
| 3613 | − struct mlx5_flow_spec *spec; | |
| 3614 | − struct mlx5_flow_destination dest_arr[2] = {}; | |
| 3615 | − struct mlx5_flow_destination *rule_dst = dest_arr; | |
| 3616 | − const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); | |
| 3617 | − unsigned int spec_index; | |
| 3618 | − u32 prev_type = 0; | |
| 3619 | − int err = 0; | |
| 3620 | − int dest_num = 0; | |
| 3621 | − bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; | |
| 3622 | − | |
| 3623 | − if (!is_valid_attr(dev->mdev, flow_attr)) | |
| 3624 | − return ERR_PTR(-EINVAL); | |
| 3625 | − | |
| 3626 | − if (dev->is_rep && is_egress) | |
| 3627 | − return ERR_PTR(-EINVAL); | |
| 3628 | − | |
| 3629 | − spec = kvzalloc(sizeof(*spec), GFP_KERNEL); | |
| 3630 | − handler = kzalloc(sizeof(*handler), GFP_KERNEL); | |
| 3631 | − if (!handler || !spec) { | |
| 3632 | − err = -ENOMEM; | |
| 3633 | − goto free; | |
| 3634 | − } | |
| 3635 | − | |
| 3636 | − INIT_LIST_HEAD(&handler->list); | |
| 3637 | − | |
| 3638 | − for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { | |
| 3639 | − err = parse_flow_attr(dev->mdev, spec, | |
| 3640 | − ib_flow, flow_attr, &flow_act, | |
| 3641 | − prev_type); | |
| 3642 | − if (err < 0) | |
| 3643 | − goto free; | |
| 3644 | − | |
| 3645 | − prev_type = ((union ib_flow_spec *)ib_flow)->type; | |
| 3646 | − ib_flow += ((union ib_flow_spec *)ib_flow)->size; | |
| 3647 | − } | |
| 3648 | − | |
| 3649 | − if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { | |
| 3650 | − memcpy(&dest_arr[0], dst, sizeof(*dst)); | |
| 3651 | − dest_num++; | |
| 3652 | − } | |
| 3653 | − | |
| 3654 | − if (!flow_is_multicast_only(flow_attr)) | |
| 3655 | − set_underlay_qp(dev, spec, underlay_qpn); | |
| 3656 | − | |
| 3657 | − if (dev->is_rep) { | |
| 3658 | − struct mlx5_eswitch_rep *rep; | |
| 3659 | − | |
| 3660 | − rep = dev->port[flow_attr->port - 1].rep; | |
| 3661 | − if (!rep) { | |
| 3662 | − err = -EINVAL; | |
| 3663 | − goto free; | |
| 3664 | − } | |
| 3665 | − | |
| 3666 | − mlx5_ib_set_rule_source_port(dev, spec, rep); | |
| 3667 | − } | |
| 3668 | − | |
| 3669 | − spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); | |
| 3670 | − | |
| 3671 | − if (is_egress && | |
| 3672 | − !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) { | |
| 3673 | − err = -EINVAL; | |
| 3674 | − goto free; | |
| 3675 | − } | |
| 3676 | − | |
| 3677 | − if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { | |
| 3678 | − struct mlx5_ib_mcounters *mcounters; | |
| 3679 | − | |
| 3680 | − err = flow_counters_set_data(flow_act.counters, ucmd); | |
| 3681 | − if (err) | |
| 3682 | − goto free; | |
| 3683 | − | |
| 3684 | − mcounters = to_mcounters(flow_act.counters); | |
| 3685 | − handler->ibcounters = flow_act.counters; | |
| 3686 | − dest_arr[dest_num].type = | |
| 3687 | − MLX5_FLOW_DESTINATION_TYPE_COUNTER; | |
| 3688 | − dest_arr[dest_num].counter_id = | |
| 3689 | − mlx5_fc_id(mcounters->hw_cntrs_hndl); | |
| 3690 | − dest_num++; | |
| 3691 | − } | |
| 3692 | − | |
| 3693 | − if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { | |
| 3694 | − if (!dest_num) | |
| 3695 | − rule_dst = NULL; | |
| 3696 | − } else { | |
| 3697 | − if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) | |
| 3698 | − flow_act.action |= | |
| 3699 | − MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; | |
| 3700 | − if (is_egress) | |
| 3701 | − flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; | |
| 3702 | − else if (dest_num) | |
| 3703 | − flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; | |
| 3704 | − } | |
| 3705 | − | |
| 3706 | − if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && | |
| 3707 | − (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || | |
| 3708 | − flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { | |
| 3709 | − mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", | |
| 3710 | − spec->flow_context.flow_tag, flow_attr->type); | |
| 3711 | − err = -EINVAL; | |
| 3712 | − goto free; | |
| 3713 | − } | |
| 3714 | − handler->rule = mlx5_add_flow_rules(ft, spec, | |
| 3715 | − &flow_act, | |
| 3716 | − rule_dst, dest_num); | |
| 3717 | − | |
| 3718 | − if (IS_ERR(handler->rule)) { | |
| 3719 | − err = PTR_ERR(handler->rule); | |
| 3720 | − goto free; | |
| 3721 | − } | |
| 3722 | − | |
| 3723 | − ft_prio->refcount++; | |
| 3724 | − handler->prio = ft_prio; | |
| 3725 | − handler->dev = dev; | |
| 3726 | − | |
| 3727 | − ft_prio->flow_table = ft; | |
| 3728 | −free: | |
| 3729 | − if (err && handler) { | |
| 3730 | − if (handler->ibcounters && | |
| 3731 | − atomic_read(&handler->ibcounters->usecnt) == 1) | |
| 3732 | − counters_clear_description(handler->ibcounters); | |
| 3733 | − kfree(handler); | |
| 3734 | − } | |
| 3735 | − kvfree(spec); | |
| 3736 | − return err ? ERR_PTR(err) : handler; | |
| 3737 | −} | |
| 3738 | − | |
| 3739 | −static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, | |
| 3740 | − struct mlx5_ib_flow_prio *ft_prio, | |
| 3741 | − const struct ib_flow_attr *flow_attr, | |
| 3742 | − struct mlx5_flow_destination *dst) | |
| 3743 | −{ | |
| 3744 | − return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); | |
| 3745 | −} | |
| 3746 | − | |
| 3747 | −enum { | |
| 3748 | − LEFTOVERS_MC, | |
| 3749 | − LEFTOVERS_UC, | |
| 3750 | −}; | |
| 3751 | − | |
| 3752 | −static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, | |
| 3753 | − struct mlx5_ib_flow_prio *ft_prio, | |
| 3754 | − struct ib_flow_attr *flow_attr, | |
| 3755 | − struct mlx5_flow_destination *dst) | |
| 3756 | −{ | |
| 3757 | − struct mlx5_ib_flow_handler *handler_ucast = NULL; | |
| 3758 | − struct mlx5_ib_flow_handler *handler = NULL; | |
| 3759 | − | |
| 3760 | − static struct { | |
| 3761 | − struct ib_flow_attr flow_attr; | |
| 3762 | − struct ib_flow_spec_eth eth_flow; | |
| 3763 | − } leftovers_specs[] = { | |
| 3764 | − [LEFTOVERS_MC] = { | |
| 3765 | − .flow_attr = { | |
| 3766 | − .num_of_specs = 1, | |
| 3767 | − .size = sizeof(leftovers_specs[0]) | |
| 3768 | − }, | |
| 3769 | − .eth_flow = { | |
| 3770 | − .type = IB_FLOW_SPEC_ETH, | |
| 3771 | − .size = sizeof(struct ib_flow_spec_eth), | |
| 3772 | − .mask = {.dst_mac = {0x1} }, | |
| 3773 | − .val = {.dst_mac = {0x1} } | |
| 3774 | − } | |
| 3775 | − }, | |
| 3776 | − [LEFTOVERS_UC] = { | |
| 3777 | − .flow_attr = { | |
| 3778 | − .num_of_specs = 1, | |
| 3779 | − .size = sizeof(leftovers_specs[0]) | |
| 3780 | − }, | |
| 3781 | − .eth_flow = { | |
| 3782 | − .type = IB_FLOW_SPEC_ETH, | |
| 3783 | − .size = sizeof(struct ib_flow_spec_eth), | |
| 3784 | − .mask = {.dst_mac = {0x1} }, | |
| 3785 | − .val = {.dst_mac = {} } | |
| 3786 | − } | |
| 3787 | − } | |
| 3788 | − }; | |
| 3789 | − | |
| 3790 | − handler = create_flow_rule(dev, ft_prio, | |
| 3791 | − &leftovers_specs[LEFTOVERS_MC].flow_attr, | |
| 3792 | − dst); | |
| 3793 | − if (!IS_ERR(handler) && | |
| 3794 | − flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { | |
| 3795 | − handler_ucast = create_flow_rule(dev, ft_prio, | |
| 3796 | − &leftovers_specs[LEFTOVERS_UC].flow_attr, | |
| 3797 | − dst); | |
| 3798 | − if (IS_ERR(handler_ucast)) { | |
| 3799 | − mlx5_del_flow_rules(handler->rule); | |
| 3800 | − ft_prio->refcount--; | |
| 3801 | − kfree(handler); | |
| 3802 | − handler = handler_ucast; | |
| 3803 | − } else { | |
| 3804 | − list_add(&handler_ucast->list, &handler->list); | |
| 3805 | − } | |
| 3806 | − } | |
| 3807 | − | |
| 3808 | − return handler; | |
| 3809 | −} | |
| 3810 | − | |
| 3811 | −static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, | |
| 3812 | − struct mlx5_ib_flow_prio *ft_rx, | |
| 3813 | − struct mlx5_ib_flow_prio *ft_tx, | |
| 3814 | − struct mlx5_flow_destination *dst) | |
| 3815 | −{ | |
| 3816 | − struct mlx5_ib_flow_handler *handler_rx; | |
| 3817 | − struct mlx5_ib_flow_handler *handler_tx; | |
| 3818 | − int err; | |
| 3819 | − static const struct ib_flow_attr flow_attr = { | |
| 3820 | − .num_of_specs = 0, | |
| 3821 | − .size = sizeof(flow_attr) | |
| 3822 | − }; | |
| 3823 | − | |
| 3824 | − handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); | |
| 3825 | − if (IS_ERR(handler_rx)) { | |
| 3826 | − err = PTR_ERR(handler_rx); | |
| 3827 | − goto err; | |
| 3828 | − } | |
| 3829 | − | |
| 3830 | − handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); | |
| 3831 | − if (IS_ERR(handler_tx)) { | |
| 3832 | − err = PTR_ERR(handler_tx); | |
| 3833 | − goto err_tx; | |
| 3834 | − } | |
| 3835 | − | |
| 3836 | − list_add(&handler_tx->list, &handler_rx->list); | |
| 3837 | − | |
| 3838 | − return handler_rx; | |
| 3839 | − | |
| 3840 | −err_tx: | |
| 3841 | − mlx5_del_flow_rules(handler_rx->rule); | |
| 3842 | − ft_rx->refcount--; | |
| 3843 | − kfree(handler_rx); | |
| 3844 | −err: | |
| 3845 | − return ERR_PTR(err); | |
| 3846 | −} | |
| 3847 | − | |
| 3848 | −static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, | |
| 3849 | − struct ib_flow_attr *flow_attr, | |
| 3850 | − int domain, | |
| 3851 | − struct ib_udata *udata) | |
| 3852 | −{ | |
| 3853 | − struct mlx5_ib_dev *dev = to_mdev(qp->device); | |
| 3854 | − struct mlx5_ib_qp *mqp = to_mqp(qp); | |
| 3855 | − struct mlx5_ib_flow_handler *handler = NULL; | |
| 3856 | − struct mlx5_flow_destination *dst = NULL; | |
| 3857 | − struct mlx5_ib_flow_prio *ft_prio_tx = NULL; | |
| 3858 | − struct mlx5_ib_flow_prio *ft_prio; | |
| 3859 | − bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; | |
| 3860 | − struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; | |
| 3861 | − size_t min_ucmd_sz, required_ucmd_sz; | |
| 3862 | − int err; | |
| 3863 | − int underlay_qpn; | |
| 3864 | − | |
| 3865 | − if (udata && udata->inlen) { | |
| 3866 | − min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) + | |
| 3867 | − sizeof(ucmd_hdr.reserved); | |
| 3868 | − if (udata->inlen < min_ucmd_sz) | |
| 3869 | − return ERR_PTR(-EOPNOTSUPP); | |
| 3870 | − | |
| 3871 | − err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); | |
| 3872 | − if (err) | |
| 3873 | − return ERR_PTR(err); | |
| 3874 | − | |
| 3875 | − /* currently supports only one counters data */ | |
| 3876 | − if (ucmd_hdr.ncounters_data > 1) | |
| 3877 | − return ERR_PTR(-EINVAL); | |
| 3878 | − | |
| 3879 | − required_ucmd_sz = min_ucmd_sz + | |
| 3880 | − sizeof(struct mlx5_ib_flow_counters_data) * | |
| 3881 | − ucmd_hdr.ncounters_data; | |
| 3882 | − if (udata->inlen > required_ucmd_sz && | |
| 3883 | − !ib_is_udata_cleared(udata, required_ucmd_sz, | |
| 3884 | − udata->inlen - required_ucmd_sz)) | |
| 3885 | − return ERR_PTR(-EOPNOTSUPP); | |
| 3886 | − | |
| 3887 | − ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); | |
| 3888 | − if (!ucmd) | |
| 3889 | − return ERR_PTR(-ENOMEM); | |
| 3890 | − | |
| 3891 | − err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); | |
| 3892 | − if (err) | |
| 3893 | − goto free_ucmd; | |
| 3894 | − } | |
| 3895 | − | |
| 3896 | − if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { | |
| 3897 | − err = -ENOMEM; | |
| 3898 | − goto free_ucmd; | |
| 3899 | − } | |
| 3900 | − | |
| 3901 | − if (domain != IB_FLOW_DOMAIN_USER || | |
| 3902 | − flow_attr->port > dev->num_ports || | |
| 3903 | − (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | | |
| 3904 | − IB_FLOW_ATTR_FLAGS_EGRESS))) { | |
| 3905 | − err = -EINVAL; | |
| 3906 | − goto free_ucmd; | |
| 3907 | − } | |
| 3908 | − | |
| 3909 | − if (is_egress && | |
| 3910 | − (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || | |
| 3911 | − flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { | |
| 3912 | − err = -EINVAL; | |
| 3913 | − goto free_ucmd; | |
| 3914 | − } | |
| 3915 | − | |
| 3916 | − dst = kzalloc(sizeof(*dst), GFP_KERNEL); | |
| 3917 | − if (!dst) { | |
| 3918 | − err = -ENOMEM; | |
| 3919 | − goto free_ucmd; | |
| 3920 | − } | |
| 3921 | − | |
| 3922 | − mutex_lock(&dev->flow_db->lock); | |
| 3923 | − | |
| 3924 | − ft_prio = get_flow_table(dev, flow_attr, | |
| 3925 | − is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); | |
| 3926 | − if (IS_ERR(ft_prio)) { | |
| 3927 | − err = PTR_ERR(ft_prio); | |
| 3928 | − goto unlock; | |
| 3929 | − } | |
| 3930 | − if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { | |
| 3931 | − ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); | |
| 3932 | − if (IS_ERR(ft_prio_tx)) { | |
| 3933 | − err = PTR_ERR(ft_prio_tx); | |
| 3934 | − ft_prio_tx = NULL; | |
| 3935 | − goto destroy_ft; | |
| 3936 | − } | |
| 3937 | − } | |
| 3938 | − | |
| 3939 | − if (is_egress) { | |
| 3940 | − dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; | |
| 3941 | − } else { | |
| 3942 | − dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; | |
| 3943 | − if (mqp->is_rss) | |
| 3944 | − dst->tir_num = mqp->rss_qp.tirn; | |
| 3945 | − else | |
| 3946 | − dst->tir_num = mqp->raw_packet_qp.rq.tirn; | |
| 3947 | − } | |
| 3948 | − | |
| 3949 | − if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { | |
| 3950 | − underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? | |
| 3951 | − mqp->underlay_qpn : | |
| 3952 | − 0; | |
| 3953 | − handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, | |
| 3954 | − underlay_qpn, ucmd); | |
| 3955 | − } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || | |
| 3956 | − flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { | |
| 3957 | − handler = create_leftovers_rule(dev, ft_prio, flow_attr, | |
| 3958 | − dst); | |
| 3959 | − } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { | |
| 3960 | − handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); | |
| 3961 | − } else { | |
| 3962 | − err = -EINVAL; | |
| 3963 | − goto destroy_ft; | |
| 3964 | − } | |
| 3965 | − | |
| 3966 | − if (IS_ERR(handler)) { | |
| 3967 | − err = PTR_ERR(handler); | |
| 3968 | − handler = NULL; | |
| 3969 | − goto destroy_ft; | |
| 3970 | − } | |
| 3971 | − | |
| 3972 | − mutex_unlock(&dev->flow_db->lock); | |
| 3973 | − kfree(dst); | |
| 3974 | − kfree(ucmd); | |
| 3975 | − | |
| 3976 | − return &handler->ibflow; | |
| 3977 | − | |
| 3978 | −destroy_ft: | |
| 3979 | − put_flow_table(dev, ft_prio, false); | |
| 3980 | − if (ft_prio_tx) | |
| 3981 | − put_flow_table(dev, ft_prio_tx, false); | |
| 3982 | −unlock: | |
| 3983 | − mutex_unlock(&dev->flow_db->lock); | |
| 3984 | − kfree(dst); | |
| 3985 | −free_ucmd: | |
| 3986 | − kfree(ucmd); | |
| 3987 | − return ERR_PTR(err); | |
| 3988 | −} | |
| 3989 | − | |
| 3990 | −static struct mlx5_ib_flow_prio * | |
| 3991 | −_get_flow_table(struct mlx5_ib_dev *dev, | |
| 3992 | − struct mlx5_ib_flow_matcher *fs_matcher, | |
| 3993 | − bool mcast) | |
| 3994 | −{ | |
| 3995 | − struct mlx5_flow_namespace *ns = NULL; | |
| 3996 | − struct mlx5_ib_flow_prio *prio = NULL; | |
| 3997 | − int max_table_size = 0; | |
| 3998 | − bool esw_encap; | |
| 3999 | − u32 flags = 0; | |
| 4000 | − int priority; | |
| 4001 | − | |
| 4002 | − if (mcast) | |
| 4003 | − priority = MLX5_IB_FLOW_MCAST_PRIO; | |
| 4004 | − else | |
| 4005 | − priority = ib_prio_to_core_prio(fs_matcher->priority, false); | |
| 4006 | − | |
| 4007 | − esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != | |
| 4008 | − DEVLINK_ESWITCH_ENCAP_MODE_NONE; | |
| 4009 | − if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { | |
| 4010 | − max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 4011 | − log_max_ft_size)); | |
| 4012 | − if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) | |
| 4013 | − flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; | |
| 4014 | − if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, | |
| 4015 | − reformat_l3_tunnel_to_l2) && | |
| 4016 | − !esw_encap) | |
| 4017 | − flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 4018 | − } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { | |
| 4019 | − max_table_size = BIT( | |
| 4020 | − MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); | |
| 4021 | − if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) | |
| 4022 | − flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 4023 | − } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { | |
| 4024 | − max_table_size = BIT( | |
| 4025 | − MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); | |
| 4026 | − if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) | |
| 4027 | − flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; | |
| 4028 | − if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && | |
| 4029 | − esw_encap) | |
| 4030 | − flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; | |
| 4031 | − priority = FDB_BYPASS_PATH; | |
| 4032 | − } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) { | |
| 4033 | − max_table_size = | |
| 4034 | − BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, | |
| 4035 | − log_max_ft_size)); | |
| 4036 | − priority = fs_matcher->priority; | |
| 4037 | − } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { | |
| 4038 | − max_table_size = | |
| 4039 | − BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, | |
| 4040 | − log_max_ft_size)); | |
| 4041 | − priority = fs_matcher->priority; | |
| 4042 | − } | |
| 4043 | − | |
| 4044 | − max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); | |
| 4045 | − | |
| 4046 | − ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type); | |
| 4047 | − if (!ns) | |
| 4048 | − return ERR_PTR(-ENOTSUPP); | |
| 4049 | − | |
| 4050 | − if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) | |
| 4051 | − prio = &dev->flow_db->prios[priority]; | |
| 4052 | − else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) | |
| 4053 | − prio = &dev->flow_db->egress_prios[priority]; | |
| 4054 | − else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) | |
| 4055 | − prio = &dev->flow_db->fdb; | |
| 4056 | − else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) | |
| 4057 | − prio = &dev->flow_db->rdma_rx[priority]; | |
| 4058 | − else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) | |
| 4059 | − prio = &dev->flow_db->rdma_tx[priority]; | |
| 4060 | − | |
| 4061 | − if (!prio) | |
| 4062 | − return ERR_PTR(-EINVAL); | |
| 4063 | − | |
| 4064 | − if (prio->flow_table) | |
| 4065 | − return prio; | |
| 4066 | − | |
| 4067 | − return _get_prio(ns, prio, priority, max_table_size, | |
| 4068 | − MLX5_FS_MAX_TYPES, flags); | |
| 4069 | −} | |
| 4070 | − | |
| 4071 | −static struct mlx5_ib_flow_handler * | |
| 4072 | −_create_raw_flow_rule(struct mlx5_ib_dev *dev, | |
| 4073 | − struct mlx5_ib_flow_prio *ft_prio, | |
| 4074 | − struct mlx5_flow_destination *dst, | |
| 4075 | − struct mlx5_ib_flow_matcher *fs_matcher, | |
| 4076 | − struct mlx5_flow_context *flow_context, | |
| 4077 | − struct mlx5_flow_act *flow_act, | |
| 4078 | − void *cmd_in, int inlen, | |
| 4079 | − int dst_num) | |
| 4080 | −{ | |
| 4081 | − struct mlx5_ib_flow_handler *handler; | |
| 4082 | − struct mlx5_flow_spec *spec; | |
| 4083 | − struct mlx5_flow_table *ft = ft_prio->flow_table; | |
| 4084 | − int err = 0; | |
| 4085 | − | |
| 4086 | − spec = kvzalloc(sizeof(*spec), GFP_KERNEL); | |
| 4087 | − handler = kzalloc(sizeof(*handler), GFP_KERNEL); | |
| 4088 | − if (!handler || !spec) { | |
| 4089 | − err = -ENOMEM; | |
| 4090 | − goto free; | |
| 4091 | − } | |
| 4092 | − | |
| 4093 | − INIT_LIST_HEAD(&handler->list); | |
| 4094 | − | |
| 4095 | − memcpy(spec->match_value, cmd_in, inlen); | |
| 4096 | − memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, | |
| 4097 | − fs_matcher->mask_len); | |
| 4098 | − spec->match_criteria_enable = fs_matcher->match_criteria_enable; | |
| 4099 | − spec->flow_context = *flow_context; | |
| 4100 | − | |
| 4101 | − handler->rule = mlx5_add_flow_rules(ft, spec, | |
| 4102 | − flow_act, dst, dst_num); | |
| 4103 | − | |
| 4104 | − if (IS_ERR(handler->rule)) { | |
| 4105 | − err = PTR_ERR(handler->rule); | |
| 4106 | − goto free; | |
| 4107 | − } | |
| 4108 | − | |
| 4109 | − ft_prio->refcount++; | |
| 4110 | − handler->prio = ft_prio; | |
| 4111 | − handler->dev = dev; | |
| 4112 | − ft_prio->flow_table = ft; | |
| 4113 | − | |
| 4114 | −free: | |
| 4115 | − if (err) | |
| 4116 | − kfree(handler); | |
| 4117 | − kvfree(spec); | |
| 4118 | − return err ? ERR_PTR(err) : handler; | |
| 4119 | −} | |
| 4120 | − | |
| 4121 | −static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, | |
| 4122 | − void *match_v) | |
| 4123 | −{ | |
| 4124 | − void *match_c; | |
| 4125 | − void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; | |
| 4126 | − void *dmac, *dmac_mask; | |
| 4127 | − void *ipv4, *ipv4_mask; | |
| 4128 | − | |
| 4129 | − if (!(fs_matcher->match_criteria_enable & | |
| 4130 | − (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) | |
| 4131 | − return false; | |
| 4132 | − | |
| 4133 | − match_c = fs_matcher->matcher_mask.match_params; | |
| 4134 | − match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, | |
| 4135 | − outer_headers); | |
| 4136 | − match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, | |
| 4137 | − outer_headers); | |
| 4138 | − | |
| 4139 | − dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, | |
| 4140 | − dmac_47_16); | |
| 4141 | − dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, | |
| 4142 | − dmac_47_16); | |
| 4143 | − | |
| 4144 | − if (is_multicast_ether_addr(dmac) && | |
| 4145 | − is_multicast_ether_addr(dmac_mask)) | |
| 4146 | − return true; | |
| 4147 | − | |
| 4148 | − ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, | |
| 4149 | − dst_ipv4_dst_ipv6.ipv4_layout.ipv4); | |
| 4150 | − | |
| 4151 | − ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, | |
| 4152 | − dst_ipv4_dst_ipv6.ipv4_layout.ipv4); | |
| 4153 | − | |
| 4154 | − if (ipv4_is_multicast(*(__be32 *)(ipv4)) && | |
| 4155 | − ipv4_is_multicast(*(__be32 *)(ipv4_mask))) | |
| 4156 | − return true; | |
| 4157 | − | |
| 4158 | − return false; | |
| 4159 | −} | |
| 4160 | − | |
| 4161 | −struct mlx5_ib_flow_handler * | |
| 4162 | −mlx5_ib_raw_fs_rule_add(struct mlx5_ib_dev *dev, | |
| 4163 | − struct mlx5_ib_flow_matcher *fs_matcher, | |
| 4164 | − struct mlx5_flow_context *flow_context, | |
| 4165 | − struct mlx5_flow_act *flow_act, | |
| 4166 | − u32 counter_id, | |
| 4167 | − void *cmd_in, int inlen, int dest_id, | |
| 4168 | − int dest_type) | |
| 4169 | −{ | |
| 4170 | − struct mlx5_flow_destination *dst; | |
| 4171 | − struct mlx5_ib_flow_prio *ft_prio; | |
| 4172 | − struct mlx5_ib_flow_handler *handler; | |
| 4173 | − int dst_num = 0; | |
| 4174 | − bool mcast; | |
| 4175 | − int err; | |
| 4176 | − | |
| 4177 | − if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) | |
| 4178 | − return ERR_PTR(-EOPNOTSUPP); | |
| 4179 | − | |
| 4180 | − if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) | |
| 4181 | − return ERR_PTR(-ENOMEM); | |
| 4182 | − | |
| 4183 | − dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); | |
| 4184 | − if (!dst) | |
| 4185 | − return ERR_PTR(-ENOMEM); | |
| 4186 | − | |
| 4187 | − mcast = raw_fs_is_multicast(fs_matcher, cmd_in); | |
| 4188 | − mutex_lock(&dev->flow_db->lock); | |
| 4189 | − | |
| 4190 | − ft_prio = _get_flow_table(dev, fs_matcher, mcast); | |
| 4191 | − if (IS_ERR(ft_prio)) { | |
| 4192 | − err = PTR_ERR(ft_prio); | |
| 4193 | − goto unlock; | |
| 4194 | − } | |
| 4195 | − | |
| 4196 | − if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) { | |
| 4197 | − dst[dst_num].type = dest_type; | |
| 4198 | − dst[dst_num++].tir_num = dest_id; | |
| 4199 | − flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; | |
| 4200 | − } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) { | |
| 4201 | − dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; | |
| 4202 | − dst[dst_num++].ft_num = dest_id; | |
| 4203 | − flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; | |
| 4204 | − } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) { | |
| 4205 | − dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; | |
| 4206 | − flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; | |
| 4207 | − } | |
| 4208 | − | |
| 4209 | − | |
| 4210 | − if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { | |
| 4211 | − dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; | |
| 4212 | − dst[dst_num].counter_id = counter_id; | |
| 4213 | − dst_num++; | |
| 4214 | − } | |
| 4215 | − | |
| 4216 | − handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher, | |
| 4217 | − flow_context, flow_act, | |
| 4218 | − cmd_in, inlen, dst_num); | |
| 4219 | − | |
| 4220 | − if (IS_ERR(handler)) { | |
| 4221 | − err = PTR_ERR(handler); | |
| 4222 | − goto destroy_ft; | |
| 4223 | − } | |
| 4224 | − | |
| 4225 | − mutex_unlock(&dev->flow_db->lock); | |
| 4226 | − atomic_inc(&fs_matcher->usecnt); | |
| 4227 | − handler->flow_matcher = fs_matcher; | |
| 4228 | − | |
| 4229 | − kfree(dst); | |
| 4230 | − | |
| 4231 | − return handler; | |
| 4232 | − | |
| 4233 | −destroy_ft: | |
| 4234 | − put_flow_table(dev, ft_prio, false); | |
| 4235 | −unlock: | |
| 4236 | − mutex_unlock(&dev->flow_db->lock); | |
| 4237 | − kfree(dst); | |
| 4238 | − | |
| 4239 | − return ERR_PTR(err); | |
| 4240 | −} | |
| 4241 | − | |
| 4242 | −static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags) | |
| 4243 | −{ | |
| 4244 | − u32 flags = 0; | |
| 4245 | − | |
| 4246 | − if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA) | |
| 4247 | − flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA; | |
| 4248 | − | |
| 4249 | − return flags; | |
| 4250 | −} | |
| 4251 | − | |
| 4252 | −#define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA | |
| 4253 | −static struct ib_flow_action * | |
| 4254 | −mlx5_ib_create_flow_action_esp(struct ib_device *device, | |
| 4255 | − const struct ib_flow_action_attrs_esp *attr, | |
| 4256 | − struct uverbs_attr_bundle *attrs) | |
| 4257 | −{ | |
| 4258 | − struct mlx5_ib_dev *mdev = to_mdev(device); | |
| 4259 | − struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm; | |
| 4260 | − struct mlx5_accel_esp_xfrm_attrs accel_attrs = {}; | |
| 4261 | − struct mlx5_ib_flow_action *action; | |
| 4262 | − u64 action_flags; | |
| 4263 | − u64 flags; | |
| 4264 | − int err = 0; | |
| 4265 | − | |
| 4266 | − err = uverbs_get_flags64( | |
| 4267 | − &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS, | |
| 4268 | − ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1)); | |
| 4269 | − if (err) | |
| 4270 | − return ERR_PTR(err); | |
| 4271 | − | |
| 4272 | − flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags); | |
| 4273 | − | |
| 4274 | − /* We current only support a subset of the standard features. Only a | |
| 4275 | − * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn | |
| 4276 | − * (with overlap). Full offload mode isn't supported. | |
| 4277 | − */ | |
| 4278 | − if (!attr->keymat || attr->replay || attr->encap || | |
| 4279 | − attr->spi || attr->seq || attr->tfc_pad || | |
| 4280 | − attr->hard_limit_pkts || | |
| 4281 | − (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | | |
| 4282 | − IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT))) | |
| 4283 | − return ERR_PTR(-EOPNOTSUPP); | |
| 4284 | − | |
| 4285 | − if (attr->keymat->protocol != | |
| 4286 | − IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM) | |
| 4287 | − return ERR_PTR(-EOPNOTSUPP); | |
| 4288 | − | |
| 4289 | − aes_gcm = &attr->keymat->keymat.aes_gcm; | |
| 4290 | − | |
| 4291 | − if (aes_gcm->icv_len != 16 || | |
| 4292 | − aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ) | |
| 4293 | − return ERR_PTR(-EOPNOTSUPP); | |
| 4294 | − | |
| 4295 | − action = kmalloc(sizeof(*action), GFP_KERNEL); | |
| 4296 | − if (!action) | |
| 4297 | − return ERR_PTR(-ENOMEM); | |
| 4298 | − | |
| 4299 | − action->esp_aes_gcm.ib_flags = attr->flags; | |
| 4300 | − memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key, | |
| 4301 | − sizeof(accel_attrs.keymat.aes_gcm.aes_key)); | |
| 4302 | − accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8; | |
| 4303 | − memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt, | |
| 4304 | − sizeof(accel_attrs.keymat.aes_gcm.salt)); | |
| 4305 | − memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv, | |
| 4306 | − sizeof(accel_attrs.keymat.aes_gcm.seq_iv)); | |
| 4307 | − accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8; | |
| 4308 | − accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ; | |
| 4309 | − accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM; | |
| 4310 | − | |
| 4311 | − accel_attrs.esn = attr->esn; | |
| 4312 | − if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) | |
| 4313 | − accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; | |
| 4314 | − if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) | |
| 4315 | − accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; | |
| 4316 | − | |
| 4317 | − if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT) | |
| 4318 | − accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT; | |
| 4319 | − | |
| 4320 | − action->esp_aes_gcm.ctx = | |
| 4321 | − mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags); | |
| 4322 | − if (IS_ERR(action->esp_aes_gcm.ctx)) { | |
| 4323 | − err = PTR_ERR(action->esp_aes_gcm.ctx); | |
| 4324 | − goto err_parse; | |
| 4325 | − } | |
| 4326 | − | |
| 4327 | − action->esp_aes_gcm.ib_flags = attr->flags; | |
| 4328 | − | |
| 4329 | − return &action->ib_action; | |
| 4330 | − | |
| 4331 | −err_parse: | |
| 4332 | − kfree(action); | |
| 4333 | − return ERR_PTR(err); | |
| 4334 | −} | |
| 4335 | − | |
| 4336 | −static int | |
| 4337 | −mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action, | |
| 4338 | − const struct ib_flow_action_attrs_esp *attr, | |
| 4339 | − struct uverbs_attr_bundle *attrs) | |
| 4340 | −{ | |
| 4341 | − struct mlx5_ib_flow_action *maction = to_mflow_act(action); | |
| 4342 | − struct mlx5_accel_esp_xfrm_attrs accel_attrs; | |
| 4343 | − int err = 0; | |
| 4344 | − | |
| 4345 | − if (attr->keymat || attr->replay || attr->encap || | |
| 4346 | − attr->spi || attr->seq || attr->tfc_pad || | |
| 4347 | − attr->hard_limit_pkts || | |
| 4348 | − (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | | |
| 4349 | − IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS | | |
| 4350 | − IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))) | |
| 4351 | − return -EOPNOTSUPP; | |
| 4352 | − | |
| 4353 | − /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can | |
| 4354 | − * be modified. | |
| 4355 | − */ | |
| 4356 | − if (!(maction->esp_aes_gcm.ib_flags & | |
| 4357 | − IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) && | |
| 4358 | − attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | | |
| 4359 | − IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)) | |
| 4360 | − return -EINVAL; | |
| 4361 | − | |
| 4362 | − memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs, | |
| 4363 | − sizeof(accel_attrs)); | |
| 4364 | − | |
| 4365 | − accel_attrs.esn = attr->esn; | |
| 4366 | − if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) | |
| 4367 | − accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; | |
| 4368 | − else | |
| 4369 | − accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; | |
| 4370 | − | |
| 4371 | − err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx, | |
| 4372 | − &accel_attrs); | |
| 4373 | − if (err) | |
| 4374 | − return err; | |
| 4375 | − | |
| 4376 | − maction->esp_aes_gcm.ib_flags &= | |
| 4377 | − ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; | |
| 4378 | − maction->esp_aes_gcm.ib_flags |= | |
| 4379 | − attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; | |
| 4380 | − | |
| 4381 | − return 0; | |
| 4382 | −} | |
| 4383 | − | |
| 4384 | −static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) | |
| 4385 | −{ | |
| 4386 | − struct mlx5_ib_flow_action *maction = to_mflow_act(action); | |
| 4387 | − | |
| 4388 | − switch (action->type) { | |
| 4389 | − case IB_FLOW_ACTION_ESP: | |
| 4390 | − /* | |
| 4391 | − * We only support aes_gcm by now, so we implicitly know this is | |
| 4392 | − * the underline crypto. | |
| 4393 | − */ | |
| 4394 | − mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx); | |
| 4395 | − break; | |
| 4396 | − case IB_FLOW_ACTION_UNSPECIFIED: | |
| 4397 | − mlx5_ib_destroy_flow_action_raw(maction); | |
| 4398 | − break; | |
| 4399 | − default: | |
| 4400 | − WARN_ON(true); | |
| 4401 | − break; | |
| 4402 | − } | |
| 4403 | − | |
| 4404 | − kfree(maction); | |
| 4405 | − return 0; | |
| 4406 | −} | |
| 4407 | − | |
| 4408 | −static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) | |
| 4409 | −{ | |
| 4410 | − struct mlx5_ib_dev *dev = to_mdev(ibqp->device); | |
| 4411 | − struct mlx5_ib_qp *mqp = to_mqp(ibqp); | |
| 4412 | − int err; | |
| 4413 | − u16 uid; | |
| 4414 | − | |
| 4415 | − uid = ibqp->pd ? | |
| 4416 | − to_mpd(ibqp->pd)->uid : 0; | |
| 4417 | − | |
| 4418 | − if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) { | |
| 4419 | − mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n"); | |
| 4420 | − return -EOPNOTSUPP; | |
| 4421 | − } | |
| 4422 | − | |
| 4423 | − err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid); | |
| 4424 | − if (err) | |
| 4425 | − mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n", | |
| 4426 | − ibqp->qp_num, gid->raw); | |
| 4427 | − | |
| 4428 | − return err; | |
| 4429 | −} | |
| 4430 | − | |
| 4431 | −static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) | |
| 4432 | −{ | |
| 4433 | − struct mlx5_ib_dev *dev = to_mdev(ibqp->device); | |
| 4434 | − int err; | |
| 4435 | − u16 uid; | |
| 4436 | − | |
| 4437 | − uid = ibqp->pd ? | |
| 4438 | − to_mpd(ibqp->pd)->uid : 0; | |
| 4439 | − err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid); | |
| 4440 | − if (err) | |
| 4441 | − mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n", | |
| 4442 | − ibqp->qp_num, gid->raw); | |
| 4443 | − | |
| 4444 | − return err; | |
| 4445 | −} | |
| 4446 | − | |
| 4447 | −static int init_node_data(struct mlx5_ib_dev *dev) | |
| 4448 | −{ | |
| 4449 | − int err; | |
| 4450 | − | |
| 4451 | − err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc); | |
| 4452 | − if (err) | |
| 4453 | − return err; | |
| 4454 | − | |
| 4455 | − dev->mdev->rev_id = dev->mdev->pdev->revision; | |
| 4456 | − | |
| 4457 | − return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid); | |
| 4458 | −} | |
| 4459 | − | |
| 4460 | −static ssize_t fw_pages_show(struct device *device, | |
| 4461 | − struct device_attribute *attr, char *buf) | |
| 4462 | −{ | |
| 4463 | − struct mlx5_ib_dev *dev = | |
| 4464 | − rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 4465 | − | |
| 4466 | − return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages); | |
| 4467 | −} | |
| 4468 | −static DEVICE_ATTR_RO(fw_pages); | |
| 4469 | − | |
| 4470 | −static ssize_t reg_pages_show(struct device *device, | |
| 4471 | − struct device_attribute *attr, char *buf) | |
| 4472 | −{ | |
| 4473 | − struct mlx5_ib_dev *dev = | |
| 4474 | − rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 4475 | − | |
| 4476 | − return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages)); | |
| 4477 | −} | |
| 4478 | −static DEVICE_ATTR_RO(reg_pages); | |
| 4479 | − | |
| 4480 | −static ssize_t hca_type_show(struct device *device, | |
| 4481 | − struct device_attribute *attr, char *buf) | |
| 4482 | −{ | |
| 4483 | − struct mlx5_ib_dev *dev = | |
| 4484 | − rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 4485 | − | |
| 4486 | − return sprintf(buf, "MT%d\n", dev->mdev->pdev->device); | |
| 4487 | −} | |
| 4488 | −static DEVICE_ATTR_RO(hca_type); | |
| 4489 | − | |
| 4490 | −static ssize_t hw_rev_show(struct device *device, | |
| 4491 | − struct device_attribute *attr, char *buf) | |
| 4492 | −{ | |
| 4493 | − struct mlx5_ib_dev *dev = | |
| 4494 | − rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 4495 | − | |
| 4496 | − return sprintf(buf, "%x\n", dev->mdev->rev_id); | |
| 4497 | −} | |
| 4498 | −static DEVICE_ATTR_RO(hw_rev); | |
| 4499 | − | |
| 4500 | −static ssize_t board_id_show(struct device *device, | |
| 4501 | − struct device_attribute *attr, char *buf) | |
| 4502 | −{ | |
| 4503 | − struct mlx5_ib_dev *dev = | |
| 4504 | − rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev); | |
| 4505 | − | |
| 4506 | − return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN, | |
| 4507 | − dev->mdev->board_id); | |
| 4508 | −} | |
| 4509 | −static DEVICE_ATTR_RO(board_id); | |
| 4510 | − | |
| 4511 | −static struct attribute *mlx5_class_attributes[] = { | |
| 4512 | − &dev_attr_hw_rev.attr, | |
| 4513 | − &dev_attr_hca_type.attr, | |
| 4514 | − &dev_attr_board_id.attr, | |
| 4515 | − &dev_attr_fw_pages.attr, | |
| 4516 | − &dev_attr_reg_pages.attr, | |
| 4517 | − NULL, | |
| 4518 | −}; | |
| 4519 | − | |
| 4520 | −static const struct attribute_group mlx5_attr_group = { | |
| 4521 | − .attrs = mlx5_class_attributes, | |
| 4522 | −}; | |
| 4523 | − | |
| 4524 | −static void pkey_change_handler(struct work_struct *work) | |
| 4525 | −{ | |
| 4526 | − struct mlx5_ib_port_resources *ports = | |
| 4527 | − container_of(work, struct mlx5_ib_port_resources, | |
| 4528 | − pkey_change_work); | |
| 4529 | − | |
| 4530 | − mutex_lock(&ports->devr->mutex); | |
| 4531 | − mlx5_ib_gsi_pkey_change(ports->gsi); | |
| 4532 | − mutex_unlock(&ports->devr->mutex); | |
| 4533 | −} | |
| 4534 | − | |
| 4535 | −static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev) | |
| 4536 | −{ | |
| 4537 | − struct mlx5_ib_qp *mqp; | |
| 4538 | − struct mlx5_ib_cq *send_mcq, *recv_mcq; | |
| 4539 | − struct mlx5_core_cq *mcq; | |
| 4540 | − struct list_head cq_armed_list; | |
| 4541 | − unsigned long flags_qp; | |
| 4542 | − unsigned long flags_cq; | |
| 4543 | − unsigned long flags; | |
| 4544 | − | |
| 4545 | − INIT_LIST_HEAD(&cq_armed_list); | |
| 4546 | − | |
| 4547 | − /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/ | |
| 4548 | − spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags); | |
| 4549 | − list_for_each_entry(mqp, &ibdev->qp_list, qps_list) { | |
| 4550 | − spin_lock_irqsave(&mqp->sq.lock, flags_qp); | |
| 4551 | − if (mqp->sq.tail != mqp->sq.head) { | |
| 4552 | − send_mcq = to_mcq(mqp->ibqp.send_cq); | |
| 4553 | − spin_lock_irqsave(&send_mcq->lock, flags_cq); | |
| 4554 | − if (send_mcq->mcq.comp && | |
| 4555 | − mqp->ibqp.send_cq->comp_handler) { | |
| 4556 | − if (!send_mcq->mcq.reset_notify_added) { | |
| 4557 | − send_mcq->mcq.reset_notify_added = 1; | |
| 4558 | − list_add_tail(&send_mcq->mcq.reset_notify, | |
| 4559 | − &cq_armed_list); | |
| 4560 | − } | |
| 4561 | − } | |
| 4562 | − spin_unlock_irqrestore(&send_mcq->lock, flags_cq); | |
| 4563 | − } | |
| 4564 | − spin_unlock_irqrestore(&mqp->sq.lock, flags_qp); | |
| 4565 | − spin_lock_irqsave(&mqp->rq.lock, flags_qp); | |
| 4566 | − /* no handling is needed for SRQ */ | |
| 4567 | − if (!mqp->ibqp.srq) { | |
| 4568 | − if (mqp->rq.tail != mqp->rq.head) { | |
| 4569 | − recv_mcq = to_mcq(mqp->ibqp.recv_cq); | |
| 4570 | − spin_lock_irqsave(&recv_mcq->lock, flags_cq); | |
| 4571 | − if (recv_mcq->mcq.comp && | |
| 4572 | − mqp->ibqp.recv_cq->comp_handler) { | |
| 4573 | − if (!recv_mcq->mcq.reset_notify_added) { | |
| 4574 | − recv_mcq->mcq.reset_notify_added = 1; | |
| 4575 | − list_add_tail(&recv_mcq->mcq.reset_notify, | |
| 4576 | − &cq_armed_list); | |
| 4577 | − } | |
| 4578 | − } | |
| 4579 | − spin_unlock_irqrestore(&recv_mcq->lock, | |
| 4580 | − flags_cq); | |
| 4581 | − } | |
| 4582 | − } | |
| 4583 | − spin_unlock_irqrestore(&mqp->rq.lock, flags_qp); | |
| 4584 | − } | |
| 4585 | − /*At that point all inflight post send were put to be executed as of we | |
| 4586 | − * lock/unlock above locks Now need to arm all involved CQs. | |
| 4587 | − */ | |
| 4588 | − list_for_each_entry(mcq, &cq_armed_list, reset_notify) { | |
| 4589 | − mcq->comp(mcq, NULL); | |
| 4590 | − } | |
| 4591 | − spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags); | |
| 4592 | −} | |
| 4593 | − | |
| 4594 | −static void delay_drop_handler(struct work_struct *work) | |
| 4595 | −{ | |
| 4596 | − int err; | |
| 4597 | − struct mlx5_ib_delay_drop *delay_drop = | |
| 4598 | − container_of(work, struct mlx5_ib_delay_drop, | |
| 4599 | − delay_drop_work); | |
| 4600 | − | |
| 4601 | − atomic_inc(&delay_drop->events_cnt); | |
| 4602 | − | |
| 4603 | − mutex_lock(&delay_drop->lock); | |
| 4604 | − err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout); | |
| 4605 | − if (err) { | |
| 4606 | − mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n", | |
| 4607 | − delay_drop->timeout); | |
| 4608 | − delay_drop->activate = false; | |
| 4609 | − } | |
| 4610 | − mutex_unlock(&delay_drop->lock); | |
| 4611 | −} | |
| 4612 | − | |
| 4613 | −static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe, | |
| 4614 | − struct ib_event *ibev) | |
| 4615 | −{ | |
| 4616 | − u8 port = (eqe->data.port.port >> 4) & 0xf; | |
| 4617 | − | |
| 4618 | − switch (eqe->sub_type) { | |
| 4619 | − case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT: | |
| 4620 | − if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) == | |
| 4621 | − IB_LINK_LAYER_ETHERNET) | |
| 4622 | − schedule_work(&ibdev->delay_drop.delay_drop_work); | |
| 4623 | − break; | |
| 4624 | − default: /* do nothing */ | |
| 4625 | − return; | |
| 4626 | − } | |
| 4627 | −} | |
| 4628 | − | |
| 4629 | −static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe, | |
| 4630 | − struct ib_event *ibev) | |
| 4631 | −{ | |
| 4632 | − u8 port = (eqe->data.port.port >> 4) & 0xf; | |
| 4633 | − | |
| 4634 | − ibev->element.port_num = port; | |
| 4635 | − | |
| 4636 | − switch (eqe->sub_type) { | |
| 4637 | − case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: | |
| 4638 | − case MLX5_PORT_CHANGE_SUBTYPE_DOWN: | |
| 4639 | − case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED: | |
| 4640 | − /* In RoCE, port up/down events are handled in | |
| 4641 | − * mlx5_netdev_event(). | |
| 4642 | − */ | |
| 4643 | − if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) == | |
| 4644 | − IB_LINK_LAYER_ETHERNET) | |
| 4645 | − return -EINVAL; | |
| 4646 | − | |
| 4647 | − ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ? | |
| 4648 | − IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; | |
| 4649 | − break; | |
| 4650 | − | |
| 4651 | − case MLX5_PORT_CHANGE_SUBTYPE_LID: | |
| 4652 | − ibev->event = IB_EVENT_LID_CHANGE; | |
| 4653 | − break; | |
| 4654 | − | |
| 4655 | − case MLX5_PORT_CHANGE_SUBTYPE_PKEY: | |
| 4656 | − ibev->event = IB_EVENT_PKEY_CHANGE; | |
| 4657 | − schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work); | |
| 4658 | − break; | |
| 4659 | − | |
| 4660 | − case MLX5_PORT_CHANGE_SUBTYPE_GUID: | |
| 4661 | − ibev->event = IB_EVENT_GID_CHANGE; | |
| 4662 | − break; | |
| 4663 | − | |
| 4664 | − case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG: | |
| 4665 | − ibev->event = IB_EVENT_CLIENT_REREGISTER; | |
| 4666 | − break; | |
| 4667 | − default: | |
| 4668 | − return -EINVAL; | |
| 4669 | − } | |
| 4670 | − | |
| 4671 | − return 0; | |
| 4672 | −} | |
| 4673 | − | |
| 4674 | −static void mlx5_ib_handle_event(struct work_struct *_work) | |
| 4675 | −{ | |
| 4676 | − struct mlx5_ib_event_work *work = | |
| 4677 | − container_of(_work, struct mlx5_ib_event_work, work); | |
| 4678 | − struct mlx5_ib_dev *ibdev; | |
| 4679 | − struct ib_event ibev; | |
| 4680 | − bool fatal = false; | |
| 4681 | − | |
| 4682 | − if (work->is_slave) { | |
| 4683 | − ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi); | |
| 4684 | − if (!ibdev) | |
| 4685 | − goto out; | |
| 4686 | − } else { | |
| 4687 | − ibdev = work->dev; | |
| 4688 | − } | |
| 4689 | − | |
| 4690 | − switch (work->event) { | |
| 4691 | − case MLX5_DEV_EVENT_SYS_ERROR: | |
| 4692 | − ibev.event = IB_EVENT_DEVICE_FATAL; | |
| 4693 | − mlx5_ib_handle_internal_error(ibdev); | |
| 4694 | − ibev.element.port_num = (u8)(unsigned long)work->param; | |
| 4695 | − fatal = true; | |
| 4696 | − break; | |
| 4697 | − case MLX5_EVENT_TYPE_PORT_CHANGE: | |
| 4698 | − if (handle_port_change(ibdev, work->param, &ibev)) | |
| 4699 | − goto out; | |
| 4700 | − break; | |
| 4701 | − case MLX5_EVENT_TYPE_GENERAL_EVENT: | |
| 4702 | − handle_general_event(ibdev, work->param, &ibev); | |
| 4703 | − /* fall through */ | |
| 4704 | − default: | |
| 4705 | − goto out; | |
| 4706 | − } | |
| 4707 | − | |
| 4708 | − ibev.device = &ibdev->ib_dev; | |
| 4709 | − | |
| 4710 | − if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) { | |
| 4711 | − mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num); | |
| 4712 | − goto out; | |
| 4713 | − } | |
| 4714 | − | |
| 4715 | − if (ibdev->ib_active) | |
| 4716 | − ib_dispatch_event(&ibev); | |
| 4717 | − | |
| 4718 | − if (fatal) | |
| 4719 | − ibdev->ib_active = false; | |
| 4720 | −out: | |
| 4721 | − kfree(work); | |
| 4722 | −} | |
| 4723 | − | |
| 4724 | −static int mlx5_ib_event(struct notifier_block *nb, | |
| 4725 | − unsigned long event, void *param) | |
| 4726 | −{ | |
| 4727 | − struct mlx5_ib_event_work *work; | |
| 4728 | − | |
| 4729 | − work = kmalloc(sizeof(*work), GFP_ATOMIC); | |
| 4730 | − if (!work) | |
| 4731 | − return NOTIFY_DONE; | |
| 4732 | − | |
| 4733 | − INIT_WORK(&work->work, mlx5_ib_handle_event); | |
| 4734 | − work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events); | |
| 4735 | − work->is_slave = false; | |
| 4736 | − work->param = param; | |
| 4737 | − work->event = event; | |
| 4738 | − | |
| 4739 | − queue_work(mlx5_ib_event_wq, &work->work); | |
| 4740 | − | |
| 4741 | − return NOTIFY_OK; | |
| 4742 | −} | |
| 4743 | − | |
| 4744 | −static int mlx5_ib_event_slave_port(struct notifier_block *nb, | |
| 4745 | − unsigned long event, void *param) | |
| 4746 | −{ | |
| 4747 | − struct mlx5_ib_event_work *work; | |
| 4748 | − | |
| 4749 | − work = kmalloc(sizeof(*work), GFP_ATOMIC); | |
| 4750 | − if (!work) | |
| 4751 | − return NOTIFY_DONE; | |
| 4752 | − | |
| 4753 | − INIT_WORK(&work->work, mlx5_ib_handle_event); | |
| 4754 | − work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events); | |
| 4755 | − work->is_slave = true; | |
| 4756 | − work->param = param; | |
| 4757 | − work->event = event; | |
| 4758 | − queue_work(mlx5_ib_event_wq, &work->work); | |
| 4759 | − | |
| 4760 | − return NOTIFY_OK; | |
| 4761 | −} | |
| 4762 | − | |
| 4763 | −static int set_has_smi_cap(struct mlx5_ib_dev *dev) | |
| 4764 | −{ | |
| 4765 | − struct mlx5_hca_vport_context vport_ctx; | |
| 4766 | − int err; | |
| 4767 | − int port; | |
| 4768 | − | |
| 4769 | − for (port = 1; port <= ARRAY_SIZE(dev->mdev->port_caps); port++) { | |
| 4770 | − dev->mdev->port_caps[port - 1].has_smi = false; | |
| 4771 | − if (MLX5_CAP_GEN(dev->mdev, port_type) == | |
| 4772 | − MLX5_CAP_PORT_TYPE_IB) { | |
| 4773 | − if (MLX5_CAP_GEN(dev->mdev, ib_virt)) { | |
| 4774 | − err = mlx5_query_hca_vport_context(dev->mdev, 0, | |
| 4775 | − port, 0, | |
| 4776 | − &vport_ctx); | |
| 4777 | − if (err) { | |
| 4778 | − mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n", | |
| 4779 | − port, err); | |
| 4780 | − return err; | |
| 4781 | − } | |
| 4782 | − dev->mdev->port_caps[port - 1].has_smi = | |
| 4783 | − vport_ctx.has_smi; | |
| 4784 | − } else { | |
| 4785 | − dev->mdev->port_caps[port - 1].has_smi = true; | |
| 4786 | − } | |
| 4787 | − } | |
| 4788 | − } | |
| 4789 | − return 0; | |
| 4790 | −} | |
| 4791 | − | |
| 4792 | −static void get_ext_port_caps(struct mlx5_ib_dev *dev) | |
| 4793 | −{ | |
| 4794 | − int port; | |
| 4795 | − | |
| 4796 | − for (port = 1; port <= dev->num_ports; port++) | |
| 4797 | − mlx5_query_ext_port_caps(dev, port); | |
| 4798 | −} | |
| 4799 | − | |
| 4800 | −static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port) | |
| 4801 | −{ | |
| 4802 | − struct ib_device_attr *dprops = NULL; | |
| 4803 | − struct ib_port_attr *pprops = NULL; | |
| 4804 | − int err = -ENOMEM; | |
| 4805 | − | |
| 4806 | − pprops = kzalloc(sizeof(*pprops), GFP_KERNEL); | |
| 4807 | − if (!pprops) | |
| 4808 | − goto out; | |
| 4809 | − | |
| 4810 | − dprops = kmalloc(sizeof(*dprops), GFP_KERNEL); | |
| 4811 | − if (!dprops) | |
| 4812 | − goto out; | |
| 4813 | − | |
| 4814 | − err = mlx5_ib_query_device(&dev->ib_dev, dprops, NULL); | |
| 4815 | − if (err) { | |
| 4816 | − mlx5_ib_warn(dev, "query_device failed %d\n", err); | |
| 4817 | − goto out; | |
| 4818 | − } | |
| 4819 | − | |
| 4820 | − err = mlx5_ib_query_port(&dev->ib_dev, port, pprops); | |
| 4821 | − if (err) { | |
| 4822 | − mlx5_ib_warn(dev, "query_port %d failed %d\n", | |
| 4823 | − port, err); | |
| 4824 | − goto out; | |
| 4825 | − } | |
| 4826 | − | |
| 4827 | − dev->mdev->port_caps[port - 1].pkey_table_len = | |
| 4828 | − dprops->max_pkeys; | |
| 4829 | − dev->mdev->port_caps[port - 1].gid_table_len = | |
| 4830 | − pprops->gid_tbl_len; | |
| 4831 | − mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n", | |
| 4832 | − port, dprops->max_pkeys, pprops->gid_tbl_len); | |
| 4833 | − | |
| 4834 | −out: | |
| 4835 | − kfree(pprops); | |
| 4836 | − kfree(dprops); | |
| 4837 | − | |
| 4838 | − return err; | |
| 4839 | −} | |
| 4840 | − | |
| 4841 | −static int get_port_caps(struct mlx5_ib_dev *dev, u8 port) | |
| 4842 | −{ | |
| 4843 | − /* For representors use port 1, is this is the only native | |
| 4844 | − * port | |
| 4845 | − */ | |
| 4846 | − if (dev->is_rep) | |
| 4847 | − return __get_port_caps(dev, 1); | |
| 4848 | − return __get_port_caps(dev, port); | |
| 4849 | −} | |
| 4850 | − | |
| 4851 | −static void destroy_umrc_res(struct mlx5_ib_dev *dev) | |
| 4852 | −{ | |
| 4853 | − int err; | |
| 4854 | − | |
| 4855 | − err = mlx5_mr_cache_cleanup(dev); | |
| 4856 | − if (err) | |
| 4857 | − mlx5_ib_warn(dev, "mr cache cleanup failed\n"); | |
| 4858 | − | |
| 4859 | − if (dev->umrc.qp) | |
| 4860 | − mlx5_ib_destroy_qp(dev->umrc.qp, NULL); | |
| 4861 | − if (dev->umrc.cq) | |
| 4862 | − ib_free_cq(dev->umrc.cq); | |
| 4863 | − if (dev->umrc.pd) | |
| 4864 | − ib_dealloc_pd(dev->umrc.pd); | |
| 4865 | −} | |
| 4866 | − | |
| 4867 | −enum { | |
| 4868 | − MAX_UMR_WR = 128, | |
| 4869 | −}; | |
| 4870 | − | |
| 4871 | −static int create_umr_res(struct mlx5_ib_dev *dev) | |
| 4872 | −{ | |
| 4873 | − struct ib_qp_init_attr *init_attr = NULL; | |
| 4874 | − struct ib_qp_attr *attr = NULL; | |
| 4875 | − struct ib_pd *pd; | |
| 4876 | − struct ib_cq *cq; | |
| 4877 | − struct ib_qp *qp; | |
| 4878 | − int ret; | |
| 4879 | − | |
| 4880 | − attr = kzalloc(sizeof(*attr), GFP_KERNEL); | |
| 4881 | − init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL); | |
| 4882 | − if (!attr || !init_attr) { | |
| 4883 | − ret = -ENOMEM; | |
| 4884 | − goto error_0; | |
| 4885 | − } | |
| 4886 | − | |
| 4887 | − pd = ib_alloc_pd(&dev->ib_dev, 0); | |
| 4888 | − if (IS_ERR(pd)) { | |
| 4889 | − mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n"); | |
| 4890 | − ret = PTR_ERR(pd); | |
| 4891 | − goto error_0; | |
| 4892 | − } | |
| 4893 | − | |
| 4894 | − cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ); | |
| 4895 | − if (IS_ERR(cq)) { | |
| 4896 | − mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n"); | |
| 4897 | − ret = PTR_ERR(cq); | |
| 4898 | − goto error_2; | |
| 4899 | − } | |
| 4900 | − | |
| 4901 | − init_attr->send_cq = cq; | |
| 4902 | − init_attr->recv_cq = cq; | |
| 4903 | − init_attr->sq_sig_type = IB_SIGNAL_ALL_WR; | |
| 4904 | − init_attr->cap.max_send_wr = MAX_UMR_WR; | |
| 4905 | − init_attr->cap.max_send_sge = 1; | |
| 4906 | − init_attr->qp_type = MLX5_IB_QPT_REG_UMR; | |
| 4907 | − init_attr->port_num = 1; | |
| 4908 | − qp = mlx5_ib_create_qp(pd, init_attr, NULL); | |
| 4909 | − if (IS_ERR(qp)) { | |
| 4910 | − mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n"); | |
| 4911 | − ret = PTR_ERR(qp); | |
| 4912 | − goto error_3; | |
| 4913 | − } | |
| 4914 | − qp->device = &dev->ib_dev; | |
| 4915 | − qp->real_qp = qp; | |
| 4916 | − qp->uobject = NULL; | |
| 4917 | − qp->qp_type = MLX5_IB_QPT_REG_UMR; | |
| 4918 | − qp->send_cq = init_attr->send_cq; | |
| 4919 | − qp->recv_cq = init_attr->recv_cq; | |
| 4920 | − | |
| 4921 | − attr->qp_state = IB_QPS_INIT; | |
| 4922 | − attr->port_num = 1; | |
| 4923 | − ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX | | |
| 4924 | − IB_QP_PORT, NULL); | |
| 4925 | − if (ret) { | |
| 4926 | − mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n"); | |
| 4927 | − goto error_4; | |
| 4928 | − } | |
| 4929 | − | |
| 4930 | − memset(attr, 0, sizeof(*attr)); | |
| 4931 | − attr->qp_state = IB_QPS_RTR; | |
| 4932 | − attr->path_mtu = IB_MTU_256; | |
| 4933 | − | |
| 4934 | − ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL); | |
| 4935 | − if (ret) { | |
| 4936 | − mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n"); | |
| 4937 | − goto error_4; | |
| 4938 | − } | |
| 4939 | − | |
| 4940 | − memset(attr, 0, sizeof(*attr)); | |
| 4941 | − attr->qp_state = IB_QPS_RTS; | |
| 4942 | − ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL); | |
| 4943 | − if (ret) { | |
| 4944 | − mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n"); | |
| 4945 | − goto error_4; | |
| 4946 | − } | |
| 4947 | − | |
| 4948 | − dev->umrc.qp = qp; | |
| 4949 | − dev->umrc.cq = cq; | |
| 4950 | − dev->umrc.pd = pd; | |
| 4951 | − | |
| 4952 | − sema_init(&dev->umrc.sem, MAX_UMR_WR); | |
| 4953 | − ret = mlx5_mr_cache_init(dev); | |
| 4954 | − if (ret) { | |
| 4955 | − mlx5_ib_warn(dev, "mr cache init failed %d\n", ret); | |
| 4956 | − goto error_4; | |
| 4957 | − } | |
| 4958 | − | |
| 4959 | − kfree(attr); | |
| 4960 | − kfree(init_attr); | |
| 4961 | − | |
| 4962 | − return 0; | |
| 4963 | − | |
| 4964 | −error_4: | |
| 4965 | − mlx5_ib_destroy_qp(qp, NULL); | |
| 4966 | − dev->umrc.qp = NULL; | |
| 4967 | − | |
| 4968 | −error_3: | |
| 4969 | − ib_free_cq(cq); | |
| 4970 | − dev->umrc.cq = NULL; | |
| 4971 | − | |
| 4972 | −error_2: | |
| 4973 | − ib_dealloc_pd(pd); | |
| 4974 | − dev->umrc.pd = NULL; | |
| 4975 | − | |
| 4976 | −error_0: | |
| 4977 | − kfree(attr); | |
| 4978 | − kfree(init_attr); | |
| 4979 | − return ret; | |
| 4980 | −} | |
| 4981 | − | |
| 4982 | −static u8 mlx5_get_umr_fence(u8 umr_fence_cap) | |
| 4983 | −{ | |
| 4984 | − switch (umr_fence_cap) { | |
| 4985 | − case MLX5_CAP_UMR_FENCE_NONE: | |
| 4986 | − return MLX5_FENCE_MODE_NONE; | |
| 4987 | − case MLX5_CAP_UMR_FENCE_SMALL: | |
| 4988 | − return MLX5_FENCE_MODE_INITIATOR_SMALL; | |
| 4989 | − default: | |
| 4990 | − return MLX5_FENCE_MODE_STRONG_ORDERING; | |
| 4991 | − } | |
| 4992 | −} | |
| 4993 | − | |
| 4994 | −static int create_dev_resources(struct mlx5_ib_resources *devr) | |
| 4995 | −{ | |
| 4996 | − struct ib_srq_init_attr attr; | |
| 4997 | − struct mlx5_ib_dev *dev; | |
| 4998 | − struct ib_device *ibdev; | |
| 4999 | − struct ib_cq_init_attr cq_attr = {.cqe = 1}; | |
| 5000 | − int port; | |
| 5001 | − int ret = 0; | |
| 5002 | − | |
| 5003 | − dev = container_of(devr, struct mlx5_ib_dev, devr); | |
| 5004 | − ibdev = &dev->ib_dev; | |
| 5005 | − | |
| 5006 | − mutex_init(&devr->mutex); | |
| 5007 | − | |
| 5008 | − devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd); | |
| 5009 | − if (!devr->p0) | |
| 5010 | − return -ENOMEM; | |
| 5011 | − | |
| 5012 | − devr->p0->device = ibdev; | |
| 5013 | − devr->p0->uobject = NULL; | |
| 5014 | − atomic_set(&devr->p0->usecnt, 0); | |
| 5015 | − | |
| 5016 | − ret = mlx5_ib_alloc_pd(devr->p0, NULL); | |
| 5017 | − if (ret) | |
| 5018 | − goto error0; | |
| 5019 | − | |
| 5020 | − devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq); | |
| 5021 | − if (!devr->c0) { | |
| 5022 | − ret = -ENOMEM; | |
| 5023 | − goto error1; | |
| 5024 | − } | |
| 5025 | − | |
| 5026 | − devr->c0->device = &dev->ib_dev; | |
| 5027 | − atomic_set(&devr->c0->usecnt, 0); | |
| 5028 | − | |
| 5029 | − ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL); | |
| 5030 | − if (ret) | |
| 5031 | − goto err_create_cq; | |
| 5032 | − | |
| 5033 | − devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL); | |
| 5034 | − if (IS_ERR(devr->x0)) { | |
| 5035 | − ret = PTR_ERR(devr->x0); | |
| 5036 | − goto error2; | |
| 5037 | − } | |
| 5038 | − devr->x0->device = &dev->ib_dev; | |
| 5039 | − devr->x0->inode = NULL; | |
| 5040 | − atomic_set(&devr->x0->usecnt, 0); | |
| 5041 | − mutex_init(&devr->x0->tgt_qp_mutex); | |
| 5042 | − INIT_LIST_HEAD(&devr->x0->tgt_qp_list); | |
| 5043 | − | |
| 5044 | − devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL); | |
| 5045 | − if (IS_ERR(devr->x1)) { | |
| 5046 | − ret = PTR_ERR(devr->x1); | |
| 5047 | − goto error3; | |
| 5048 | − } | |
| 5049 | − devr->x1->device = &dev->ib_dev; | |
| 5050 | − devr->x1->inode = NULL; | |
| 5051 | − atomic_set(&devr->x1->usecnt, 0); | |
| 5052 | − mutex_init(&devr->x1->tgt_qp_mutex); | |
| 5053 | − INIT_LIST_HEAD(&devr->x1->tgt_qp_list); | |
| 5054 | − | |
| 5055 | − memset(&attr, 0, sizeof(attr)); | |
| 5056 | − attr.attr.max_sge = 1; | |
| 5057 | − attr.attr.max_wr = 1; | |
| 5058 | − attr.srq_type = IB_SRQT_XRC; | |
| 5059 | − attr.ext.cq = devr->c0; | |
| 5060 | − attr.ext.xrc.xrcd = devr->x0; | |
| 5061 | − | |
| 5062 | − devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq); | |
| 5063 | − if (!devr->s0) { | |
| 5064 | − ret = -ENOMEM; | |
| 5065 | − goto error4; | |
| 5066 | − } | |
| 5067 | − | |
| 5068 | − devr->s0->device = &dev->ib_dev; | |
| 5069 | − devr->s0->pd = devr->p0; | |
| 5070 | − devr->s0->srq_type = IB_SRQT_XRC; | |
| 5071 | − devr->s0->ext.xrc.xrcd = devr->x0; | |
| 5072 | − devr->s0->ext.cq = devr->c0; | |
| 5073 | − ret = mlx5_ib_create_srq(devr->s0, &attr, NULL); | |
| 5074 | − if (ret) | |
| 5075 | − goto err_create; | |
| 5076 | − | |
| 5077 | − atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt); | |
| 5078 | − atomic_inc(&devr->s0->ext.cq->usecnt); | |
| 5079 | − atomic_inc(&devr->p0->usecnt); | |
| 5080 | − atomic_set(&devr->s0->usecnt, 0); | |
| 5081 | − | |
| 5082 | − memset(&attr, 0, sizeof(attr)); | |
| 5083 | − attr.attr.max_sge = 1; | |
| 5084 | − attr.attr.max_wr = 1; | |
| 5085 | − attr.srq_type = IB_SRQT_BASIC; | |
| 5086 | − devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq); | |
| 5087 | − if (!devr->s1) { | |
| 5088 | − ret = -ENOMEM; | |
| 5089 | − goto error5; | |
| 5090 | − } | |
| 5091 | − | |
| 5092 | − devr->s1->device = &dev->ib_dev; | |
| 5093 | − devr->s1->pd = devr->p0; | |
| 5094 | − devr->s1->srq_type = IB_SRQT_BASIC; | |
| 5095 | − devr->s1->ext.cq = devr->c0; | |
| 5096 | − | |
| 5097 | − ret = mlx5_ib_create_srq(devr->s1, &attr, NULL); | |
| 5098 | − if (ret) | |
| 5099 | − goto error6; | |
| 5100 | − | |
| 5101 | − atomic_inc(&devr->p0->usecnt); | |
| 5102 | − atomic_set(&devr->s1->usecnt, 0); | |
| 5103 | − | |
| 5104 | − for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) { | |
| 5105 | − INIT_WORK(&devr->ports[port].pkey_change_work, | |
| 5106 | − pkey_change_handler); | |
| 5107 | − devr->ports[port].devr = devr; | |
| 5108 | − } | |
| 5109 | − | |
| 5110 | − return 0; | |
| 5111 | − | |
| 5112 | −error6: | |
| 5113 | − kfree(devr->s1); | |
| 5114 | −error5: | |
| 5115 | − mlx5_ib_destroy_srq(devr->s0, NULL); | |
| 5116 | −err_create: | |
| 5117 | − kfree(devr->s0); | |
| 5118 | −error4: | |
| 5119 | − mlx5_ib_dealloc_xrcd(devr->x1, NULL); | |
| 5120 | −error3: | |
| 5121 | − mlx5_ib_dealloc_xrcd(devr->x0, NULL); | |
| 5122 | −error2: | |
| 5123 | − mlx5_ib_destroy_cq(devr->c0, NULL); | |
| 5124 | −err_create_cq: | |
| 5125 | − kfree(devr->c0); | |
| 5126 | −error1: | |
| 5127 | − mlx5_ib_dealloc_pd(devr->p0, NULL); | |
| 5128 | −error0: | |
| 5129 | − kfree(devr->p0); | |
| 5130 | − return ret; | |
| 5131 | −} | |
| 5132 | − | |
| 5133 | −static void destroy_dev_resources(struct mlx5_ib_resources *devr) | |
| 5134 | −{ | |
| 5135 | − int port; | |
| 5136 | − | |
| 5137 | − mlx5_ib_destroy_srq(devr->s1, NULL); | |
| 5138 | − kfree(devr->s1); | |
| 5139 | − mlx5_ib_destroy_srq(devr->s0, NULL); | |
| 5140 | − kfree(devr->s0); | |
| 5141 | − mlx5_ib_dealloc_xrcd(devr->x0, NULL); | |
| 5142 | − mlx5_ib_dealloc_xrcd(devr->x1, NULL); | |
| 5143 | − mlx5_ib_destroy_cq(devr->c0, NULL); | |
| 5144 | − kfree(devr->c0); | |
| 5145 | − mlx5_ib_dealloc_pd(devr->p0, NULL); | |
| 5146 | − kfree(devr->p0); | |
| 5147 | − | |
| 5148 | − /* Make sure no change P_Key work items are still executing */ | |
| 5149 | − for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) | |
| 5150 | − cancel_work_sync(&devr->ports[port].pkey_change_work); | |
| 5151 | −} | |
| 5152 | − | |
| 5153 | −static u32 get_core_cap_flags(struct ib_device *ibdev, | |
| 5154 | − struct mlx5_hca_vport_context *rep) | |
| 5155 | −{ | |
| 5156 | − struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 5157 | − enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1); | |
| 5158 | − u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type); | |
| 5159 | − u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version); | |
| 5160 | − bool raw_support = !mlx5_core_mp_enabled(dev->mdev); | |
| 5161 | − u32 ret = 0; | |
| 5162 | − | |
| 5163 | − if (rep->grh_required) | |
| 5164 | − ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED; | |
| 5165 | − | |
| 5166 | − if (ll == IB_LINK_LAYER_INFINIBAND) | |
| 5167 | − return ret | RDMA_CORE_PORT_IBA_IB; | |
| 5168 | − | |
| 5169 | − if (raw_support) | |
| 5170 | − ret |= RDMA_CORE_PORT_RAW_PACKET; | |
| 5171 | − | |
| 5172 | − if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP)) | |
| 5173 | − return ret; | |
| 5174 | − | |
| 5175 | − if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP)) | |
| 5176 | − return ret; | |
| 5177 | − | |
| 5178 | − if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP) | |
| 5179 | − ret |= RDMA_CORE_PORT_IBA_ROCE; | |
| 5180 | − | |
| 5181 | − if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP) | |
| 5182 | − ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; | |
| 5183 | − | |
| 5184 | − return ret; | |
| 5185 | −} | |
| 5186 | − | |
| 5187 | −static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num, | |
| 5188 | − struct ib_port_immutable *immutable) | |
| 5189 | −{ | |
| 5190 | − struct ib_port_attr attr; | |
| 5191 | − struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 5192 | − enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num); | |
| 5193 | − struct mlx5_hca_vport_context rep = {0}; | |
| 5194 | − int err; | |
| 5195 | − | |
| 5196 | − err = ib_query_port(ibdev, port_num, &attr); | |
| 5197 | − if (err) | |
| 5198 | − return err; | |
| 5199 | − | |
| 5200 | − if (ll == IB_LINK_LAYER_INFINIBAND) { | |
| 5201 | − err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0, | |
| 5202 | − &rep); | |
| 5203 | − if (err) | |
| 5204 | − return err; | |
| 5205 | − } | |
| 5206 | − | |
| 5207 | − immutable->pkey_tbl_len = attr.pkey_tbl_len; | |
| 5208 | − immutable->gid_tbl_len = attr.gid_tbl_len; | |
| 5209 | − immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep); | |
| 5210 | − immutable->max_mad_size = IB_MGMT_MAD_SIZE; | |
| 5211 | − | |
| 5212 | − return 0; | |
| 5213 | −} | |
| 5214 | − | |
| 5215 | −static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num, | |
| 5216 | − struct ib_port_immutable *immutable) | |
| 5217 | −{ | |
| 5218 | − struct ib_port_attr attr; | |
| 5219 | − int err; | |
| 5220 | − | |
| 5221 | − immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; | |
| 5222 | − | |
| 5223 | − err = ib_query_port(ibdev, port_num, &attr); | |
| 5224 | − if (err) | |
| 5225 | − return err; | |
| 5226 | − | |
| 5227 | − immutable->pkey_tbl_len = attr.pkey_tbl_len; | |
| 5228 | − immutable->gid_tbl_len = attr.gid_tbl_len; | |
| 5229 | − immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; | |
| 5230 | − | |
| 5231 | − return 0; | |
| 5232 | −} | |
| 5233 | − | |
| 5234 | −static void get_dev_fw_str(struct ib_device *ibdev, char *str) | |
| 5235 | −{ | |
| 5236 | − struct mlx5_ib_dev *dev = | |
| 5237 | − container_of(ibdev, struct mlx5_ib_dev, ib_dev); | |
| 5238 | − snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d", | |
| 5239 | − fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev), | |
| 5240 | − fw_rev_sub(dev->mdev)); | |
| 5241 | −} | |
| 5242 | − | |
| 5243 | −static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev) | |
| 5244 | −{ | |
| 5245 | − struct mlx5_core_dev *mdev = dev->mdev; | |
| 5246 | − struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev, | |
| 5247 | − MLX5_FLOW_NAMESPACE_LAG); | |
| 5248 | − struct mlx5_flow_table *ft; | |
| 5249 | − int err; | |
| 5250 | − | |
| 5251 | − if (!ns || !mlx5_lag_is_roce(mdev)) | |
| 5252 | − return 0; | |
| 5253 | − | |
| 5254 | − err = mlx5_cmd_create_vport_lag(mdev); | |
| 5255 | − if (err) | |
| 5256 | − return err; | |
| 5257 | − | |
| 5258 | − ft = mlx5_create_lag_demux_flow_table(ns, 0, 0); | |
| 5259 | − if (IS_ERR(ft)) { | |
| 5260 | − err = PTR_ERR(ft); | |
| 5261 | − goto err_destroy_vport_lag; | |
| 5262 | − } | |
| 5263 | − | |
| 5264 | − dev->flow_db->lag_demux_ft = ft; | |
| 5265 | − dev->lag_active = true; | |
| 5266 | − return 0; | |
| 5267 | − | |
| 5268 | −err_destroy_vport_lag: | |
| 5269 | − mlx5_cmd_destroy_vport_lag(mdev); | |
| 5270 | − return err; | |
| 5271 | −} | |
| 5272 | − | |
| 5273 | −static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev) | |
| 5274 | −{ | |
| 5275 | − struct mlx5_core_dev *mdev = dev->mdev; | |
| 5276 | − | |
| 5277 | − if (dev->lag_active) { | |
| 5278 | − dev->lag_active = false; | |
| 2919 | + struct mlx5_ib_event_work *work; | |
| 5279 | 2920 | |
| 5280 | − mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft); | |
| 5281 | − dev->flow_db->lag_demux_ft = NULL; | |
| 2921 | + work = kmalloc(sizeof(*work), GFP_ATOMIC); | |
| 2922 | + if (!work) | |
| 2923 | + return NOTIFY_DONE; | |
| 5282 | 2924 | |
| 5283 | − mlx5_cmd_destroy_vport_lag(mdev); | |
| 5284 | − } | |
| 2925 | + INIT_WORK(&work->work, mlx5_ib_handle_event); | |
| 2926 | + work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events); | |
| 2927 | + work->is_slave = true; | |
| 2928 | + work->param = param; | |
| 2929 | + work->event = event; | |
| 2930 | + queue_work(mlx5_ib_event_wq, &work->work); | |
| 2931 | + | |
| 2932 | + return NOTIFY_OK; | |
| 5285 | 2933 | } |
| 5286 | 2934 | |
| 5287 | −static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num) | |
| 2935 | +static int set_has_smi_cap(struct mlx5_ib_dev *dev) | |
| 5288 | 2936 | { |
| 2937 | + struct mlx5_hca_vport_context vport_ctx; | |
| 5289 | 2938 | int err; |
| 2939 | + int port; | |
| 5290 | 2940 | |
| 5291 | − dev->port[port_num].roce.nb.notifier_call = mlx5_netdev_event; | |
| 5292 | − err = register_netdevice_notifier(&dev->port[port_num].roce.nb); | |
| 5293 | − if (err) { | |
| 5294 | − dev->port[port_num].roce.nb.notifier_call = NULL; | |
| 5295 | − return err; | |
| 2941 | + for (port = 1; port <= ARRAY_SIZE(dev->mdev->port_caps); port++) { | |
| 2942 | + dev->mdev->port_caps[port - 1].has_smi = false; | |
| 2943 | + if (MLX5_CAP_GEN(dev->mdev, port_type) == | |
| 2944 | + MLX5_CAP_PORT_TYPE_IB) { | |
| 2945 | + if (MLX5_CAP_GEN(dev->mdev, ib_virt)) { | |
| 2946 | + err = mlx5_query_hca_vport_context(dev->mdev, 0, | |
| 2947 | + port, 0, | |
| 2948 | + &vport_ctx); | |
| 2949 | + if (err) { | |
| 2950 | + mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n", | |
| 2951 | + port, err); | |
| 2952 | + return err; | |
| 2953 | + } | |
| 2954 | + dev->mdev->port_caps[port - 1].has_smi = | |
| 2955 | + vport_ctx.has_smi; | |
| 2956 | + } else { | |
| 2957 | + dev->mdev->port_caps[port - 1].has_smi = true; | |
| 2958 | + } | |
| 2959 | + } | |
| 5296 | 2960 | } |
| 5297 | − | |
| 5298 | 2961 | return 0; |
| 5299 | 2962 | } |
| 5300 | 2963 | |
| 5301 | −static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num) | |
| 2964 | +static void get_ext_port_caps(struct mlx5_ib_dev *dev) | |
| 5302 | 2965 | { |
| 5303 | − if (dev->port[port_num].roce.nb.notifier_call) { | |
| 5304 | − unregister_netdevice_notifier(&dev->port[port_num].roce.nb); | |
| 5305 | − dev->port[port_num].roce.nb.notifier_call = NULL; | |
| 5306 | − } | |
| 2966 | + int port; | |
| 2967 | + | |
| 2968 | + for (port = 1; port <= dev->num_ports; port++) | |
| 2969 | + mlx5_query_ext_port_caps(dev, port); | |
| 5307 | 2970 | } |
| 5308 | 2971 | |
| 5309 | −static int mlx5_enable_eth(struct mlx5_ib_dev *dev) | |
| 2972 | +static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port) | |
| 5310 | 2973 | { |
| 5311 | − int err; | |
| 2974 | + struct ib_device_attr *dprops = NULL; | |
| 2975 | + struct ib_port_attr *pprops = NULL; | |
| 2976 | + int err = -ENOMEM; | |
| 5312 | 2977 | |
| 5313 | − err = mlx5_nic_vport_enable_roce(dev->mdev); | |
| 5314 | − if (err) | |
| 5315 | − return err; | |
| 2978 | + pprops = kzalloc(sizeof(*pprops), GFP_KERNEL); | |
| 2979 | + if (!pprops) | |
| 2980 | + goto out; | |
| 5316 | 2981 | |
| 5317 | − err = mlx5_eth_lag_init(dev); | |
| 5318 | − if (err) | |
| 5319 | − goto err_disable_roce; | |
| 2982 | + dprops = kmalloc(sizeof(*dprops), GFP_KERNEL); | |
| 2983 | + if (!dprops) | |
| 2984 | + goto out; | |
| 5320 | 2985 | |
| 5321 | − return 0; | |
| 2986 | + err = mlx5_ib_query_device(&dev->ib_dev, dprops, NULL); | |
| 2987 | + if (err) { | |
| 2988 | + mlx5_ib_warn(dev, "query_device failed %d\n", err); | |
| 2989 | + goto out; | |
| 2990 | + } | |
| 5322 | 2991 | |
| 5323 | −err_disable_roce: | |
| 5324 | − mlx5_nic_vport_disable_roce(dev->mdev); | |
| 2992 | + err = mlx5_ib_query_port(&dev->ib_dev, port, pprops); | |
| 2993 | + if (err) { | |
| 2994 | + mlx5_ib_warn(dev, "query_port %d failed %d\n", | |
| 2995 | + port, err); | |
| 2996 | + goto out; | |
| 2997 | + } | |
| 2998 | + | |
| 2999 | + dev->mdev->port_caps[port - 1].pkey_table_len = | |
| 3000 | + dprops->max_pkeys; | |
| 3001 | + dev->mdev->port_caps[port - 1].gid_table_len = | |
| 3002 | + pprops->gid_tbl_len; | |
| 3003 | + mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n", | |
| 3004 | + port, dprops->max_pkeys, pprops->gid_tbl_len); | |
| 3005 | + | |
| 3006 | +out: | |
| 3007 | + kfree(pprops); | |
| 3008 | + kfree(dprops); | |
| 5325 | 3009 | |
| 5326 | 3010 | return err; |
| 5327 | 3011 | } |
| 5328 | 3012 | |
| 5329 | −static void mlx5_disable_eth(struct mlx5_ib_dev *dev) | |
| 3013 | +static int get_port_caps(struct mlx5_ib_dev *dev, u8 port) | |
| 5330 | 3014 | { |
| 5331 | − mlx5_eth_lag_cleanup(dev); | |
| 5332 | − mlx5_nic_vport_disable_roce(dev->mdev); | |
| 3015 | + /* For representors use port 1, is this is the only native | |
| 3016 | + * port | |
| 3017 | + */ | |
| 3018 | + if (dev->is_rep) | |
| 3019 | + return __get_port_caps(dev, 1); | |
| 3020 | + return __get_port_caps(dev, port); | |
| 5333 | 3021 | } |
| 5334 | 3022 | |
| 5335 | −struct mlx5_ib_counter { | |
| 5336 | − const char *name; | |
| 5337 | − size_t offset; | |
| 5338 | −}; | |
| 5339 | − | |
| 5340 | −#define INIT_Q_COUNTER(_name) \ | |
| 5341 | − { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)} | |
| 3023 | +static u8 mlx5_get_umr_fence(u8 umr_fence_cap) | |
| 3024 | +{ | |
| 3025 | + switch (umr_fence_cap) { | |
| 3026 | + case MLX5_CAP_UMR_FENCE_NONE: | |
| 3027 | + return MLX5_FENCE_MODE_NONE; | |
| 3028 | + case MLX5_CAP_UMR_FENCE_SMALL: | |
| 3029 | + return MLX5_FENCE_MODE_INITIATOR_SMALL; | |
| 3030 | + default: | |
| 3031 | + return MLX5_FENCE_MODE_STRONG_ORDERING; | |
| 3032 | + } | |
| 3033 | +} | |
| 5342 | 3034 | |
| 5343 | −static const struct mlx5_ib_counter basic_q_cnts[] = { | |
| 5344 | − INIT_Q_COUNTER(rx_write_requests), | |
| 5345 | − INIT_Q_COUNTER(rx_read_requests), | |
| 5346 | − INIT_Q_COUNTER(rx_atomic_requests), | |
| 5347 | − INIT_Q_COUNTER(out_of_buffer), | |
| 5348 | −}; | |
| 3035 | +static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) | |
| 3036 | +{ | |
| 3037 | + struct mlx5_ib_resources *devr = &dev->devr; | |
| 3038 | + struct ib_srq_init_attr attr; | |
| 3039 | + struct ib_device *ibdev; | |
| 3040 | + struct ib_cq_init_attr cq_attr = {.cqe = 1}; | |
| 3041 | + int port; | |
| 3042 | + int ret = 0; | |
| 5349 | 3043 | |
| 5350 | −static const struct mlx5_ib_counter out_of_seq_q_cnts[] = { | |
| 5351 | − INIT_Q_COUNTER(out_of_sequence), | |
| 5352 | −}; | |
| 3044 | + ibdev = &dev->ib_dev; | |
| 5353 | 3045 | |
| 5354 | −static const struct mlx5_ib_counter retrans_q_cnts[] = { | |
| 5355 | − INIT_Q_COUNTER(duplicate_request), | |
| 5356 | − INIT_Q_COUNTER(rnr_nak_retry_err), | |
| 5357 | − INIT_Q_COUNTER(packet_seq_err), | |
| 5358 | − INIT_Q_COUNTER(implied_nak_seq_err), | |
| 5359 | − INIT_Q_COUNTER(local_ack_timeout_err), | |
| 5360 | −}; | |
| 3046 | + if (!MLX5_CAP_GEN(dev->mdev, xrc)) | |
| 3047 | + return -EOPNOTSUPP; | |
| 5361 | 3048 | |
| 5362 | −#define INIT_CONG_COUNTER(_name) \ | |
| 5363 | − { .name = #_name, .offset = \ | |
| 5364 | − MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)} | |
| 3049 | + mutex_init(&devr->mutex); | |
| 5365 | 3050 | |
| 5366 | −static const struct mlx5_ib_counter cong_cnts[] = { | |
| 5367 | − INIT_CONG_COUNTER(rp_cnp_ignored), | |
| 5368 | − INIT_CONG_COUNTER(rp_cnp_handled), | |
| 5369 | − INIT_CONG_COUNTER(np_ecn_marked_roce_packets), | |
| 5370 | − INIT_CONG_COUNTER(np_cnp_sent), | |
| 5371 | −}; | |
| 3051 | + devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd); | |
| 3052 | + if (!devr->p0) | |
| 3053 | + return -ENOMEM; | |
| 5372 | 3054 | |
| 5373 | −static const struct mlx5_ib_counter extended_err_cnts[] = { | |
| 5374 | − INIT_Q_COUNTER(resp_local_length_error), | |
| 5375 | − INIT_Q_COUNTER(resp_cqe_error), | |
| 5376 | − INIT_Q_COUNTER(req_cqe_error), | |
| 5377 | − INIT_Q_COUNTER(req_remote_invalid_request), | |
| 5378 | − INIT_Q_COUNTER(req_remote_access_errors), | |
| 5379 | − INIT_Q_COUNTER(resp_remote_access_errors), | |
| 5380 | − INIT_Q_COUNTER(resp_cqe_flush_error), | |
| 5381 | − INIT_Q_COUNTER(req_cqe_flush_error), | |
| 5382 | −}; | |
| 3055 | + devr->p0->device = ibdev; | |
| 3056 | + devr->p0->uobject = NULL; | |
| 3057 | + atomic_set(&devr->p0->usecnt, 0); | |
| 5383 | 3058 | |
| 5384 | −static const struct mlx5_ib_counter roce_accl_cnts[] = { | |
| 5385 | − INIT_Q_COUNTER(roce_adp_retrans), | |
| 5386 | − INIT_Q_COUNTER(roce_adp_retrans_to), | |
| 5387 | − INIT_Q_COUNTER(roce_slow_restart), | |
| 5388 | − INIT_Q_COUNTER(roce_slow_restart_cnps), | |
| 5389 | − INIT_Q_COUNTER(roce_slow_restart_trans), | |
| 5390 | −}; | |
| 3059 | + ret = mlx5_ib_alloc_pd(devr->p0, NULL); | |
| 3060 | + if (ret) | |
| 3061 | + goto error0; | |
| 5391 | 3062 | |
| 5392 | −#define INIT_EXT_PPCNT_COUNTER(_name) \ | |
| 5393 | − { .name = #_name, .offset = \ | |
| 5394 | − MLX5_BYTE_OFF(ppcnt_reg, \ | |
| 5395 | − counter_set.eth_extended_cntrs_grp_data_layout._name##_high)} | |
| 3063 | + devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq); | |
| 3064 | + if (!devr->c0) { | |
| 3065 | + ret = -ENOMEM; | |
| 3066 | + goto error1; | |
| 3067 | + } | |
| 5396 | 3068 | |
| 5397 | −static const struct mlx5_ib_counter ext_ppcnt_cnts[] = { | |
| 5398 | − INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated), | |
| 5399 | −}; | |
| 3069 | + devr->c0->device = &dev->ib_dev; | |
| 3070 | + atomic_set(&devr->c0->usecnt, 0); | |
| 5400 | 3071 | |
| 5401 | −static bool is_mdev_switchdev_mode(const struct mlx5_core_dev *mdev) | |
| 5402 | −{ | |
| 5403 | − return MLX5_ESWITCH_MANAGER(mdev) && | |
| 5404 | − mlx5_ib_eswitch_mode(mdev->priv.eswitch) == | |
| 5405 | − MLX5_ESWITCH_OFFLOADS; | |
| 5406 | −} | |
| 3072 | + ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL); | |
| 3073 | + if (ret) | |
| 3074 | + goto err_create_cq; | |
| 5407 | 3075 | |
| 5408 | −static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev) | |
| 5409 | −{ | |
| 5410 | − u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; | |
| 5411 | − int num_cnt_ports; | |
| 5412 | − int i; | |
| 3076 | + ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0); | |
| 3077 | + if (ret) | |
| 3078 | + goto error2; | |
| 5413 | 3079 | |
| 5414 | − num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; | |
| 3080 | + ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0); | |
| 3081 | + if (ret) | |
| 3082 | + goto error3; | |
| 5415 | 3083 | |
| 5416 | − MLX5_SET(dealloc_q_counter_in, in, opcode, | |
| 5417 | − MLX5_CMD_OP_DEALLOC_Q_COUNTER); | |
| 3084 | + memset(&attr, 0, sizeof(attr)); | |
| 3085 | + attr.attr.max_sge = 1; | |
| 3086 | + attr.attr.max_wr = 1; | |
| 3087 | + attr.srq_type = IB_SRQT_XRC; | |
| 3088 | + attr.ext.cq = devr->c0; | |
| 5418 | 3089 | |
| 5419 | − for (i = 0; i < num_cnt_ports; i++) { | |
| 5420 | − if (dev->port[i].cnts.set_id) { | |
| 5421 | − MLX5_SET(dealloc_q_counter_in, in, counter_set_id, | |
| 5422 | − dev->port[i].cnts.set_id); | |
| 5423 | − mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); | |
| 5424 | − } | |
| 5425 | − kfree(dev->port[i].cnts.names); | |
| 5426 | − kfree(dev->port[i].cnts.offsets); | |
| 3090 | + devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq); | |
| 3091 | + if (!devr->s0) { | |
| 3092 | + ret = -ENOMEM; | |
| 3093 | + goto error4; | |
| 5427 | 3094 | } |
| 5428 | −} | |
| 5429 | − | |
| 5430 | −static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, | |
| 5431 | − struct mlx5_ib_counters *cnts) | |
| 5432 | −{ | |
| 5433 | − u32 num_counters; | |
| 5434 | 3095 | |
| 5435 | − num_counters = ARRAY_SIZE(basic_q_cnts); | |
| 3096 | + devr->s0->device = &dev->ib_dev; | |
| 3097 | + devr->s0->pd = devr->p0; | |
| 3098 | + devr->s0->srq_type = IB_SRQT_XRC; | |
| 3099 | + devr->s0->ext.cq = devr->c0; | |
| 3100 | + ret = mlx5_ib_create_srq(devr->s0, &attr, NULL); | |
| 3101 | + if (ret) | |
| 3102 | + goto err_create; | |
| 5436 | 3103 | |
| 5437 | − if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) | |
| 5438 | − num_counters += ARRAY_SIZE(out_of_seq_q_cnts); | |
| 3104 | + atomic_inc(&devr->s0->ext.cq->usecnt); | |
| 3105 | + atomic_inc(&devr->p0->usecnt); | |
| 3106 | + atomic_set(&devr->s0->usecnt, 0); | |
| 5439 | 3107 | |
| 5440 | − if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) | |
| 5441 | − num_counters += ARRAY_SIZE(retrans_q_cnts); | |
| 3108 | + memset(&attr, 0, sizeof(attr)); | |
| 3109 | + attr.attr.max_sge = 1; | |
| 3110 | + attr.attr.max_wr = 1; | |
| 3111 | + attr.srq_type = IB_SRQT_BASIC; | |
| 3112 | + devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq); | |
| 3113 | + if (!devr->s1) { | |
| 3114 | + ret = -ENOMEM; | |
| 3115 | + goto error5; | |
| 3116 | + } | |
| 5442 | 3117 | |
| 5443 | − if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) | |
| 5444 | − num_counters += ARRAY_SIZE(extended_err_cnts); | |
| 3118 | + devr->s1->device = &dev->ib_dev; | |
| 3119 | + devr->s1->pd = devr->p0; | |
| 3120 | + devr->s1->srq_type = IB_SRQT_BASIC; | |
| 3121 | + devr->s1->ext.cq = devr->c0; | |
| 5445 | 3122 | |
| 5446 | − if (MLX5_CAP_GEN(dev->mdev, roce_accl)) | |
| 5447 | − num_counters += ARRAY_SIZE(roce_accl_cnts); | |
| 3123 | + ret = mlx5_ib_create_srq(devr->s1, &attr, NULL); | |
| 3124 | + if (ret) | |
| 3125 | + goto error6; | |
| 5448 | 3126 | |
| 5449 | − cnts->num_q_counters = num_counters; | |
| 3127 | + atomic_inc(&devr->p0->usecnt); | |
| 3128 | + atomic_set(&devr->s1->usecnt, 0); | |
| 5450 | 3129 | |
| 5451 | − if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { | |
| 5452 | − cnts->num_cong_counters = ARRAY_SIZE(cong_cnts); | |
| 5453 | − num_counters += ARRAY_SIZE(cong_cnts); | |
| 5454 | − } | |
| 5455 | − if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { | |
| 5456 | − cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts); | |
| 5457 | − num_counters += ARRAY_SIZE(ext_ppcnt_cnts); | |
| 3130 | + for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) { | |
| 3131 | + INIT_WORK(&devr->ports[port].pkey_change_work, | |
| 3132 | + pkey_change_handler); | |
| 3133 | + devr->ports[port].devr = devr; | |
| 5458 | 3134 | } |
| 5459 | − cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL); | |
| 5460 | − if (!cnts->names) | |
| 5461 | − return -ENOMEM; | |
| 5462 | − | |
| 5463 | − cnts->offsets = kcalloc(num_counters, | |
| 5464 | − sizeof(cnts->offsets), GFP_KERNEL); | |
| 5465 | − if (!cnts->offsets) | |
| 5466 | − goto err_names; | |
| 5467 | 3135 | |
| 5468 | 3136 | return 0; |
| 5469 | 3137 | |
| 5470 | −err_names: | |
| 5471 | − kfree(cnts->names); | |
| 5472 | − cnts->names = NULL; | |
| 5473 | − return -ENOMEM; | |
| 3138 | +error6: | |
| 3139 | + kfree(devr->s1); | |
| 3140 | +error5: | |
| 3141 | + mlx5_ib_destroy_srq(devr->s0, NULL); | |
| 3142 | +err_create: | |
| 3143 | + kfree(devr->s0); | |
| 3144 | +error4: | |
| 3145 | + mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0); | |
| 3146 | +error3: | |
| 3147 | + mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0); | |
| 3148 | +error2: | |
| 3149 | + mlx5_ib_destroy_cq(devr->c0, NULL); | |
| 3150 | +err_create_cq: | |
| 3151 | + kfree(devr->c0); | |
| 3152 | +error1: | |
| 3153 | + mlx5_ib_dealloc_pd(devr->p0, NULL); | |
| 3154 | +error0: | |
| 3155 | + kfree(devr->p0); | |
| 3156 | + return ret; | |
| 5474 | 3157 | } |
| 5475 | 3158 | |
| 5476 | −static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev, | |
| 5477 | − const char **names, | |
| 5478 | − size_t *offsets) | |
| 3159 | +static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev) | |
| 5479 | 3160 | { |
| 5480 | − int i; | |
| 5481 | − int j = 0; | |
| 5482 | − | |
| 5483 | − for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) { | |
| 5484 | − names[j] = basic_q_cnts[i].name; | |
| 5485 | − offsets[j] = basic_q_cnts[i].offset; | |
| 5486 | − } | |
| 5487 | − | |
| 5488 | − if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) { | |
| 5489 | − for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) { | |
| 5490 | − names[j] = out_of_seq_q_cnts[i].name; | |
| 5491 | − offsets[j] = out_of_seq_q_cnts[i].offset; | |
| 5492 | − } | |
| 5493 | − } | |
| 5494 | − | |
| 5495 | − if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) { | |
| 5496 | − for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) { | |
| 5497 | − names[j] = retrans_q_cnts[i].name; | |
| 5498 | − offsets[j] = retrans_q_cnts[i].offset; | |
| 5499 | − } | |
| 5500 | − } | |
| 5501 | − | |
| 5502 | − if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) { | |
| 5503 | − for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) { | |
| 5504 | − names[j] = extended_err_cnts[i].name; | |
| 5505 | − offsets[j] = extended_err_cnts[i].offset; | |
| 5506 | − } | |
| 5507 | − } | |
| 5508 | − | |
| 5509 | − if (MLX5_CAP_GEN(dev->mdev, roce_accl)) { | |
| 5510 | − for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) { | |
| 5511 | − names[j] = roce_accl_cnts[i].name; | |
| 5512 | − offsets[j] = roce_accl_cnts[i].offset; | |
| 5513 | − } | |
| 5514 | − } | |
| 3161 | + struct mlx5_ib_resources *devr = &dev->devr; | |
| 3162 | + int port; | |
| 5515 | 3163 | |
| 5516 | − if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { | |
| 5517 | − for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) { | |
| 5518 | − names[j] = cong_cnts[i].name; | |
| 5519 | − offsets[j] = cong_cnts[i].offset; | |
| 5520 | − } | |
| 5521 | − } | |
| 3164 | + mlx5_ib_destroy_srq(devr->s1, NULL); | |
| 3165 | + kfree(devr->s1); | |
| 3166 | + mlx5_ib_destroy_srq(devr->s0, NULL); | |
| 3167 | + kfree(devr->s0); | |
| 3168 | + mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0); | |
| 3169 | + mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0); | |
| 3170 | + mlx5_ib_destroy_cq(devr->c0, NULL); | |
| 3171 | + kfree(devr->c0); | |
| 3172 | + mlx5_ib_dealloc_pd(devr->p0, NULL); | |
| 3173 | + kfree(devr->p0); | |
| 5522 | 3174 | |
| 5523 | − if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { | |
| 5524 | − for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) { | |
| 5525 | − names[j] = ext_ppcnt_cnts[i].name; | |
| 5526 | − offsets[j] = ext_ppcnt_cnts[i].offset; | |
| 5527 | − } | |
| 5528 | − } | |
| 3175 | + /* Make sure no change P_Key work items are still executing */ | |
| 3176 | + for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) | |
| 3177 | + cancel_work_sync(&devr->ports[port].pkey_change_work); | |
| 5529 | 3178 | } |
| 5530 | 3179 | |
| 5531 | −static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev) | |
| 3180 | +static u32 get_core_cap_flags(struct ib_device *ibdev, | |
| 3181 | + struct mlx5_hca_vport_context *rep) | |
| 5532 | 3182 | { |
| 5533 | − u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; | |
| 5534 | − u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; | |
| 5535 | − int num_cnt_ports; | |
| 5536 | − int err = 0; | |
| 5537 | − int i; | |
| 5538 | − bool is_shared; | |
| 5539 | − | |
| 5540 | − MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); | |
| 5541 | − is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0; | |
| 5542 | − num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; | |
| 5543 | − | |
| 5544 | − for (i = 0; i < num_cnt_ports; i++) { | |
| 5545 | − err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts); | |
| 5546 | − if (err) | |
| 5547 | − goto err_alloc; | |
| 3183 | + struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 3184 | + enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1); | |
| 3185 | + u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type); | |
| 3186 | + u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version); | |
| 3187 | + bool raw_support = !mlx5_core_mp_enabled(dev->mdev); | |
| 3188 | + u32 ret = 0; | |
| 5548 | 3189 | |
| 5549 | − mlx5_ib_fill_counters(dev, dev->port[i].cnts.names, | |
| 5550 | − dev->port[i].cnts.offsets); | |
| 3190 | + if (rep->grh_required) | |
| 3191 | + ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED; | |
| 5551 | 3192 | |
| 5552 | − MLX5_SET(alloc_q_counter_in, in, uid, | |
| 5553 | − is_shared ? MLX5_SHARED_RESOURCE_UID : 0); | |
| 3193 | + if (ll == IB_LINK_LAYER_INFINIBAND) | |
| 3194 | + return ret | RDMA_CORE_PORT_IBA_IB; | |
| 5554 | 3195 | |
| 5555 | − err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); | |
| 5556 | − if (err) { | |
| 5557 | − mlx5_ib_warn(dev, | |
| 5558 | − "couldn't allocate queue counter for port %d, err %d\n", | |
| 5559 | − i + 1, err); | |
| 5560 | − goto err_alloc; | |
| 5561 | − } | |
| 3196 | + if (raw_support) | |
| 3197 | + ret |= RDMA_CORE_PORT_RAW_PACKET; | |
| 5562 | 3198 | |
| 5563 | − dev->port[i].cnts.set_id = | |
| 5564 | − MLX5_GET(alloc_q_counter_out, out, counter_set_id); | |
| 5565 | − } | |
| 5566 | − return 0; | |
| 3199 | + if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP)) | |
| 3200 | + return ret; | |
| 5567 | 3201 | |
| 5568 | −err_alloc: | |
| 5569 | − mlx5_ib_dealloc_counters(dev); | |
| 5570 | − return err; | |
| 5571 | −} | |
| 3202 | + if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP)) | |
| 3203 | + return ret; | |
| 5572 | 3204 | |
| 5573 | −static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, | |
| 5574 | − u8 port_num) | |
| 5575 | −{ | |
| 5576 | − return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts : | |
| 5577 | − &dev->port[port_num].cnts; | |
| 5578 | −} | |
| 3205 | + if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP) | |
| 3206 | + ret |= RDMA_CORE_PORT_IBA_ROCE; | |
| 5579 | 3207 | |
| 5580 | −/** | |
| 5581 | − * mlx5_ib_get_counters_id - Returns counters id to use for device+port | |
| 5582 | − * @dev: Pointer to mlx5 IB device | |
| 5583 | − * @port_num: Zero based port number | |
| 5584 | − * | |
| 5585 | − * mlx5_ib_get_counters_id() Returns counters set id to use for given | |
| 5586 | − * device port combination in switchdev and non switchdev mode of the | |
| 5587 | − * parent device. | |
| 5588 | − */ | |
| 5589 | −u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num) | |
| 5590 | −{ | |
| 5591 | − const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); | |
| 3208 | + if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP) | |
| 3209 | + ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; | |
| 5592 | 3210 | |
| 5593 | − return cnts->set_id; | |
| 3211 | + return ret; | |
| 5594 | 3212 | } |
| 5595 | 3213 | |
| 5596 | −static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev, | |
| 5597 | − u8 port_num) | |
| 3214 | +static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num, | |
| 3215 | + struct ib_port_immutable *immutable) | |
| 5598 | 3216 | { |
| 3217 | + struct ib_port_attr attr; | |
| 5599 | 3218 | struct mlx5_ib_dev *dev = to_mdev(ibdev); |
| 5600 | − const struct mlx5_ib_counters *cnts; | |
| 5601 | − bool is_switchdev = is_mdev_switchdev_mode(dev->mdev); | |
| 3219 | + enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num); | |
| 3220 | + struct mlx5_hca_vport_context rep = {0}; | |
| 3221 | + int err; | |
| 5602 | 3222 | |
| 5603 | − if ((is_switchdev && port_num) || (!is_switchdev && !port_num)) | |
| 5604 | − return NULL; | |
| 3223 | + err = ib_query_port(ibdev, port_num, &attr); | |
| 3224 | + if (err) | |
| 3225 | + return err; | |
| 3226 | + | |
| 3227 | + if (ll == IB_LINK_LAYER_INFINIBAND) { | |
| 3228 | + err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0, | |
| 3229 | + &rep); | |
| 3230 | + if (err) | |
| 3231 | + return err; | |
| 3232 | + } | |
| 5605 | 3233 | |
| 5606 | − cnts = get_counters(dev, port_num - 1); | |
| 3234 | + immutable->pkey_tbl_len = attr.pkey_tbl_len; | |
| 3235 | + immutable->gid_tbl_len = attr.gid_tbl_len; | |
| 3236 | + immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep); | |
| 3237 | + immutable->max_mad_size = IB_MGMT_MAD_SIZE; | |
| 5607 | 3238 | |
| 5608 | − return rdma_alloc_hw_stats_struct(cnts->names, | |
| 5609 | − cnts->num_q_counters + | |
| 5610 | − cnts->num_cong_counters + | |
| 5611 | − cnts->num_ext_ppcnt_counters, | |
| 5612 | − RDMA_HW_STATS_DEFAULT_LIFESPAN); | |
| 3239 | + return 0; | |
| 5613 | 3240 | } |
| 5614 | 3241 | |
| 5615 | −static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, | |
| 5616 | − const struct mlx5_ib_counters *cnts, | |
| 5617 | − struct rdma_hw_stats *stats, | |
| 5618 | − u16 set_id) | |
| 3242 | +static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num, | |
| 3243 | + struct ib_port_immutable *immutable) | |
| 5619 | 3244 | { |
| 5620 | − u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; | |
| 5621 | − u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; | |
| 5622 | − __be32 val; | |
| 5623 | − int ret, i; | |
| 3245 | + struct ib_port_attr attr; | |
| 3246 | + int err; | |
| 5624 | 3247 | |
| 5625 | − MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); | |
| 5626 | − MLX5_SET(query_q_counter_in, in, counter_set_id, set_id); | |
| 5627 | − ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); | |
| 5628 | − if (ret) | |
| 5629 | − return ret; | |
| 3248 | + immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; | |
| 5630 | 3249 | |
| 5631 | − for (i = 0; i < cnts->num_q_counters; i++) { | |
| 5632 | − val = *(__be32 *)((void *)out + cnts->offsets[i]); | |
| 5633 | − stats->value[i] = (u64)be32_to_cpu(val); | |
| 5634 | − } | |
| 3250 | + err = ib_query_port(ibdev, port_num, &attr); | |
| 3251 | + if (err) | |
| 3252 | + return err; | |
| 3253 | + | |
| 3254 | + immutable->pkey_tbl_len = attr.pkey_tbl_len; | |
| 3255 | + immutable->gid_tbl_len = attr.gid_tbl_len; | |
| 3256 | + immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; | |
| 5635 | 3257 | |
| 5636 | 3258 | return 0; |
| 5637 | 3259 | } |
| 5638 | 3260 | |
| 5639 | −static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev, | |
| 5640 | − const struct mlx5_ib_counters *cnts, | |
| 5641 | − struct rdma_hw_stats *stats) | |
| 3261 | +static void get_dev_fw_str(struct ib_device *ibdev, char *str) | |
| 5642 | 3262 | { |
| 5643 | − int offset = cnts->num_q_counters + cnts->num_cong_counters; | |
| 5644 | − int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); | |
| 5645 | − int ret, i; | |
| 5646 | − void *out; | |
| 5647 | − | |
| 5648 | − out = kvzalloc(sz, GFP_KERNEL); | |
| 5649 | − if (!out) | |
| 5650 | − return -ENOMEM; | |
| 5651 | − | |
| 5652 | − ret = mlx5_cmd_query_ext_ppcnt_counters(dev->mdev, out); | |
| 5653 | − if (ret) | |
| 5654 | − goto free; | |
| 5655 | − | |
| 5656 | − for (i = 0; i < cnts->num_ext_ppcnt_counters; i++) | |
| 5657 | − stats->value[i + offset] = | |
| 5658 | − be64_to_cpup((__be64 *)(out + | |
| 5659 | − cnts->offsets[i + offset])); | |
| 5660 | −free: | |
| 5661 | − kvfree(out); | |
| 5662 | − return ret; | |
| 3263 | + struct mlx5_ib_dev *dev = | |
| 3264 | + container_of(ibdev, struct mlx5_ib_dev, ib_dev); | |
| 3265 | + snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d", | |
| 3266 | + fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev), | |
| 3267 | + fw_rev_sub(dev->mdev)); | |
| 5663 | 3268 | } |
| 5664 | 3269 | |
| 5665 | −static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, | |
| 5666 | − struct rdma_hw_stats *stats, | |
| 5667 | − u8 port_num, int index) | |
| 3270 | +static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev) | |
| 5668 | 3271 | { |
| 5669 | − struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 5670 | − const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1); | |
| 5671 | − struct mlx5_core_dev *mdev; | |
| 5672 | − int ret, num_counters; | |
| 5673 | − u8 mdev_port_num; | |
| 5674 | − | |
| 5675 | − if (!stats) | |
| 5676 | − return -EINVAL; | |
| 3272 | + struct mlx5_core_dev *mdev = dev->mdev; | |
| 3273 | + struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev, | |
| 3274 | + MLX5_FLOW_NAMESPACE_LAG); | |
| 3275 | + struct mlx5_flow_table *ft; | |
| 3276 | + int err; | |
| 5677 | 3277 | |
| 5678 | − num_counters = cnts->num_q_counters + | |
| 5679 | − cnts->num_cong_counters + | |
| 5680 | − cnts->num_ext_ppcnt_counters; | |
| 3278 | + if (!ns || !mlx5_lag_is_roce(mdev)) | |
| 3279 | + return 0; | |
| 5681 | 3280 | |
| 5682 | − /* q_counters are per IB device, query the master mdev */ | |
| 5683 | − ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id); | |
| 5684 | − if (ret) | |
| 5685 | − return ret; | |
| 3281 | + err = mlx5_cmd_create_vport_lag(mdev); | |
| 3282 | + if (err) | |
| 3283 | + return err; | |
| 5686 | 3284 | |
| 5687 | − if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { | |
| 5688 | − ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats); | |
| 5689 | − if (ret) | |
| 5690 | − return ret; | |
| 3285 | + ft = mlx5_create_lag_demux_flow_table(ns, 0, 0); | |
| 3286 | + if (IS_ERR(ft)) { | |
| 3287 | + err = PTR_ERR(ft); | |
| 3288 | + goto err_destroy_vport_lag; | |
| 5691 | 3289 | } |
| 5692 | 3290 | |
| 5693 | − if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { | |
| 5694 | − mdev = mlx5_ib_get_native_port_mdev(dev, port_num, | |
| 5695 | − &mdev_port_num); | |
| 5696 | − if (!mdev) { | |
| 5697 | − /* If port is not affiliated yet, its in down state | |
| 5698 | − * which doesn't have any counters yet, so it would be | |
| 5699 | − * zero. So no need to read from the HCA. | |
| 5700 | − */ | |
| 5701 | − goto done; | |
| 5702 | − } | |
| 5703 | − ret = mlx5_lag_query_cong_counters(dev->mdev, | |
| 5704 | − stats->value + | |
| 5705 | − cnts->num_q_counters, | |
| 5706 | − cnts->num_cong_counters, | |
| 5707 | − cnts->offsets + | |
| 5708 | − cnts->num_q_counters); | |
| 5709 | − | |
| 5710 | − mlx5_ib_put_native_port_mdev(dev, port_num); | |
| 5711 | − if (ret) | |
| 5712 | − return ret; | |
| 5713 | − } | |
| 3291 | + dev->flow_db->lag_demux_ft = ft; | |
| 3292 | + dev->lag_active = true; | |
| 3293 | + return 0; | |
| 5714 | 3294 | |
| 5715 | −done: | |
| 5716 | − return num_counters; | |
| 3295 | +err_destroy_vport_lag: | |
| 3296 | + mlx5_cmd_destroy_vport_lag(mdev); | |
| 3297 | + return err; | |
| 5717 | 3298 | } |
| 5718 | 3299 | |
| 5719 | −static struct rdma_hw_stats * | |
| 5720 | −mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) | |
| 3300 | +static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev) | |
| 5721 | 3301 | { |
| 5722 | − struct mlx5_ib_dev *dev = to_mdev(counter->device); | |
| 5723 | − const struct mlx5_ib_counters *cnts = | |
| 5724 | − get_counters(dev, counter->port - 1); | |
| 3302 | + struct mlx5_core_dev *mdev = dev->mdev; | |
| 5725 | 3303 | |
| 5726 | − return rdma_alloc_hw_stats_struct(cnts->names, | |
| 5727 | − cnts->num_q_counters + | |
| 5728 | − cnts->num_cong_counters + | |
| 5729 | − cnts->num_ext_ppcnt_counters, | |
| 5730 | − RDMA_HW_STATS_DEFAULT_LIFESPAN); | |
| 5731 | −} | |
| 3304 | + if (dev->lag_active) { | |
| 3305 | + dev->lag_active = false; | |
| 5732 | 3306 | |
| 5733 | −static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) | |
| 5734 | −{ | |
| 5735 | − struct mlx5_ib_dev *dev = to_mdev(counter->device); | |
| 5736 | − const struct mlx5_ib_counters *cnts = | |
| 5737 | − get_counters(dev, counter->port - 1); | |
| 3307 | + mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft); | |
| 3308 | + dev->flow_db->lag_demux_ft = NULL; | |
| 5738 | 3309 | |
| 5739 | − return mlx5_ib_query_q_counters(dev->mdev, cnts, | |
| 5740 | − counter->stats, counter->id); | |
| 3310 | + mlx5_cmd_destroy_vport_lag(mdev); | |
| 3311 | + } | |
| 5741 | 3312 | } |
| 5742 | 3313 | |
| 5743 | −static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) | |
| 3314 | +static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num) | |
| 5744 | 3315 | { |
| 5745 | − struct mlx5_ib_dev *dev = to_mdev(counter->device); | |
| 5746 | − u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; | |
| 3316 | + int err; | |
| 5747 | 3317 | |
| 5748 | − if (!counter->id) | |
| 5749 | − return 0; | |
| 3318 | + dev->port[port_num].roce.nb.notifier_call = mlx5_netdev_event; | |
| 3319 | + err = register_netdevice_notifier(&dev->port[port_num].roce.nb); | |
| 3320 | + if (err) { | |
| 3321 | + dev->port[port_num].roce.nb.notifier_call = NULL; | |
| 3322 | + return err; | |
| 3323 | + } | |
| 5750 | 3324 | |
| 5751 | − MLX5_SET(dealloc_q_counter_in, in, opcode, | |
| 5752 | − MLX5_CMD_OP_DEALLOC_Q_COUNTER); | |
| 5753 | − MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id); | |
| 5754 | − return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); | |
| 3325 | + return 0; | |
| 5755 | 3326 | } |
| 5756 | 3327 | |
| 5757 | −static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, | |
| 5758 | − struct ib_qp *qp) | |
| 3328 | +static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num) | |
| 5759 | 3329 | { |
| 5760 | − struct mlx5_ib_dev *dev = to_mdev(qp->device); | |
| 5761 | − int err; | |
| 3330 | + if (dev->port[port_num].roce.nb.notifier_call) { | |
| 3331 | + unregister_netdevice_notifier(&dev->port[port_num].roce.nb); | |
| 3332 | + dev->port[port_num].roce.nb.notifier_call = NULL; | |
| 3333 | + } | |
| 3334 | +} | |
| 5762 | 3335 | |
| 5763 | − if (!counter->id) { | |
| 5764 | − u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; | |
| 5765 | − u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; | |
| 3336 | +static int mlx5_enable_eth(struct mlx5_ib_dev *dev) | |
| 3337 | +{ | |
| 3338 | + int err; | |
| 5766 | 3339 | |
| 5767 | − MLX5_SET(alloc_q_counter_in, in, opcode, | |
| 5768 | − MLX5_CMD_OP_ALLOC_Q_COUNTER); | |
| 5769 | − MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID); | |
| 5770 | − err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); | |
| 5771 | − if (err) | |
| 5772 | − return err; | |
| 5773 | − counter->id = | |
| 5774 | − MLX5_GET(alloc_q_counter_out, out, counter_set_id); | |
| 5775 | − } | |
| 3340 | + err = mlx5_nic_vport_enable_roce(dev->mdev); | |
| 3341 | + if (err) | |
| 3342 | + return err; | |
| 5776 | 3343 | |
| 5777 | − err = mlx5_ib_qp_set_counter(qp, counter); | |
| 3344 | + err = mlx5_eth_lag_init(dev); | |
| 5778 | 3345 | if (err) |
| 5779 | − goto fail_set_counter; | |
| 3346 | + goto err_disable_roce; | |
| 5780 | 3347 | |
| 5781 | 3348 | return 0; |
| 5782 | 3349 | |
| 5783 | −fail_set_counter: | |
| 5784 | − mlx5_ib_counter_dealloc(counter); | |
| 5785 | − counter->id = 0; | |
| 3350 | +err_disable_roce: | |
| 3351 | + mlx5_nic_vport_disable_roce(dev->mdev); | |
| 5786 | 3352 | |
| 5787 | 3353 | return err; |
| 5788 | 3354 | } |
| 5789 | 3355 | |
| 5790 | −static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) | |
| 3356 | +static void mlx5_disable_eth(struct mlx5_ib_dev *dev) | |
| 5791 | 3357 | { |
| 5792 | − return mlx5_ib_qp_set_counter(qp, NULL); | |
| 3358 | + mlx5_eth_lag_cleanup(dev); | |
| 3359 | + mlx5_nic_vport_disable_roce(dev->mdev); | |
| 5793 | 3360 | } |
| 5794 | 3361 | |
| 5795 | 3362 | static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num, |
| @@ -5802,23 +3369,6 @@ static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num, · 6 unmodified lines | ||
| 5802 | 3369 | return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params); |
| 5803 | 3370 | } |
| 5804 | 3371 | |
| 5805 | −static void delay_drop_debugfs_cleanup(struct mlx5_ib_dev *dev) | |
| 5806 | −{ | |
| 5807 | − if (!dev->delay_drop.dir_debugfs) | |
| 5808 | − return; | |
| 5809 | − debugfs_remove_recursive(dev->delay_drop.dir_debugfs); | |
| 5810 | − dev->delay_drop.dir_debugfs = NULL; | |
| 5811 | −} | |
| 5812 | − | |
| 5813 | −static void cancel_delay_drop(struct mlx5_ib_dev *dev) | |
| 5814 | −{ | |
| 5815 | − if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP)) | |
| 5816 | − return; | |
| 5817 | − | |
| 5818 | − cancel_work_sync(&dev->delay_drop.delay_drop_work); | |
| 5819 | − delay_drop_debugfs_cleanup(dev); | |
| 5820 | −} | |
| 5821 | − | |
| 5822 | 3372 | static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf, |
| 5823 | 3373 | size_t count, loff_t *pos) |
| 5824 | 3374 | { |
| @@ -5858,40 +3408,6 @@ static const struct file_operations fops_delay_drop_timeout = { · 33 unmodified lines | ||
| 5858 | 3408 | .read = delay_drop_timeout_read, |
| 5859 | 3409 | }; |
| 5860 | 3410 | |
| 5861 | −static void delay_drop_debugfs_init(struct mlx5_ib_dev *dev) | |
| 5862 | −{ | |
| 5863 | − struct dentry *root; | |
| 5864 | − | |
| 5865 | − if (!mlx5_debugfs_root) | |
| 5866 | − return; | |
| 5867 | − | |
| 5868 | − root = debugfs_create_dir("delay_drop", dev->mdev->priv.dbg_root); | |
| 5869 | − dev->delay_drop.dir_debugfs = root; | |
| 5870 | − | |
| 5871 | − debugfs_create_atomic_t("num_timeout_events", 0400, root, | |
| 5872 | − &dev->delay_drop.events_cnt); | |
| 5873 | − debugfs_create_atomic_t("num_rqs", 0400, root, | |
| 5874 | − &dev->delay_drop.rqs_cnt); | |
| 5875 | − debugfs_create_file("timeout", 0600, root, &dev->delay_drop, | |
| 5876 | − &fops_delay_drop_timeout); | |
| 5877 | −} | |
| 5878 | − | |
| 5879 | −static void init_delay_drop(struct mlx5_ib_dev *dev) | |
| 5880 | −{ | |
| 5881 | − if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP)) | |
| 5882 | − return; | |
| 5883 | − | |
| 5884 | − mutex_init(&dev->delay_drop.lock); | |
| 5885 | − dev->delay_drop.dev = dev; | |
| 5886 | − dev->delay_drop.activate = false; | |
| 5887 | − dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000; | |
| 5888 | − INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler); | |
| 5889 | − atomic_set(&dev->delay_drop.rqs_cnt, 0); | |
| 5890 | − atomic_set(&dev->delay_drop.events_cnt, 0); | |
| 5891 | − | |
| 5892 | − delay_drop_debugfs_init(dev); | |
| 5893 | −} | |
| 5894 | − | |
| 5895 | 3411 | static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev, |
| 5896 | 3412 | struct mlx5_ib_multiport_info *mpi) |
| 5897 | 3413 | { |
| @@ -6385,90 +3901,32 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE( · 487 unmodified lines | ||
| 6385 | 3901 | UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS, |
| 6386 | 3902 | enum mlx5_ib_uapi_flow_action_flags)); |
| 6387 | 3903 | |
| 3904 | +ADD_UVERBS_ATTRIBUTES_SIMPLE( | |
| 3905 | + mlx5_ib_query_context, | |
| 3906 | + UVERBS_OBJECT_DEVICE, | |
| 3907 | + UVERBS_METHOD_QUERY_CONTEXT, | |
| 3908 | + UVERBS_ATTR_PTR_OUT( | |
| 3909 | + MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX, | |
| 3910 | + UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp, | |
| 3911 | + dump_fill_mkey), | |
| 3912 | + UA_MANDATORY)); | |
| 3913 | + | |
| 6388 | 3914 | static const struct uapi_definition mlx5_ib_defs[] = { |
| 6389 | 3915 | UAPI_DEF_CHAIN(mlx5_ib_devx_defs), |
| 6390 | 3916 | UAPI_DEF_CHAIN(mlx5_ib_flow_defs), |
| 6391 | 3917 | UAPI_DEF_CHAIN(mlx5_ib_qos_defs), |
| 3918 | + UAPI_DEF_CHAIN(mlx5_ib_std_types_defs), | |
| 6392 | 3919 | |
| 6393 | 3920 | UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, |
| 6394 | 3921 | &mlx5_ib_flow_action), |
| 6395 | 3922 | UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm), |
| 3923 | + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context), | |
| 6396 | 3924 | UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR, |
| 6397 | 3925 | UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)), |
| 6398 | 3926 | UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR), |
| 6399 | 3927 | {} |
| 6400 | 3928 | }; |
| 6401 | 3929 | |
| 6402 | −static int mlx5_ib_read_counters(struct ib_counters *counters, | |
| 6403 | − struct ib_counters_read_attr *read_attr, | |
| 6404 | − struct uverbs_attr_bundle *attrs) | |
| 6405 | −{ | |
| 6406 | − struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); | |
| 6407 | − struct mlx5_read_counters_attr mread_attr = {}; | |
| 6408 | − struct mlx5_ib_flow_counters_desc *desc; | |
| 6409 | − int ret, i; | |
| 6410 | − | |
| 6411 | − mutex_lock(&mcounters->mcntrs_mutex); | |
| 6412 | − if (mcounters->cntrs_max_index > read_attr->ncounters) { | |
| 6413 | − ret = -EINVAL; | |
| 6414 | − goto err_bound; | |
| 6415 | − } | |
| 6416 | − | |
| 6417 | − mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64), | |
| 6418 | − GFP_KERNEL); | |
| 6419 | − if (!mread_attr.out) { | |
| 6420 | − ret = -ENOMEM; | |
| 6421 | − goto err_bound; | |
| 6422 | − } | |
| 6423 | − | |
| 6424 | − mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl; | |
| 6425 | − mread_attr.flags = read_attr->flags; | |
| 6426 | − ret = mcounters->read_counters(counters->device, &mread_attr); | |
| 6427 | − if (ret) | |
| 6428 | − goto err_read; | |
| 6429 | − | |
| 6430 | − /* do the pass over the counters data array to assign according to the | |
| 6431 | − * descriptions and indexing pairs | |
| 6432 | − */ | |
| 6433 | − desc = mcounters->counters_data; | |
| 6434 | − for (i = 0; i < mcounters->ncounters; i++) | |
| 6435 | − read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description]; | |
| 6436 | − | |
| 6437 | −err_read: | |
| 6438 | − kfree(mread_attr.out); | |
| 6439 | −err_bound: | |
| 6440 | − mutex_unlock(&mcounters->mcntrs_mutex); | |
| 6441 | − return ret; | |
| 6442 | −} | |
| 6443 | − | |
| 6444 | −static int mlx5_ib_destroy_counters(struct ib_counters *counters) | |
| 6445 | −{ | |
| 6446 | − struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); | |
| 6447 | − | |
| 6448 | − counters_clear_description(counters); | |
| 6449 | − if (mcounters->hw_cntrs_hndl) | |
| 6450 | − mlx5_fc_destroy(to_mdev(counters->device)->mdev, | |
| 6451 | − mcounters->hw_cntrs_hndl); | |
| 6452 | − | |
| 6453 | − kfree(mcounters); | |
| 6454 | − | |
| 6455 | − return 0; | |
| 6456 | −} | |
| 6457 | − | |
| 6458 | −static struct ib_counters *mlx5_ib_create_counters(struct ib_device *device, | |
| 6459 | − struct uverbs_attr_bundle *attrs) | |
| 6460 | −{ | |
| 6461 | − struct mlx5_ib_mcounters *mcounters; | |
| 6462 | − | |
| 6463 | − mcounters = kzalloc(sizeof(*mcounters), GFP_KERNEL); | |
| 6464 | − if (!mcounters) | |
| 6465 | − return ERR_PTR(-ENOMEM); | |
| 6466 | − | |
| 6467 | − mutex_init(&mcounters->mcntrs_mutex); | |
| 6468 | − | |
| 6469 | − return &mcounters->ibcntrs; | |
| 6470 | −} | |
| 6471 | − | |
| 6472 | 3930 | static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev) |
| 6473 | 3931 | { |
| 6474 | 3932 | mlx5_ib_cleanup_multiport_master(dev); |
| @@ -6547,21 +4005,16 @@ err_mp: · 72 unmodified lines | ||
| 6547 | 4005 | return -ENOMEM; |
| 6548 | 4006 | } |
| 6549 | 4007 | |
| 6550 | −static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev) | |
| 4008 | +static int mlx5_ib_enable_driver(struct ib_device *dev) | |
| 6551 | 4009 | { |
| 6552 | − dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); | |
| 6553 | − | |
| 6554 | − if (!dev->flow_db) | |
| 6555 | − return -ENOMEM; | |
| 6556 | − | |
| 6557 | − mutex_init(&dev->flow_db->lock); | |
| 4010 | + struct mlx5_ib_dev *mdev = to_mdev(dev); | |
| 4011 | + int ret; | |
| 6558 | 4012 | |
| 6559 | − return 0; | |
| 6560 | −} | |
| 4013 | + ret = mlx5_ib_test_wc(mdev); | |
| 4014 | + mlx5_ib_dbg(mdev, "Write-Combining %s", | |
| 4015 | + mdev->wc_support ? "supported" : "not supported"); | |
| 6561 | 4016 | |
| 6562 | −static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev) | |
| 6563 | −{ | |
| 6564 | − kfree(dev->flow_db); | |
| 4017 | + return ret; | |
| 6565 | 4018 | } |
| 6566 | 4019 | |
| 6567 | 4020 | static const struct ib_device_ops mlx5_ib_dev_ops = { |
| @@ -6577,9 +4030,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 9 unmodified lines | ||
| 6577 | 4030 | .attach_mcast = mlx5_ib_mcg_attach, |
| 6578 | 4031 | .check_mr_status = mlx5_ib_check_mr_status, |
| 6579 | 4032 | .create_ah = mlx5_ib_create_ah, |
| 6580 | − .create_counters = mlx5_ib_create_counters, | |
| 6581 | 4033 | .create_cq = mlx5_ib_create_cq, |
| 6582 | − .create_flow = mlx5_ib_create_flow, | |
| 6583 | 4034 | .create_qp = mlx5_ib_create_qp, |
| 6584 | 4035 | .create_srq = mlx5_ib_create_srq, |
| 6585 | 4036 | .dealloc_pd = mlx5_ib_dealloc_pd, |
| @@ -6587,10 +4038,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 1 unmodified line | ||
| 6587 | 4038 | .del_gid = mlx5_ib_del_gid, |
| 6588 | 4039 | .dereg_mr = mlx5_ib_dereg_mr, |
| 6589 | 4040 | .destroy_ah = mlx5_ib_destroy_ah, |
| 6590 | − .destroy_counters = mlx5_ib_destroy_counters, | |
| 6591 | 4041 | .destroy_cq = mlx5_ib_destroy_cq, |
| 6592 | − .destroy_flow = mlx5_ib_destroy_flow, | |
| 6593 | − .destroy_flow_action = mlx5_ib_destroy_flow_action, | |
| 6594 | 4042 | .destroy_qp = mlx5_ib_destroy_qp, |
| 6595 | 4043 | .destroy_srq = mlx5_ib_destroy_srq, |
| 6596 | 4044 | .detach_mcast = mlx5_ib_mcg_detach, |
| @@ -6598,8 +4046,6 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 1 unmodified line | ||
| 6598 | 4046 | .drain_rq = mlx5_ib_drain_rq, |
| 6599 | 4047 | .drain_sq = mlx5_ib_drain_sq, |
| 6600 | 4048 | .enable_driver = mlx5_ib_enable_driver, |
| 6601 | − .fill_res_entry = mlx5_ib_fill_res_entry, | |
| 6602 | − .fill_stat_entry = mlx5_ib_fill_stat_entry, | |
| 6603 | 4049 | .get_dev_fw_str = get_dev_fw_str, |
| 6604 | 4050 | .get_dma_mr = mlx5_ib_get_dma_mr, |
| 6605 | 4051 | .get_link_layer = mlx5_ib_port_link_layer, |
| @@ -6623,24 +4069,20 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 17 unmodified lines | ||
| 6623 | 4069 | .query_pkey = mlx5_ib_query_pkey, |
| 6624 | 4070 | .query_qp = mlx5_ib_query_qp, |
| 6625 | 4071 | .query_srq = mlx5_ib_query_srq, |
| 6626 | − .read_counters = mlx5_ib_read_counters, | |
| 4072 | + .query_ucontext = mlx5_ib_query_ucontext, | |
| 6627 | 4073 | .reg_user_mr = mlx5_ib_reg_user_mr, |
| 6628 | 4074 | .req_notify_cq = mlx5_ib_arm_cq, |
| 6629 | 4075 | .rereg_user_mr = mlx5_ib_rereg_user_mr, |
| 6630 | 4076 | .resize_cq = mlx5_ib_resize_cq, |
| 6631 | 4077 | |
| 6632 | 4078 | INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah), |
| 4079 | + INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs), | |
| 6633 | 4080 | INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq), |
| 6634 | 4081 | INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd), |
| 6635 | 4082 | INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq), |
| 6636 | 4083 | INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext), |
| 6637 | 4084 | }; |
| 6638 | 4085 | |
| 6639 | −static const struct ib_device_ops mlx5_ib_dev_flow_ipsec_ops = { | |
| 6640 | − .create_flow_action_esp = mlx5_ib_create_flow_action_esp, | |
| 6641 | − .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp, | |
| 6642 | −}; | |
| 6643 | − | |
| 6644 | 4086 | static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = { |
| 6645 | 4087 | .rdma_netdev_get_params = mlx5_ib_rn_get_params, |
| 6646 | 4088 | }; |
| @@ -6661,6 +4103,8 @@ static const struct ib_device_ops mlx5_ib_dev_mw_ops = { · 14 unmodified lines | ||
| 6661 | 4103 | static const struct ib_device_ops mlx5_ib_dev_xrc_ops = { |
| 6662 | 4104 | .alloc_xrcd = mlx5_ib_alloc_xrcd, |
| 6663 | 4105 | .dealloc_xrcd = mlx5_ib_dealloc_xrcd, |
| 4106 | + | |
| 4107 | + INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd), | |
| 6664 | 4108 | }; |
| 6665 | 4109 | |
| 6666 | 4110 | static const struct ib_device_ops mlx5_ib_dev_dm_ops = { |
| @@ -6769,9 +4213,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) · 102 unmodified lines | ||
| 6769 | 4213 | MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM) |
| 6770 | 4214 | ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops); |
| 6771 | 4215 | |
| 6772 | − if (mlx5_accel_ipsec_device_caps(dev->mdev) & | |
| 6773 | − MLX5_ACCEL_IPSEC_CAP_DEVICE) | |
| 6774 | − ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_flow_ipsec_ops); | |
| 6775 | 4216 | ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops); |
| 6776 | 4217 | |
| 6777 | 4218 | if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)) |
| @@ -6829,65 +4270,36 @@ static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = { · 51 unmodified lines | ||
| 6829 | 4270 | .modify_wq = mlx5_ib_modify_wq, |
| 6830 | 4271 | }; |
| 6831 | 4272 | |
| 6832 | −static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev) | |
| 6833 | −{ | |
| 6834 | − u8 port_num; | |
| 6835 | − | |
| 6836 | − dev->ib_dev.uverbs_ex_cmd_mask |= | |
| 6837 | − (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) | | |
| 6838 | − (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) | | |
| 6839 | − (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) | | |
| 6840 | − (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) | | |
| 6841 | − (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL); | |
| 6842 | − ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops); | |
| 6843 | − | |
| 6844 | − port_num = mlx5_core_native_port_num(dev->mdev) - 1; | |
| 6845 | − | |
| 6846 | − /* Register only for native ports */ | |
| 6847 | − return mlx5_add_netdev_notifier(dev, port_num); | |
| 6848 | −} | |
| 6849 | − | |
| 6850 | −static void mlx5_ib_stage_common_roce_cleanup(struct mlx5_ib_dev *dev) | |
| 6851 | −{ | |
| 6852 | − u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1; | |
| 6853 | − | |
| 6854 | − mlx5_remove_netdev_notifier(dev, port_num); | |
| 6855 | −} | |
| 6856 | − | |
| 6857 | −static int mlx5_ib_stage_raw_eth_roce_init(struct mlx5_ib_dev *dev) | |
| 6858 | −{ | |
| 6859 | − struct mlx5_core_dev *mdev = dev->mdev; | |
| 6860 | − enum rdma_link_layer ll; | |
| 6861 | − int port_type_cap; | |
| 6862 | − int err = 0; | |
| 6863 | − | |
| 6864 | − port_type_cap = MLX5_CAP_GEN(mdev, port_type); | |
| 6865 | − ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap); | |
| 6866 | − | |
| 6867 | − if (ll == IB_LINK_LAYER_ETHERNET) | |
| 6868 | − err = mlx5_ib_stage_common_roce_init(dev); | |
| 6869 | − | |
| 6870 | − return err; | |
| 6871 | −} | |
| 6872 | − | |
| 6873 | −static void mlx5_ib_stage_raw_eth_roce_cleanup(struct mlx5_ib_dev *dev) | |
| 6874 | −{ | |
| 6875 | − mlx5_ib_stage_common_roce_cleanup(dev); | |
| 6876 | −} | |
| 6877 | − | |
| 6878 | −static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev) | |
| 4273 | +static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev) | |
| 6879 | 4274 | { |
| 6880 | 4275 | struct mlx5_core_dev *mdev = dev->mdev; |
| 6881 | 4276 | enum rdma_link_layer ll; |
| 6882 | 4277 | int port_type_cap; |
| 4278 | + u8 port_num = 0; | |
| 6883 | 4279 | int err; |
| 6884 | 4280 | |
| 6885 | 4281 | port_type_cap = MLX5_CAP_GEN(mdev, port_type); |
| 6886 | 4282 | ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap); |
| 6887 | 4283 | |
| 6888 | 4284 | if (ll == IB_LINK_LAYER_ETHERNET) { |
| 6889 | − err = mlx5_ib_stage_common_roce_init(dev); | |
| 6890 | − if (err) | |
| 4285 | + dev->ib_dev.uverbs_ex_cmd_mask |= | |
| 4286 | + (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) | | |
| 4287 | + (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) | | |
| 4288 | + (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) | | |
| 4289 | + (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) | | |
| 4290 | + (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL); | |
| 4291 | + ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops); | |
| 4292 | + | |
| 4293 | + port_num = mlx5_core_native_port_num(dev->mdev) - 1; | |
| 4294 | + | |
| 4295 | + /* Register only for native ports */ | |
| 4296 | + err = mlx5_add_netdev_notifier(dev, port_num); | |
| 4297 | + if (err || dev->is_rep || !mlx5_is_roce_enabled(mdev)) | |
| 4298 | + /* | |
| 4299 | + * We don't enable ETH interface for | |
| 4300 | + * 1. IB representors | |
| 4301 | + * 2. User disabled ROCE through devlink interface | |
| 4302 | + */ | |
| 6891 | 4303 | return err; |
| 6892 | 4304 | |
| 6893 | 4305 | err = mlx5_enable_eth(dev); |
| @@ -6897,71 +4309,27 @@ static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev) · 3 unmodified lines | ||
| 6897 | 4309 | |
| 6898 | 4310 | return 0; |
| 6899 | 4311 | cleanup: |
| 6900 | − mlx5_ib_stage_common_roce_cleanup(dev); | |
| 6901 | − | |
| 4312 | + mlx5_remove_netdev_notifier(dev, port_num); | |
| 6902 | 4313 | return err; |
| 6903 | 4314 | } |
| 6904 | 4315 | |
| 6905 | −static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev) | |
| 4316 | +static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev) | |
| 6906 | 4317 | { |
| 6907 | 4318 | struct mlx5_core_dev *mdev = dev->mdev; |
| 6908 | 4319 | enum rdma_link_layer ll; |
| 6909 | 4320 | int port_type_cap; |
| 4321 | + u8 port_num; | |
| 6910 | 4322 | |
| 6911 | 4323 | port_type_cap = MLX5_CAP_GEN(mdev, port_type); |
| 6912 | 4324 | ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap); |
| 6913 | 4325 | |
| 6914 | 4326 | if (ll == IB_LINK_LAYER_ETHERNET) { |
| 6915 | − mlx5_disable_eth(dev); | |
| 6916 | − mlx5_ib_stage_common_roce_cleanup(dev); | |
| 6917 | − } | |
| 6918 | −} | |
| 6919 | − | |
| 6920 | −static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev) | |
| 6921 | −{ | |
| 6922 | − return create_dev_resources(&dev->devr); | |
| 6923 | −} | |
| 6924 | − | |
| 6925 | −static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev) | |
| 6926 | −{ | |
| 6927 | − destroy_dev_resources(&dev->devr); | |
| 6928 | −} | |
| 6929 | − | |
| 6930 | −static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev) | |
| 6931 | −{ | |
| 6932 | − return mlx5_ib_odp_init_one(dev); | |
| 6933 | −} | |
| 6934 | − | |
| 6935 | −static void mlx5_ib_stage_odp_cleanup(struct mlx5_ib_dev *dev) | |
| 6936 | −{ | |
| 6937 | − mlx5_ib_odp_cleanup_one(dev); | |
| 6938 | −} | |
| 6939 | − | |
| 6940 | −static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = { | |
| 6941 | − .alloc_hw_stats = mlx5_ib_alloc_hw_stats, | |
| 6942 | − .get_hw_stats = mlx5_ib_get_hw_stats, | |
| 6943 | − .counter_bind_qp = mlx5_ib_counter_bind_qp, | |
| 6944 | − .counter_unbind_qp = mlx5_ib_counter_unbind_qp, | |
| 6945 | − .counter_dealloc = mlx5_ib_counter_dealloc, | |
| 6946 | − .counter_alloc_stats = mlx5_ib_counter_alloc_stats, | |
| 6947 | − .counter_update_stats = mlx5_ib_counter_update_stats, | |
| 6948 | −}; | |
| 6949 | − | |
| 6950 | −static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev) | |
| 6951 | −{ | |
| 6952 | − if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) { | |
| 6953 | − ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_hw_stats_ops); | |
| 4327 | + if (!dev->is_rep) | |
| 4328 | + mlx5_disable_eth(dev); | |
| 6954 | 4329 | |
| 6955 | − return mlx5_ib_alloc_counters(dev); | |
| 4330 | + port_num = mlx5_core_native_port_num(dev->mdev) - 1; | |
| 4331 | + mlx5_remove_netdev_notifier(dev, port_num); | |
| 6956 | 4332 | } |
| 6957 | − | |
| 6958 | − return 0; | |
| 6959 | −} | |
| 6960 | − | |
| 6961 | −static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev) | |
| 6962 | −{ | |
| 6963 | − if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) | |
| 6964 | − mlx5_ib_dealloc_counters(dev); | |
| 6965 | 4333 | } |
| 6966 | 4334 | |
| 6967 | 4335 | static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev) |
| @@ -7023,7 +4391,18 @@ static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev) · 55 unmodified lines | ||
| 7023 | 4391 | |
| 7024 | 4392 | static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev) |
| 7025 | 4393 | { |
| 7026 | − destroy_umrc_res(dev); | |
| 4394 | + int err; | |
| 4395 | + | |
| 4396 | + err = mlx5_mr_cache_cleanup(dev); | |
| 4397 | + if (err) | |
| 4398 | + mlx5_ib_warn(dev, "mr cache cleanup failed\n"); | |
| 4399 | + | |
| 4400 | + if (dev->umrc.qp) | |
| 4401 | + mlx5_ib_destroy_qp(dev->umrc.qp, NULL); | |
| 4402 | + if (dev->umrc.cq) | |
| 4403 | + ib_free_cq(dev->umrc.cq); | |
| 4404 | + if (dev->umrc.pd) | |
| 4405 | + ib_dealloc_pd(dev->umrc.pd); | |
| 7027 | 4406 | } |
| 7028 | 4407 | |
| 7029 | 4408 | static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev) |
| @@ -7031,21 +4410,162 @@ static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev) · 1 unmodified line | ||
| 7031 | 4410 | ib_unregister_device(&dev->ib_dev); |
| 7032 | 4411 | } |
| 7033 | 4412 | |
| 4413 | +enum { | |
| 4414 | + MAX_UMR_WR = 128, | |
| 4415 | +}; | |
| 4416 | + | |
| 7034 | 4417 | static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev) |
| 7035 | 4418 | { |
| 7036 | − return create_umr_res(dev); | |
| 4419 | + struct ib_qp_init_attr *init_attr = NULL; | |
| 4420 | + struct ib_qp_attr *attr = NULL; | |
| 4421 | + struct ib_pd *pd; | |
| 4422 | + struct ib_cq *cq; | |
| 4423 | + struct ib_qp *qp; | |
| 4424 | + int ret; | |
| 4425 | + | |
| 4426 | + attr = kzalloc(sizeof(*attr), GFP_KERNEL); | |
| 4427 | + init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL); | |
| 4428 | + if (!attr || !init_attr) { | |
| 4429 | + ret = -ENOMEM; | |
| 4430 | + goto error_0; | |
| 4431 | + } | |
| 4432 | + | |
| 4433 | + pd = ib_alloc_pd(&dev->ib_dev, 0); | |
| 4434 | + if (IS_ERR(pd)) { | |
| 4435 | + mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n"); | |
| 4436 | + ret = PTR_ERR(pd); | |
| 4437 | + goto error_0; | |
| 4438 | + } | |
| 4439 | + | |
| 4440 | + cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ); | |
| 4441 | + if (IS_ERR(cq)) { | |
| 4442 | + mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n"); | |
| 4443 | + ret = PTR_ERR(cq); | |
| 4444 | + goto error_2; | |
| 4445 | + } | |
| 4446 | + | |
| 4447 | + init_attr->send_cq = cq; | |
| 4448 | + init_attr->recv_cq = cq; | |
| 4449 | + init_attr->sq_sig_type = IB_SIGNAL_ALL_WR; | |
| 4450 | + init_attr->cap.max_send_wr = MAX_UMR_WR; | |
| 4451 | + init_attr->cap.max_send_sge = 1; | |
| 4452 | + init_attr->qp_type = MLX5_IB_QPT_REG_UMR; | |
| 4453 | + init_attr->port_num = 1; | |
| 4454 | + qp = mlx5_ib_create_qp(pd, init_attr, NULL); | |
| 4455 | + if (IS_ERR(qp)) { | |
| 4456 | + mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n"); | |
| 4457 | + ret = PTR_ERR(qp); | |
| 4458 | + goto error_3; | |
| 4459 | + } | |
| 4460 | + qp->device = &dev->ib_dev; | |
| 4461 | + qp->real_qp = qp; | |
| 4462 | + qp->uobject = NULL; | |
| 4463 | + qp->qp_type = MLX5_IB_QPT_REG_UMR; | |
| 4464 | + qp->send_cq = init_attr->send_cq; | |
| 4465 | + qp->recv_cq = init_attr->recv_cq; | |
| 4466 | + | |
| 4467 | + attr->qp_state = IB_QPS_INIT; | |
| 4468 | + attr->port_num = 1; | |
| 4469 | + ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX | | |
| 4470 | + IB_QP_PORT, NULL); | |
| 4471 | + if (ret) { | |
| 4472 | + mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n"); | |
| 4473 | + goto error_4; | |
| 4474 | + } | |
| 4475 | + | |
| 4476 | + memset(attr, 0, sizeof(*attr)); | |
| 4477 | + attr->qp_state = IB_QPS_RTR; | |
| 4478 | + attr->path_mtu = IB_MTU_256; | |
| 4479 | + | |
| 4480 | + ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL); | |
| 4481 | + if (ret) { | |
| 4482 | + mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n"); | |
| 4483 | + goto error_4; | |
| 4484 | + } | |
| 4485 | + | |
| 4486 | + memset(attr, 0, sizeof(*attr)); | |
| 4487 | + attr->qp_state = IB_QPS_RTS; | |
| 4488 | + ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL); | |
| 4489 | + if (ret) { | |
| 4490 | + mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n"); | |
| 4491 | + goto error_4; | |
| 4492 | + } | |
| 4493 | + | |
| 4494 | + dev->umrc.qp = qp; | |
| 4495 | + dev->umrc.cq = cq; | |
| 4496 | + dev->umrc.pd = pd; | |
| 4497 | + | |
| 4498 | + sema_init(&dev->umrc.sem, MAX_UMR_WR); | |
| 4499 | + ret = mlx5_mr_cache_init(dev); | |
| 4500 | + if (ret) { | |
| 4501 | + mlx5_ib_warn(dev, "mr cache init failed %d\n", ret); | |
| 4502 | + goto error_4; | |
| 4503 | + } | |
| 4504 | + | |
| 4505 | + kfree(attr); | |
| 4506 | + kfree(init_attr); | |
| 4507 | + | |
| 4508 | + return 0; | |
| 4509 | + | |
| 4510 | +error_4: | |
| 4511 | + mlx5_ib_destroy_qp(qp, NULL); | |
| 4512 | + dev->umrc.qp = NULL; | |
| 4513 | + | |
| 4514 | +error_3: | |
| 4515 | + ib_free_cq(cq); | |
| 4516 | + dev->umrc.cq = NULL; | |
| 4517 | + | |
| 4518 | +error_2: | |
| 4519 | + ib_dealloc_pd(pd); | |
| 4520 | + dev->umrc.pd = NULL; | |
| 4521 | + | |
| 4522 | +error_0: | |
| 4523 | + kfree(attr); | |
| 4524 | + kfree(init_attr); | |
| 4525 | + return ret; | |
| 7037 | 4526 | } |
| 7038 | 4527 | |
| 7039 | 4528 | static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev) |
| 7040 | 4529 | { |
| 7041 | − init_delay_drop(dev); | |
| 4530 | + struct dentry *root; | |
| 4531 | + | |
| 4532 | + if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP)) | |
| 4533 | + return 0; | |
| 4534 | + | |
| 4535 | + mutex_init(&dev->delay_drop.lock); | |
| 4536 | + dev->delay_drop.dev = dev; | |
| 4537 | + dev->delay_drop.activate = false; | |
| 4538 | + dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000; | |
| 4539 | + INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler); | |
| 4540 | + atomic_set(&dev->delay_drop.rqs_cnt, 0); | |
| 4541 | + atomic_set(&dev->delay_drop.events_cnt, 0); | |
| 4542 | + | |
| 4543 | + if (!mlx5_debugfs_root) | |
| 4544 | + return 0; | |
| 4545 | + | |
| 4546 | + root = debugfs_create_dir("delay_drop", dev->mdev->priv.dbg_root); | |
| 4547 | + dev->delay_drop.dir_debugfs = root; | |
| 7042 | 4548 | |
| 4549 | + debugfs_create_atomic_t("num_timeout_events", 0400, root, | |
| 4550 | + &dev->delay_drop.events_cnt); | |
| 4551 | + debugfs_create_atomic_t("num_rqs", 0400, root, | |
| 4552 | + &dev->delay_drop.rqs_cnt); | |
| 4553 | + debugfs_create_file("timeout", 0600, root, &dev->delay_drop, | |
| 4554 | + &fops_delay_drop_timeout); | |
| 7043 | 4555 | return 0; |
| 7044 | 4556 | } |
| 7045 | 4557 | |
| 7046 | 4558 | static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev) |
| 7047 | 4559 | { |
| 7048 | − cancel_delay_drop(dev); | |
| 4560 | + if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP)) | |
| 4561 | + return; | |
| 4562 | + | |
| 4563 | + cancel_work_sync(&dev->delay_drop.delay_drop_work); | |
| 4564 | + if (!dev->delay_drop.dir_debugfs) | |
| 4565 | + return; | |
| 4566 | + | |
| 4567 | + debugfs_remove_recursive(dev->delay_drop.dir_debugfs); | |
| 4568 | + dev->delay_drop.dir_debugfs = NULL; | |
| 7049 | 4569 | } |
| 7050 | 4570 | |
| 7051 | 4571 | static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev) |
| @@ -7060,38 +4580,6 @@ static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev) · 8 unmodified lines | ||
| 7060 | 4580 | mlx5_notifier_unregister(dev->mdev, &dev->mdev_events); |
| 7061 | 4581 | } |
| 7062 | 4582 | |
| 7063 | −static int mlx5_ib_stage_devx_init(struct mlx5_ib_dev *dev) | |
| 7064 | −{ | |
| 7065 | − int uid; | |
| 7066 | − | |
| 7067 | − uid = mlx5_ib_devx_create(dev, false); | |
| 7068 | − if (uid > 0) { | |
| 7069 | − dev->devx_whitelist_uid = uid; | |
| 7070 | − mlx5_ib_devx_init_event_table(dev); | |
| 7071 | − } | |
| 7072 | − | |
| 7073 | − return 0; | |
| 7074 | −} | |
| 7075 | −static void mlx5_ib_stage_devx_cleanup(struct mlx5_ib_dev *dev) | |
| 7076 | −{ | |
| 7077 | − if (dev->devx_whitelist_uid) { | |
| 7078 | − mlx5_ib_devx_cleanup_event_table(dev); | |
| 7079 | − mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid); | |
| 7080 | − } | |
| 7081 | −} | |
| 7082 | − | |
| 7083 | −int mlx5_ib_enable_driver(struct ib_device *dev) | |
| 7084 | −{ | |
| 7085 | − struct mlx5_ib_dev *mdev = to_mdev(dev); | |
| 7086 | − int ret; | |
| 7087 | − | |
| 7088 | − ret = mlx5_ib_test_wc(mdev); | |
| 7089 | − mlx5_ib_dbg(mdev, "Write-Combining %s", | |
| 7090 | − mdev->wc_support ? "supported" : "not supported"); | |
| 7091 | − | |
| 7092 | − return ret; | |
| 7093 | −} | |
| 7094 | − | |
| 7095 | 4583 | void __mlx5_ib_remove(struct mlx5_ib_dev *dev, |
| 7096 | 4584 | const struct mlx5_ib_profile *profile, |
| 7097 | 4585 | int stage) |
| @@ -7139,9 +4627,9 @@ static const struct mlx5_ib_profile pf_profile = { · 41 unmodified lines | ||
| 7139 | 4627 | STAGE_CREATE(MLX5_IB_STAGE_INIT, |
| 7140 | 4628 | mlx5_ib_stage_init_init, |
| 7141 | 4629 | mlx5_ib_stage_init_cleanup), |
| 7142 | − STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB, | |
| 7143 | − mlx5_ib_stage_flow_db_init, | |
| 7144 | − mlx5_ib_stage_flow_db_cleanup), | |
| 4630 | + STAGE_CREATE(MLX5_IB_STAGE_FS, | |
| 4631 | + mlx5_ib_fs_init, | |
| 4632 | + mlx5_ib_fs_cleanup), | |
| 7145 | 4633 | STAGE_CREATE(MLX5_IB_STAGE_CAPS, |
| 7146 | 4634 | mlx5_ib_stage_caps_init, |
| 7147 | 4635 | mlx5_ib_stage_caps_cleanup), |
| @@ -7149,8 +4637,8 @@ static const struct mlx5_ib_profile pf_profile = { · 1 unmodified line | ||
| 7149 | 4637 | mlx5_ib_stage_non_default_cb, |
| 7150 | 4638 | NULL), |
| 7151 | 4639 | STAGE_CREATE(MLX5_IB_STAGE_ROCE, |
| 7152 | − mlx5_ib_stage_roce_init, | |
| 7153 | − mlx5_ib_stage_roce_cleanup), | |
| 4640 | + mlx5_ib_roce_init, | |
| 4641 | + mlx5_ib_roce_cleanup), | |
| 7154 | 4642 | STAGE_CREATE(MLX5_IB_STAGE_QP, |
| 7155 | 4643 | mlx5_init_qp_table, |
| 7156 | 4644 | mlx5_cleanup_qp_table), |
| @@ -7158,17 +4646,17 @@ static const struct mlx5_ib_profile pf_profile = { · 1 unmodified line | ||
| 7158 | 4646 | mlx5_init_srq_table, |
| 7159 | 4647 | mlx5_cleanup_srq_table), |
| 7160 | 4648 | STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES, |
| 7161 | − mlx5_ib_stage_dev_res_init, | |
| 7162 | − mlx5_ib_stage_dev_res_cleanup), | |
| 4649 | + mlx5_ib_dev_res_init, | |
| 4650 | + mlx5_ib_dev_res_cleanup), | |
| 7163 | 4651 | STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER, |
| 7164 | 4652 | mlx5_ib_stage_dev_notifier_init, |
| 7165 | 4653 | mlx5_ib_stage_dev_notifier_cleanup), |
| 7166 | 4654 | STAGE_CREATE(MLX5_IB_STAGE_ODP, |
| 7167 | − mlx5_ib_stage_odp_init, | |
| 7168 | − mlx5_ib_stage_odp_cleanup), | |
| 4655 | + mlx5_ib_odp_init_one, | |
| 4656 | + mlx5_ib_odp_cleanup_one), | |
| 7169 | 4657 | STAGE_CREATE(MLX5_IB_STAGE_COUNTERS, |
| 7170 | − mlx5_ib_stage_counters_init, | |
| 7171 | − mlx5_ib_stage_counters_cleanup), | |
| 4658 | + mlx5_ib_counters_init, | |
| 4659 | + mlx5_ib_counters_cleanup), | |
| 7172 | 4660 | STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS, |
| 7173 | 4661 | mlx5_ib_stage_cong_debugfs_init, |
| 7174 | 4662 | mlx5_ib_stage_cong_debugfs_cleanup), |
| @@ -7182,8 +4670,8 @@ static const struct mlx5_ib_profile pf_profile = { · 7 unmodified lines | ||
| 7182 | 4670 | NULL, |
| 7183 | 4671 | mlx5_ib_stage_pre_ib_reg_umr_cleanup), |
| 7184 | 4672 | STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, |
| 7185 | − mlx5_ib_stage_devx_init, | |
| 7186 | − mlx5_ib_stage_devx_cleanup), | |
| 4673 | + mlx5_ib_devx_init, | |
| 4674 | + mlx5_ib_devx_cleanup), | |
| 7187 | 4675 | STAGE_CREATE(MLX5_IB_STAGE_IB_REG, |
| 7188 | 4676 | mlx5_ib_stage_ib_reg_init, |
| 7189 | 4677 | mlx5_ib_stage_ib_reg_cleanup), |
| @@ -7193,15 +4681,18 @@ static const struct mlx5_ib_profile pf_profile = { · 3 unmodified lines | ||
| 7193 | 4681 | STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP, |
| 7194 | 4682 | mlx5_ib_stage_delay_drop_init, |
| 7195 | 4683 | mlx5_ib_stage_delay_drop_cleanup), |
| 4684 | + STAGE_CREATE(MLX5_IB_STAGE_RESTRACK, | |
| 4685 | + mlx5_ib_restrack_init, | |
| 4686 | + NULL), | |
| 7196 | 4687 | }; |
| 7197 | 4688 | |
| 7198 | 4689 | const struct mlx5_ib_profile raw_eth_profile = { |
| 7199 | 4690 | STAGE_CREATE(MLX5_IB_STAGE_INIT, |
| 7200 | 4691 | mlx5_ib_stage_init_init, |
| 7201 | 4692 | mlx5_ib_stage_init_cleanup), |
| 7202 | − STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB, | |
| 7203 | − mlx5_ib_stage_flow_db_init, | |
| 7204 | − mlx5_ib_stage_flow_db_cleanup), | |
| 4693 | + STAGE_CREATE(MLX5_IB_STAGE_FS, | |
| 4694 | + mlx5_ib_fs_init, | |
| 4695 | + mlx5_ib_fs_cleanup), | |
| 7205 | 4696 | STAGE_CREATE(MLX5_IB_STAGE_CAPS, |
| 7206 | 4697 | mlx5_ib_stage_caps_init, |
| 7207 | 4698 | mlx5_ib_stage_caps_cleanup), |
| @@ -7209,8 +4700,8 @@ const struct mlx5_ib_profile raw_eth_profile = { · 1 unmodified line | ||
| 7209 | 4700 | mlx5_ib_stage_raw_eth_non_default_cb, |
| 7210 | 4701 | NULL), |
| 7211 | 4702 | STAGE_CREATE(MLX5_IB_STAGE_ROCE, |
| 7212 | − mlx5_ib_stage_raw_eth_roce_init, | |
| 7213 | − mlx5_ib_stage_raw_eth_roce_cleanup), | |
| 4703 | + mlx5_ib_roce_init, | |
| 4704 | + mlx5_ib_roce_cleanup), | |
| 7214 | 4705 | STAGE_CREATE(MLX5_IB_STAGE_QP, |
| 7215 | 4706 | mlx5_init_qp_table, |
| 7216 | 4707 | mlx5_cleanup_qp_table), |
| @@ -7218,14 +4709,14 @@ const struct mlx5_ib_profile raw_eth_profile = { · 1 unmodified line | ||
| 7218 | 4709 | mlx5_init_srq_table, |
| 7219 | 4710 | mlx5_cleanup_srq_table), |
| 7220 | 4711 | STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES, |
| 7221 | − mlx5_ib_stage_dev_res_init, | |
| 7222 | − mlx5_ib_stage_dev_res_cleanup), | |
| 4712 | + mlx5_ib_dev_res_init, | |
| 4713 | + mlx5_ib_dev_res_cleanup), | |
| 7223 | 4714 | STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER, |
| 7224 | 4715 | mlx5_ib_stage_dev_notifier_init, |
| 7225 | 4716 | mlx5_ib_stage_dev_notifier_cleanup), |
| 7226 | 4717 | STAGE_CREATE(MLX5_IB_STAGE_COUNTERS, |
| 7227 | − mlx5_ib_stage_counters_init, | |
| 7228 | − mlx5_ib_stage_counters_cleanup), | |
| 4718 | + mlx5_ib_counters_init, | |
| 4719 | + mlx5_ib_counters_cleanup), | |
| 7229 | 4720 | STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS, |
| 7230 | 4721 | mlx5_ib_stage_cong_debugfs_init, |
| 7231 | 4722 | mlx5_ib_stage_cong_debugfs_cleanup), |
| @@ -7239,14 +4730,17 @@ const struct mlx5_ib_profile raw_eth_profile = { · 7 unmodified lines | ||
| 7239 | 4730 | NULL, |
| 7240 | 4731 | mlx5_ib_stage_pre_ib_reg_umr_cleanup), |
| 7241 | 4732 | STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, |
| 7242 | − mlx5_ib_stage_devx_init, | |
| 7243 | − mlx5_ib_stage_devx_cleanup), | |
| 4733 | + mlx5_ib_devx_init, | |
| 4734 | + mlx5_ib_devx_cleanup), | |
| 7244 | 4735 | STAGE_CREATE(MLX5_IB_STAGE_IB_REG, |
| 7245 | 4736 | mlx5_ib_stage_ib_reg_init, |
| 7246 | 4737 | mlx5_ib_stage_ib_reg_cleanup), |
| 7247 | 4738 | STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR, |
| 7248 | 4739 | mlx5_ib_stage_post_ib_reg_umr_init, |
| 7249 | 4740 | NULL), |
| 4741 | + STAGE_CREATE(MLX5_IB_STAGE_RESTRACK, | |
| 4742 | + mlx5_ib_restrack_init, | |
| 4743 | + NULL), | |
| 7250 | 4744 | }; |
| 7251 | 4745 | |
| 7252 | 4746 | static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev) |
| @@ -1,33 +1,6 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | − * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. | |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 3 | + * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | 6 | #ifndef MLX5_IB_H |
| @@ -730,8 +703,8 @@ struct mlx5_ib_port_resources { · 696 unmodified lines | ||
| 730 | 703 | |
| 731 | 704 | struct mlx5_ib_resources { |
| 732 | 705 | struct ib_cq *c0; |
| 733 | − struct ib_xrcd *x0; | |
| 734 | − struct ib_xrcd *x1; | |
| 706 | + u32 xrcdn0; | |
| 707 | + u32 xrcdn1; | |
| 735 | 708 | struct ib_pd *p0; |
| 736 | 709 | struct ib_srq *s0; |
| 737 | 710 | struct ib_srq *s1; |
| @@ -832,7 +805,7 @@ struct mlx5_ib_delay_drop { · 94 unmodified lines | ||
| 832 | 805 | |
| 833 | 806 | enum mlx5_ib_stages { |
| 834 | 807 | MLX5_IB_STAGE_INIT, |
| 835 | − MLX5_IB_STAGE_FLOW_DB, | |
| 808 | + MLX5_IB_STAGE_FS, | |
| 836 | 809 | MLX5_IB_STAGE_CAPS, |
| 837 | 810 | MLX5_IB_STAGE_NON_DEFAULT_CB, |
| 838 | 811 | MLX5_IB_STAGE_ROCE, |
| @@ -850,7 +823,7 @@ enum mlx5_ib_stages { · 11 unmodified lines | ||
| 850 | 823 | MLX5_IB_STAGE_IB_REG, |
| 851 | 824 | MLX5_IB_STAGE_POST_IB_REG_UMR, |
| 852 | 825 | MLX5_IB_STAGE_DELAY_DROP, |
| 853 | − MLX5_IB_STAGE_CLASS_ATTR, | |
| 826 | + MLX5_IB_STAGE_RESTRACK, | |
| 854 | 827 | MLX5_IB_STAGE_MAX, |
| 855 | 828 | }; |
| 856 | 829 | |
| @@ -1078,11 +1051,6 @@ static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp) · 221 unmodified lines | ||
| 1078 | 1051 | return container_of(core_qp, struct mlx5_ib_rwq, core_qp); |
| 1079 | 1052 | } |
| 1080 | 1053 | |
| 1081 | −static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey) | |
| 1082 | −{ | |
| 1083 | − return container_of(mmkey, struct mlx5_ib_mr, mmkey); | |
| 1084 | −} | |
| 1085 | − | |
| 1086 | 1054 | static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd) |
| 1087 | 1055 | { |
| 1088 | 1056 | return container_of(ibpd, struct mlx5_ib_pd, ibpd); |
| @@ -1210,7 +1178,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, · 121 unmodified lines | ||
| 1210 | 1178 | struct ib_pd *pd, struct ib_udata *udata); |
| 1211 | 1179 | int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); |
| 1212 | 1180 | struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 1213 | − u32 max_num_sg, struct ib_udata *udata); | |
| 1181 | + u32 max_num_sg); | |
| 1214 | 1182 | struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, |
| 1215 | 1183 | u32 max_num_sg, |
| 1216 | 1184 | u32 max_num_meta_sg); |
| @@ -1224,9 +1192,8 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, · 7 unmodified lines | ||
| 1224 | 1192 | const struct ib_wc *in_wc, const struct ib_grh *in_grh, |
| 1225 | 1193 | const struct ib_mad *in, struct ib_mad *out, |
| 1226 | 1194 | size_t *out_mad_size, u16 *out_mad_pkey_index); |
| 1227 | −struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev, | |
| 1228 | − struct ib_udata *udata); | |
| 1229 | −int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata); | |
| 1195 | +int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata); | |
| 1196 | +void mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata); | |
| 1230 | 1197 | int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset); |
| 1231 | 1198 | int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port); |
| 1232 | 1199 | int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev, |
| @@ -1375,46 +1342,12 @@ struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *dev, · 142 unmodified lines | ||
| 1375 | 1342 | u8 *native_port_num); |
| 1376 | 1343 | void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev, |
| 1377 | 1344 | u8 port_num); |
| 1378 | −int mlx5_ib_fill_res_entry(struct sk_buff *msg, | |
| 1379 | − struct rdma_restrack_entry *res); | |
| 1380 | −int mlx5_ib_fill_stat_entry(struct sk_buff *msg, | |
| 1381 | − struct rdma_restrack_entry *res); | |
| 1382 | 1345 | |
| 1383 | 1346 | extern const struct uapi_definition mlx5_ib_devx_defs[]; |
| 1384 | 1347 | extern const struct uapi_definition mlx5_ib_flow_defs[]; |
| 1385 | 1348 | extern const struct uapi_definition mlx5_ib_qos_defs[]; |
| 1349 | +extern const struct uapi_definition mlx5_ib_std_types_defs[]; | |
| 1386 | 1350 | |
| 1387 | −#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) | |
| 1388 | −int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user); | |
| 1389 | −void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid); | |
| 1390 | −void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev); | |
| 1391 | −void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev); | |
| 1392 | −struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add( | |
| 1393 | − struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, | |
| 1394 | − struct mlx5_flow_context *flow_context, | |
| 1395 | − struct mlx5_flow_act *flow_act, u32 counter_id, | |
| 1396 | − void *cmd_in, int inlen, int dest_id, int dest_type); | |
| 1397 | −bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type); | |
| 1398 | −bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id); | |
| 1399 | −void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction); | |
| 1400 | −#else | |
| 1401 | −static inline int | |
| 1402 | −mlx5_ib_devx_create(struct mlx5_ib_dev *dev, | |
| 1403 | − bool is_user) { return -EOPNOTSUPP; } | |
| 1404 | −static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {} | |
| 1405 | −static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {} | |
| 1406 | −static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {} | |
| 1407 | −static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, | |
| 1408 | − int *dest_type) | |
| 1409 | −{ | |
| 1410 | − return false; | |
| 1411 | −} | |
| 1412 | −static inline void | |
| 1413 | −mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) | |
| 1414 | −{ | |
| 1415 | − return; | |
| 1416 | −}; | |
| 1417 | −#endif | |
| 1418 | 1351 | static inline void init_query_mad(struct ib_smp *mad) |
| 1419 | 1352 | { |
| 1420 | 1353 | mad->base_version = 1; |
| @@ -1423,15 +1356,6 @@ static inline void init_query_mad(struct ib_smp *mad) · 2 unmodified lines | ||
| 1423 | 1356 | mad->method = IB_MGMT_METHOD_GET; |
| 1424 | 1357 | } |
| 1425 | 1358 | |
| 1426 | −static inline u8 convert_access(int acc) | |
| 1427 | −{ | |
| 1428 | − return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) | | |
| 1429 | − (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) | | |
| 1430 | − (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) | | |
| 1431 | − (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) | | |
| 1432 | − MLX5_PERM_LOCAL_READ; | |
| 1433 | −} | |
| 1434 | − | |
| 1435 | 1359 | static inline int is_qp1(enum ib_qp_type qp_type) |
| 1436 | 1360 | { |
| 1437 | 1361 | return qp_type == MLX5_IB_QPT_HW_GSI; |
| @@ -1518,9 +1442,6 @@ int bfregn_to_uar_index(struct mlx5_ib_dev *dev, · 80 unmodified lines | ||
| 1518 | 1442 | struct mlx5_bfreg_info *bfregi, u32 bfregn, |
| 1519 | 1443 | bool dyn_bfreg); |
| 1520 | 1444 | |
| 1521 | −int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter); | |
| 1522 | −u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num); | |
| 1523 | − | |
| 1524 | 1445 | static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, |
| 1525 | 1446 | bool do_modify_atomic, int access_flags) |
| 1526 | 1447 | { |
| @@ -1533,14 +1454,18 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, · 6 unmodified lines | ||
| 1533 | 1454 | return false; |
| 1534 | 1455 | |
| 1535 | 1456 | if (access_flags & IB_ACCESS_RELAXED_ORDERING && |
| 1536 | − (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) || | |
| 1537 | − MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))) | |
| 1457 | + MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) && | |
| 1458 | + !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) | |
| 1459 | + return false; | |
| 1460 | + | |
| 1461 | + if (access_flags & IB_ACCESS_RELAXED_ORDERING && | |
| 1462 | + MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) && | |
| 1463 | + !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) | |
| 1538 | 1464 | return false; |
| 1539 | 1465 | |
| 1540 | 1466 | return true; |
| 1541 | 1467 | } |
| 1542 | 1468 | |
| 1543 | −int mlx5_ib_enable_driver(struct ib_device *dev); | |
| 1544 | 1469 | int mlx5_ib_test_wc(struct mlx5_ib_dev *dev); |
| 1545 | 1470 | |
| 1546 | 1471 | static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev) |
| @@ -1961,7 +1961,7 @@ err_free: · 1960 unmodified lines | ||
| 1961 | 1961 | } |
| 1962 | 1962 | |
| 1963 | 1963 | struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 1964 | − u32 max_num_sg, struct ib_udata *udata) | |
| 1964 | + u32 max_num_sg) | |
| 1965 | 1965 | { |
| 1966 | 1966 | return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0); |
| 1967 | 1967 | } |
| @@ -816,6 +816,7 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, · 815 unmodified lines | ||
| 816 | 816 | { |
| 817 | 817 | struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); |
| 818 | 818 | |
| 819 | + lockdep_assert_held(&mr->dev->odp_srcu); | |
| 819 | 820 | if (unlikely(io_virt < mr->mmkey.iova)) |
| 820 | 821 | return -EFAULT; |
| 821 | 822 | |
| @@ -929,11 +930,6 @@ next_mr: · 107 unmodified lines | ||
| 929 | 930 | if (ret < 0) |
| 930 | 931 | goto srcu_unlock; |
| 931 | 932 | |
| 932 | − /* | |
| 933 | − * When prefetching a page, page fault is generated | |
| 934 | − * in order to bring the page to the main memory. | |
| 935 | − * In the current flow, page faults are being counted. | |
| 936 | − */ | |
| 937 | 933 | mlx5_update_odp_stats(mr, faults, ret); |
| 938 | 934 | |
| 939 | 935 | npages += ret; |
| @@ -1770,13 +1766,26 @@ static void mlx5_ib_prefetch_mr_work(struct work_struct *w) · 830 unmodified lines | ||
| 1770 | 1766 | { |
| 1771 | 1767 | struct prefetch_mr_work *work = |
| 1772 | 1768 | container_of(w, struct prefetch_mr_work, work); |
| 1769 | + struct mlx5_ib_dev *dev; | |
| 1773 | 1770 | u32 bytes_mapped = 0; |
| 1771 | + int srcu_key; | |
| 1772 | + int ret; | |
| 1774 | 1773 | u32 i; |
| 1775 | 1774 | |
| 1776 | − for (i = 0; i < work->num_sge; ++i) | |
| 1777 | − pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, | |
| 1778 | − work->frags[i].length, &bytes_mapped, | |
| 1779 | − work->pf_flags); | |
| 1775 | + /* We rely on IB/core that work is executed if we have num_sge != 0 only. */ | |
| 1776 | + WARN_ON(!work->num_sge); | |
| 1777 | + dev = work->frags[0].mr->dev; | |
| 1778 | + /* SRCU should be held when calling to mlx5_odp_populate_xlt() */ | |
| 1779 | + srcu_key = srcu_read_lock(&dev->odp_srcu); | |
| 1780 | + for (i = 0; i < work->num_sge; ++i) { | |
| 1781 | + ret = pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, | |
| 1782 | + work->frags[i].length, &bytes_mapped, | |
| 1783 | + work->pf_flags); | |
| 1784 | + if (ret <= 0) | |
| 1785 | + continue; | |
| 1786 | + mlx5_update_odp_stats(work->frags[i].mr, prefetch, ret); | |
| 1787 | + } | |
| 1788 | + srcu_read_unlock(&dev->odp_srcu, srcu_key); | |
| 1780 | 1789 | |
| 1781 | 1790 | destroy_prefetch_work(work); |
| 1782 | 1791 | } |
| @@ -1832,6 +1841,7 @@ static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, · 49 unmodified lines | ||
| 1832 | 1841 | &bytes_mapped, pf_flags); |
| 1833 | 1842 | if (ret < 0) |
| 1834 | 1843 | goto out; |
| 1844 | + mlx5_update_odp_stats(mr, prefetch, ret); | |
| 1835 | 1845 | } |
| 1836 | 1846 | ret = 0; |
| 1837 | 1847 | |
| @@ -38,6 +38,7 @@ · 37 unmodified lines | ||
| 38 | 38 | #include <linux/mlx5/fs.h> |
| 39 | 39 | #include "mlx5_ib.h" |
| 40 | 40 | #include "ib_rep.h" |
| 41 | +#include "counters.h" | |
| 41 | 42 | #include "cmd.h" |
| 42 | 43 | #include "qp.h" |
| 43 | 44 | #include "wr.h" |
| @@ -2031,15 +2032,15 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, · 1987 unmodified lines | ||
| 2031 | 2032 | switch (init_attr->qp_type) { |
| 2032 | 2033 | case IB_QPT_XRC_INI: |
| 2033 | 2034 | MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn); |
| 2034 | − MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn); | |
| 2035 | + MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1); | |
| 2035 | 2036 | MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn); |
| 2036 | 2037 | break; |
| 2037 | 2038 | default: |
| 2038 | 2039 | if (init_attr->srq) { |
| 2039 | − MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn); | |
| 2040 | + MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0); | |
| 2040 | 2041 | MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn); |
| 2041 | 2042 | } else { |
| 2042 | − MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn); | |
| 2043 | + MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1); | |
| 2043 | 2044 | MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn); |
| 2044 | 2045 | } |
| 2045 | 2046 | } |
| @@ -2178,11 +2179,11 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, · 132 unmodified lines | ||
| 2178 | 2179 | MLX5_SET(qpc, qpc, no_sq, 1); |
| 2179 | 2180 | |
| 2180 | 2181 | if (attr->srq) { |
| 2181 | − MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn); | |
| 2182 | + MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0); | |
| 2182 | 2183 | MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, |
| 2183 | 2184 | to_msrq(attr->srq)->msrq.srqn); |
| 2184 | 2185 | } else { |
| 2185 | − MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn); | |
| 2186 | + MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1); | |
| 2186 | 2187 | MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, |
| 2187 | 2188 | to_msrq(devr->s1)->msrq.srqn); |
| 2188 | 2189 | } |
| @@ -3554,7 +3555,7 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, · 1365 unmodified lines | ||
| 3554 | 3555 | switch (raw_qp_param->operation) { |
| 3555 | 3556 | case MLX5_CMD_OP_RST2INIT_QP: |
| 3556 | 3557 | rq_state = MLX5_RQC_STATE_RDY; |
| 3557 | − sq_state = MLX5_SQC_STATE_RDY; | |
| 3558 | + sq_state = MLX5_SQC_STATE_RST; | |
| 3558 | 3559 | break; |
| 3559 | 3560 | case MLX5_CMD_OP_2ERR_QP: |
| 3560 | 3561 | rq_state = MLX5_RQC_STATE_ERR; |
| @@ -3566,13 +3567,11 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, · 5 unmodified lines | ||
| 3566 | 3567 | break; |
| 3567 | 3568 | case MLX5_CMD_OP_RTR2RTS_QP: |
| 3568 | 3569 | case MLX5_CMD_OP_RTS2RTS_QP: |
| 3569 | − if (raw_qp_param->set_mask == | |
| 3570 | − MLX5_RAW_QP_RATE_LIMIT) { | |
| 3571 | − modify_rq = 0; | |
| 3572 | − sq_state = sq->state; | |
| 3573 | − } else { | |
| 3574 | − return raw_qp_param->set_mask ? -EINVAL : 0; | |
| 3575 | − } | |
| 3570 | + if (raw_qp_param->set_mask & ~MLX5_RAW_QP_RATE_LIMIT) | |
| 3571 | + return -EINVAL; | |
| 3572 | + | |
| 3573 | + modify_rq = 0; | |
| 3574 | + sq_state = MLX5_SQC_STATE_RDY; | |
| 3576 | 3575 | break; |
| 3577 | 3576 | case MLX5_CMD_OP_INIT2INIT_QP: |
| 3578 | 3577 | case MLX5_CMD_OP_INIT2RTR_QP: |
| @@ -4114,9 +4113,9 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 535 unmodified lines | ||
| 4114 | 4113 | struct mlx5_ib_qp *qp = to_mqp(ibqp); |
| 4115 | 4114 | struct mlx5_ib_dev *dev = to_mdev(ibqp->device); |
| 4116 | 4115 | enum ib_qp_state cur_state, new_state; |
| 4117 | − int err = 0; | |
| 4118 | 4116 | int required = IB_QP_STATE; |
| 4119 | 4117 | void *dctc; |
| 4118 | + int err; | |
| 4120 | 4119 | |
| 4121 | 4120 | if (!(attr_mask & IB_QP_STATE)) |
| 4122 | 4121 | return -EINVAL; |
| @@ -4208,11 +4207,9 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 85 unmodified lines | ||
| 4208 | 4207 | mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state); |
| 4209 | 4208 | return -EINVAL; |
| 4210 | 4209 | } |
| 4211 | − if (err) | |
| 4212 | − qp->state = IB_QPS_ERR; | |
| 4213 | − else | |
| 4214 | − qp->state = new_state; | |
| 4215 | − return err; | |
| 4210 | + | |
| 4211 | + qp->state = new_state; | |
| 4212 | + return 0; | |
| 4216 | 4213 | } |
| 4217 | 4214 | |
| 4218 | 4215 | int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, |
| @@ -4450,7 +4447,7 @@ static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state, · 231 unmodified lines | ||
| 4450 | 4447 | [MLX5_SQ_STATE_NA] = IB_QPS_RESET, |
| 4451 | 4448 | }, |
| 4452 | 4449 | [MLX5_RQC_STATE_RDY] = { |
| 4453 | − [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD, | |
| 4450 | + [MLX5_SQC_STATE_RST] = MLX5_QP_STATE, | |
| 4454 | 4451 | [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE, |
| 4455 | 4452 | [MLX5_SQC_STATE_ERR] = IB_QPS_SQE, |
| 4456 | 4453 | [MLX5_SQ_STATE_NA] = MLX5_QP_STATE, |
| @@ -4462,7 +4459,7 @@ static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state, · 5 unmodified lines | ||
| 4462 | 4459 | [MLX5_SQ_STATE_NA] = IB_QPS_ERR, |
| 4463 | 4460 | }, |
| 4464 | 4461 | [MLX5_RQ_STATE_NA] = { |
| 4465 | − [MLX5_SQC_STATE_RST] = IB_QPS_RESET, | |
| 4462 | + [MLX5_SQC_STATE_RST] = MLX5_QP_STATE, | |
| 4466 | 4463 | [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE, |
| 4467 | 4464 | [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE, |
| 4468 | 4465 | [MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD, |
| @@ -4708,41 +4705,23 @@ out: · 239 unmodified lines | ||
| 4708 | 4705 | return err; |
| 4709 | 4706 | } |
| 4710 | 4707 | |
| 4711 | −struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev, | |
| 4712 | − struct ib_udata *udata) | |
| 4708 | +int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) | |
| 4713 | 4709 | { |
| 4714 | − struct mlx5_ib_dev *dev = to_mdev(ibdev); | |
| 4715 | − struct mlx5_ib_xrcd *xrcd; | |
| 4716 | − int err; | |
| 4710 | + struct mlx5_ib_dev *dev = to_mdev(ibxrcd->device); | |
| 4711 | + struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd); | |
| 4717 | 4712 | |
| 4718 | 4713 | if (!MLX5_CAP_GEN(dev->mdev, xrc)) |
| 4719 | − return ERR_PTR(-ENOSYS); | |
| 4720 | − | |
| 4721 | − xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL); | |
| 4722 | − if (!xrcd) | |
| 4723 | − return ERR_PTR(-ENOMEM); | |
| 4724 | − | |
| 4725 | − err = mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0); | |
| 4726 | − if (err) { | |
| 4727 | − kfree(xrcd); | |
| 4728 | − return ERR_PTR(-ENOMEM); | |
| 4729 | − } | |
| 4714 | + return -EOPNOTSUPP; | |
| 4730 | 4715 | |
| 4731 | − return &xrcd->ibxrcd; | |
| 4716 | + return mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0); | |
| 4732 | 4717 | } |
| 4733 | 4718 | |
| 4734 | −int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) | |
| 4719 | +void mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata) | |
| 4735 | 4720 | { |
| 4736 | 4721 | struct mlx5_ib_dev *dev = to_mdev(xrcd->device); |
| 4737 | 4722 | u32 xrcdn = to_mxrcd(xrcd)->xrcdn; |
| 4738 | − int err; | |
| 4739 | 4723 | |
| 4740 | − err = mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0); | |
| 4741 | − if (err) | |
| 4742 | − mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn); | |
| 4743 | − | |
| 4744 | − kfree(xrcd); | |
| 4745 | − return 0; | |
| 4724 | + mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0); | |
| 4746 | 4725 | } |
| 4747 | 4726 | |
| 4748 | 4727 | static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type) |
| @@ -43,4 +43,5 @@ void mlx5_core_res_put(struct mlx5_core_rsc_common *res); · 42 unmodified lines | ||
| 43 | 43 | |
| 44 | 44 | int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn); |
| 45 | 45 | int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn); |
| 46 | +int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter); | |
| 46 | 47 | #endif /* _MLX5_IB_QP_H */ |
| @@ -1,17 +1,85 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
| 2 | 2 | /* |
| 3 | − * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. | |
| 3 | + * Copyright (c) 2019-2020, Mellanox Technologies Ltd. All rights reserved. | |
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | 6 | #include <uapi/rdma/rdma_netlink.h> |
| 7 | +#include <linux/mlx5/rsc_dump.h> | |
| 7 | 8 | #include <rdma/ib_umem_odp.h> |
| 8 | 9 | #include <rdma/restrack.h> |
| 9 | 10 | #include "mlx5_ib.h" |
| 11 | +#include "restrack.h" | |
| 10 | 12 | |
| 11 | −static int fill_stat_mr_entry(struct sk_buff *msg, | |
| 12 | − struct rdma_restrack_entry *res) | |
| 13 | +#define MAX_DUMP_SIZE 1024 | |
| 14 | + | |
| 15 | +static int dump_rsc(struct mlx5_core_dev *dev, enum mlx5_sgmt_type type, | |
| 16 | + int index, void *data, int *data_len) | |
| 17 | +{ | |
| 18 | + struct mlx5_core_dev *mdev = dev; | |
| 19 | + struct mlx5_rsc_dump_cmd *cmd; | |
| 20 | + struct mlx5_rsc_key key = {}; | |
| 21 | + struct page *page; | |
| 22 | + int offset = 0; | |
| 23 | + int err = 0; | |
| 24 | + int cmd_err; | |
| 25 | + int size; | |
| 26 | + | |
| 27 | + page = alloc_page(GFP_KERNEL); | |
| 28 | + if (!page) | |
| 29 | + return -ENOMEM; | |
| 30 | + | |
| 31 | + key.size = PAGE_SIZE; | |
| 32 | + key.rsc = type; | |
| 33 | + key.index1 = index; | |
| 34 | + key.num_of_obj1 = 1; | |
| 35 | + | |
| 36 | + cmd = mlx5_rsc_dump_cmd_create(mdev, &key); | |
| 37 | + if (IS_ERR(cmd)) { | |
| 38 | + err = PTR_ERR(cmd); | |
| 39 | + goto free_page; | |
| 40 | + } | |
| 41 | + | |
| 42 | + do { | |
| 43 | + cmd_err = mlx5_rsc_dump_next(mdev, cmd, page, &size); | |
| 44 | + if (cmd_err < 0 || size + offset > MAX_DUMP_SIZE) { | |
| 45 | + err = cmd_err; | |
| 46 | + goto destroy_cmd; | |
| 47 | + } | |
| 48 | + memcpy(data + offset, page_address(page), size); | |
| 49 | + offset += size; | |
| 50 | + } while (cmd_err > 0); | |
| 51 | + *data_len = offset; | |
| 52 | + | |
| 53 | +destroy_cmd: | |
| 54 | + mlx5_rsc_dump_cmd_destroy(cmd); | |
| 55 | +free_page: | |
| 56 | + __free_page(page); | |
| 57 | + return err; | |
| 58 | +} | |
| 59 | + | |
| 60 | +static int fill_res_raw(struct sk_buff *msg, struct mlx5_ib_dev *dev, | |
| 61 | + enum mlx5_sgmt_type type, u32 key) | |
| 62 | +{ | |
| 63 | + int len = 0; | |
| 64 | + void *data; | |
| 65 | + int err; | |
| 66 | + | |
| 67 | + data = kzalloc(MAX_DUMP_SIZE, GFP_KERNEL); | |
| 68 | + if (!data) | |
| 69 | + return -ENOMEM; | |
| 70 | + | |
| 71 | + err = dump_rsc(dev->mdev, type, key, data, &len); | |
| 72 | + if (err) | |
| 73 | + goto out; | |
| 74 | + | |
| 75 | + err = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, len, data); | |
| 76 | +out: | |
| 77 | + kfree(data); | |
| 78 | + return err; | |
| 79 | +} | |
| 80 | + | |
| 81 | +static int fill_stat_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr) | |
| 13 | 82 | { |
| 14 | − struct ib_mr *ibmr = container_of(res, struct ib_mr, res); | |
| 15 | 83 | struct mlx5_ib_mr *mr = to_mmr(ibmr); |
| 16 | 84 | struct nlattr *table_attr; |
| 17 | 85 | |
| @@ -31,6 +99,9 @@ static int fill_stat_mr_entry(struct sk_buff *msg, · 13 unmodified lines | ||
| 31 | 99 | msg, "page_invalidations", |
| 32 | 100 | atomic64_read(&mr->odp_stats.invalidations))) |
| 33 | 101 | goto err_table; |
| 102 | + if (rdma_nl_stat_hwcounter_entry(msg, "page_prefetch", | |
| 103 | + atomic64_read(&mr->odp_stats.prefetch))) | |
| 104 | + goto err_table; | |
| 34 | 105 | |
| 35 | 106 | nla_nest_end(msg, table_attr); |
| 36 | 107 | return 0; |
| @@ -41,10 +112,16 @@ err: · 4 unmodified lines | ||
| 41 | 112 | return -EMSGSIZE; |
| 42 | 113 | } |
| 43 | 114 | |
| 44 | −static int fill_res_mr_entry(struct sk_buff *msg, | |
| 45 | − struct rdma_restrack_entry *res) | |
| 115 | +static int fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ibmr) | |
| 116 | +{ | |
| 117 | + struct mlx5_ib_mr *mr = to_mmr(ibmr); | |
| 118 | + | |
| 119 | + return fill_res_raw(msg, mr->dev, MLX5_SGMT_TYPE_PRM_QUERY_MKEY, | |
| 120 | + mlx5_mkey_to_idx(mr->mmkey.key)); | |
| 121 | +} | |
| 122 | + | |
| 123 | +static int fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr) | |
| 46 | 124 | { |
| 47 | − struct ib_mr *ibmr = container_of(res, struct ib_mr, res); | |
| 48 | 125 | struct mlx5_ib_mr *mr = to_mmr(ibmr); |
| 49 | 126 | struct nlattr *table_attr; |
| 50 | 127 | |
| @@ -71,20 +148,32 @@ err: · 20 unmodified lines | ||
| 71 | 148 | return -EMSGSIZE; |
| 72 | 149 | } |
| 73 | 150 | |
| 74 | −int mlx5_ib_fill_res_entry(struct sk_buff *msg, | |
| 75 | − struct rdma_restrack_entry *res) | |
| 151 | +static int fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ibcq) | |
| 76 | 152 | { |
| 77 | − if (res->type == RDMA_RESTRACK_MR) | |
| 78 | − return fill_res_mr_entry(msg, res); | |
| 153 | + struct mlx5_ib_dev *dev = to_mdev(ibcq->device); | |
| 154 | + struct mlx5_ib_cq *cq = to_mcq(ibcq); | |
| 79 | 155 | |
| 80 | − return 0; | |
| 156 | + return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_CQ, cq->mcq.cqn); | |
| 81 | 157 | } |
| 82 | 158 | |
| 83 | −int mlx5_ib_fill_stat_entry(struct sk_buff *msg, | |
| 84 | − struct rdma_restrack_entry *res) | |
| 159 | +static int fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ibqp) | |
| 85 | 160 | { |
| 86 | − if (res->type == RDMA_RESTRACK_MR) | |
| 87 | − return fill_stat_mr_entry(msg, res); | |
| 161 | + struct mlx5_ib_dev *dev = to_mdev(ibqp->device); | |
| 162 | + | |
| 163 | + return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_QP, | |
| 164 | + ibqp->qp_num); | |
| 165 | +} | |
| 88 | 166 | |
| 167 | +static const struct ib_device_ops restrack_ops = { | |
| 168 | + .fill_res_cq_entry_raw = fill_res_cq_entry_raw, | |
| 169 | + .fill_res_mr_entry = fill_res_mr_entry, | |
| 170 | + .fill_res_mr_entry_raw = fill_res_mr_entry_raw, | |
| 171 | + .fill_res_qp_entry_raw = fill_res_qp_entry_raw, | |
| 172 | + .fill_stat_mr_entry = fill_stat_mr_entry, | |
| 173 | +}; | |
| 174 | + | |
| 175 | +int mlx5_ib_restrack_init(struct mlx5_ib_dev *dev) | |
| 176 | +{ | |
| 177 | + ib_set_device_ops(&dev->ib_dev, &restrack_ops); | |
| 89 | 178 | return 0; |
| 90 | 179 | } |
| @@ -0,0 +1,13 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 2 | +/* | |
| 3 | + * Copyright (c) 2013-2020, Mellanox Technologies Ltd. All rights reserved. | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#ifndef _MLX5_IB_RESTRACK_H | |
| 7 | +#define _MLX5_IB_RESTRACK_H | |
| 8 | + | |
| 9 | +#include "mlx5_ib.h" | |
| 10 | + | |
| 11 | +int mlx5_ib_restrack_init(struct mlx5_ib_dev *dev); | |
| 12 | + | |
| 13 | +#endif /* _MLX5_IB_RESTRACK_H */ | |
| @@ -274,10 +274,10 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, · 273 unmodified lines | ||
| 274 | 274 | if (srq->wq_sig) |
| 275 | 275 | in.flags |= MLX5_SRQ_FLAG_WQ_SIG; |
| 276 | 276 | |
| 277 | − if (init_attr->srq_type == IB_SRQT_XRC) | |
| 277 | + if (init_attr->srq_type == IB_SRQT_XRC && init_attr->ext.xrc.xrcd) | |
| 278 | 278 | in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn; |
| 279 | 279 | else |
| 280 | − in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn; | |
| 280 | + in.xrcd = dev->devr.xrcdn0; | |
| 281 | 281 | |
| 282 | 282 | if (init_attr->srq_type == IB_SRQT_TM) { |
| 283 | 283 | in.tm_log_list_size = |
| @@ -0,0 +1,45 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB | |
| 2 | +/* | |
| 3 | + * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#include <rdma/uverbs_ioctl.h> | |
| 7 | +#include <rdma/mlx5_user_ioctl_cmds.h> | |
| 8 | +#include <rdma/mlx5_user_ioctl_verbs.h> | |
| 9 | +#include <linux/mlx5/driver.h> | |
| 10 | +#include "mlx5_ib.h" | |
| 11 | + | |
| 12 | +#define UVERBS_MODULE_NAME mlx5_ib | |
| 13 | +#include <rdma/uverbs_named_ioctl.h> | |
| 14 | + | |
| 15 | +static int UVERBS_HANDLER(MLX5_IB_METHOD_PD_QUERY)( | |
| 16 | + struct uverbs_attr_bundle *attrs) | |
| 17 | +{ | |
| 18 | + struct ib_pd *pd = | |
| 19 | + uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_PD_HANDLE); | |
| 20 | + struct mlx5_ib_pd *mpd = to_mpd(pd); | |
| 21 | + | |
| 22 | + return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_PD_RESP_PDN, | |
| 23 | + &mpd->pdn, sizeof(mpd->pdn)); | |
| 24 | +} | |
| 25 | + | |
| 26 | +DECLARE_UVERBS_NAMED_METHOD( | |
| 27 | + MLX5_IB_METHOD_PD_QUERY, | |
| 28 | + UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_PD_HANDLE, | |
| 29 | + UVERBS_OBJECT_PD, | |
| 30 | + UVERBS_ACCESS_READ, | |
| 31 | + UA_MANDATORY), | |
| 32 | + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_PD_RESP_PDN, | |
| 33 | + UVERBS_ATTR_TYPE(u32), | |
| 34 | + UA_MANDATORY)); | |
| 35 | + | |
| 36 | +ADD_UVERBS_METHODS(mlx5_ib_pd, | |
| 37 | + UVERBS_OBJECT_PD, | |
| 38 | + &UVERBS_METHOD(MLX5_IB_METHOD_PD_QUERY)); | |
| 39 | + | |
| 40 | +const struct uapi_definition mlx5_ib_std_types_defs[] = { | |
| 41 | + UAPI_DEF_CHAIN_OBJ_TREE( | |
| 42 | + UVERBS_OBJECT_PD, | |
| 43 | + &mlx5_ib_pd), | |
| 44 | + {}, | |
| 45 | +}; | |
| @@ -263,7 +263,9 @@ static __be64 get_umr_update_translation_mask(void) · 262 unmodified lines | ||
| 263 | 263 | return cpu_to_be64(result); |
| 264 | 264 | } |
| 265 | 265 | |
| 266 | −static __be64 get_umr_update_access_mask(int atomic) | |
| 266 | +static __be64 get_umr_update_access_mask(int atomic, | |
| 267 | + int relaxed_ordering_write, | |
| 268 | + int relaxed_ordering_read) | |
| 267 | 269 | { |
| 268 | 270 | u64 result; |
| 269 | 271 | |
| @@ -275,6 +277,12 @@ static __be64 get_umr_update_access_mask(int atomic) · 5 unmodified lines | ||
| 275 | 277 | if (atomic) |
| 276 | 278 | result |= MLX5_MKEY_MASK_A; |
| 277 | 279 | |
| 280 | + if (relaxed_ordering_write) | |
| 281 | + result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE; | |
| 282 | + | |
| 283 | + if (relaxed_ordering_read) | |
| 284 | + result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ; | |
| 285 | + | |
| 278 | 286 | return cpu_to_be64(result); |
| 279 | 287 | } |
| 280 | 288 | |
| @@ -289,17 +297,28 @@ static __be64 get_umr_update_pd_mask(void) · 8 unmodified lines | ||
| 289 | 297 | |
| 290 | 298 | static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask) |
| 291 | 299 | { |
| 292 | − if ((mask & MLX5_MKEY_MASK_PAGE_SIZE && | |
| 293 | − MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) || | |
| 294 | − (mask & MLX5_MKEY_MASK_A && | |
| 295 | − MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))) | |
| 300 | + if (mask & MLX5_MKEY_MASK_PAGE_SIZE && | |
| 301 | + MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) | |
| 296 | 302 | return -EPERM; |
| 303 | + | |
| 304 | + if (mask & MLX5_MKEY_MASK_A && | |
| 305 | + MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)) | |
| 306 | + return -EPERM; | |
| 307 | + | |
| 308 | + if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE && | |
| 309 | + !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) | |
| 310 | + return -EPERM; | |
| 311 | + | |
| 312 | + if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ && | |
| 313 | + !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) | |
| 314 | + return -EPERM; | |
| 315 | + | |
| 297 | 316 | return 0; |
| 298 | 317 | } |
| 299 | 318 | |
| 300 | 319 | static int set_reg_umr_segment(struct mlx5_ib_dev *dev, |
| 301 | 320 | struct mlx5_wqe_umr_ctrl_seg *umr, |
| 302 | − const struct ib_send_wr *wr, int atomic) | |
| 321 | + const struct ib_send_wr *wr) | |
| 303 | 322 | { |
| 304 | 323 | const struct mlx5_umr_wr *umrwr = umr_wr(wr); |
| 305 | 324 | |
| @@ -325,7 +344,10 @@ static int set_reg_umr_segment(struct mlx5_ib_dev *dev, · 19 unmodified lines | ||
| 325 | 344 | if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION) |
| 326 | 345 | umr->mkey_mask |= get_umr_update_translation_mask(); |
| 327 | 346 | if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) { |
| 328 | − umr->mkey_mask |= get_umr_update_access_mask(atomic); | |
| 347 | + umr->mkey_mask |= get_umr_update_access_mask( | |
| 348 | + !!(MLX5_CAP_GEN(dev->mdev, atomic)), | |
| 349 | + !!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)), | |
| 350 | + !!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))); | |
| 329 | 351 | umr->mkey_mask |= get_umr_update_pd_mask(); |
| 330 | 352 | } |
| 331 | 353 | if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR) |
| @@ -383,20 +405,31 @@ static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, · 51 unmodified lines | ||
| 383 | 405 | |
| 384 | 406 | memset(seg, 0, sizeof(*seg)); |
| 385 | 407 | if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR) |
| 386 | − seg->status = MLX5_MKEY_STATUS_FREE; | |
| 408 | + MLX5_SET(mkc, seg, free, 1); | |
| 409 | + | |
| 410 | + MLX5_SET(mkc, seg, a, | |
| 411 | + !!(umrwr->access_flags & IB_ACCESS_REMOTE_ATOMIC)); | |
| 412 | + MLX5_SET(mkc, seg, rw, | |
| 413 | + !!(umrwr->access_flags & IB_ACCESS_REMOTE_WRITE)); | |
| 414 | + MLX5_SET(mkc, seg, rr, !!(umrwr->access_flags & IB_ACCESS_REMOTE_READ)); | |
| 415 | + MLX5_SET(mkc, seg, lw, !!(umrwr->access_flags & IB_ACCESS_LOCAL_WRITE)); | |
| 416 | + MLX5_SET(mkc, seg, lr, 1); | |
| 417 | + MLX5_SET(mkc, seg, relaxed_ordering_write, | |
| 418 | + !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING)); | |
| 419 | + MLX5_SET(mkc, seg, relaxed_ordering_read, | |
| 420 | + !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING)); | |
| 387 | 421 | |
| 388 | − seg->flags = convert_access(umrwr->access_flags); | |
| 389 | 422 | if (umrwr->pd) |
| 390 | − seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn); | |
| 423 | + MLX5_SET(mkc, seg, pd, to_mpd(umrwr->pd)->pdn); | |
| 391 | 424 | if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION && |
| 392 | 425 | !umrwr->length) |
| 393 | − seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64); | |
| 426 | + MLX5_SET(mkc, seg, length64, 1); | |
| 394 | 427 | |
| 395 | − seg->start_addr = cpu_to_be64(umrwr->virt_addr); | |
| 396 | − seg->len = cpu_to_be64(umrwr->length); | |
| 397 | − seg->log2_page_size = umrwr->page_shift; | |
| 398 | − seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 | | |
| 399 | − mlx5_mkey_variant(umrwr->mkey)); | |
| 428 | + MLX5_SET64(mkc, seg, start_addr, umrwr->virt_addr); | |
| 429 | + MLX5_SET64(mkc, seg, len, umrwr->length); | |
| 430 | + MLX5_SET(mkc, seg, log_page_size, umrwr->page_shift); | |
| 431 | + MLX5_SET(mkc, seg, qpn, 0xffffff); | |
| 432 | + MLX5_SET(mkc, seg, mkey_7_0, mlx5_mkey_variant(umrwr->mkey)); | |
| 400 | 433 | } |
| 401 | 434 | |
| 402 | 435 | static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg, |
| @@ -1224,8 +1257,7 @@ static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, · 821 unmodified lines | ||
| 1224 | 1257 | |
| 1225 | 1258 | qp->sq.wr_data[idx] = MLX5_IB_WR_UMR; |
| 1226 | 1259 | (*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey); |
| 1227 | − err = set_reg_umr_segment(dev, *seg, wr, | |
| 1228 | − !!(MLX5_CAP_GEN(dev->mdev, atomic))); | |
| 1260 | + err = set_reg_umr_segment(dev, *seg, wr); | |
| 1229 | 1261 | if (unlikely(err)) |
| 1230 | 1262 | goto out; |
| 1231 | 1263 | *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); |
| @@ -2901,7 +2901,7 @@ int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) · 2900 unmodified lines | ||
| 2901 | 2901 | } |
| 2902 | 2902 | |
| 2903 | 2903 | struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, |
| 2904 | − u32 max_num_sg, struct ib_udata *udata) | |
| 2904 | + u32 max_num_sg) | |
| 2905 | 2905 | { |
| 2906 | 2906 | int status; |
| 2907 | 2907 | struct ocrdma_mr *mr; |
| @@ -101,7 +101,7 @@ struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *, int acc); · 100 unmodified lines | ||
| 101 | 101 | struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *, u64 start, u64 length, |
| 102 | 102 | u64 virt, int acc, struct ib_udata *); |
| 103 | 103 | struct ib_mr *ocrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 104 | − u32 max_num_sg, struct ib_udata *udata); | |
| 104 | + u32 max_num_sg); | |
| 105 | 105 | int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, |
| 106 | 106 | unsigned int *sg_offset); |
| 107 | 107 | |
| @@ -110,7 +110,6 @@ static int qedr_iw_port_immutable(struct ib_device *ibdev, u8 port_num, · 109 unmodified lines | ||
| 110 | 110 | if (err) |
| 111 | 111 | return err; |
| 112 | 112 | |
| 113 | − immutable->pkey_tbl_len = 1; | |
| 114 | 113 | immutable->gid_tbl_len = 1; |
| 115 | 114 | immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; |
| 116 | 115 | immutable->max_mad_size = 0; |
| @@ -179,6 +178,7 @@ static int qedr_iw_register_device(struct qedr_dev *dev) · 62 unmodified lines | ||
| 179 | 178 | |
| 180 | 179 | static const struct ib_device_ops qedr_roce_dev_ops = { |
| 181 | 180 | .get_port_immutable = qedr_roce_port_immutable, |
| 181 | + .query_pkey = qedr_query_pkey, | |
| 182 | 182 | }; |
| 183 | 183 | |
| 184 | 184 | static void qedr_roce_register_device(struct qedr_dev *dev) |
| @@ -221,7 +221,6 @@ static const struct ib_device_ops qedr_dev_ops = { · 36 unmodified lines | ||
| 221 | 221 | .post_srq_recv = qedr_post_srq_recv, |
| 222 | 222 | .process_mad = qedr_process_mad, |
| 223 | 223 | .query_device = qedr_query_device, |
| 224 | − .query_pkey = qedr_query_pkey, | |
| 225 | 224 | .query_port = qedr_query_port, |
| 226 | 225 | .query_qp = qedr_query_qp, |
| 227 | 226 | .query_srq = qedr_query_srq, |
| @@ -235,6 +235,7 @@ struct qedr_ucontext { · 234 unmodified lines | ||
| 235 | 235 | u32 dpi_size; |
| 236 | 236 | u16 dpi; |
| 237 | 237 | bool db_rec; |
| 238 | + u8 edpm_mode; | |
| 238 | 239 | }; |
| 239 | 240 | |
| 240 | 241 | union db_prod32 { |
| @@ -344,10 +345,10 @@ struct qedr_srq_hwq_info { · 103 unmodified lines | ||
| 344 | 345 | u32 wqe_prod; |
| 345 | 346 | u32 sge_prod; |
| 346 | 347 | u32 wr_prod_cnt; |
| 347 | − u32 wr_cons_cnt; | |
| 348 | + atomic_t wr_cons_cnt; | |
| 348 | 349 | u32 num_elems; |
| 349 | 350 | |
| 350 | − u32 *virt_prod_pair_addr; | |
| 351 | + struct rdma_srq_producers *virt_prod_pair_addr; | |
| 351 | 352 | dma_addr_t phy_prod_pair_addr; |
| 352 | 353 | }; |
| 353 | 354 | |
| @@ -239,7 +239,6 @@ int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) · 238 unmodified lines | ||
| 239 | 239 | attr->ip_gids = true; |
| 240 | 240 | if (rdma_protocol_iwarp(&dev->ibdev, 1)) { |
| 241 | 241 | attr->gid_tbl_len = 1; |
| 242 | − attr->pkey_tbl_len = 1; | |
| 243 | 242 | } else { |
| 244 | 243 | attr->gid_tbl_len = QEDR_MAX_SGID; |
| 245 | 244 | attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; |
| @@ -275,7 +274,8 @@ int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) · 29 unmodified lines | ||
| 275 | 274 | DP_ERR(dev, "Problem copying data from user space\n"); |
| 276 | 275 | return -EFAULT; |
| 277 | 276 | } |
| 278 | − | |
| 277 | + ctx->edpm_mode = !!(ureq.context_flags & | |
| 278 | + QEDR_ALLOC_UCTX_EDPM_MODE); | |
| 279 | 279 | ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC); |
| 280 | 280 | } |
| 281 | 281 | |
| @@ -316,11 +316,15 @@ int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) · 34 unmodified lines | ||
| 316 | 316 | uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY; |
| 317 | 317 | else |
| 318 | 318 | uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED | |
| 319 | − QEDR_DPM_TYPE_ROCE_LEGACY; | |
| 319 | + QEDR_DPM_TYPE_ROCE_LEGACY | | |
| 320 | + QEDR_DPM_TYPE_ROCE_EDPM_MODE; | |
| 320 | 321 | |
| 321 | − uresp.dpm_flags |= QEDR_DPM_SIZES_SET; | |
| 322 | − uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE; | |
| 323 | − uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE; | |
| 322 | + if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) { | |
| 323 | + uresp.dpm_flags |= QEDR_DPM_SIZES_SET; | |
| 324 | + uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE; | |
| 325 | + uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE; | |
| 326 | + uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE; | |
| 327 | + } | |
| 324 | 328 | |
| 325 | 329 | uresp.wids_enabled = 1; |
| 326 | 330 | uresp.wid_count = oparams.wid_count; |
| @@ -1754,7 +1758,7 @@ static int qedr_create_user_qp(struct qedr_dev *dev, · 1427 unmodified lines | ||
| 1754 | 1758 | struct qed_rdma_create_qp_out_params out_params; |
| 1755 | 1759 | struct qedr_pd *pd = get_qedr_pd(ibpd); |
| 1756 | 1760 | struct qedr_create_qp_uresp uresp; |
| 1757 | − struct qedr_ucontext *ctx = NULL; | |
| 1761 | + struct qedr_ucontext *ctx = pd ? pd->uctx : NULL; | |
| 1758 | 1762 | struct qedr_create_qp_ureq ureq; |
| 1759 | 1763 | int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); |
| 1760 | 1764 | int rc = -EINVAL; |
| @@ -1792,6 +1796,9 @@ static int qedr_create_user_qp(struct qedr_dev *dev, · 31 unmodified lines | ||
| 1792 | 1796 | in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; |
| 1793 | 1797 | } |
| 1794 | 1798 | |
| 1799 | + if (ctx) | |
| 1800 | + SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode); | |
| 1801 | + | |
| 1795 | 1802 | qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, |
| 1796 | 1803 | &in_params, &out_params); |
| 1797 | 1804 | |
| @@ -3004,7 +3011,7 @@ err0: · 1206 unmodified lines | ||
| 3004 | 3011 | } |
| 3005 | 3012 | |
| 3006 | 3013 | struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, |
| 3007 | − u32 max_num_sg, struct ib_udata *udata) | |
| 3014 | + u32 max_num_sg) | |
| 3008 | 3015 | { |
| 3009 | 3016 | struct qedr_mr *mr; |
| 3010 | 3017 | |
| @@ -3687,7 +3694,7 @@ static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) · 676 unmodified lines | ||
| 3687 | 3694 | * count and consumer count and subtract it from max |
| 3688 | 3695 | * work request supported so that we get elements left. |
| 3689 | 3696 | */ |
| 3690 | − used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; | |
| 3697 | + used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt); | |
| 3691 | 3698 | |
| 3692 | 3699 | return hw_srq->max_wr - used; |
| 3693 | 3700 | } |
| @@ -3702,7 +3709,6 @@ int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, · 8 unmodified lines | ||
| 3702 | 3709 | unsigned long flags; |
| 3703 | 3710 | int status = 0; |
| 3704 | 3711 | u32 num_sge; |
| 3705 | − u32 offset; | |
| 3706 | 3712 | |
| 3707 | 3713 | spin_lock_irqsave(&srq->lock, flags); |
| 3708 | 3714 | |
| @@ -3715,7 +3721,8 @@ int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, · 6 unmodified lines | ||
| 3715 | 3721 | if (!qedr_srq_elem_left(hw_srq) || |
| 3716 | 3722 | wr->num_sge > srq->hw_srq.max_sges) { |
| 3717 | 3723 | DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", |
| 3718 | − hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, | |
| 3724 | + hw_srq->wr_prod_cnt, | |
| 3725 | + atomic_read(&hw_srq->wr_cons_cnt), | |
| 3719 | 3726 | wr->num_sge, srq->hw_srq.max_sges); |
| 3720 | 3727 | status = -ENOMEM; |
| 3721 | 3728 | *bad_wr = wr; |
| @@ -3749,22 +3756,20 @@ int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, · 27 unmodified lines | ||
| 3749 | 3756 | hw_srq->sge_prod++; |
| 3750 | 3757 | } |
| 3751 | 3758 | |
| 3752 | − /* Flush WQE and SGE information before | |
| 3759 | + /* Update WQE and SGE information before | |
| 3753 | 3760 | * updating producer. |
| 3754 | 3761 | */ |
| 3755 | − wmb(); | |
| 3762 | + dma_wmb(); | |
| 3756 | 3763 | |
| 3757 | 3764 | /* SRQ producer is 8 bytes. Need to update SGE producer index |
| 3758 | 3765 | * in first 4 bytes and need to update WQE producer in |
| 3759 | 3766 | * next 4 bytes. |
| 3760 | 3767 | */ |
| 3761 | − *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod; | |
| 3762 | − offset = offsetof(struct rdma_srq_producers, wqe_prod); | |
| 3763 | − *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = | |
| 3764 | − hw_srq->wqe_prod; | |
| 3768 | + srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod; | |
| 3769 | + /* Make sure sge producer is updated first */ | |
| 3770 | + dma_wmb(); | |
| 3771 | + srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod; | |
| 3765 | 3772 | |
| 3766 | − /* Flush producer after updating it. */ | |
| 3767 | − wmb(); | |
| 3768 | 3773 | wr = wr->next; |
| 3769 | 3774 | } |
| 3770 | 3775 | |
| @@ -4183,7 +4188,7 @@ static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, · 412 unmodified lines | ||
| 4183 | 4188 | } else { |
| 4184 | 4189 | __process_resp_one(dev, qp, cq, wc, resp, wr_id); |
| 4185 | 4190 | } |
| 4186 | − srq->hw_srq.wr_cons_cnt++; | |
| 4191 | + atomic_inc(&srq->hw_srq.wr_cons_cnt); | |
| 4187 | 4192 | |
| 4188 | 4193 | return 1; |
| 4189 | 4194 | } |
| @@ -84,7 +84,7 @@ int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, · 83 unmodified lines | ||
| 84 | 84 | int sg_nents, unsigned int *sg_offset); |
| 85 | 85 | |
| 86 | 86 | struct ib_mr *qedr_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 87 | − u32 max_num_sg, struct ib_udata *udata); | |
| 87 | + u32 max_num_sg); | |
| 88 | 88 | int qedr_poll_cq(struct ib_cq *, int num_entries, struct ib_wc *wc); |
| 89 | 89 | int qedr_post_send(struct ib_qp *, const struct ib_send_wr *, |
| 90 | 90 | const struct ib_send_wr **bad_wr); |
| @@ -214,7 +214,7 @@ usnic_fwd_alloc_flow(struct usnic_fwd_dev *ufdev, struct filter *filter, · 213 unmodified lines | ||
| 214 | 214 | if (!flow) |
| 215 | 215 | return ERR_PTR(-ENOMEM); |
| 216 | 216 | |
| 217 | − tlv = pci_alloc_consistent(pdev, tlv_size, &tlv_pa); | |
| 217 | + tlv = dma_alloc_coherent(&pdev->dev, tlv_size, &tlv_pa, GFP_ATOMIC); | |
| 218 | 218 | if (!tlv) { |
| 219 | 219 | usnic_err("Failed to allocate memory\n"); |
| 220 | 220 | status = -ENOMEM; |
| @@ -258,7 +258,7 @@ usnic_fwd_alloc_flow(struct usnic_fwd_dev *ufdev, struct filter *filter, · 37 unmodified lines | ||
| 258 | 258 | |
| 259 | 259 | out_free_tlv: |
| 260 | 260 | spin_unlock(&ufdev->lock); |
| 261 | − pci_free_consistent(pdev, tlv_size, tlv, tlv_pa); | |
| 261 | + dma_free_coherent(&pdev->dev, tlv_size, tlv, tlv_pa); | |
| 262 | 262 | if (!status) |
| 263 | 263 | return flow; |
| 264 | 264 | out_free_flow: |
| @@ -202,7 +202,7 @@ err_umem: · 201 unmodified lines | ||
| 202 | 202 | * @return: ib_mr pointer on success, otherwise returns an errno. |
| 203 | 203 | */ |
| 204 | 204 | struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 205 | − u32 max_num_sg, struct ib_udata *udata) | |
| 205 | + u32 max_num_sg) | |
| 206 | 206 | { |
| 207 | 207 | struct pvrdma_dev *dev = to_vdev(pd->device); |
| 208 | 208 | struct pvrdma_user_mr *mr; |
| @@ -406,7 +406,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · 405 unmodified lines | ||
| 406 | 406 | struct ib_udata *udata); |
| 407 | 407 | int pvrdma_dereg_mr(struct ib_mr *mr, struct ib_udata *udata); |
| 408 | 408 | struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 409 | − u32 max_num_sg, struct ib_udata *udata); | |
| 409 | + u32 max_num_sg); | |
| 410 | 410 | int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, |
| 411 | 411 | int sg_nents, unsigned int *sg_offset); |
| 412 | 412 | int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, |
| @@ -90,8 +90,7 @@ EXPORT_SYMBOL(rvt_check_ah); · 89 unmodified lines | ||
| 90 | 90 | /** |
| 91 | 91 | * rvt_create_ah - create an address handle |
| 92 | 92 | * @ibah: the IB address handle |
| 93 | − * @ah_attr: the attributes of the AH | |
| 94 | − * @create_flags: create address handle flags (see enum rdma_create_ah_flags) | |
| 93 | + * @init_attr: the attributes of the AH | |
| 95 | 94 | * @udata: pointer to user's input output buffer information. |
| 96 | 95 | * |
| 97 | 96 | * This may be called from interrupt context. |
| @@ -576,7 +576,7 @@ out: · 575 unmodified lines | ||
| 576 | 576 | * Return: the memory region on success, otherwise return an errno. |
| 577 | 577 | */ |
| 578 | 578 | struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 579 | − u32 max_num_sg, struct ib_udata *udata) | |
| 579 | + u32 max_num_sg) | |
| 580 | 580 | { |
| 581 | 581 | struct rvt_mr *mr; |
| 582 | 582 | |
| @@ -71,7 +71,7 @@ struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · 70 unmodified lines | ||
| 71 | 71 | struct ib_udata *udata); |
| 72 | 72 | int rvt_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); |
| 73 | 73 | struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 74 | − u32 max_num_sg, struct ib_udata *udata); | |
| 74 | + u32 max_num_sg); | |
| 75 | 75 | int rvt_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, |
| 76 | 76 | int sg_nents, unsigned int *sg_offset); |
| 77 | 77 | |
| @@ -40,14 +40,6 @@ MODULE_AUTHOR("Bob Pearson, Frank Zago, John Groves, Kamal Heib"); · 39 unmodified lines | ||
| 40 | 40 | MODULE_DESCRIPTION("Soft RDMA transport"); |
| 41 | 41 | MODULE_LICENSE("Dual BSD/GPL"); |
| 42 | 42 | |
| 43 | −/* free resources for all ports on a device */ | |
| 44 | −static void rxe_cleanup_ports(struct rxe_dev *rxe) | |
| 45 | −{ | |
| 46 | − kfree(rxe->port.pkey_tbl); | |
| 47 | − rxe->port.pkey_tbl = NULL; | |
| 48 | − | |
| 49 | −} | |
| 50 | − | |
| 51 | 43 | /* free resources for a rxe device all objects created for this device must |
| 52 | 44 | * have been destroyed |
| 53 | 45 | */ |
| @@ -66,8 +58,6 @@ void rxe_dealloc(struct ib_device *ib_dev) · 12 unmodified lines | ||
| 66 | 58 | rxe_pool_cleanup(&rxe->mc_grp_pool); |
| 67 | 59 | rxe_pool_cleanup(&rxe->mc_elem_pool); |
| 68 | 60 | |
| 69 | − rxe_cleanup_ports(rxe); | |
| 70 | − | |
| 71 | 61 | if (rxe->tfm) |
| 72 | 62 | crypto_free_shash(rxe->tfm); |
| 73 | 63 | } |
| @@ -111,7 +101,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe) · 37 unmodified lines | ||
| 111 | 101 | } |
| 112 | 102 | |
| 113 | 103 | /* initialize port attributes */ |
| 114 | −static int rxe_init_port_param(struct rxe_port *port) | |
| 104 | +static void rxe_init_port_param(struct rxe_port *port) | |
| 115 | 105 | { |
| 116 | 106 | port->attr.state = IB_PORT_DOWN; |
| 117 | 107 | port->attr.max_mtu = IB_MTU_4096; |
| @@ -134,35 +124,19 @@ static int rxe_init_port_param(struct rxe_port *port) · 16 unmodified lines | ||
| 134 | 124 | port->attr.phys_state = RXE_PORT_PHYS_STATE; |
| 135 | 125 | port->mtu_cap = ib_mtu_enum_to_int(IB_MTU_256); |
| 136 | 126 | port->subnet_prefix = cpu_to_be64(RXE_PORT_SUBNET_PREFIX); |
| 137 | − | |
| 138 | − return 0; | |
| 139 | 127 | } |
| 140 | 128 | |
| 141 | 129 | /* initialize port state, note IB convention that HCA ports are always |
| 142 | 130 | * numbered from 1 |
| 143 | 131 | */ |
| 144 | −static int rxe_init_ports(struct rxe_dev *rxe) | |
| 132 | +static void rxe_init_ports(struct rxe_dev *rxe) | |
| 145 | 133 | { |
| 146 | 134 | struct rxe_port *port = &rxe->port; |
| 147 | 135 | |
| 148 | 136 | rxe_init_port_param(port); |
| 149 | − | |
| 150 | − if (!port->attr.pkey_tbl_len || !port->attr.gid_tbl_len) | |
| 151 | − return -EINVAL; | |
| 152 | − | |
| 153 | − port->pkey_tbl = kcalloc(port->attr.pkey_tbl_len, | |
| 154 | − sizeof(*port->pkey_tbl), GFP_KERNEL); | |
| 155 | − | |
| 156 | − if (!port->pkey_tbl) | |
| 157 | − return -ENOMEM; | |
| 158 | − | |
| 159 | − port->pkey_tbl[0] = 0xffff; | |
| 160 | 137 | addrconf_addr_eui48((unsigned char *)&port->port_guid, |
| 161 | 138 | rxe->ndev->dev_addr); |
| 162 | − | |
| 163 | 139 | spin_lock_init(&port->port_lock); |
| 164 | − | |
| 165 | − return 0; | |
| 166 | 140 | } |
| 167 | 141 | |
| 168 | 142 | /* init pools of managed objects */ |
| @@ -252,13 +226,11 @@ static int rxe_init(struct rxe_dev *rxe) · 83 unmodified lines | ||
| 252 | 226 | /* init default device parameters */ |
| 253 | 227 | rxe_init_device_param(rxe); |
| 254 | 228 | |
| 255 | − err = rxe_init_ports(rxe); | |
| 256 | − if (err) | |
| 257 | − goto err1; | |
| 229 | + rxe_init_ports(rxe); | |
| 258 | 230 | |
| 259 | 231 | err = rxe_init_pools(rxe); |
| 260 | 232 | if (err) |
| 261 | − goto err2; | |
| 233 | + return err; | |
| 262 | 234 | |
| 263 | 235 | /* init pending mmap list */ |
| 264 | 236 | spin_lock_init(&rxe->mmap_offset_lock); |
| @@ -268,11 +240,6 @@ static int rxe_init(struct rxe_dev *rxe) · 3 unmodified lines | ||
| 268 | 240 | mutex_init(&rxe->usdev_lock); |
| 269 | 241 | |
| 270 | 242 | return 0; |
| 271 | − | |
| 272 | −err2: | |
| 273 | − rxe_cleanup_ports(rxe); | |
| 274 | −err1: | |
| 275 | − return err; | |
| 276 | 243 | } |
| 277 | 244 | |
| 278 | 245 | void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu) |
| @@ -103,8 +103,8 @@ enum copy_direction { · 102 unmodified lines | ||
| 103 | 103 | from_mem_obj, |
| 104 | 104 | }; |
| 105 | 105 | |
| 106 | −int rxe_mem_init_dma(struct rxe_pd *pd, | |
| 107 | − int access, struct rxe_mem *mem); | |
| 106 | +void rxe_mem_init_dma(struct rxe_pd *pd, | |
| 107 | + int access, struct rxe_mem *mem); | |
| 108 | 108 | |
| 109 | 109 | int rxe_mem_init_user(struct rxe_pd *pd, u64 start, |
| 110 | 110 | u64 length, u64 iova, int access, struct ib_udata *udata, |
| @@ -132,9 +132,6 @@ struct rxe_mem *lookup_mem(struct rxe_pd *pd, int access, u32 key, · 21 unmodified lines | ||
| 132 | 132 | |
| 133 | 133 | int mem_check_range(struct rxe_mem *mem, u64 iova, size_t length); |
| 134 | 134 | |
| 135 | −int rxe_mem_map_pages(struct rxe_dev *rxe, struct rxe_mem *mem, | |
| 136 | − u64 *page, int num_pages, u64 iova); | |
| 137 | − | |
| 138 | 135 | void rxe_mem_cleanup(struct rxe_pool_entry *arg); |
| 139 | 136 | |
| 140 | 137 | int advance_dma_data(struct rxe_dma_info *dma, unsigned int length); |
| @@ -145,7 +142,6 @@ int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb); · 4 unmodified lines | ||
| 145 | 142 | struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, |
| 146 | 143 | int paylen, struct rxe_pkt_info *pkt); |
| 147 | 144 | int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb, u32 *crc); |
| 148 | −enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num); | |
| 149 | 145 | const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num); |
| 150 | 146 | struct device *rxe_dma_device(struct rxe_dev *rxe); |
| 151 | 147 | int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid); |
| @@ -144,8 +144,8 @@ err1: · 143 unmodified lines | ||
| 144 | 144 | return -ENOMEM; |
| 145 | 145 | } |
| 146 | 146 | |
| 147 | −int rxe_mem_init_dma(struct rxe_pd *pd, | |
| 148 | − int access, struct rxe_mem *mem) | |
| 147 | +void rxe_mem_init_dma(struct rxe_pd *pd, | |
| 148 | + int access, struct rxe_mem *mem) | |
| 149 | 149 | { |
| 150 | 150 | rxe_mem_init(access, mem); |
| 151 | 151 | |
| @@ -153,8 +153,6 @@ int rxe_mem_init_dma(struct rxe_pd *pd, · 1 unmodified line | ||
| 153 | 153 | mem->access = access; |
| 154 | 154 | mem->state = RXE_MEM_STATE_VALID; |
| 155 | 155 | mem->type = RXE_MEM_TYPE_DMA; |
| 156 | − | |
| 157 | − return 0; | |
| 158 | 156 | } |
| 159 | 157 | |
| 160 | 158 | int rxe_mem_init_user(struct rxe_pd *pd, u64 start, |
| @@ -587,47 +585,3 @@ struct rxe_mem *lookup_mem(struct rxe_pd *pd, int access, u32 key, · 426 unmodified lines | ||
| 587 | 585 | |
| 588 | 586 | return mem; |
| 589 | 587 | } |
| 590 | − | |
| 591 | −int rxe_mem_map_pages(struct rxe_dev *rxe, struct rxe_mem *mem, | |
| 592 | − u64 *page, int num_pages, u64 iova) | |
| 593 | −{ | |
| 594 | − int i; | |
| 595 | − int num_buf; | |
| 596 | − int err; | |
| 597 | − struct rxe_map **map; | |
| 598 | − struct rxe_phys_buf *buf; | |
| 599 | − int page_size; | |
| 600 | − | |
| 601 | − if (num_pages > mem->max_buf) { | |
| 602 | − err = -EINVAL; | |
| 603 | − goto err1; | |
| 604 | − } | |
| 605 | − | |
| 606 | − num_buf = 0; | |
| 607 | − page_size = 1 << mem->page_shift; | |
| 608 | − map = mem->map; | |
| 609 | − buf = map[0]->buf; | |
| 610 | − | |
| 611 | − for (i = 0; i < num_pages; i++) { | |
| 612 | − buf->addr = *page++; | |
| 613 | − buf->size = page_size; | |
| 614 | − buf++; | |
| 615 | − num_buf++; | |
| 616 | − | |
| 617 | − if (num_buf == RXE_BUF_PER_MAP) { | |
| 618 | − map++; | |
| 619 | − buf = map[0]->buf; | |
| 620 | − num_buf = 0; | |
| 621 | − } | |
| 622 | − } | |
| 623 | − | |
| 624 | − mem->iova = iova; | |
| 625 | − mem->va = iova; | |
| 626 | − mem->length = num_pages << mem->page_shift; | |
| 627 | − mem->state = RXE_MEM_STATE_VALID; | |
| 628 | − | |
| 629 | − return 0; | |
| 630 | − | |
| 631 | −err1: | |
| 632 | − return err; | |
| 633 | −} | |
| @@ -520,11 +520,6 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num) · 519 unmodified lines | ||
| 520 | 520 | return rxe->ndev->name; |
| 521 | 521 | } |
| 522 | 522 | |
| 523 | −enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num) | |
| 524 | −{ | |
| 525 | − return IB_LINK_LAYER_ETHERNET; | |
| 526 | −} | |
| 527 | − | |
| 528 | 523 | int rxe_net_add(const char *ibdev_name, struct net_device *ndev) |
| 529 | 524 | { |
| 530 | 525 | int err; |
| @@ -100,7 +100,7 @@ enum rxe_device_param { · 99 unmodified lines | ||
| 100 | 100 | RXE_MAX_SRQ_SGE = 27, |
| 101 | 101 | RXE_MIN_SRQ_SGE = 1, |
| 102 | 102 | RXE_MAX_FMR_PAGE_LIST_LEN = 512, |
| 103 | − RXE_MAX_PKEYS = 64, | |
| 103 | + RXE_MAX_PKEYS = 1, | |
| 104 | 104 | RXE_LOCAL_CA_ACK_DELAY = 15, |
| 105 | 105 | |
| 106 | 106 | RXE_MAX_UCONTEXT = 512, |
| @@ -148,7 +148,7 @@ enum rxe_port_param { · 41 unmodified lines | ||
| 148 | 148 | RXE_PORT_INIT_TYPE_REPLY = 0, |
| 149 | 149 | RXE_PORT_ACTIVE_WIDTH = IB_WIDTH_1X, |
| 150 | 150 | RXE_PORT_ACTIVE_SPEED = 1, |
| 151 | − RXE_PORT_PKEY_TBL_LEN = 64, | |
| 151 | + RXE_PORT_PKEY_TBL_LEN = 1, | |
| 152 | 152 | RXE_PORT_PHYS_STATE = IB_PORT_PHYS_STATE_POLLING, |
| 153 | 153 | RXE_PORT_SUBNET_PREFIX = 0xfe80000000000000ULL, |
| 154 | 154 | }; |
| @@ -101,36 +101,15 @@ static void set_qkey_viol_cntr(struct rxe_port *port) · 100 unmodified lines | ||
| 101 | 101 | static int check_keys(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, |
| 102 | 102 | u32 qpn, struct rxe_qp *qp) |
| 103 | 103 | { |
| 104 | − int i; | |
| 105 | − int found_pkey = 0; | |
| 106 | 104 | struct rxe_port *port = &rxe->port; |
| 107 | 105 | u16 pkey = bth_pkey(pkt); |
| 108 | 106 | |
| 109 | 107 | pkt->pkey_index = 0; |
| 110 | 108 | |
| 111 | − if (qpn == 1) { | |
| 112 | − for (i = 0; i < port->attr.pkey_tbl_len; i++) { | |
| 113 | − if (pkey_match(pkey, port->pkey_tbl[i])) { | |
| 114 | − pkt->pkey_index = i; | |
| 115 | − found_pkey = 1; | |
| 116 | − break; | |
| 117 | − } | |
| 118 | − } | |
| 119 | − | |
| 120 | − if (!found_pkey) { | |
| 121 | − pr_warn_ratelimited("bad pkey = 0x%x\n", pkey); | |
| 122 | − set_bad_pkey_cntr(port); | |
| 123 | − goto err1; | |
| 124 | − } | |
| 125 | − } else { | |
| 126 | − if (unlikely(!pkey_match(pkey, | |
| 127 | − port->pkey_tbl[qp->attr.pkey_index] | |
| 128 | − ))) { | |
| 129 | − pr_warn_ratelimited("bad pkey = 0x%0x\n", pkey); | |
| 130 | − set_bad_pkey_cntr(port); | |
| 131 | − goto err1; | |
| 132 | − } | |
| 133 | − pkt->pkey_index = qp->attr.pkey_index; | |
| 109 | + if (!pkey_match(pkey, IB_DEFAULT_PKEY_FULL)) { | |
| 110 | + pr_warn_ratelimited("bad pkey = 0x%x\n", pkey); | |
| 111 | + set_bad_pkey_cntr(port); | |
| 112 | + goto err1; | |
| 134 | 113 | } |
| 135 | 114 | |
| 136 | 115 | if ((qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_GSI) && |
| @@ -330,10 +309,14 @@ err1: · 193 unmodified lines | ||
| 330 | 309 | |
| 331 | 310 | static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb) |
| 332 | 311 | { |
| 312 | + struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); | |
| 333 | 313 | const struct ib_gid_attr *gid_attr; |
| 334 | 314 | union ib_gid dgid; |
| 335 | 315 | union ib_gid *pdgid; |
| 336 | 316 | |
| 317 | + if (pkt->mask & RXE_LOOPBACK_MASK) | |
| 318 | + return 0; | |
| 319 | + | |
| 337 | 320 | if (skb->protocol == htons(ETH_P_IP)) { |
| 338 | 321 | ipv6_addr_set_v4mapped(ip_hdr(skb)->daddr, |
| 339 | 322 | (struct in6_addr *)&dgid); |
| @@ -366,7 +349,7 @@ void rxe_rcv(struct sk_buff *skb) · 26 unmodified lines | ||
| 366 | 349 | if (unlikely(skb->len < pkt->offset + RXE_BTH_BYTES)) |
| 367 | 350 | goto drop; |
| 368 | 351 | |
| 369 | − if (unlikely(rxe_match_dgid(rxe, skb) < 0)) { | |
| 352 | + if (rxe_match_dgid(rxe, skb) < 0) { | |
| 370 | 353 | pr_warn_ratelimited("failed matching dgid\n"); |
| 371 | 354 | goto drop; |
| 372 | 355 | } |
| @@ -381,7 +381,6 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, · 380 unmodified lines | ||
| 381 | 381 | struct rxe_pkt_info *pkt) |
| 382 | 382 | { |
| 383 | 383 | struct rxe_dev *rxe = to_rdev(qp->ibqp.device); |
| 384 | − struct rxe_port *port = &rxe->port; | |
| 385 | 384 | struct sk_buff *skb; |
| 386 | 385 | struct rxe_send_wr *ibwr = &wqe->wr; |
| 387 | 386 | struct rxe_av *av; |
| @@ -419,9 +418,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, · 31 unmodified lines | ||
| 419 | 418 | (pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) == |
| 420 | 419 | (RXE_WRITE_MASK | RXE_IMMDT_MASK)); |
| 421 | 420 | |
| 422 | − pkey = (qp_type(qp) == IB_QPT_GSI) ? | |
| 423 | − port->pkey_tbl[ibwr->wr.ud.pkey_index] : | |
| 424 | − port->pkey_tbl[qp->attr.pkey_index]; | |
| 421 | + pkey = IB_DEFAULT_PKEY_FULL; | |
| 425 | 422 | |
| 426 | 423 | qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn : |
| 427 | 424 | qp->attr.dest_qp_num; |
| @@ -83,22 +83,11 @@ static int rxe_query_port(struct ib_device *dev, · 82 unmodified lines | ||
| 83 | 83 | static int rxe_query_pkey(struct ib_device *device, |
| 84 | 84 | u8 port_num, u16 index, u16 *pkey) |
| 85 | 85 | { |
| 86 | − struct rxe_dev *rxe = to_rdev(device); | |
| 87 | − struct rxe_port *port; | |
| 88 | − | |
| 89 | − port = &rxe->port; | |
| 90 | − | |
| 91 | − if (unlikely(index >= port->attr.pkey_tbl_len)) { | |
| 92 | − dev_warn(device->dev.parent, "invalid index = %d\n", | |
| 93 | − index); | |
| 94 | − goto err1; | |
| 95 | − } | |
| 86 | + if (index > 0) | |
| 87 | + return -EINVAL; | |
| 96 | 88 | |
| 97 | − *pkey = port->pkey_tbl[index]; | |
| 89 | + *pkey = IB_DEFAULT_PKEY_FULL; | |
| 98 | 90 | return 0; |
| 99 | − | |
| 100 | −err1: | |
| 101 | − return -EINVAL; | |
| 102 | 91 | } |
| 103 | 92 | |
| 104 | 93 | static int rxe_modify_device(struct ib_device *dev, |
| @@ -141,9 +130,7 @@ static int rxe_modify_port(struct ib_device *dev, · 36 unmodified lines | ||
| 141 | 130 | static enum rdma_link_layer rxe_get_link_layer(struct ib_device *dev, |
| 142 | 131 | u8 port_num) |
| 143 | 132 | { |
| 144 | − struct rxe_dev *rxe = to_rdev(dev); | |
| 145 | − | |
| 146 | − return rxe_link_layer(rxe, port_num); | |
| 133 | + return IB_LINK_LAYER_ETHERNET; | |
| 147 | 134 | } |
| 148 | 135 | |
| 149 | 136 | static int rxe_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) |
| @@ -684,6 +671,7 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, const struct ib_send_wr *wr, · 534 unmodified lines | ||
| 684 | 671 | unsigned int mask; |
| 685 | 672 | unsigned int length = 0; |
| 686 | 673 | int i; |
| 674 | + struct ib_send_wr *next; | |
| 687 | 675 | |
| 688 | 676 | while (wr) { |
| 689 | 677 | mask = wr_opcode_mask(wr->opcode, qp); |
| @@ -700,6 +688,8 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, const struct ib_send_wr *wr, · 10 unmodified lines | ||
| 700 | 688 | break; |
| 701 | 689 | } |
| 702 | 690 | |
| 691 | + next = wr->next; | |
| 692 | + | |
| 703 | 693 | length = 0; |
| 704 | 694 | for (i = 0; i < wr->num_sge; i++) |
| 705 | 695 | length += wr->sg_list[i].length; |
| @@ -710,7 +700,7 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, const struct ib_send_wr *wr, · 4 unmodified lines | ||
| 710 | 700 | *bad_wr = wr; |
| 711 | 701 | break; |
| 712 | 702 | } |
| 713 | − wr = wr->next; | |
| 703 | + wr = next; | |
| 714 | 704 | } |
| 715 | 705 | |
| 716 | 706 | rxe_run_task(&qp->req.task, 1); |
| @@ -901,30 +891,16 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access) · 184 unmodified lines | ||
| 901 | 891 | struct rxe_dev *rxe = to_rdev(ibpd->device); |
| 902 | 892 | struct rxe_pd *pd = to_rpd(ibpd); |
| 903 | 893 | struct rxe_mem *mr; |
| 904 | − int err; | |
| 905 | 894 | |
| 906 | 895 | mr = rxe_alloc(&rxe->mr_pool); |
| 907 | − if (!mr) { | |
| 908 | − err = -ENOMEM; | |
| 909 | − goto err1; | |
| 910 | − } | |
| 896 | + if (!mr) | |
| 897 | + return ERR_PTR(-ENOMEM); | |
| 911 | 898 | |
| 912 | 899 | rxe_add_index(mr); |
| 913 | − | |
| 914 | 900 | rxe_add_ref(pd); |
| 915 | − | |
| 916 | − err = rxe_mem_init_dma(pd, access, mr); | |
| 917 | − if (err) | |
| 918 | − goto err2; | |
| 901 | + rxe_mem_init_dma(pd, access, mr); | |
| 919 | 902 | |
| 920 | 903 | return &mr->ibmr; |
| 921 | − | |
| 922 | −err2: | |
| 923 | − rxe_drop_ref(pd); | |
| 924 | − rxe_drop_index(mr); | |
| 925 | − rxe_drop_ref(mr); | |
| 926 | −err1: | |
| 927 | − return ERR_PTR(err); | |
| 928 | 904 | } |
| 929 | 905 | |
| 930 | 906 | static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, |
| @@ -975,7 +951,7 @@ static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) · 44 unmodified lines | ||
| 975 | 951 | } |
| 976 | 952 | |
| 977 | 953 | static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, |
| 978 | − u32 max_num_sg, struct ib_udata *udata) | |
| 954 | + u32 max_num_sg) | |
| 979 | 955 | { |
| 980 | 956 | struct rxe_dev *rxe = to_rdev(ibpd->device); |
| 981 | 957 | struct rxe_pd *pd = to_rpd(ibpd); |
| @@ -371,7 +371,6 @@ struct rxe_mc_elem { · 370 unmodified lines | ||
| 371 | 371 | |
| 372 | 372 | struct rxe_port { |
| 373 | 373 | struct ib_port_attr attr; |
| 374 | − u16 *pkey_tbl; | |
| 375 | 374 | __be64 port_guid; |
| 376 | 375 | __be64 subnet_prefix; |
| 377 | 376 | spinlock_t port_lock; /* guard port */ |
| @@ -289,7 +289,6 @@ static const struct ib_device_ops siw_device_ops = { · 288 unmodified lines | ||
| 289 | 289 | .post_srq_recv = siw_post_srq_recv, |
| 290 | 290 | .query_device = siw_query_device, |
| 291 | 291 | .query_gid = siw_query_gid, |
| 292 | − .query_pkey = siw_query_pkey, | |
| 293 | 292 | .query_port = siw_query_port, |
| 294 | 293 | .query_qp = siw_query_qp, |
| 295 | 294 | .query_srq = siw_query_srq, |
| @@ -176,7 +176,6 @@ int siw_query_port(struct ib_device *base_dev, u8 port, · 175 unmodified lines | ||
| 176 | 176 | attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu); |
| 177 | 177 | attr->phys_state = sdev->state == IB_PORT_ACTIVE ? |
| 178 | 178 | IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED; |
| 179 | − attr->pkey_tbl_len = 1; | |
| 180 | 179 | attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP; |
| 181 | 180 | attr->state = sdev->state; |
| 182 | 181 | /* |
| @@ -204,20 +203,12 @@ int siw_get_port_immutable(struct ib_device *base_dev, u8 port, · 21 unmodified lines | ||
| 204 | 203 | if (rv) |
| 205 | 204 | return rv; |
| 206 | 205 | |
| 207 | − port_immutable->pkey_tbl_len = attr.pkey_tbl_len; | |
| 208 | 206 | port_immutable->gid_tbl_len = attr.gid_tbl_len; |
| 209 | 207 | port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; |
| 210 | 208 | |
| 211 | 209 | return 0; |
| 212 | 210 | } |
| 213 | 211 | |
| 214 | −int siw_query_pkey(struct ib_device *base_dev, u8 port, u16 idx, u16 *pkey) | |
| 215 | −{ | |
| 216 | − /* Report the default pkey */ | |
| 217 | − *pkey = 0xffff; | |
| 218 | − return 0; | |
| 219 | −} | |
| 220 | − | |
| 221 | 212 | int siw_query_gid(struct ib_device *base_dev, u8 port, int idx, |
| 222 | 213 | union ib_gid *gid) |
| 223 | 214 | { |
| @@ -1373,7 +1364,7 @@ err_out: · 1149 unmodified lines | ||
| 1373 | 1364 | } |
| 1374 | 1365 | |
| 1375 | 1366 | struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 1376 | − u32 max_sge, struct ib_udata *udata) | |
| 1367 | + u32 max_sge) | |
| 1377 | 1368 | { |
| 1378 | 1369 | struct siw_device *sdev = to_siw_dev(pd->device); |
| 1379 | 1370 | struct siw_mr *mr = NULL; |
| @@ -46,7 +46,6 @@ int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr, · 45 unmodified lines | ||
| 46 | 46 | struct ib_udata *udata); |
| 47 | 47 | int siw_query_port(struct ib_device *base_dev, u8 port, |
| 48 | 48 | struct ib_port_attr *attr); |
| 49 | −int siw_query_pkey(struct ib_device *base_dev, u8 port, u16 idx, u16 *pkey); | |
| 50 | 49 | int siw_query_gid(struct ib_device *base_dev, u8 port, int idx, |
| 51 | 50 | union ib_gid *gid); |
| 52 | 51 | int siw_alloc_pd(struct ib_pd *base_pd, struct ib_udata *udata); |
| @@ -69,7 +68,7 @@ int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags); · 16 unmodified lines | ||
| 69 | 68 | struct ib_mr *siw_reg_user_mr(struct ib_pd *base_pd, u64 start, u64 len, |
| 70 | 69 | u64 rnic_va, int rights, struct ib_udata *udata); |
| 71 | 70 | struct ib_mr *siw_alloc_mr(struct ib_pd *base_pd, enum ib_mr_type mr_type, |
| 72 | − u32 max_sge, struct ib_udata *udata); | |
| 71 | + u32 max_sge); | |
| 73 | 72 | struct ib_mr *siw_get_dma_mr(struct ib_pd *base_pd, int rights); |
| 74 | 73 | int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle, |
| 75 | 74 | unsigned int *sg_off); |
| @@ -515,7 +515,7 @@ void ipoib_ib_dev_cleanup(struct net_device *dev); · 514 unmodified lines | ||
| 515 | 515 | |
| 516 | 516 | int ipoib_ib_dev_open_default(struct net_device *dev); |
| 517 | 517 | int ipoib_ib_dev_open(struct net_device *dev); |
| 518 | −int ipoib_ib_dev_stop(struct net_device *dev); | |
| 518 | +void ipoib_ib_dev_stop(struct net_device *dev); | |
| 519 | 519 | void ipoib_ib_dev_up(struct net_device *dev); |
| 520 | 520 | void ipoib_ib_dev_down(struct net_device *dev); |
| 521 | 521 | int ipoib_ib_dev_stop_default(struct net_device *dev); |
| @@ -527,7 +527,7 @@ void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb); · 5 unmodified lines | ||
| 527 | 527 | |
| 528 | 528 | void ipoib_mcast_restart_task(struct work_struct *work); |
| 529 | 529 | void ipoib_mcast_start_thread(struct net_device *dev); |
| 530 | −int ipoib_mcast_stop_thread(struct net_device *dev); | |
| 530 | +void ipoib_mcast_stop_thread(struct net_device *dev); | |
| 531 | 531 | |
| 532 | 532 | void ipoib_mcast_dev_down(struct net_device *dev); |
| 533 | 533 | void ipoib_mcast_dev_flush(struct net_device *dev); |
| @@ -670,13 +670,12 @@ int ipoib_send(struct net_device *dev, struct sk_buff *skb, · 669 unmodified lines | ||
| 670 | 670 | return rc; |
| 671 | 671 | } |
| 672 | 672 | |
| 673 | −static void __ipoib_reap_ah(struct net_device *dev) | |
| 673 | +static void ipoib_reap_dead_ahs(struct ipoib_dev_priv *priv) | |
| 674 | 674 | { |
| 675 | − struct ipoib_dev_priv *priv = ipoib_priv(dev); | |
| 676 | 675 | struct ipoib_ah *ah, *tah; |
| 677 | 676 | unsigned long flags; |
| 678 | 677 | |
| 679 | − netif_tx_lock_bh(dev); | |
| 678 | + netif_tx_lock_bh(priv->dev); | |
| 680 | 679 | spin_lock_irqsave(&priv->lock, flags); |
| 681 | 680 | |
| 682 | 681 | list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list) |
| @@ -687,37 +686,37 @@ static void __ipoib_reap_ah(struct net_device *dev) · 4 unmodified lines | ||
| 687 | 686 | } |
| 688 | 687 | |
| 689 | 688 | spin_unlock_irqrestore(&priv->lock, flags); |
| 690 | − netif_tx_unlock_bh(dev); | |
| 689 | + netif_tx_unlock_bh(priv->dev); | |
| 691 | 690 | } |
| 692 | 691 | |
| 693 | 692 | void ipoib_reap_ah(struct work_struct *work) |
| 694 | 693 | { |
| 695 | 694 | struct ipoib_dev_priv *priv = |
| 696 | 695 | container_of(work, struct ipoib_dev_priv, ah_reap_task.work); |
| 697 | − struct net_device *dev = priv->dev; | |
| 698 | 696 | |
| 699 | − __ipoib_reap_ah(dev); | |
| 697 | + ipoib_reap_dead_ahs(priv); | |
| 700 | 698 | |
| 701 | 699 | if (!test_bit(IPOIB_STOP_REAPER, &priv->flags)) |
| 702 | 700 | queue_delayed_work(priv->wq, &priv->ah_reap_task, |
| 703 | 701 | round_jiffies_relative(HZ)); |
| 704 | 702 | } |
| 705 | 703 | |
| 706 | −static void ipoib_flush_ah(struct net_device *dev) | |
| 704 | +static void ipoib_start_ah_reaper(struct ipoib_dev_priv *priv) | |
| 707 | 705 | { |
| 708 | − struct ipoib_dev_priv *priv = ipoib_priv(dev); | |
| 709 | − | |
| 710 | − cancel_delayed_work(&priv->ah_reap_task); | |
| 711 | − flush_workqueue(priv->wq); | |
| 712 | − ipoib_reap_ah(&priv->ah_reap_task.work); | |
| 706 | + clear_bit(IPOIB_STOP_REAPER, &priv->flags); | |
| 707 | + queue_delayed_work(priv->wq, &priv->ah_reap_task, | |
| 708 | + round_jiffies_relative(HZ)); | |
| 713 | 709 | } |
| 714 | 710 | |
| 715 | −static void ipoib_stop_ah(struct net_device *dev) | |
| 711 | +static void ipoib_stop_ah_reaper(struct ipoib_dev_priv *priv) | |
| 716 | 712 | { |
| 717 | − struct ipoib_dev_priv *priv = ipoib_priv(dev); | |
| 718 | − | |
| 719 | 713 | set_bit(IPOIB_STOP_REAPER, &priv->flags); |
| 720 | − ipoib_flush_ah(dev); | |
| 714 | + cancel_delayed_work(&priv->ah_reap_task); | |
| 715 | + /* | |
| 716 | + * After ipoib_stop_ah_reaper() we always go through | |
| 717 | + * ipoib_reap_dead_ahs() which ensures the work is really stopped and | |
| 718 | + * does a final flush out of the dead_ah's list | |
| 719 | + */ | |
| 721 | 720 | } |
| 722 | 721 | |
| 723 | 722 | static int recvs_pending(struct net_device *dev) |
| @@ -846,18 +845,6 @@ timeout: · 122 unmodified lines | ||
| 846 | 845 | return 0; |
| 847 | 846 | } |
| 848 | 847 | |
| 849 | −int ipoib_ib_dev_stop(struct net_device *dev) | |
| 850 | −{ | |
| 851 | − struct ipoib_dev_priv *priv = ipoib_priv(dev); | |
| 852 | − | |
| 853 | − priv->rn_ops->ndo_stop(dev); | |
| 854 | − | |
| 855 | − clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags); | |
| 856 | − ipoib_flush_ah(dev); | |
| 857 | − | |
| 858 | − return 0; | |
| 859 | −} | |
| 860 | − | |
| 861 | 848 | int ipoib_ib_dev_open_default(struct net_device *dev) |
| 862 | 849 | { |
| 863 | 850 | struct ipoib_dev_priv *priv = ipoib_priv(dev); |
| @@ -901,10 +888,7 @@ int ipoib_ib_dev_open(struct net_device *dev) · 37 unmodified lines | ||
| 901 | 888 | return -1; |
| 902 | 889 | } |
| 903 | 890 | |
| 904 | − clear_bit(IPOIB_STOP_REAPER, &priv->flags); | |
| 905 | − queue_delayed_work(priv->wq, &priv->ah_reap_task, | |
| 906 | − round_jiffies_relative(HZ)); | |
| 907 | − | |
| 891 | + ipoib_start_ah_reaper(priv); | |
| 908 | 892 | if (priv->rn_ops->ndo_open(dev)) { |
| 909 | 893 | pr_warn("%s: Failed to open dev\n", dev->name); |
| 910 | 894 | goto dev_stop; |
| @@ -915,13 +899,20 @@ int ipoib_ib_dev_open(struct net_device *dev) · 4 unmodified lines | ||
| 915 | 899 | return 0; |
| 916 | 900 | |
| 917 | 901 | dev_stop: |
| 918 | − set_bit(IPOIB_STOP_REAPER, &priv->flags); | |
| 919 | − cancel_delayed_work(&priv->ah_reap_task); | |
| 920 | − set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags); | |
| 921 | − ipoib_ib_dev_stop(dev); | |
| 902 | + ipoib_stop_ah_reaper(priv); | |
| 922 | 903 | return -1; |
| 923 | 904 | } |
| 924 | 905 | |
| 906 | +void ipoib_ib_dev_stop(struct net_device *dev) | |
| 907 | +{ | |
| 908 | + struct ipoib_dev_priv *priv = ipoib_priv(dev); | |
| 909 | + | |
| 910 | + priv->rn_ops->ndo_stop(dev); | |
| 911 | + | |
| 912 | + clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags); | |
| 913 | + ipoib_stop_ah_reaper(priv); | |
| 914 | +} | |
| 915 | + | |
| 925 | 916 | void ipoib_pkey_dev_check_presence(struct net_device *dev) |
| 926 | 917 | { |
| 927 | 918 | struct ipoib_dev_priv *priv = ipoib_priv(dev); |
| @@ -1232,7 +1223,7 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, · 304 unmodified lines | ||
| 1232 | 1223 | ipoib_mcast_dev_flush(dev); |
| 1233 | 1224 | if (oper_up) |
| 1234 | 1225 | set_bit(IPOIB_FLAG_OPER_UP, &priv->flags); |
| 1235 | − ipoib_flush_ah(dev); | |
| 1226 | + ipoib_reap_dead_ahs(priv); | |
| 1236 | 1227 | } |
| 1237 | 1228 | |
| 1238 | 1229 | if (level >= IPOIB_FLUSH_NORMAL) |
| @@ -1307,7 +1298,7 @@ void ipoib_ib_dev_cleanup(struct net_device *dev) · 68 unmodified lines | ||
| 1307 | 1298 | * the neighbor garbage collection is stopped and reaped. |
| 1308 | 1299 | * That should all be done now, so make a final ah flush. |
| 1309 | 1300 | */ |
| 1310 | − ipoib_stop_ah(dev); | |
| 1301 | + ipoib_reap_dead_ahs(priv); | |
| 1311 | 1302 | |
| 1312 | 1303 | clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); |
| 1313 | 1304 | |
| @@ -1892,8 +1892,15 @@ static void ipoib_child_init(struct net_device *ndev) · 1891 unmodified lines | ||
| 1892 | 1892 | |
| 1893 | 1893 | priv->max_ib_mtu = ppriv->max_ib_mtu; |
| 1894 | 1894 | set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags); |
| 1895 | − memcpy(priv->dev->dev_addr, ppriv->dev->dev_addr, INFINIBAND_ALEN); | |
| 1896 | − memcpy(&priv->local_gid, &ppriv->local_gid, sizeof(priv->local_gid)); | |
| 1895 | + if (memchr_inv(priv->dev->dev_addr, 0, INFINIBAND_ALEN)) | |
| 1896 | + memcpy(&priv->local_gid, priv->dev->dev_addr + 4, | |
| 1897 | + sizeof(priv->local_gid)); | |
| 1898 | + else { | |
| 1899 | + memcpy(priv->dev->dev_addr, ppriv->dev->dev_addr, | |
| 1900 | + INFINIBAND_ALEN); | |
| 1901 | + memcpy(&priv->local_gid, &ppriv->local_gid, | |
| 1902 | + sizeof(priv->local_gid)); | |
| 1903 | + } | |
| 1897 | 1904 | } |
| 1898 | 1905 | |
| 1899 | 1906 | static int ipoib_ndo_init(struct net_device *ndev) |
| @@ -1976,6 +1983,8 @@ static void ipoib_ndo_uninit(struct net_device *dev) · 76 unmodified lines | ||
| 1976 | 1983 | |
| 1977 | 1984 | /* no more works over the priv->wq */ |
| 1978 | 1985 | if (priv->wq) { |
| 1986 | + /* See ipoib_mcast_carrier_on_task() */ | |
| 1987 | + WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)); | |
| 1979 | 1988 | flush_workqueue(priv->wq); |
| 1980 | 1989 | destroy_workqueue(priv->wq); |
| 1981 | 1990 | priv->wq = NULL; |
| @@ -680,15 +680,13 @@ void ipoib_mcast_start_thread(struct net_device *dev) · 679 unmodified lines | ||
| 680 | 680 | spin_unlock_irqrestore(&priv->lock, flags); |
| 681 | 681 | } |
| 682 | 682 | |
| 683 | −int ipoib_mcast_stop_thread(struct net_device *dev) | |
| 683 | +void ipoib_mcast_stop_thread(struct net_device *dev) | |
| 684 | 684 | { |
| 685 | 685 | struct ipoib_dev_priv *priv = ipoib_priv(dev); |
| 686 | 686 | |
| 687 | 687 | ipoib_dbg_mcast(priv, "stopping multicast thread\n"); |
| 688 | 688 | |
| 689 | 689 | cancel_delayed_work_sync(&priv->mcast_task); |
| 690 | − | |
| 691 | − return 0; | |
| 692 | 690 | } |
| 693 | 691 | |
| 694 | 692 | static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast) |
| @@ -299,18 +299,6 @@ struct iser_conn; · 298 unmodified lines | ||
| 299 | 299 | struct ib_conn; |
| 300 | 300 | struct iscsi_iser_task; |
| 301 | 301 | |
| 302 | −/** | |
| 303 | − * struct iser_comp - iSER completion context | |
| 304 | − * | |
| 305 | − * @cq: completion queue | |
| 306 | − * @active_qps: Number of active QPs attached | |
| 307 | − * to completion context | |
| 308 | − */ | |
| 309 | −struct iser_comp { | |
| 310 | − struct ib_cq *cq; | |
| 311 | − int active_qps; | |
| 312 | −}; | |
| 313 | − | |
| 314 | 302 | /** |
| 315 | 303 | * struct iser_device - iSER device handle |
| 316 | 304 | * |
| @@ -320,9 +308,6 @@ struct iser_comp { · 3 unmodified lines | ||
| 320 | 308 | * @event_handler: IB events handle routine |
| 321 | 309 | * @ig_list: entry in devices list |
| 322 | 310 | * @refcount: Reference counter, dominated by open iser connections |
| 323 | − * @comps_used: Number of completion contexts used, Min between online | |
| 324 | − * cpus and device max completion vectors | |
| 325 | − * @comps: Dinamically allocated array of completion handlers | |
| 326 | 311 | */ |
| 327 | 312 | struct iser_device { |
| 328 | 313 | struct ib_device *ib_device; |
| @@ -330,8 +315,6 @@ struct iser_device { · 1 unmodified line | ||
| 330 | 315 | struct ib_event_handler event_handler; |
| 331 | 316 | struct list_head ig_list; |
| 332 | 317 | int refcount; |
| 333 | − int comps_used; | |
| 334 | − struct iser_comp *comps; | |
| 335 | 318 | }; |
| 336 | 319 | |
| 337 | 320 | /** |
| @@ -353,6 +336,7 @@ struct iser_reg_resources { · 15 unmodified lines | ||
| 353 | 336 | * @list: entry in connection fastreg pool |
| 354 | 337 | * @rsc: data buffer registration resources |
| 355 | 338 | * @sig_protected: is region protected indicator |
| 339 | + * @all_list: first and last list members | |
| 356 | 340 | */ |
| 357 | 341 | struct iser_fr_desc { |
| 358 | 342 | struct list_head list; |
| @@ -367,6 +351,7 @@ struct iser_fr_desc { · 8 unmodified lines | ||
| 367 | 351 | * @list: list of fastreg descriptors |
| 368 | 352 | * @lock: protects fastreg pool |
| 369 | 353 | * @size: size of the pool |
| 354 | + * @all_list: first and last list members | |
| 370 | 355 | */ |
| 371 | 356 | struct iser_fr_pool { |
| 372 | 357 | struct list_head list; |
| @@ -380,11 +365,12 @@ struct iser_fr_pool { · 7 unmodified lines | ||
| 380 | 365 | * |
| 381 | 366 | * @cma_id: rdma_cm connection maneger handle |
| 382 | 367 | * @qp: Connection Queue-pair |
| 368 | + * @cq: Connection completion queue | |
| 369 | + * @cq_size: The number of max outstanding completions | |
| 383 | 370 | * @post_recv_buf_count: post receive counter |
| 384 | 371 | * @sig_count: send work request signal count |
| 385 | 372 | * @rx_wr: receive work request for batch posts |
| 386 | 373 | * @device: reference to iser device |
| 387 | − * @comp: iser completion context | |
| 388 | 374 | * @fr_pool: connection fast registration poool |
| 389 | 375 | * @pi_support: Indicate device T10-PI support |
| 390 | 376 | * @reg_cqe: completion handler |
| @@ -392,11 +378,12 @@ struct iser_fr_pool { · 1 unmodified line | ||
| 392 | 378 | struct ib_conn { |
| 393 | 379 | struct rdma_cm_id *cma_id; |
| 394 | 380 | struct ib_qp *qp; |
| 381 | + struct ib_cq *cq; | |
| 382 | + u32 cq_size; | |
| 395 | 383 | int post_recv_buf_count; |
| 396 | 384 | u8 sig_count; |
| 397 | 385 | struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX]; |
| 398 | 386 | struct iser_device *device; |
| 399 | − struct iser_comp *comp; | |
| 400 | 387 | struct iser_fr_pool fr_pool; |
| 401 | 388 | bool pi_support; |
| 402 | 389 | struct ib_cqe reg_cqe; |
| @@ -68,59 +68,23 @@ static void iser_event_handler(struct ib_event_handler *handler, · 67 unmodified lines | ||
| 68 | 68 | static int iser_create_device_ib_res(struct iser_device *device) |
| 69 | 69 | { |
| 70 | 70 | struct ib_device *ib_dev = device->ib_device; |
| 71 | − int i, max_cqe; | |
| 72 | 71 | |
| 73 | 72 | if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) { |
| 74 | 73 | iser_err("IB device does not support memory registrations\n"); |
| 75 | 74 | return -1; |
| 76 | 75 | } |
| 77 | 76 | |
| 78 | − device->comps_used = min_t(int, num_online_cpus(), | |
| 79 | − ib_dev->num_comp_vectors); | |
| 80 | − | |
| 81 | − device->comps = kcalloc(device->comps_used, sizeof(*device->comps), | |
| 82 | − GFP_KERNEL); | |
| 83 | − if (!device->comps) | |
| 84 | − goto comps_err; | |
| 85 | − | |
| 86 | − max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe); | |
| 87 | − | |
| 88 | − iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n", | |
| 89 | − device->comps_used, dev_name(&ib_dev->dev), | |
| 90 | − ib_dev->num_comp_vectors, max_cqe); | |
| 91 | − | |
| 92 | 77 | device->pd = ib_alloc_pd(ib_dev, |
| 93 | 78 | iser_always_reg ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY); |
| 94 | 79 | if (IS_ERR(device->pd)) |
| 95 | 80 | goto pd_err; |
| 96 | 81 | |
| 97 | − for (i = 0; i < device->comps_used; i++) { | |
| 98 | − struct iser_comp *comp = &device->comps[i]; | |
| 99 | − | |
| 100 | − comp->cq = ib_alloc_cq(ib_dev, comp, max_cqe, i, | |
| 101 | − IB_POLL_SOFTIRQ); | |
| 102 | − if (IS_ERR(comp->cq)) { | |
| 103 | − comp->cq = NULL; | |
| 104 | − goto cq_err; | |
| 105 | − } | |
| 106 | − } | |
| 107 | − | |
| 108 | 82 | INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev, |
| 109 | 83 | iser_event_handler); |
| 110 | 84 | ib_register_event_handler(&device->event_handler); |
| 111 | 85 | return 0; |
| 112 | 86 | |
| 113 | −cq_err: | |
| 114 | − for (i = 0; i < device->comps_used; i++) { | |
| 115 | − struct iser_comp *comp = &device->comps[i]; | |
| 116 | − | |
| 117 | − if (comp->cq) | |
| 118 | − ib_free_cq(comp->cq); | |
| 119 | − } | |
| 120 | − ib_dealloc_pd(device->pd); | |
| 121 | 87 | pd_err: |
| 122 | − kfree(device->comps); | |
| 123 | −comps_err: | |
| 124 | 88 | iser_err("failed to allocate an IB resource\n"); |
| 125 | 89 | return -1; |
| 126 | 90 | } |
| @@ -131,20 +95,9 @@ comps_err: · 4 unmodified lines | ||
| 131 | 95 | */ |
| 132 | 96 | static void iser_free_device_ib_res(struct iser_device *device) |
| 133 | 97 | { |
| 134 | − int i; | |
| 135 | − | |
| 136 | − for (i = 0; i < device->comps_used; i++) { | |
| 137 | − struct iser_comp *comp = &device->comps[i]; | |
| 138 | − | |
| 139 | − ib_free_cq(comp->cq); | |
| 140 | − comp->cq = NULL; | |
| 141 | − } | |
| 142 | − | |
| 143 | 98 | ib_unregister_event_handler(&device->event_handler); |
| 144 | 99 | ib_dealloc_pd(device->pd); |
| 145 | 100 | |
| 146 | − kfree(device->comps); | |
| 147 | − device->comps = NULL; | |
| 148 | 101 | device->pd = NULL; |
| 149 | 102 | } |
| 150 | 103 | |
| @@ -287,70 +240,57 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) · 136 unmodified lines | ||
| 287 | 240 | struct ib_device *ib_dev; |
| 288 | 241 | struct ib_qp_init_attr init_attr; |
| 289 | 242 | int ret = -ENOMEM; |
| 290 | − int index, min_index = 0; | |
| 243 | + unsigned int max_send_wr, cq_size; | |
| 291 | 244 | |
| 292 | 245 | BUG_ON(ib_conn->device == NULL); |
| 293 | 246 | |
| 294 | 247 | device = ib_conn->device; |
| 295 | 248 | ib_dev = device->ib_device; |
| 296 | 249 | |
| 297 | − memset(&init_attr, 0, sizeof init_attr); | |
| 250 | + if (ib_conn->pi_support) | |
| 251 | + max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1; | |
| 252 | + else | |
| 253 | + max_send_wr = ISER_QP_MAX_REQ_DTOS + 1; | |
| 254 | + max_send_wr = min_t(unsigned int, max_send_wr, | |
| 255 | + (unsigned int)ib_dev->attrs.max_qp_wr); | |
| 298 | 256 | |
| 299 | − mutex_lock(&ig.connlist_mutex); | |
| 300 | − /* select the CQ with the minimal number of usages */ | |
| 301 | − for (index = 0; index < device->comps_used; index++) { | |
| 302 | − if (device->comps[index].active_qps < | |
| 303 | − device->comps[min_index].active_qps) | |
| 304 | − min_index = index; | |
| 257 | + cq_size = max_send_wr + ISER_QP_MAX_RECV_DTOS; | |
| 258 | + ib_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_SOFTIRQ); | |
| 259 | + if (IS_ERR(ib_conn->cq)) { | |
| 260 | + ret = PTR_ERR(ib_conn->cq); | |
| 261 | + goto cq_err; | |
| 305 | 262 | } |
| 306 | − ib_conn->comp = &device->comps[min_index]; | |
| 307 | − ib_conn->comp->active_qps++; | |
| 308 | − mutex_unlock(&ig.connlist_mutex); | |
| 309 | − iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn); | |
| 263 | + ib_conn->cq_size = cq_size; | |
| 264 | + | |
| 265 | + memset(&init_attr, 0, sizeof(init_attr)); | |
| 310 | 266 | |
| 311 | 267 | init_attr.event_handler = iser_qp_event_callback; |
| 312 | 268 | init_attr.qp_context = (void *)ib_conn; |
| 313 | − init_attr.send_cq = ib_conn->comp->cq; | |
| 314 | − init_attr.recv_cq = ib_conn->comp->cq; | |
| 269 | + init_attr.send_cq = ib_conn->cq; | |
| 270 | + init_attr.recv_cq = ib_conn->cq; | |
| 315 | 271 | init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; |
| 316 | 272 | init_attr.cap.max_send_sge = 2; |
| 317 | 273 | init_attr.cap.max_recv_sge = 1; |
| 318 | 274 | init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; |
| 319 | 275 | init_attr.qp_type = IB_QPT_RC; |
| 320 | − if (ib_conn->pi_support) { | |
| 321 | − init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1; | |
| 276 | + init_attr.cap.max_send_wr = max_send_wr; | |
| 277 | + if (ib_conn->pi_support) | |
| 322 | 278 | init_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; |
| 323 | − iser_conn->max_cmds = | |
| 324 | − ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS); | |
| 325 | − } else { | |
| 326 | − if (ib_dev->attrs.max_qp_wr > ISER_QP_MAX_REQ_DTOS) { | |
| 327 | − init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS + 1; | |
| 328 | − iser_conn->max_cmds = | |
| 329 | − ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS); | |
| 330 | − } else { | |
| 331 | − init_attr.cap.max_send_wr = ib_dev->attrs.max_qp_wr; | |
| 332 | − iser_conn->max_cmds = | |
| 333 | − ISER_GET_MAX_XMIT_CMDS(ib_dev->attrs.max_qp_wr); | |
| 334 | − iser_dbg("device %s supports max_send_wr %d\n", | |
| 335 | − dev_name(&device->ib_device->dev), | |
| 336 | − ib_dev->attrs.max_qp_wr); | |
| 337 | − } | |
| 338 | − } | |
| 279 | + iser_conn->max_cmds = ISER_GET_MAX_XMIT_CMDS(max_send_wr - 1); | |
| 339 | 280 | |
| 340 | 281 | ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr); |
| 341 | 282 | if (ret) |
| 342 | 283 | goto out_err; |
| 343 | 284 | |
| 344 | 285 | ib_conn->qp = ib_conn->cma_id->qp; |
| 345 | − iser_info("setting conn %p cma_id %p qp %p\n", | |
| 286 | + iser_info("setting conn %p cma_id %p qp %p max_send_wr %d\n", | |
| 346 | 287 | ib_conn, ib_conn->cma_id, |
| 347 | − ib_conn->cma_id->qp); | |
| 288 | + ib_conn->cma_id->qp, max_send_wr); | |
| 348 | 289 | return ret; |
| 349 | 290 | |
| 350 | 291 | out_err: |
| 351 | − mutex_lock(&ig.connlist_mutex); | |
| 352 | − ib_conn->comp->active_qps--; | |
| 353 | − mutex_unlock(&ig.connlist_mutex); | |
| 292 | + ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size); | |
| 293 | +cq_err: | |
| 354 | 294 | iser_err("unable to alloc mem or create resource, err %d\n", ret); |
| 355 | 295 | |
| 356 | 296 | return ret; |
| @@ -462,10 +402,8 @@ static void iser_free_ib_conn_res(struct iser_conn *iser_conn, · 105 unmodified lines | ||
| 462 | 402 | iser_conn, ib_conn->cma_id, ib_conn->qp); |
| 463 | 403 | |
| 464 | 404 | if (ib_conn->qp != NULL) { |
| 465 | − mutex_lock(&ig.connlist_mutex); | |
| 466 | − ib_conn->comp->active_qps--; | |
| 467 | − mutex_unlock(&ig.connlist_mutex); | |
| 468 | 405 | rdma_destroy_qp(ib_conn->cma_id); |
| 406 | + ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size); | |
| 469 | 407 | ib_conn->qp = NULL; |
| 470 | 408 | } |
| 471 | 409 | |
| @@ -24,13 +24,6 @@ · 23 unmodified lines | ||
| 24 | 24 | |
| 25 | 25 | #include "ib_isert.h" |
| 26 | 26 | |
| 27 | −#define ISERT_MAX_CONN 8 | |
| 28 | −#define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) | |
| 29 | −#define ISER_MAX_TX_CQ_LEN \ | |
| 30 | − ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN) | |
| 31 | −#define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ | |
| 32 | − ISERT_MAX_CONN) | |
| 33 | − | |
| 34 | 27 | static int isert_debug_level; |
| 35 | 28 | module_param_named(debug_level, isert_debug_level, int, 0644); |
| 36 | 29 | MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); |
| @@ -82,53 +75,34 @@ isert_qp_event_callback(struct ib_event *e, void *context) · 45 unmodified lines | ||
| 82 | 75 | } |
| 83 | 76 | } |
| 84 | 77 | |
| 85 | −static struct isert_comp * | |
| 86 | −isert_comp_get(struct isert_conn *isert_conn) | |
| 87 | −{ | |
| 88 | − struct isert_device *device = isert_conn->device; | |
| 89 | − struct isert_comp *comp; | |
| 90 | − int i, min = 0; | |
| 91 | − | |
| 92 | − mutex_lock(&device_list_mutex); | |
| 93 | − for (i = 0; i < device->comps_used; i++) | |
| 94 | − if (device->comps[i].active_qps < | |
| 95 | − device->comps[min].active_qps) | |
| 96 | − min = i; | |
| 97 | − comp = &device->comps[min]; | |
| 98 | − comp->active_qps++; | |
| 99 | − mutex_unlock(&device_list_mutex); | |
| 100 | − | |
| 101 | − isert_info("conn %p, using comp %p min_index: %d\n", | |
| 102 | − isert_conn, comp, min); | |
| 103 | − | |
| 104 | − return comp; | |
| 105 | −} | |
| 106 | − | |
| 107 | −static void | |
| 108 | −isert_comp_put(struct isert_comp *comp) | |
| 109 | −{ | |
| 110 | − mutex_lock(&device_list_mutex); | |
| 111 | − comp->active_qps--; | |
| 112 | − mutex_unlock(&device_list_mutex); | |
| 113 | −} | |
| 114 | − | |
| 115 | 78 | static struct ib_qp * |
| 116 | 79 | isert_create_qp(struct isert_conn *isert_conn, |
| 117 | − struct isert_comp *comp, | |
| 118 | 80 | struct rdma_cm_id *cma_id) |
| 119 | 81 | { |
| 82 | + u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2; | |
| 120 | 83 | struct isert_device *device = isert_conn->device; |
| 84 | + struct ib_device *ib_dev = device->ib_device; | |
| 121 | 85 | struct ib_qp_init_attr attr; |
| 122 | − int ret; | |
| 86 | + int ret, factor; | |
| 87 | + | |
| 88 | + isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE); | |
| 89 | + if (IS_ERR(isert_conn->cq)) { | |
| 90 | + isert_err("Unable to allocate cq\n"); | |
| 91 | + ret = PTR_ERR(isert_conn->cq); | |
| 92 | + return ERR_PTR(ret); | |
| 93 | + } | |
| 94 | + isert_conn->cq_size = cq_size; | |
| 123 | 95 | |
| 124 | 96 | memset(&attr, 0, sizeof(struct ib_qp_init_attr)); |
| 125 | 97 | attr.event_handler = isert_qp_event_callback; |
| 126 | 98 | attr.qp_context = isert_conn; |
| 127 | − attr.send_cq = comp->cq; | |
| 128 | − attr.recv_cq = comp->cq; | |
| 99 | + attr.send_cq = isert_conn->cq; | |
| 100 | + attr.recv_cq = isert_conn->cq; | |
| 129 | 101 | attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; |
| 130 | 102 | attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; |
| 131 | − attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX; | |
| 103 | + factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num, | |
| 104 | + ISCSI_ISER_MAX_SG_TABLESIZE); | |
| 105 | + attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor; | |
| 132 | 106 | attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; |
| 133 | 107 | attr.cap.max_recv_sge = 1; |
| 134 | 108 | attr.sq_sig_type = IB_SIGNAL_REQ_WR; |
| @@ -139,31 +113,14 @@ isert_create_qp(struct isert_conn *isert_conn, · 4 unmodified lines | ||
| 139 | 113 | ret = rdma_create_qp(cma_id, device->pd, &attr); |
| 140 | 114 | if (ret) { |
| 141 | 115 | isert_err("rdma_create_qp failed for cma_id %d\n", ret); |
| 116 | + ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); | |
| 117 | + | |
| 142 | 118 | return ERR_PTR(ret); |
| 143 | 119 | } |
| 144 | 120 | |
| 145 | 121 | return cma_id->qp; |
| 146 | 122 | } |
| 147 | 123 | |
| 148 | −static int | |
| 149 | −isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) | |
| 150 | −{ | |
| 151 | − struct isert_comp *comp; | |
| 152 | − int ret; | |
| 153 | − | |
| 154 | − comp = isert_comp_get(isert_conn); | |
| 155 | − isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); | |
| 156 | − if (IS_ERR(isert_conn->qp)) { | |
| 157 | − ret = PTR_ERR(isert_conn->qp); | |
| 158 | − goto err; | |
| 159 | − } | |
| 160 | − | |
| 161 | − return 0; | |
| 162 | −err: | |
| 163 | − isert_comp_put(comp); | |
| 164 | − return ret; | |
| 165 | −} | |
| 166 | − | |
| 167 | 124 | static int |
| 168 | 125 | isert_alloc_rx_descriptors(struct isert_conn *isert_conn) |
| 169 | 126 | { |
| @@ -231,61 +188,6 @@ isert_free_rx_descriptors(struct isert_conn *isert_conn) · 61 unmodified lines | ||
| 231 | 188 | isert_conn->rx_descs = NULL; |
| 232 | 189 | } |
| 233 | 190 | |
| 234 | −static void | |
| 235 | −isert_free_comps(struct isert_device *device) | |
| 236 | −{ | |
| 237 | − int i; | |
| 238 | − | |
| 239 | − for (i = 0; i < device->comps_used; i++) { | |
| 240 | − struct isert_comp *comp = &device->comps[i]; | |
| 241 | − | |
| 242 | − if (comp->cq) | |
| 243 | − ib_free_cq(comp->cq); | |
| 244 | − } | |
| 245 | − kfree(device->comps); | |
| 246 | −} | |
| 247 | − | |
| 248 | −static int | |
| 249 | −isert_alloc_comps(struct isert_device *device) | |
| 250 | −{ | |
| 251 | − int i, max_cqe, ret = 0; | |
| 252 | − | |
| 253 | − device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), | |
| 254 | − device->ib_device->num_comp_vectors)); | |
| 255 | − | |
| 256 | − isert_info("Using %d CQs, %s supports %d vectors support " | |
| 257 | − "pi_capable %d\n", | |
| 258 | − device->comps_used, dev_name(&device->ib_device->dev), | |
| 259 | − device->ib_device->num_comp_vectors, | |
| 260 | − device->pi_capable); | |
| 261 | − | |
| 262 | − device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), | |
| 263 | − GFP_KERNEL); | |
| 264 | − if (!device->comps) | |
| 265 | − return -ENOMEM; | |
| 266 | − | |
| 267 | − max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); | |
| 268 | − | |
| 269 | − for (i = 0; i < device->comps_used; i++) { | |
| 270 | − struct isert_comp *comp = &device->comps[i]; | |
| 271 | − | |
| 272 | − comp->device = device; | |
| 273 | − comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, | |
| 274 | − IB_POLL_WORKQUEUE); | |
| 275 | − if (IS_ERR(comp->cq)) { | |
| 276 | − isert_err("Unable to allocate cq\n"); | |
| 277 | − ret = PTR_ERR(comp->cq); | |
| 278 | − comp->cq = NULL; | |
| 279 | − goto out_cq; | |
| 280 | − } | |
| 281 | − } | |
| 282 | − | |
| 283 | − return 0; | |
| 284 | −out_cq: | |
| 285 | − isert_free_comps(device); | |
| 286 | − return ret; | |
| 287 | −} | |
| 288 | − | |
| 289 | 191 | static int |
| 290 | 192 | isert_create_device_ib_res(struct isert_device *device) |
| 291 | 193 | { |
| @@ -296,16 +198,12 @@ isert_create_device_ib_res(struct isert_device *device) · 4 unmodified lines | ||
| 296 | 198 | ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); |
| 297 | 199 | isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); |
| 298 | 200 | |
| 299 | − ret = isert_alloc_comps(device); | |
| 300 | − if (ret) | |
| 301 | − goto out; | |
| 302 | − | |
| 303 | 201 | device->pd = ib_alloc_pd(ib_dev, 0); |
| 304 | 202 | if (IS_ERR(device->pd)) { |
| 305 | 203 | ret = PTR_ERR(device->pd); |
| 306 | 204 | isert_err("failed to allocate pd, device %p, ret=%d\n", |
| 307 | 205 | device, ret); |
| 308 | − goto out_cq; | |
| 206 | + return ret; | |
| 309 | 207 | } |
| 310 | 208 | |
| 311 | 209 | /* Check signature cap */ |
| @@ -313,13 +211,6 @@ isert_create_device_ib_res(struct isert_device *device) · 1 unmodified line | ||
| 313 | 211 | IB_DEVICE_INTEGRITY_HANDOVER ? true : false; |
| 314 | 212 | |
| 315 | 213 | return 0; |
| 316 | − | |
| 317 | −out_cq: | |
| 318 | − isert_free_comps(device); | |
| 319 | −out: | |
| 320 | − if (ret > 0) | |
| 321 | − ret = -EINVAL; | |
| 322 | − return ret; | |
| 323 | 214 | } |
| 324 | 215 | |
| 325 | 216 | static void |
| @@ -328,7 +219,6 @@ isert_free_device_ib_res(struct isert_device *device) · 2 unmodified lines | ||
| 328 | 219 | isert_info("device %p\n", device); |
| 329 | 220 | |
| 330 | 221 | ib_dealloc_pd(device->pd); |
| 331 | − isert_free_comps(device); | |
| 332 | 222 | } |
| 333 | 223 | |
| 334 | 224 | static void |
| @@ -490,6 +380,13 @@ isert_set_nego_params(struct isert_conn *isert_conn, · 155 unmodified lines | ||
| 490 | 380 | } |
| 491 | 381 | } |
| 492 | 382 | |
| 383 | +static void | |
| 384 | +isert_destroy_qp(struct isert_conn *isert_conn) | |
| 385 | +{ | |
| 386 | + ib_destroy_qp(isert_conn->qp); | |
| 387 | + ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); | |
| 388 | +} | |
| 389 | + | |
| 493 | 390 | static int |
| 494 | 391 | isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) |
| 495 | 392 | { |
| @@ -530,17 +427,19 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) · 34 unmodified lines | ||
| 530 | 427 | |
| 531 | 428 | isert_set_nego_params(isert_conn, &event->param.conn); |
| 532 | 429 | |
| 533 | − ret = isert_conn_setup_qp(isert_conn, cma_id); | |
| 534 | − if (ret) | |
| 430 | + isert_conn->qp = isert_create_qp(isert_conn, cma_id); | |
| 431 | + if (IS_ERR(isert_conn->qp)) { | |
| 432 | + ret = PTR_ERR(isert_conn->qp); | |
| 535 | 433 | goto out_conn_dev; |
| 434 | + } | |
| 536 | 435 | |
| 537 | 436 | ret = isert_login_post_recv(isert_conn); |
| 538 | 437 | if (ret) |
| 539 | − goto out_conn_dev; | |
| 438 | + goto out_destroy_qp; | |
| 540 | 439 | |
| 541 | 440 | ret = isert_rdma_accept(isert_conn); |
| 542 | 441 | if (ret) |
| 543 | − goto out_conn_dev; | |
| 442 | + goto out_destroy_qp; | |
| 544 | 443 | |
| 545 | 444 | mutex_lock(&isert_np->mutex); |
| 546 | 445 | list_add_tail(&isert_conn->node, &isert_np->accepted); |
| @@ -548,6 +447,8 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) · 1 unmodified line | ||
| 548 | 447 | |
| 549 | 448 | return 0; |
| 550 | 449 | |
| 450 | +out_destroy_qp: | |
| 451 | + isert_destroy_qp(isert_conn); | |
| 551 | 452 | out_conn_dev: |
| 552 | 453 | isert_device_put(device); |
| 553 | 454 | out_rsp_dma_map: |
| @@ -572,12 +473,8 @@ isert_connect_release(struct isert_conn *isert_conn) · 18 unmodified lines | ||
| 572 | 473 | !isert_conn->dev_removed) |
| 573 | 474 | rdma_destroy_id(isert_conn->cm_id); |
| 574 | 475 | |
| 575 | − if (isert_conn->qp) { | |
| 576 | − struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; | |
| 577 | − | |
| 578 | − isert_comp_put(comp); | |
| 579 | − ib_destroy_qp(isert_conn->qp); | |
| 580 | − } | |
| 476 | + if (isert_conn->qp) | |
| 477 | + isert_destroy_qp(isert_conn); | |
| 581 | 478 | |
| 582 | 479 | if (isert_conn->login_req_buf) |
| 583 | 480 | isert_free_login_buf(isert_conn); |
| @@ -63,7 +63,8 @@ · 62 unmodified lines | ||
| 63 | 63 | (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge) + \ |
| 64 | 64 | sizeof(struct ib_cqe) + sizeof(bool))) |
| 65 | 65 | |
| 66 | −#define ISCSI_ISER_SG_TABLESIZE 256 | |
| 66 | +/* Maximum support is 16MB I/O size */ | |
| 67 | +#define ISCSI_ISER_MAX_SG_TABLESIZE 4096 | |
| 67 | 68 | |
| 68 | 69 | enum isert_desc_type { |
| 69 | 70 | ISCSI_TX_CONTROL, |
| @@ -155,6 +156,8 @@ struct isert_conn { · 85 unmodified lines | ||
| 155 | 156 | struct iser_tx_desc login_tx_desc; |
| 156 | 157 | struct rdma_cm_id *cm_id; |
| 157 | 158 | struct ib_qp *qp; |
| 159 | + struct ib_cq *cq; | |
| 160 | + u32 cq_size; | |
| 158 | 161 | struct isert_device *device; |
| 159 | 162 | struct mutex mutex; |
| 160 | 163 | struct kref kref; |
| @@ -165,22 +168,6 @@ struct isert_conn { · 4 unmodified lines | ||
| 165 | 168 | bool dev_removed; |
| 166 | 169 | }; |
| 167 | 170 | |
| 168 | −#define ISERT_MAX_CQ 64 | |
| 169 | − | |
| 170 | −/** | |
| 171 | − * struct isert_comp - iSER completion context | |
| 172 | − * | |
| 173 | − * @device: pointer to device handle | |
| 174 | − * @cq: completion queue | |
| 175 | − * @active_qps: Number of active QPs attached | |
| 176 | − * to completion context | |
| 177 | − */ | |
| 178 | −struct isert_comp { | |
| 179 | − struct isert_device *device; | |
| 180 | − struct ib_cq *cq; | |
| 181 | − int active_qps; | |
| 182 | −}; | |
| 183 | − | |
| 184 | 171 | struct isert_device { |
| 185 | 172 | bool pi_capable; |
| 186 | 173 | int refcount; |
| @@ -118,12 +118,17 @@ · 117 unmodified lines | ||
| 118 | 118 | * struct opa_vesw_info - OPA vnic switch information |
| 119 | 119 | * @fabric_id: 10-bit fabric id |
| 120 | 120 | * @vesw_id: 12-bit virtual ethernet switch id |
| 121 | + * @rsvd0: reserved bytes | |
| 121 | 122 | * @def_port_mask: bitmask of default ports |
| 123 | + * @rsvd1: reserved bytes | |
| 122 | 124 | * @pkey: partition key |
| 125 | + * @rsvd2: reserved bytes | |
| 123 | 126 | * @u_mcast_dlid: unknown multicast dlid |
| 124 | 127 | * @u_ucast_dlid: array of unknown unicast dlids |
| 128 | + * @rsvd3: reserved bytes | |
| 125 | 129 | * @rc: routing control |
| 126 | 130 | * @eth_mtu: Ethernet MTU |
| 131 | + * @rsvd4: reserved bytes | |
| 127 | 132 | */ |
| 128 | 133 | struct opa_vesw_info { |
| 129 | 134 | __be16 fabric_id; |
| @@ -150,12 +155,14 @@ struct opa_vesw_info { · 20 unmodified lines | ||
| 150 | 155 | * struct opa_per_veswport_info - OPA vnic per port information |
| 151 | 156 | * @port_num: port number |
| 152 | 157 | * @eth_link_status: current ethernet link state |
| 158 | + * @rsvd0: reserved bytes | |
| 153 | 159 | * @base_mac_addr: base mac address |
| 154 | 160 | * @config_state: configured port state |
| 155 | 161 | * @oper_state: operational port state |
| 156 | 162 | * @max_mac_tbl_ent: max number of mac table entries |
| 157 | 163 | * @max_smac_ent: max smac entries in mac table |
| 158 | 164 | * @mac_tbl_digest: mac table digest |
| 165 | + * @rsvd1: reserved bytes | |
| 159 | 166 | * @encap_slid: base slid for the port |
| 160 | 167 | * @pcp_to_sc_uc: sc by pcp index for unicast ethernet packets |
| 161 | 168 | * @pcp_to_vl_uc: vl by pcp index for unicast ethernet packets |
| @@ -165,8 +172,10 @@ struct opa_vesw_info { · 3 unmodified lines | ||
| 165 | 172 | * @non_vlan_vl_uc: vl for non-vlan unicast ethernet packets |
| 166 | 173 | * @non_vlan_sc_mc: sc for non-vlan multicast ethernet packets |
| 167 | 174 | * @non_vlan_vl_mc: vl for non-vlan multicast ethernet packets |
| 175 | + * @rsvd2: reserved bytes | |
| 168 | 176 | * @uc_macs_gen_count: generation count for unicast macs list |
| 169 | 177 | * @mc_macs_gen_count: generation count for multicast macs list |
| 178 | + * @rsvd3: reserved bytes | |
| 170 | 179 | */ |
| 171 | 180 | struct opa_per_veswport_info { |
| 172 | 181 | __be32 port_num; |
| @@ -294,6 +303,7 @@ struct opa_veswport_mactable { · 121 unmodified lines | ||
| 294 | 303 | * @rx_512_1023: received packet length is >=512 and < 1023 bytes |
| 295 | 304 | * @rx_1024_1518: received packet length is >=1024 and < 1518 bytes |
| 296 | 305 | * @rx_1519_max: received packet length >= 1519 bytes |
| 306 | + * @reserved: reserved bytes | |
| 297 | 307 | * |
| 298 | 308 | * All the above are counters of corresponding conditions. |
| 299 | 309 | */ |
| @@ -347,16 +357,26 @@ struct opa_veswport_summary_counters { · 47 unmodified lines | ||
| 347 | 357 | * @veswport_num: virtual ethernet switch port number |
| 348 | 358 | * @tx_errors: transmit errors |
| 349 | 359 | * @rx_errors: receive errors |
| 360 | + * @rsvd0: reserved bytes | |
| 350 | 361 | * @tx_smac_filt: smac filter errors |
| 362 | + * @rsvd1: reserved bytes | |
| 363 | + * @rsvd2: reserved bytes | |
| 364 | + * @rsvd3: reserved bytes | |
| 351 | 365 | * @tx_dlid_zero: transmit packets with invalid dlid |
| 366 | + * @rsvd4: reserved bytes | |
| 352 | 367 | * @tx_logic: other transmit errors |
| 368 | + * @rsvd5: reserved bytes | |
| 353 | 369 | * @tx_drop_state: packet tansmission in non-forward port state |
| 354 | 370 | * @rx_bad_veswid: received packet with invalid vesw id |
| 371 | + * @rsvd6: reserved bytes | |
| 355 | 372 | * @rx_runt: received ethernet packet with length < 64 bytes |
| 356 | 373 | * @rx_oversize: received ethernet packet with length > MTU size |
| 374 | + * @rsvd7: reserved bytes | |
| 357 | 375 | * @rx_eth_down: received packets when interface is down |
| 358 | 376 | * @rx_drop_state: received packets in non-forwarding port state |
| 359 | 377 | * @rx_logic: other receive errors |
| 378 | + * @rsvd8: reserved bytes | |
| 379 | + * @rsvd9: reserved bytes | |
| 360 | 380 | * |
| 361 | 381 | * All the above are counters of corresponding error conditions. |
| 362 | 382 | */ |
| @@ -447,6 +467,7 @@ struct opa_veswport_iface_macs { · 84 unmodified lines | ||
| 447 | 467 | * struct opa_vnic_vema_mad - Generic VEMA MAD |
| 448 | 468 | * @mad_hdr: Generic MAD header |
| 449 | 469 | * @rmpp_hdr: RMPP header for vendor specific MADs |
| 470 | + * @reserved: reserved bytes | |
| 450 | 471 | * @oui: Unique org identifier |
| 451 | 472 | * @data: MAD data |
| 452 | 473 | */ |
| @@ -467,6 +488,7 @@ struct opa_vnic_vema_mad { · 14 unmodified lines | ||
| 467 | 488 | * @trap_num: Trap number |
| 468 | 489 | * @toggle_count: Notice toggle bit and count value |
| 469 | 490 | * @issuer_lid: Trap issuer's lid |
| 491 | + * @reserved: reserved bytes | |
| 470 | 492 | * @issuer_gid: Issuer GID (only if Report method) |
| 471 | 493 | * @raw_data: Trap message body |
| 472 | 494 | */ |
| @@ -487,6 +509,7 @@ struct opa_vnic_notice_attr { · 14 unmodified lines | ||
| 487 | 509 | * struct opa_vnic_vema_mad_trap - Generic VEMA MAD Trap |
| 488 | 510 | * @mad_hdr: Generic MAD header |
| 489 | 511 | * @rmpp_hdr: RMPP header for vendor specific MADs |
| 512 | + * @reserved: reserved bytes | |
| 490 | 513 | * @oui: Unique org identifier |
| 491 | 514 | * @notice: Notice structure |
| 492 | 515 | */ |
| @@ -12,6 +12,7 @@ · 11 unmodified lines | ||
| 12 | 12 | |
| 13 | 13 | #include <linux/module.h> |
| 14 | 14 | #include <linux/rculist.h> |
| 15 | +#include <linux/random.h> | |
| 15 | 16 | |
| 16 | 17 | #include "rtrs-clt.h" |
| 17 | 18 | #include "rtrs-log.h" |
| @@ -23,6 +24,12 @@ · 5 unmodified lines | ||
| 23 | 24 | * leads to "false positives" failed reconnect attempts |
| 24 | 25 | */ |
| 25 | 26 | #define RTRS_RECONNECT_BACKOFF 1000 |
| 27 | +/* | |
| 28 | + * Wait for additional random time between 0 and 8 seconds | |
| 29 | + * before starting to reconnect to avoid clients reconnecting | |
| 30 | + * all at once in case of a major network outage | |
| 31 | + */ | |
| 32 | +#define RTRS_RECONNECT_SEED 8 | |
| 26 | 33 | |
| 27 | 34 | MODULE_DESCRIPTION("RDMA Transport Client"); |
| 28 | 35 | MODULE_LICENSE("GPL"); |
| @@ -306,7 +313,8 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con) · 277 unmodified lines | ||
| 306 | 313 | */ |
| 307 | 314 | delay_ms = clt->reconnect_delay_sec * 1000; |
| 308 | 315 | queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, |
| 309 | − msecs_to_jiffies(delay_ms)); | |
| 316 | + msecs_to_jiffies(delay_ms + | |
| 317 | + prandom_u32() % RTRS_RECONNECT_SEED)); | |
| 310 | 318 | } else { |
| 311 | 319 | /* |
| 312 | 320 | * Error can happen just on establishing new connection, |
| @@ -2503,7 +2511,9 @@ reconnect_again: · 2190 unmodified lines | ||
| 2503 | 2511 | sess->stats->reconnects.fail_cnt++; |
| 2504 | 2512 | delay_ms = clt->reconnect_delay_sec * 1000; |
| 2505 | 2513 | queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, |
| 2506 | − msecs_to_jiffies(delay_ms)); | |
| 2514 | + msecs_to_jiffies(delay_ms + | |
| 2515 | + prandom_u32() % | |
| 2516 | + RTRS_RECONNECT_SEED)); | |
| 2507 | 2517 | } |
| 2508 | 2518 | } |
| 2509 | 2519 | |
| @@ -2972,7 +2982,7 @@ static int __init rtrs_client_init(void) · 462 unmodified lines | ||
| 2972 | 2982 | pr_err("Failed to create rtrs-client dev class\n"); |
| 2973 | 2983 | return PTR_ERR(rtrs_clt_dev_class); |
| 2974 | 2984 | } |
| 2975 | − rtrs_wq = alloc_workqueue("rtrs_client_wq", WQ_MEM_RECLAIM, 0); | |
| 2985 | + rtrs_wq = alloc_workqueue("rtrs_client_wq", 0, 0); | |
| 2976 | 2986 | if (!rtrs_wq) { |
| 2977 | 2987 | class_destroy(rtrs_clt_dev_class); |
| 2978 | 2988 | return -ENOMEM; |
| @@ -2150,7 +2150,7 @@ static int __init rtrs_server_init(void) · 2149 unmodified lines | ||
| 2150 | 2150 | err = PTR_ERR(rtrs_dev_class); |
| 2151 | 2151 | goto out_chunk_pool; |
| 2152 | 2152 | } |
| 2153 | − rtrs_wq = alloc_workqueue("rtrs_server_wq", WQ_MEM_RECLAIM, 0); | |
| 2153 | + rtrs_wq = alloc_workqueue("rtrs_server_wq", 0, 0); | |
| 2154 | 2154 | if (!rtrs_wq) { |
| 2155 | 2155 | err = -ENOMEM; |
| 2156 | 2156 | goto out_dev_class; |
| @@ -869,7 +869,7 @@ static int srpt_zerolength_write(struct srpt_rdma_ch *ch) · 868 unmodified lines | ||
| 869 | 869 | |
| 870 | 870 | static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc) |
| 871 | 871 | { |
| 872 | − struct srpt_rdma_ch *ch = cq->cq_context; | |
| 872 | + struct srpt_rdma_ch *ch = wc->qp->qp_context; | |
| 873 | 873 | |
| 874 | 874 | pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num, |
| 875 | 875 | wc->status); |
| @@ -1322,7 +1322,7 @@ static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx) · 446 unmodified lines | ||
| 1322 | 1322 | */ |
| 1323 | 1323 | static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) |
| 1324 | 1324 | { |
| 1325 | − struct srpt_rdma_ch *ch = cq->cq_context; | |
| 1325 | + struct srpt_rdma_ch *ch = wc->qp->qp_context; | |
| 1326 | 1326 | struct srpt_send_ioctx *ioctx = |
| 1327 | 1327 | container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe); |
| 1328 | 1328 | |
| @@ -1683,7 +1683,7 @@ push: · 354 unmodified lines | ||
| 1683 | 1683 | |
| 1684 | 1684 | static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc) |
| 1685 | 1685 | { |
| 1686 | − struct srpt_rdma_ch *ch = cq->cq_context; | |
| 1686 | + struct srpt_rdma_ch *ch = wc->qp->qp_context; | |
| 1687 | 1687 | struct srpt_recv_ioctx *ioctx = |
| 1688 | 1688 | container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe); |
| 1689 | 1689 | |
| @@ -1744,7 +1744,7 @@ static void srpt_process_wait_list(struct srpt_rdma_ch *ch) · 54 unmodified lines | ||
| 1744 | 1744 | */ |
| 1745 | 1745 | static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc) |
| 1746 | 1746 | { |
| 1747 | − struct srpt_rdma_ch *ch = cq->cq_context; | |
| 1747 | + struct srpt_rdma_ch *ch = wc->qp->qp_context; | |
| 1748 | 1748 | struct srpt_send_ioctx *ioctx = |
| 1749 | 1749 | container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe); |
| 1750 | 1750 | enum srpt_command_state state; |
| @@ -1791,7 +1791,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) · 40 unmodified lines | ||
| 1791 | 1791 | goto out; |
| 1792 | 1792 | |
| 1793 | 1793 | retry: |
| 1794 | − ch->cq = ib_alloc_cq_any(sdev->device, ch, ch->rq_size + sq_size, | |
| 1794 | + ch->cq = ib_cq_pool_get(sdev->device, ch->rq_size + sq_size, -1, | |
| 1795 | 1795 | IB_POLL_WORKQUEUE); |
| 1796 | 1796 | if (IS_ERR(ch->cq)) { |
| 1797 | 1797 | ret = PTR_ERR(ch->cq); |
| @@ -1799,6 +1799,7 @@ retry: · 1 unmodified line | ||
| 1799 | 1799 | ch->rq_size + sq_size, ret); |
| 1800 | 1800 | goto out; |
| 1801 | 1801 | } |
| 1802 | + ch->cq_size = ch->rq_size + sq_size; | |
| 1802 | 1803 | |
| 1803 | 1804 | qp_init->qp_context = (void *)ch; |
| 1804 | 1805 | qp_init->event_handler |
| @@ -1843,7 +1844,7 @@ retry: · 38 unmodified lines | ||
| 1843 | 1844 | if (retry) { |
| 1844 | 1845 | pr_debug("failed to create queue pair with sq_size = %d (%d) - retrying\n", |
| 1845 | 1846 | sq_size, ret); |
| 1846 | − ib_free_cq(ch->cq); | |
| 1847 | + ib_cq_pool_put(ch->cq, ch->cq_size); | |
| 1847 | 1848 | sq_size = max(sq_size / 2, MIN_SRPT_SQ_SIZE); |
| 1848 | 1849 | goto retry; |
| 1849 | 1850 | } else { |
| @@ -1869,14 +1870,14 @@ out: · 19 unmodified lines | ||
| 1869 | 1870 | |
| 1870 | 1871 | err_destroy_cq: |
| 1871 | 1872 | ch->qp = NULL; |
| 1872 | − ib_free_cq(ch->cq); | |
| 1873 | + ib_cq_pool_put(ch->cq, ch->cq_size); | |
| 1873 | 1874 | goto out; |
| 1874 | 1875 | } |
| 1875 | 1876 | |
| 1876 | 1877 | static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch) |
| 1877 | 1878 | { |
| 1878 | 1879 | ib_destroy_qp(ch->qp); |
| 1879 | − ib_free_cq(ch->cq); | |
| 1880 | + ib_cq_pool_put(ch->cq, ch->cq_size); | |
| 1880 | 1881 | } |
| 1881 | 1882 | |
| 1882 | 1883 | /** |
| @@ -2156,9 +2157,6 @@ static int srpt_cm_req_recv(struct srpt_device *const sdev, · 273 unmodified lines | ||
| 2156 | 2157 | |
| 2157 | 2158 | WARN_ON_ONCE(irqs_disabled()); |
| 2158 | 2159 | |
| 2159 | − if (WARN_ON(!sdev || !req)) | |
| 2160 | − return -EINVAL; | |
| 2161 | − | |
| 2162 | 2160 | it_iu_len = be32_to_cpu(req->req_it_iu_len); |
| 2163 | 2161 | |
| 2164 | 2162 | pr_info("Received SRP_LOGIN_REQ with i_port_id %pI6, t_port_id %pI6 and it_iu_len %d on port %d (guid=%pI6); pkey %#04x\n", |
| @@ -300,6 +300,7 @@ struct srpt_rdma_ch { · 299 unmodified lines | ||
| 300 | 300 | } rdma_cm; |
| 301 | 301 | }; |
| 302 | 302 | struct ib_cq *cq; |
| 303 | + u32 cq_size; | |
| 303 | 304 | struct ib_cqe zw_cqe; |
| 304 | 305 | struct rcu_head rcu; |
| 305 | 306 | struct kref kref; |
| @@ -299,11 +299,18 @@ void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas) · 298 unmodified lines | ||
| 299 | 299 | } |
| 300 | 300 | EXPORT_SYMBOL_GPL(mlx5_fill_page_array); |
| 301 | 301 | |
| 302 | −void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) | |
| 302 | +void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm) | |
| 303 | 303 | { |
| 304 | 304 | int i; |
| 305 | 305 | |
| 306 | + WARN_ON(perm & 0xfc); | |
| 306 | 307 | for (i = 0; i < buf->npages; i++) |
| 307 | − pas[i] = cpu_to_be64(buf->frags[i].map); | |
| 308 | + pas[i] = cpu_to_be64(buf->frags[i].map | perm); | |
| 309 | +} | |
| 310 | +EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm); | |
| 311 | + | |
| 312 | +void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) | |
| 313 | +{ | |
| 314 | + mlx5_fill_page_frag_array_perm(buf, pas, 0); | |
| 308 | 315 | } |
| 309 | 316 | EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array); |
| @@ -1577,6 +1577,7 @@ static struct mlx5_flow_handle *add_rule_fg(struct mlx5_flow_group *fg, · 1576 unmodified lines | ||
| 1577 | 1577 | static bool counter_is_valid(u32 action) |
| 1578 | 1578 | { |
| 1579 | 1579 | return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP | |
| 1580 | + MLX5_FLOW_CONTEXT_ACTION_ALLOW | | |
| 1580 | 1581 | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST)); |
| 1581 | 1582 | } |
| 1582 | 1583 | |
| @@ -226,13 +226,20 @@ void mlx5i_uninit_underlay_qp(struct mlx5e_priv *priv) · 225 unmodified lines | ||
| 226 | 226 | |
| 227 | 227 | int mlx5i_create_underlay_qp(struct mlx5e_priv *priv) |
| 228 | 228 | { |
| 229 | + unsigned char *dev_addr = priv->netdev->dev_addr; | |
| 229 | 230 | u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; |
| 230 | 231 | u32 in[MLX5_ST_SZ_DW(create_qp_in)] = {}; |
| 231 | 232 | struct mlx5i_priv *ipriv = priv->ppriv; |
| 232 | 233 | void *addr_path; |
| 234 | + int qpn = 0; | |
| 233 | 235 | int ret = 0; |
| 234 | 236 | void *qpc; |
| 235 | 237 | |
| 238 | + if (MLX5_CAP_GEN(priv->mdev, mkey_by_name)) { | |
| 239 | + qpn = (dev_addr[1] << 16) + (dev_addr[2] << 8) + dev_addr[3]; | |
| 240 | + MLX5_SET(create_qp_in, in, input_qpn, qpn); | |
| 241 | + } | |
| 242 | + | |
| 236 | 243 | qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); |
| 237 | 244 | MLX5_SET(qpc, qpc, st, MLX5_QP_ST_UD); |
| 238 | 245 | MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); |
| @@ -557,6 +557,9 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) · 556 unmodified lines | ||
| 557 | 557 | if (MLX5_CAP_GEN_MAX(dev, release_all_pages)) |
| 558 | 558 | MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1); |
| 559 | 559 | |
| 560 | + if (MLX5_CAP_GEN_MAX(dev, mkey_by_name)) | |
| 561 | + MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1); | |
| 562 | + | |
| 560 | 563 | return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); |
| 561 | 564 | } |
| 562 | 565 | |
| @@ -276,7 +276,9 @@ enum { · 275 unmodified lines | ||
| 276 | 276 | MLX5_MKEY_MASK_RW = 1ull << 20, |
| 277 | 277 | MLX5_MKEY_MASK_A = 1ull << 21, |
| 278 | 278 | MLX5_MKEY_MASK_SMALL_FENCE = 1ull << 23, |
| 279 | − MLX5_MKEY_MASK_FREE = 1ull << 29, | |
| 279 | + MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE = 1ull << 25, | |
| 280 | + MLX5_MKEY_MASK_FREE = 1ull << 29, | |
| 281 | + MLX5_MKEY_MASK_RELAXED_ORDERING_READ = 1ull << 47, | |
| 280 | 282 | }; |
| 281 | 283 | |
| 282 | 284 | enum { |
| @@ -1007,7 +1009,6 @@ enum { · 724 unmodified lines | ||
| 1007 | 1009 | MLX5_MKEY_REMOTE_INVAL = 1 << 24, |
| 1008 | 1010 | MLX5_MKEY_FLAG_SYNC_UMR = 1 << 29, |
| 1009 | 1011 | MLX5_MKEY_BSF_EN = 1 << 30, |
| 1010 | − MLX5_MKEY_LEN64 = 1 << 31, | |
| 1011 | 1012 | }; |
| 1012 | 1013 | |
| 1013 | 1014 | struct mlx5_mkey_seg { |
| @@ -1361,11 +1362,11 @@ enum mlx5_qcam_feature_groups { · 347 unmodified lines | ||
| 1361 | 1362 | MLX5_ADDR_OF(device_event_cap, (mdev)->caps.hca_cur[MLX5_CAP_DEV_EVENT], cap) |
| 1362 | 1363 | |
| 1363 | 1364 | #define MLX5_CAP_DEV_VDPA_EMULATION(mdev, cap)\ |
| 1364 | − MLX5_GET(device_virtio_emulation_cap, \ | |
| 1365 | + MLX5_GET(virtio_emulation_cap, \ | |
| 1365 | 1366 | (mdev)->caps.hca_cur[MLX5_CAP_VDPA_EMULATION], cap) |
| 1366 | 1367 | |
| 1367 | 1368 | #define MLX5_CAP64_DEV_VDPA_EMULATION(mdev, cap)\ |
| 1368 | − MLX5_GET64(device_virtio_emulation_cap, \ | |
| 1369 | + MLX5_GET64(virtio_emulation_cap, \ | |
| 1369 | 1370 | (mdev)->caps.hca_cur[MLX5_CAP_VDPA_EMULATION], cap) |
| 1370 | 1371 | |
| 1371 | 1372 | #define MLX5_CAP_IPSEC(mdev, cap)\ |
| @@ -975,6 +975,7 @@ void mlx5_register_debugfs(void); · 974 unmodified lines | ||
| 975 | 975 | void mlx5_unregister_debugfs(void); |
| 976 | 976 | |
| 977 | 977 | void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas); |
| 978 | +void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm); | |
| 978 | 979 | void mlx5_fill_page_frag_array(struct mlx5_frag_buf *frag_buf, __be64 *pas); |
| 979 | 980 | int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, |
| 980 | 981 | unsigned int *irqn); |
| @@ -1057,6 +1058,7 @@ enum { · 76 unmodified lines | ||
| 1057 | 1058 | enum { |
| 1058 | 1059 | MLX5_INTERFACE_PROTOCOL_IB = 0, |
| 1059 | 1060 | MLX5_INTERFACE_PROTOCOL_ETH = 1, |
| 1061 | + MLX5_INTERFACE_PROTOCOL_VDPA = 2, | |
| 1060 | 1062 | }; |
| 1061 | 1063 | |
| 1062 | 1064 | struct mlx5_interface { |
| @@ -93,6 +93,7 @@ enum { · 92 unmodified lines | ||
| 93 | 93 | |
| 94 | 94 | enum { |
| 95 | 95 | MLX5_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b, |
| 96 | + MLX5_OBJ_TYPE_VIRTIO_NET_Q = 0x000d, | |
| 96 | 97 | MLX5_OBJ_TYPE_MKEY = 0xff01, |
| 97 | 98 | MLX5_OBJ_TYPE_QP = 0xff02, |
| 98 | 99 | MLX5_OBJ_TYPE_PSV = 0xff03, |
| @@ -985,17 +986,40 @@ struct mlx5_ifc_device_event_cap_bits { · 886 unmodified lines | ||
| 985 | 986 | u8 user_unaffiliated_events[4][0x40]; |
| 986 | 987 | }; |
| 987 | 988 | |
| 988 | −struct mlx5_ifc_device_virtio_emulation_cap_bits { | |
| 989 | − u8 reserved_at_0[0x20]; | |
| 989 | +struct mlx5_ifc_virtio_emulation_cap_bits { | |
| 990 | + u8 desc_tunnel_offload_type[0x1]; | |
| 991 | + u8 eth_frame_offload_type[0x1]; | |
| 992 | + u8 virtio_version_1_0[0x1]; | |
| 993 | + u8 device_features_bits_mask[0xd]; | |
| 994 | + u8 event_mode[0x8]; | |
| 995 | + u8 virtio_queue_type[0x8]; | |
| 990 | 996 | |
| 991 | − u8 reserved_at_20[0x13]; | |
| 997 | + u8 max_tunnel_desc[0x10]; | |
| 998 | + u8 reserved_at_30[0x3]; | |
| 992 | 999 | u8 log_doorbell_stride[0x5]; |
| 993 | 1000 | u8 reserved_at_38[0x3]; |
| 994 | 1001 | u8 log_doorbell_bar_size[0x5]; |
| 995 | 1002 | |
| 996 | 1003 | u8 doorbell_bar_offset[0x40]; |
| 997 | 1004 | |
| 998 | − u8 reserved_at_80[0x780]; | |
| 1005 | + u8 max_emulated_devices[0x8]; | |
| 1006 | + u8 max_num_virtio_queues[0x18]; | |
| 1007 | + | |
| 1008 | + u8 reserved_at_a0[0x60]; | |
| 1009 | + | |
| 1010 | + u8 umem_1_buffer_param_a[0x20]; | |
| 1011 | + | |
| 1012 | + u8 umem_1_buffer_param_b[0x20]; | |
| 1013 | + | |
| 1014 | + u8 umem_2_buffer_param_a[0x20]; | |
| 1015 | + | |
| 1016 | + u8 umem_2_buffer_param_b[0x20]; | |
| 1017 | + | |
| 1018 | + u8 umem_3_buffer_param_a[0x20]; | |
| 1019 | + | |
| 1020 | + u8 umem_3_buffer_param_b[0x20]; | |
| 1021 | + | |
| 1022 | + u8 reserved_at_1c0[0x640]; | |
| 999 | 1023 | }; |
| 1000 | 1024 | |
| 1001 | 1025 | enum { |
| @@ -1220,7 +1244,11 @@ struct mlx5_ifc_cmd_hca_cap_bits { · 218 unmodified lines | ||
| 1220 | 1244 | |
| 1221 | 1245 | u8 max_sgl_for_optimized_performance[0x8]; |
| 1222 | 1246 | u8 log_max_cq_sz[0x8]; |
| 1223 | − u8 reserved_at_d0[0xb]; | |
| 1247 | + u8 relaxed_ordering_write_umr[0x1]; | |
| 1248 | + u8 relaxed_ordering_read_umr[0x1]; | |
| 1249 | + u8 reserved_at_d2[0x7]; | |
| 1250 | + u8 virtio_net_device_emualtion_manager[0x1]; | |
| 1251 | + u8 virtio_blk_device_emualtion_manager[0x1]; | |
| 1224 | 1252 | u8 log_max_cq[0x5]; |
| 1225 | 1253 | |
| 1226 | 1254 | u8 log_max_eq_sz[0x8]; |
| @@ -1396,7 +1424,10 @@ struct mlx5_ifc_cmd_hca_cap_bits { · 169 unmodified lines | ||
| 1396 | 1424 | u8 bf[0x1]; |
| 1397 | 1425 | u8 driver_version[0x1]; |
| 1398 | 1426 | u8 pad_tx_eth_packet[0x1]; |
| 1399 | − u8 reserved_at_263[0x8]; | |
| 1427 | + u8 reserved_at_263[0x3]; | |
| 1428 | + u8 mkey_by_name[0x1]; | |
| 1429 | + u8 reserved_at_267[0x4]; | |
| 1430 | + | |
| 1400 | 1431 | u8 log_bf_reg_size[0x5]; |
| 1401 | 1432 | |
| 1402 | 1433 | u8 reserved_at_270[0x8]; |
| @@ -2953,7 +2984,7 @@ union mlx5_ifc_hca_cap_union_bits { · 1550 unmodified lines | ||
| 2953 | 2984 | struct mlx5_ifc_fpga_cap_bits fpga_cap; |
| 2954 | 2985 | struct mlx5_ifc_tls_cap_bits tls_cap; |
| 2955 | 2986 | struct mlx5_ifc_device_mem_cap_bits device_mem_cap; |
| 2956 | − struct mlx5_ifc_device_virtio_emulation_cap_bits virtio_emulation_cap; | |
| 2987 | + struct mlx5_ifc_virtio_emulation_cap_bits virtio_emulation_cap; | |
| 2957 | 2988 | u8 reserved_at_0[0x8000]; |
| 2958 | 2989 | }; |
| 2959 | 2990 | |
| @@ -3302,15 +3333,18 @@ struct mlx5_ifc_scheduling_context_bits { · 342 unmodified lines | ||
| 3302 | 3333 | }; |
| 3303 | 3334 | |
| 3304 | 3335 | struct mlx5_ifc_rqtc_bits { |
| 3305 | − u8 reserved_at_0[0xa0]; | |
| 3336 | + u8 reserved_at_0[0xa0]; | |
| 3306 | 3337 | |
| 3307 | − u8 reserved_at_a0[0x10]; | |
| 3308 | − u8 rqt_max_size[0x10]; | |
| 3338 | + u8 reserved_at_a0[0x5]; | |
| 3339 | + u8 list_q_type[0x3]; | |
| 3340 | + u8 reserved_at_a8[0x8]; | |
| 3341 | + u8 rqt_max_size[0x10]; | |
| 3309 | 3342 | |
| 3310 | − u8 reserved_at_c0[0x10]; | |
| 3311 | − u8 rqt_actual_size[0x10]; | |
| 3343 | + u8 rq_vhca_id_format[0x1]; | |
| 3344 | + u8 reserved_at_c1[0xf]; | |
| 3345 | + u8 rqt_actual_size[0x10]; | |
| 3312 | 3346 | |
| 3313 | − u8 reserved_at_e0[0x6a0]; | |
| 3347 | + u8 reserved_at_e0[0x6a0]; | |
| 3314 | 3348 | |
| 3315 | 3349 | struct mlx5_ifc_rq_num_bits rq_num[]; |
| 3316 | 3350 | }; |
| @@ -7092,7 +7126,7 @@ struct mlx5_ifc_destroy_mkey_out_bits { · 3775 unmodified lines | ||
| 7092 | 7126 | |
| 7093 | 7127 | struct mlx5_ifc_destroy_mkey_in_bits { |
| 7094 | 7128 | u8 opcode[0x10]; |
| 7095 | − u8 reserved_at_10[0x10]; | |
| 7129 | + u8 uid[0x10]; | |
| 7096 | 7130 | |
| 7097 | 7131 | u8 reserved_at_20[0x10]; |
| 7098 | 7132 | u8 op_mod[0x10]; |
| @@ -7723,8 +7757,10 @@ struct mlx5_ifc_create_qp_in_bits { · 624 unmodified lines | ||
| 7723 | 7757 | u8 reserved_at_20[0x10]; |
| 7724 | 7758 | u8 op_mod[0x10]; |
| 7725 | 7759 | |
| 7726 | − u8 reserved_at_40[0x40]; | |
| 7760 | + u8 reserved_at_40[0x8]; | |
| 7761 | + u8 input_qpn[0x18]; | |
| 7727 | 7762 | |
| 7763 | + u8 reserved_at_60[0x20]; | |
| 7728 | 7764 | u8 opt_param_mask[0x20]; |
| 7729 | 7765 | |
| 7730 | 7766 | u8 ece[0x20]; |
| @@ -7788,7 +7824,7 @@ struct mlx5_ifc_create_mkey_out_bits { · 57 unmodified lines | ||
| 7788 | 7824 | |
| 7789 | 7825 | struct mlx5_ifc_create_mkey_in_bits { |
| 7790 | 7826 | u8 opcode[0x10]; |
| 7791 | − u8 reserved_at_10[0x10]; | |
| 7827 | + u8 uid[0x10]; | |
| 7792 | 7828 | |
| 7793 | 7829 | u8 reserved_at_20[0x10]; |
| 7794 | 7830 | u8 op_mod[0x10]; |
| @@ -10346,6 +10382,40 @@ struct mlx5_ifc_create_umem_in_bits { · 2551 unmodified lines | ||
| 10346 | 10382 | struct mlx5_ifc_umem_bits umem; |
| 10347 | 10383 | }; |
| 10348 | 10384 | |
| 10385 | +struct mlx5_ifc_create_umem_out_bits { | |
| 10386 | + u8 status[0x8]; | |
| 10387 | + u8 reserved_at_8[0x18]; | |
| 10388 | + | |
| 10389 | + u8 syndrome[0x20]; | |
| 10390 | + | |
| 10391 | + u8 reserved_at_40[0x8]; | |
| 10392 | + u8 umem_id[0x18]; | |
| 10393 | + | |
| 10394 | + u8 reserved_at_60[0x20]; | |
| 10395 | +}; | |
| 10396 | + | |
| 10397 | +struct mlx5_ifc_destroy_umem_in_bits { | |
| 10398 | + u8 opcode[0x10]; | |
| 10399 | + u8 uid[0x10]; | |
| 10400 | + | |
| 10401 | + u8 reserved_at_20[0x10]; | |
| 10402 | + u8 op_mod[0x10]; | |
| 10403 | + | |
| 10404 | + u8 reserved_at_40[0x8]; | |
| 10405 | + u8 umem_id[0x18]; | |
| 10406 | + | |
| 10407 | + u8 reserved_at_60[0x20]; | |
| 10408 | +}; | |
| 10409 | + | |
| 10410 | +struct mlx5_ifc_destroy_umem_out_bits { | |
| 10411 | + u8 status[0x8]; | |
| 10412 | + u8 reserved_at_8[0x18]; | |
| 10413 | + | |
| 10414 | + u8 syndrome[0x20]; | |
| 10415 | + | |
| 10416 | + u8 reserved_at_40[0x40]; | |
| 10417 | +}; | |
| 10418 | + | |
| 10349 | 10419 | struct mlx5_ifc_create_uctx_in_bits { |
| 10350 | 10420 | u8 opcode[0x10]; |
| 10351 | 10421 | u8 reserved_at_10[0x10]; |
| @@ -10358,6 +10428,18 @@ struct mlx5_ifc_create_uctx_in_bits { · 6 unmodified lines | ||
| 10358 | 10428 | struct mlx5_ifc_uctx_bits uctx; |
| 10359 | 10429 | }; |
| 10360 | 10430 | |
| 10431 | +struct mlx5_ifc_create_uctx_out_bits { | |
| 10432 | + u8 status[0x8]; | |
| 10433 | + u8 reserved_at_8[0x18]; | |
| 10434 | + | |
| 10435 | + u8 syndrome[0x20]; | |
| 10436 | + | |
| 10437 | + u8 reserved_at_40[0x10]; | |
| 10438 | + u8 uid[0x10]; | |
| 10439 | + | |
| 10440 | + u8 reserved_at_60[0x20]; | |
| 10441 | +}; | |
| 10442 | + | |
| 10361 | 10443 | struct mlx5_ifc_destroy_uctx_in_bits { |
| 10362 | 10444 | u8 opcode[0x10]; |
| 10363 | 10445 | u8 reserved_at_10[0x10]; |
| @@ -10371,6 +10453,15 @@ struct mlx5_ifc_destroy_uctx_in_bits { · 7 unmodified lines | ||
| 10371 | 10453 | u8 reserved_at_60[0x20]; |
| 10372 | 10454 | }; |
| 10373 | 10455 | |
| 10456 | +struct mlx5_ifc_destroy_uctx_out_bits { | |
| 10457 | + u8 status[0x8]; | |
| 10458 | + u8 reserved_at_8[0x18]; | |
| 10459 | + | |
| 10460 | + u8 syndrome[0x20]; | |
| 10461 | + | |
| 10462 | + u8 reserved_at_40[0x40]; | |
| 10463 | +}; | |
| 10464 | + | |
| 10374 | 10465 | struct mlx5_ifc_create_sw_icm_in_bits { |
| 10375 | 10466 | struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr; |
| 10376 | 10467 | struct mlx5_ifc_sw_icm_bits sw_icm; |
| @@ -10687,4 +10778,10 @@ struct mlx5_ifc_tls_progress_params_bits { · 310 unmodified lines | ||
| 10687 | 10778 | u8 hw_offset_record_number[0x18]; |
| 10688 | 10779 | }; |
| 10689 | 10780 | |
| 10781 | +enum { | |
| 10782 | + MLX5_MTT_PERM_READ = 1 << 0, | |
| 10783 | + MLX5_MTT_PERM_WRITE = 1 << 1, | |
| 10784 | + MLX5_MTT_PERM_RW = MLX5_MTT_PERM_READ | MLX5_MTT_PERM_WRITE, | |
| 10785 | +}; | |
| 10786 | + | |
| 10690 | 10787 | #endif /* MLX5_IFC_H */ |
| @@ -1,36 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2010 Intel Corporation. All rights reserved. |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | −#if !defined(_RDMA_IB_H) | |
| 6 | +#ifndef _RDMA_IB_H | |
| 34 | 7 | #define _RDMA_IB_H |
| 35 | 8 | |
| 36 | 9 | #include <linux/types.h> |
| @@ -1,37 +1,10 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2005 Voltaire Inc. All rights reserved. |
| 3 | 4 | * Copyright (c) 2005 Intel Corporation. All rights reserved. |
| 4 | − * | |
| 5 | − * This software is available to you under a choice of one of two | |
| 6 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 7 | − * General Public License (GPL) Version 2, available from the file | |
| 8 | − * COPYING in the main directory of this source tree, or the | |
| 9 | − * OpenIB.org BSD license below: | |
| 10 | − * | |
| 11 | − * Redistribution and use in source and binary forms, with or | |
| 12 | − * without modification, are permitted provided that the following | |
| 13 | − * conditions are met: | |
| 14 | − * | |
| 15 | − * - Redistributions of source code must retain the above | |
| 16 | − * copyright notice, this list of conditions and the following | |
| 17 | − * disclaimer. | |
| 18 | − * | |
| 19 | − * - Redistributions in binary form must reproduce the above | |
| 20 | − * copyright notice, this list of conditions and the following | |
| 21 | − * disclaimer in the documentation and/or other materials | |
| 22 | − * provided with the distribution. | |
| 23 | − * | |
| 24 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 25 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 26 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 27 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 28 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 29 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 30 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 31 | − * SOFTWARE. | |
| 32 | 5 | */ |
| 33 | 6 | |
| 34 | −#if !defined(IB_ADDR_H) | |
| 7 | +#ifndef IB_ADDR_H | |
| 35 | 8 | #define IB_ADDR_H |
| 36 | 9 | |
| 37 | 10 | #include <linux/in.h> |
| @@ -1,35 +1,8 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2004 Topspin Communications. All rights reserved. |
| 3 | 4 | * Copyright (c) 2005 Intel Corporation. All rights reserved. |
| 4 | 5 | * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. |
| 5 | − * | |
| 6 | − * This software is available to you under a choice of one of two | |
| 7 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 8 | − * General Public License (GPL) Version 2, available from the file | |
| 9 | − * COPYING in the main directory of this source tree, or the | |
| 10 | − * OpenIB.org BSD license below: | |
| 11 | − * | |
| 12 | − * Redistribution and use in source and binary forms, with or | |
| 13 | − * without modification, are permitted provided that the following | |
| 14 | − * conditions are met: | |
| 15 | − * | |
| 16 | − * - Redistributions of source code must retain the above | |
| 17 | − * copyright notice, this list of conditions and the following | |
| 18 | − * disclaimer. | |
| 19 | − * | |
| 20 | − * - Redistributions in binary form must reproduce the above | |
| 21 | − * copyright notice, this list of conditions and the following | |
| 22 | − * disclaimer in the documentation and/or other materials | |
| 23 | − * provided with the distribution. | |
| 24 | − * | |
| 25 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 26 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 27 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 28 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 29 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 30 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 31 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 32 | − * SOFTWARE. | |
| 33 | 6 | */ |
| 34 | 7 | |
| 35 | 8 | #ifndef _IB_CACHE_H |
| @@ -6,6 +6,7 @@ · 5 unmodified lines | ||
| 6 | 6 | * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. |
| 7 | 7 | * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. |
| 8 | 8 | */ |
| 9 | + | |
| 9 | 10 | #ifndef IB_CM_H |
| 10 | 11 | #define IB_CM_H |
| 11 | 12 | |
| @@ -1,48 +1,6 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright(c) 2016 - 2018 Intel Corporation. |
| 3 | − * | |
| 4 | − * This file is provided under a dual BSD/GPLv2 license. When using or | |
| 5 | − * redistributing this file, you may do so under either license. | |
| 6 | − * | |
| 7 | − * GPL LICENSE SUMMARY | |
| 8 | − * | |
| 9 | − * This program is free software; you can redistribute it and/or modify | |
| 10 | − * it under the terms of version 2 of the GNU General Public License as | |
| 11 | − * published by the Free Software Foundation. | |
| 12 | − * | |
| 13 | − * This program is distributed in the hope that it will be useful, but | |
| 14 | − * WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 15 | − * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 16 | − * General Public License for more details. | |
| 17 | − * | |
| 18 | − * BSD LICENSE | |
| 19 | − * | |
| 20 | − * Redistribution and use in source and binary forms, with or without | |
| 21 | − * modification, are permitted provided that the following conditions | |
| 22 | − * are met: | |
| 23 | − * | |
| 24 | − * - Redistributions of source code must retain the above copyright | |
| 25 | − * notice, this list of conditions and the following disclaimer. | |
| 26 | − * - Redistributions in binary form must reproduce the above copyright | |
| 27 | − * notice, this list of conditions and the following disclaimer in | |
| 28 | − * the documentation and/or other materials provided with the | |
| 29 | − * distribution. | |
| 30 | − * - Neither the name of Intel Corporation nor the names of its | |
| 31 | − * contributors may be used to endorse or promote products derived | |
| 32 | − * from this software without specific prior written permission. | |
| 33 | − * | |
| 34 | − * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 35 | − * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 36 | − * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 37 | − * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 38 | − * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 39 | − * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 40 | − * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 41 | − * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 42 | − * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 43 | − * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 44 | − * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 45 | − * | |
| 46 | 4 | */ |
| 47 | 5 | |
| 48 | 6 | #ifndef IB_HDRS_H |
| @@ -1,40 +1,13 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. |
| 3 | 4 | * Copyright (c) 2004 Infinicon Corporation. All rights reserved. |
| 4 | 5 | * Copyright (c) 2004 Intel Corporation. All rights reserved. |
| 5 | 6 | * Copyright (c) 2004 Topspin Corporation. All rights reserved. |
| 6 | 7 | * Copyright (c) 2004-2006 Voltaire Corporation. All rights reserved. |
| 7 | − * | |
| 8 | − * This software is available to you under a choice of one of two | |
| 9 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 10 | − * General Public License (GPL) Version 2, available from the file | |
| 11 | − * COPYING in the main directory of this source tree, or the | |
| 12 | − * OpenIB.org BSD license below: | |
| 13 | − * | |
| 14 | − * Redistribution and use in source and binary forms, with or | |
| 15 | − * without modification, are permitted provided that the following | |
| 16 | − * conditions are met: | |
| 17 | − * | |
| 18 | − * - Redistributions of source code must retain the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer. | |
| 21 | − * | |
| 22 | − * - Redistributions in binary form must reproduce the above | |
| 23 | − * copyright notice, this list of conditions and the following | |
| 24 | − * disclaimer in the documentation and/or other materials | |
| 25 | − * provided with the distribution. | |
| 26 | − * | |
| 27 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 28 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 29 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 30 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 31 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 32 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 33 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 34 | − * SOFTWARE. | |
| 35 | 8 | */ |
| 36 | 9 | |
| 37 | −#if !defined(IB_MAD_H) | |
| 10 | +#ifndef IB_MAD_H | |
| 38 | 11 | #define IB_MAD_H |
| 39 | 12 | |
| 40 | 13 | #include <linux/list.h> |
| @@ -1,36 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | −#if !defined(IB_USER_MARSHALL_H) | |
| 6 | +#ifndef IB_USER_MARSHALL_H | |
| 34 | 7 | #define IB_USER_MARSHALL_H |
| 35 | 8 | |
| 36 | 9 | #include <rdma/ib_verbs.h> |
| @@ -1,33 +1,6 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2004 Topspin Corporation. All rights reserved. |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | 6 | #ifndef IB_PACK_H |
| @@ -1,38 +1,11 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation. |
| 3 | 4 | * All rights reserved. |
| 4 | 5 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. |
| 5 | − * | |
| 6 | − * This software is available to you under a choice of one of two | |
| 7 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 8 | − * General Public License (GPL) Version 2, available from the file | |
| 9 | − * COPYING in the main directory of this source tree, or the | |
| 10 | − * OpenIB.org BSD license below: | |
| 11 | − * | |
| 12 | − * Redistribution and use in source and binary forms, with or | |
| 13 | − * without modification, are permitted provided that the following | |
| 14 | − * conditions are met: | |
| 15 | − * | |
| 16 | − * - Redistributions of source code must retain the above | |
| 17 | − * copyright notice, this list of conditions and the following | |
| 18 | − * disclaimer. | |
| 19 | − * | |
| 20 | − * - Redistributions in binary form must reproduce the above | |
| 21 | − * copyright notice, this list of conditions and the following | |
| 22 | − * disclaimer in the documentation and/or other materials | |
| 23 | − * provided with the distribution. | |
| 24 | − * | |
| 25 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 26 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 27 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 28 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 29 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 30 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 31 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 32 | − * SOFTWARE. | |
| 33 | 6 | */ |
| 34 | 7 | |
| 35 | −#if !defined(IB_PMA_H) | |
| 8 | +#ifndef IB_PMA_H | |
| 36 | 9 | #define IB_PMA_H |
| 37 | 10 | |
| 38 | 11 | #include <rdma/ib_mad.h> |
| @@ -1,35 +1,8 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2004 Topspin Communications. All rights reserved. |
| 3 | 4 | * Copyright (c) 2005 Voltaire, Inc. All rights reserved. |
| 4 | 5 | * Copyright (c) 2006 Intel Corporation. All rights reserved. |
| 5 | − * | |
| 6 | − * This software is available to you under a choice of one of two | |
| 7 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 8 | − * General Public License (GPL) Version 2, available from the file | |
| 9 | − * COPYING in the main directory of this source tree, or the | |
| 10 | − * OpenIB.org BSD license below: | |
| 11 | − * | |
| 12 | − * Redistribution and use in source and binary forms, with or | |
| 13 | − * without modification, are permitted provided that the following | |
| 14 | − * conditions are met: | |
| 15 | − * | |
| 16 | − * - Redistributions of source code must retain the above | |
| 17 | − * copyright notice, this list of conditions and the following | |
| 18 | − * disclaimer. | |
| 19 | − * | |
| 20 | − * - Redistributions in binary form must reproduce the above | |
| 21 | − * copyright notice, this list of conditions and the following | |
| 22 | − * disclaimer in the documentation and/or other materials | |
| 23 | − * provided with the distribution. | |
| 24 | − * | |
| 25 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 26 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 27 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 28 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 29 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 30 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 31 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 32 | − * SOFTWARE. | |
| 33 | 6 | */ |
| 34 | 7 | |
| 35 | 8 | #ifndef IB_SA_H |
| @@ -1,40 +1,13 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. |
| 3 | 4 | * Copyright (c) 2004 Infinicon Corporation. All rights reserved. |
| 4 | 5 | * Copyright (c) 2004 Intel Corporation. All rights reserved. |
| 5 | 6 | * Copyright (c) 2004 Topspin Corporation. All rights reserved. |
| 6 | 7 | * Copyright (c) 2004 Voltaire Corporation. All rights reserved. |
| 7 | − * | |
| 8 | − * This software is available to you under a choice of one of two | |
| 9 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 10 | − * General Public License (GPL) Version 2, available from the file | |
| 11 | − * COPYING in the main directory of this source tree, or the | |
| 12 | − * OpenIB.org BSD license below: | |
| 13 | − * | |
| 14 | − * Redistribution and use in source and binary forms, with or | |
| 15 | − * without modification, are permitted provided that the following | |
| 16 | − * conditions are met: | |
| 17 | − * | |
| 18 | − * - Redistributions of source code must retain the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer. | |
| 21 | − * | |
| 22 | − * - Redistributions in binary form must reproduce the above | |
| 23 | − * copyright notice, this list of conditions and the following | |
| 24 | − * disclaimer in the documentation and/or other materials | |
| 25 | − * provided with the distribution. | |
| 26 | − * | |
| 27 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 28 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 29 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 30 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 31 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 32 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 33 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 34 | − * SOFTWARE. | |
| 35 | 8 | */ |
| 36 | 9 | |
| 37 | −#if !defined(IB_SMI_H) | |
| 10 | +#ifndef IB_SMI_H | |
| 38 | 11 | #define IB_SMI_H |
| 39 | 12 | |
| 40 | 13 | #include <rdma/ib_mad.h> |
| @@ -1,33 +1,6 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2007 Cisco Systems. All rights reserved. |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | 6 | #ifndef IB_UMEM_H |
| @@ -1,33 +1,6 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2014 Mellanox Technologies. All rights reserved. |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | 6 | #ifndef IB_UMEM_ODP_H |
| @@ -1,3 +1,4 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. |
| 3 | 4 | * Copyright (c) 2004 Infinicon Corporation. All rights reserved. |
| @@ -6,37 +7,9 @@ · 2 unmodified lines | ||
| 6 | 7 | * Copyright (c) 2004 Voltaire Corporation. All rights reserved. |
| 7 | 8 | * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. |
| 8 | 9 | * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. |
| 9 | − * | |
| 10 | − * This software is available to you under a choice of one of two | |
| 11 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 12 | − * General Public License (GPL) Version 2, available from the file | |
| 13 | − * COPYING in the main directory of this source tree, or the | |
| 14 | − * OpenIB.org BSD license below: | |
| 15 | − * | |
| 16 | − * Redistribution and use in source and binary forms, with or | |
| 17 | − * without modification, are permitted provided that the following | |
| 18 | − * conditions are met: | |
| 19 | − * | |
| 20 | − * - Redistributions of source code must retain the above | |
| 21 | − * copyright notice, this list of conditions and the following | |
| 22 | − * disclaimer. | |
| 23 | − * | |
| 24 | − * - Redistributions in binary form must reproduce the above | |
| 25 | − * copyright notice, this list of conditions and the following | |
| 26 | − * disclaimer in the documentation and/or other materials | |
| 27 | − * provided with the distribution. | |
| 28 | − * | |
| 29 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 30 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 31 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 32 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 33 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 34 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 35 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 36 | − * SOFTWARE. | |
| 37 | 10 | */ |
| 38 | 11 | |
| 39 | −#if !defined(IB_VERBS_H) | |
| 12 | +#ifndef IB_VERBS_H | |
| 40 | 13 | #define IB_VERBS_H |
| 41 | 14 | |
| 42 | 15 | #include <linux/types.h> |
| @@ -75,6 +48,7 @@ struct ib_umem_odp; · 32 unmodified lines | ||
| 75 | 48 | struct ib_uqp_object; |
| 76 | 49 | struct ib_usrq_object; |
| 77 | 50 | struct ib_uwq_object; |
| 51 | +struct rdma_cm_id; | |
| 78 | 52 | |
| 79 | 53 | extern struct workqueue_struct *ib_wq; |
| 80 | 54 | extern struct workqueue_struct *ib_comp_wq; |
| @@ -1566,9 +1540,8 @@ struct ib_xrcd { · 1485 unmodified lines | ||
| 1566 | 1540 | struct ib_device *device; |
| 1567 | 1541 | atomic_t usecnt; /* count all exposed resources */ |
| 1568 | 1542 | struct inode *inode; |
| 1569 | − | |
| 1570 | − struct mutex tgt_qp_mutex; | |
| 1571 | − struct list_head tgt_qp_list; | |
| 1543 | + struct rw_semaphore tgt_qps_rwsem; | |
| 1544 | + struct xarray tgt_qps; | |
| 1572 | 1545 | }; |
| 1573 | 1546 | |
| 1574 | 1547 | struct ib_ah { |
| @@ -2270,6 +2243,7 @@ struct rdma_netdev_alloc_params { · 695 unmodified lines | ||
| 2270 | 2243 | struct ib_odp_counters { |
| 2271 | 2244 | atomic64_t faults; |
| 2272 | 2245 | atomic64_t invalidations; |
| 2246 | + atomic64_t prefetch; | |
| 2273 | 2247 | }; |
| 2274 | 2248 | |
| 2275 | 2249 | struct ib_counters { |
| @@ -2476,7 +2450,7 @@ struct ib_device_ops { · 200 unmodified lines | ||
| 2476 | 2450 | struct ib_pd *pd, struct ib_udata *udata); |
| 2477 | 2451 | int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata); |
| 2478 | 2452 | struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 2479 | − u32 max_num_sg, struct ib_udata *udata); | |
| 2453 | + u32 max_num_sg); | |
| 2480 | 2454 | struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd, |
| 2481 | 2455 | u32 max_num_data_sg, |
| 2482 | 2456 | u32 max_num_meta_sg); |
| @@ -2493,9 +2467,8 @@ struct ib_device_ops { · 10 unmodified lines | ||
| 2493 | 2467 | int (*dealloc_mw)(struct ib_mw *mw); |
| 2494 | 2468 | int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid); |
| 2495 | 2469 | int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid); |
| 2496 | − struct ib_xrcd *(*alloc_xrcd)(struct ib_device *device, | |
| 2497 | − struct ib_udata *udata); | |
| 2498 | − int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata); | |
| 2470 | + int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata); | |
| 2471 | + void (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata); | |
| 2499 | 2472 | struct ib_flow *(*create_flow)(struct ib_qp *qp, |
| 2500 | 2473 | struct ib_flow_attr *flow_attr, |
| 2501 | 2474 | int domain, struct ib_udata *udata); |
| @@ -2539,9 +2512,9 @@ struct ib_device_ops { · 37 unmodified lines | ||
| 2539 | 2512 | struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm, |
| 2540 | 2513 | struct ib_dm_mr_attr *attr, |
| 2541 | 2514 | struct uverbs_attr_bundle *attrs); |
| 2542 | − struct ib_counters *(*create_counters)( | |
| 2543 | − struct ib_device *device, struct uverbs_attr_bundle *attrs); | |
| 2544 | − int (*destroy_counters)(struct ib_counters *counters); | |
| 2515 | + int (*create_counters)(struct ib_counters *counters, | |
| 2516 | + struct uverbs_attr_bundle *attrs); | |
| 2517 | + void (*destroy_counters)(struct ib_counters *counters); | |
| 2545 | 2518 | int (*read_counters)(struct ib_counters *counters, |
| 2546 | 2519 | struct ib_counters_read_attr *counters_read_attr, |
| 2547 | 2520 | struct uverbs_attr_bundle *attrs); |
| @@ -2581,8 +2554,13 @@ struct ib_device_ops { · 33 unmodified lines | ||
| 2581 | 2554 | /** |
| 2582 | 2555 | * Allows rdma drivers to add their own restrack attributes. |
| 2583 | 2556 | */ |
| 2584 | − int (*fill_res_entry)(struct sk_buff *msg, | |
| 2585 | − struct rdma_restrack_entry *entry); | |
| 2557 | + int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr); | |
| 2558 | + int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr); | |
| 2559 | + int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq); | |
| 2560 | + int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq); | |
| 2561 | + int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp); | |
| 2562 | + int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp); | |
| 2563 | + int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id); | |
| 2586 | 2564 | |
| 2587 | 2565 | /* Device lifecycle callbacks */ |
| 2588 | 2566 | /* |
| @@ -2637,14 +2615,19 @@ struct ib_device_ops { · 48 unmodified lines | ||
| 2637 | 2615 | * Allows rdma drivers to add their own restrack attributes |
| 2638 | 2616 | * dumped via 'rdma stat' iproute2 command. |
| 2639 | 2617 | */ |
| 2640 | − int (*fill_stat_entry)(struct sk_buff *msg, | |
| 2641 | − struct rdma_restrack_entry *entry); | |
| 2618 | + int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr); | |
| 2619 | + | |
| 2620 | + /* query driver for its ucontext properties */ | |
| 2621 | + int (*query_ucontext)(struct ib_ucontext *context, | |
| 2622 | + struct uverbs_attr_bundle *attrs); | |
| 2642 | 2623 | |
| 2643 | 2624 | DECLARE_RDMA_OBJ_SIZE(ib_ah); |
| 2625 | + DECLARE_RDMA_OBJ_SIZE(ib_counters); | |
| 2644 | 2626 | DECLARE_RDMA_OBJ_SIZE(ib_cq); |
| 2645 | 2627 | DECLARE_RDMA_OBJ_SIZE(ib_pd); |
| 2646 | 2628 | DECLARE_RDMA_OBJ_SIZE(ib_srq); |
| 2647 | 2629 | DECLARE_RDMA_OBJ_SIZE(ib_ucontext); |
| 2630 | + DECLARE_RDMA_OBJ_SIZE(ib_xrcd); | |
| 2648 | 2631 | }; |
| 2649 | 2632 | |
| 2650 | 2633 | struct ib_core_device { |
| @@ -4263,14 +4246,8 @@ static inline int ib_dereg_mr(struct ib_mr *mr) · 1612 unmodified lines | ||
| 4263 | 4246 | return ib_dereg_mr_user(mr, NULL); |
| 4264 | 4247 | } |
| 4265 | 4248 | |
| 4266 | −struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type, | |
| 4267 | − u32 max_num_sg, struct ib_udata *udata); | |
| 4268 | − | |
| 4269 | −static inline struct ib_mr *ib_alloc_mr(struct ib_pd *pd, | |
| 4270 | − enum ib_mr_type mr_type, u32 max_num_sg) | |
| 4271 | −{ | |
| 4272 | − return ib_alloc_mr_user(pd, mr_type, max_num_sg, NULL); | |
| 4273 | −} | |
| 4249 | +struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, | |
| 4250 | + u32 max_num_sg); | |
| 4274 | 4251 | |
| 4275 | 4252 | struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd, |
| 4276 | 4253 | u32 max_num_data_sg, |
| @@ -4321,21 +4298,9 @@ int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); · 44 unmodified lines | ||
| 4321 | 4298 | */ |
| 4322 | 4299 | int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); |
| 4323 | 4300 | |
| 4324 | −/** | |
| 4325 | − * ib_alloc_xrcd - Allocates an XRC domain. | |
| 4326 | − * @device: The device on which to allocate the XRC domain. | |
| 4327 | − * @caller: Module name for kernel consumers | |
| 4328 | − */ | |
| 4329 | −struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller); | |
| 4330 | −#define ib_alloc_xrcd(device) \ | |
| 4331 | − __ib_alloc_xrcd((device), KBUILD_MODNAME) | |
| 4332 | − | |
| 4333 | −/** | |
| 4334 | − * ib_dealloc_xrcd - Deallocates an XRC domain. | |
| 4335 | − * @xrcd: The XRC domain to deallocate. | |
| 4336 | − * @udata: Valid user data or NULL for kernel object | |
| 4337 | − */ | |
| 4338 | −int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata); | |
| 4301 | +struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device, | |
| 4302 | + struct inode *inode, struct ib_udata *udata); | |
| 4303 | +int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata); | |
| 4339 | 4304 | |
| 4340 | 4305 | static inline int ib_check_mr_access(int flags) |
| 4341 | 4306 | { |
| @@ -4417,9 +4382,6 @@ struct ib_wq *ib_create_wq(struct ib_pd *pd, · 75 unmodified lines | ||
| 4417 | 4382 | int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); |
| 4418 | 4383 | int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *attr, |
| 4419 | 4384 | u32 wq_attr_mask); |
| 4420 | −struct ib_rwq_ind_table *ib_create_rwq_ind_table(struct ib_device *device, | |
| 4421 | − struct ib_rwq_ind_table_init_attr* | |
| 4422 | − wq_ind_table_init_attr); | |
| 4423 | 4385 | int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table); |
| 4424 | 4386 | |
| 4425 | 4387 | int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents, |
| @@ -1,35 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2005 Network Appliance, Inc. All rights reserved. |
| 3 | 4 | * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. |
| 4 | − * | |
| 5 | − * This software is available to you under a choice of one of two | |
| 6 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 7 | − * General Public License (GPL) Version 2, available from the file | |
| 8 | − * COPYING in the main directory of this source tree, or the | |
| 9 | − * OpenIB.org BSD license below: | |
| 10 | − * | |
| 11 | − * Redistribution and use in source and binary forms, with or | |
| 12 | − * without modification, are permitted provided that the following | |
| 13 | − * conditions are met: | |
| 14 | − * | |
| 15 | − * - Redistributions of source code must retain the above | |
| 16 | − * copyright notice, this list of conditions and the following | |
| 17 | − * disclaimer. | |
| 18 | − * | |
| 19 | − * - Redistributions in binary form must reproduce the above | |
| 20 | − * copyright notice, this list of conditions and the following | |
| 21 | − * disclaimer in the documentation and/or other materials | |
| 22 | − * provided with the distribution. | |
| 23 | − * | |
| 24 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 25 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 26 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 27 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 28 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 29 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 30 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 31 | − * SOFTWARE. | |
| 32 | 5 | */ |
| 6 | + | |
| 33 | 7 | #ifndef IW_CM_H |
| 34 | 8 | #define IW_CM_H |
| 35 | 9 | |
| @@ -1,35 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2014 Intel Corporation. All rights reserved. |
| 3 | 4 | * Copyright (c) 2014 Chelsio, Inc. All rights reserved. |
| 4 | − * | |
| 5 | − * This software is available to you under a choice of one of two | |
| 6 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 7 | − * General Public License (GPL) Version 2, available from the file | |
| 8 | − * COPYING in the main directory of this source tree, or the | |
| 9 | − * OpenIB.org BSD license below: | |
| 10 | − * | |
| 11 | − * Redistribution and use in source and binary forms, with or | |
| 12 | − * without modification, are permitted provided that the following | |
| 13 | − * conditions are met: | |
| 14 | − * | |
| 15 | − * - Redistributions of source code must retain the above | |
| 16 | − * copyright notice, this list of conditions and the following | |
| 17 | − * disclaimer. | |
| 18 | − * | |
| 19 | − * - Redistributions in binary form must reproduce the above | |
| 20 | − * copyright notice, this list of conditions and the following | |
| 21 | − * disclaimer in the documentation and/or other materials | |
| 22 | − * provided with the distribution. | |
| 23 | − * | |
| 24 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 25 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 26 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 27 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 28 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 29 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 30 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 31 | − * SOFTWARE. | |
| 32 | 5 | */ |
| 6 | + | |
| 33 | 7 | #ifndef _IW_PORTMAP_H |
| 34 | 8 | #define _IW_PORTMAP_H |
| 35 | 9 | |
| @@ -1,48 +1,6 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright(c) 2017 Intel Corporation. |
| 3 | − * | |
| 4 | − * This file is provided under a dual BSD/GPLv2 license. When using or | |
| 5 | − * redistributing this file, you may do so under either license. | |
| 6 | − * | |
| 7 | − * GPL LICENSE SUMMARY | |
| 8 | − * | |
| 9 | − * This program is free software; you can redistribute it and/or modify | |
| 10 | − * it under the terms of version 2 of the GNU General Public License as | |
| 11 | − * published by the Free Software Foundation. | |
| 12 | − * | |
| 13 | − * This program is distributed in the hope that it will be useful, but | |
| 14 | − * WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 15 | − * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 16 | − * General Public License for more details. | |
| 17 | − * | |
| 18 | − * BSD LICENSE | |
| 19 | − * | |
| 20 | − * Redistribution and use in source and binary forms, with or without | |
| 21 | − * modification, are permitted provided that the following conditions | |
| 22 | − * are met: | |
| 23 | − * | |
| 24 | − * - Redistributions of source code must retain the above copyright | |
| 25 | − * notice, this list of conditions and the following disclaimer. | |
| 26 | − * - Redistributions in binary form must reproduce the above copyright | |
| 27 | − * notice, this list of conditions and the following disclaimer in | |
| 28 | − * the documentation and/or other materials provided with the | |
| 29 | − * distribution. | |
| 30 | − * - Neither the name of Intel Corporation nor the names of its | |
| 31 | − * contributors may be used to endorse or promote products derived | |
| 32 | − * from this software without specific prior written permission. | |
| 33 | − * | |
| 34 | − * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 35 | − * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 36 | − * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 37 | − * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 38 | − * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 39 | − * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 40 | − * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 41 | − * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 42 | − * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 43 | − * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 44 | − * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 45 | − * | |
| 46 | 4 | */ |
| 47 | 5 | |
| 48 | 6 | #ifndef OPA_ADDR_H |
| @@ -1,36 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2014-2020 Intel Corporation. All rights reserved. |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | −#if !defined(OPA_PORT_INFO_H) | |
| 6 | +#ifndef OPA_PORT_INFO_H | |
| 34 | 7 | #define OPA_PORT_INFO_H |
| 35 | 8 | |
| 36 | 9 | #include <rdma/opa_smi.h> |
| @@ -1,36 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2014 Intel Corporation. All rights reserved. |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | −#if !defined(OPA_SMI_H) | |
| 6 | +#ifndef OPA_SMI_H | |
| 34 | 7 | #define OPA_SMI_H |
| 35 | 8 | |
| 36 | 9 | #include <rdma/ib_mad.h> |
| @@ -1,52 +1,11 @@ | ||
| 1 | −#ifndef _OPA_VNIC_H | |
| 2 | −#define _OPA_VNIC_H | |
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ | |
| 3 | 2 | /* |
| 4 | 3 | * Copyright(c) 2017 - 2020 Intel Corporation. |
| 5 | − * | |
| 6 | − * This file is provided under a dual BSD/GPLv2 license. When using or | |
| 7 | − * redistributing this file, you may do so under either license. | |
| 8 | − * | |
| 9 | − * GPL LICENSE SUMMARY | |
| 10 | − * | |
| 11 | − * This program is free software; you can redistribute it and/or modify | |
| 12 | − * it under the terms of version 2 of the GNU General Public License as | |
| 13 | − * published by the Free Software Foundation. | |
| 14 | − * | |
| 15 | − * This program is distributed in the hope that it will be useful, but | |
| 16 | − * WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 17 | − * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 18 | − * General Public License for more details. | |
| 19 | − * | |
| 20 | − * BSD LICENSE | |
| 21 | − * | |
| 22 | − * Redistribution and use in source and binary forms, with or without | |
| 23 | − * modification, are permitted provided that the following conditions | |
| 24 | − * are met: | |
| 25 | − * | |
| 26 | − * - Redistributions of source code must retain the above copyright | |
| 27 | − * notice, this list of conditions and the following disclaimer. | |
| 28 | − * - Redistributions in binary form must reproduce the above copyright | |
| 29 | − * notice, this list of conditions and the following disclaimer in | |
| 30 | − * the documentation and/or other materials provided with the | |
| 31 | − * distribution. | |
| 32 | − * - Neither the name of Intel Corporation nor the names of its | |
| 33 | − * contributors may be used to endorse or promote products derived | |
| 34 | − * from this software without specific prior written permission. | |
| 35 | − * | |
| 36 | − * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 37 | − * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 38 | − * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 39 | − * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 40 | − * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 41 | − * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 42 | − * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 43 | − * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 44 | − * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 45 | − * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 46 | − * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 47 | − * | |
| 48 | 4 | */ |
| 49 | 5 | |
| 6 | +#ifndef _OPA_VNIC_H | |
| 7 | +#define _OPA_VNIC_H | |
| 8 | + | |
| 50 | 9 | /* |
| 51 | 10 | * This file contains Intel Omni-Path (OPA) Virtual Network Interface |
| 52 | 11 | * Controller (VNIC) specific declarations. |
| @@ -1,37 +1,10 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2005 Voltaire Inc. All rights reserved. |
| 3 | 4 | * Copyright (c) 2005 Intel Corporation. All rights reserved. |
| 4 | − * | |
| 5 | − * This software is available to you under a choice of one of two | |
| 6 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 7 | − * General Public License (GPL) Version 2, available from the file | |
| 8 | − * COPYING in the main directory of this source tree, or the | |
| 9 | − * OpenIB.org BSD license below: | |
| 10 | − * | |
| 11 | − * Redistribution and use in source and binary forms, with or | |
| 12 | − * without modification, are permitted provided that the following | |
| 13 | − * conditions are met: | |
| 14 | − * | |
| 15 | − * - Redistributions of source code must retain the above | |
| 16 | − * copyright notice, this list of conditions and the following | |
| 17 | − * disclaimer. | |
| 18 | − * | |
| 19 | − * - Redistributions in binary form must reproduce the above | |
| 20 | − * copyright notice, this list of conditions and the following | |
| 21 | − * disclaimer in the documentation and/or other materials | |
| 22 | − * provided with the distribution. | |
| 23 | − * | |
| 24 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 25 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 26 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 27 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 28 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 29 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 30 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 31 | − * SOFTWARE. | |
| 32 | 5 | */ |
| 33 | 6 | |
| 34 | −#if !defined(RDMA_CM_H) | |
| 7 | +#ifndef RDMA_CM_H | |
| 35 | 8 | #define RDMA_CM_H |
| 36 | 9 | |
| 37 | 10 | #include <linux/socket.h> |
| @@ -1,36 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ | |
| 1 | 2 | /* |
| 2 | 3 | * Copyright (c) 2006 Intel Corporation. All rights reserved. |
| 3 | − * | |
| 4 | − * This software is available to you under a choice of one of two | |
| 5 | − * licenses. You may choose to be licensed under the terms of the GNU | |
| 6 | − * General Public License (GPL) Version 2, available from the file | |
| 7 | − * COPYING in the main directory of this source tree, or the | |
| 8 | − * OpenIB.org BSD license below: | |
| 9 | − * | |
| 10 | − * Redistribution and use in source and binary forms, with or | |
| 11 | − * without modification, are permitted provided that the following | |
| 12 | − * conditions are met: | |
| 13 | − * | |
| 14 | − * - Redistributions of source code must retain the above | |
| 15 | − * copyright notice, this list of conditions and the following | |
| 16 | − * disclaimer. | |
| 17 | − * | |
| 18 | − * - Redistributions in binary form must reproduce the above | |
| 19 | − * copyright notice, this list of conditions and the following | |
| 20 | − * disclaimer in the documentation and/or other materials | |
| 21 | − * provided with the distribution. | |
| 22 | − * | |
| 23 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| 24 | − * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| 25 | − * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| 26 | − * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| 27 | − * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| 28 | − * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| 29 | − * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| 30 | − * SOFTWARE. | |
| 31 | 4 | */ |
| 32 | 5 | |
| 33 | −#if !defined(RDMA_CM_IB_H) | |
| 6 | +#ifndef RDMA_CM_IB_H | |
| 34 | 7 | #define RDMA_CM_IB_H |
| 35 | 8 | |
| 36 | 9 | #include <rdma/rdma_cm.h> |
| @@ -1,8 +1,8 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | + | |
| 2 | 3 | #ifndef _RDMA_NETLINK_H |
| 3 | 4 | #define _RDMA_NETLINK_H |
| 4 | 5 | |
| 5 | − | |
| 6 | 6 | #include <linux/netlink.h> |
| 7 | 7 | #include <uapi/rdma/rdma_netlink.h> |
| 8 | 8 | |
| @@ -1,53 +1,11 @@ | ||
| 1 | −#ifndef DEF_RDMA_VT_H | |
| 2 | −#define DEF_RDMA_VT_H | |
| 3 | − | |
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ | |
| 4 | 2 | /* |
| 5 | 3 | * Copyright(c) 2016 - 2019 Intel Corporation. |
| 6 | − * | |
| 7 | − * This file is provided under a dual BSD/GPLv2 license. When using or | |
| 8 | − * redistributing this file, you may do so under either license. | |
| 9 | − * | |
| 10 | − * GPL LICENSE SUMMARY | |
| 11 | − * | |
| 12 | − * This program is free software; you can redistribute it and/or modify | |
| 13 | − * it under the terms of version 2 of the GNU General Public License as | |
| 14 | − * published by the Free Software Foundation. | |
| 15 | − * | |
| 16 | − * This program is distributed in the hope that it will be useful, but | |
| 17 | − * WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 18 | − * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 19 | − * General Public License for more details. | |
| 20 | − * | |
| 21 | − * BSD LICENSE | |
| 22 | − * | |
| 23 | − * Redistribution and use in source and binary forms, with or without | |
| 24 | − * modification, are permitted provided that the following conditions | |
| 25 | − * are met: | |
| 26 | − * | |
| 27 | − * - Redistributions of source code must retain the above copyright | |
| 28 | − * notice, this list of conditions and the following disclaimer. | |
28 more lines aren’t shown here.
This diff is large: its 61 lines aren’t shown here.
This diff is large: its 58 lines aren’t shown here.
This diff is large: its 58 lines aren’t shown here.
This diff is large: its 43 lines aren’t shown here.
This diff is large: its 35 lines aren’t shown here.
This diff is large: its 56 lines aren’t shown here.
This diff is large: its 35 lines aren’t shown here.
This diff is large: its 35 lines aren’t shown here.
This diff is large: its 43 lines aren’t shown here.
This diff is large: its 35 lines aren’t shown here.
This diff is large: its 37 lines aren’t shown here.
This diff is large: its 29 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.