History

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull rdma updates from Jason Gunthorpe: "A smaller set of patches, nothing stands out as being particularly major this cycle. The biggest item would be the new HIP09 HW support from HNS, otherwise it was pretty quiet for new work here: - Driver bug fixes and updates: bnxt_re, cxgb4, rxe, hns, i40iw, cxgb4, mlx4 and mlx5 - Bug fixes and polishing for the new rts ULP - Cleanup of uverbs checking for allowed driver operations - Use sysfs_emit all over the place - Lots of bug fixes and clarity improvements for hns - hip09 support for hns - NDR and 50/100Gb signaling rates - Remove dma_virt_ops and go back to using the IB DMA wrappers - mlx5 optimizations for contiguous DMA regions" * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (147 commits) RDMA/cma: Don't overwrite sgid_attr after device is released RDMA/mlx5: Fix MR cache memory leak RDMA/rxe: Use acquire/release for memory ordering RDMA/hns: Simplify AEQE process for different types of queue RDMA/hns: Fix inaccurate prints RDMA/hns: Fix incorrect symbol types RDMA/hns: Clear redundant variable initialization RDMA/hns: Fix coding style issues RDMA/hns: Remove unnecessary access right set during INIT2INIT RDMA/hns: WARN_ON if get a reserved sl from users RDMA/hns: Avoid filling sl in high 3 bits of vlan_id RDMA/hns: Do shift on traffic class when using RoCEv2 RDMA/hns: Normalization the judgment of some features RDMA/hns: Limit the length of data copied between kernel and userspace RDMA/mlx4: Remove bogus dev_base_lock usage RDMA/uverbs: Fix incorrect variable type RDMA/core: Do not indicate device ready when device enablement fails RDMA/core: Clean up cq pool mechanism RDMA/core: Update kernel documentation for ib_create_named_qp() MAINTAINERS: SOFT-ROCE: Change Zhu Yanjun's email address ...

Linus Torvalds committed 5 years ago

Commit 009bd552 parents 60f7c50 e246b7c175 files · +3845 −3915Browse files

Unified · Split

175 files
  1. .mailmap +1 −0
  2. MAINTAINERS +1 −1
  3. drivers/infiniband/core/cm.c +5 −4
  4. drivers/infiniband/core/cma.c +117 −78
  5. drivers/infiniband/core/cma_configfs.c +2 −2
  6. drivers/infiniband/core/core_priv.h +9 −19
  7. drivers/infiniband/core/counters.c +63 −75
  8. drivers/infiniband/core/cq.c +4 −12
  9. drivers/infiniband/core/device.c +66 −26
  10. drivers/infiniband/core/iwpm_util.h +1 −1
  11. drivers/infiniband/core/rdma_core.c +68 −33
  12. drivers/infiniband/core/restrack.c +19 −4
  13. drivers/infiniband/core/rw.c +4 −1
  14. drivers/infiniband/core/sa_query.c +2 −1
  15. drivers/infiniband/core/sysfs.c +97 −69
  16. drivers/infiniband/core/ucma.c +1 −1
  17. drivers/infiniband/core/umem.c +13 −4
  18. drivers/infiniband/core/user_mad.c +3 −3
  19. drivers/infiniband/core/uverbs_cmd.c +91 −58
  20. drivers/infiniband/core/uverbs_main.c +2 −2
  21. drivers/infiniband/core/uverbs_std_types.c +7 −11
  22. drivers/infiniband/core/uverbs_std_types_async_fd.c +2 −3
  23. drivers/infiniband/core/uverbs_std_types_counters.c +2 −3
  24. drivers/infiniband/core/uverbs_std_types_cq.c +2 −2
  25. drivers/infiniband/core/uverbs_std_types_device.c +5 −9
  26. drivers/infiniband/core/uverbs_std_types_dm.c +2 −4
  27. drivers/infiniband/core/uverbs_std_types_flow_action.c +2 −4
  28. drivers/infiniband/core/uverbs_std_types_mr.c +5 −1
  29. drivers/infiniband/core/uverbs_std_types_qp.c +4 −4
  30. drivers/infiniband/core/uverbs_std_types_srq.c +2 −2
  31. drivers/infiniband/core/uverbs_std_types_wq.c +2 −2
  32. drivers/infiniband/core/uverbs_uapi.c +1 −4
  33. drivers/infiniband/core/verbs.c +17 −10
  34. drivers/infiniband/hw/bnxt_re/ib_verbs.c +12 −3
  35. drivers/infiniband/hw/bnxt_re/main.c +3 −31
  36. drivers/infiniband/hw/bnxt_re/qplib_sp.c +1 −1
  37. drivers/infiniband/hw/cxgb4/cq.c +3 −0
  38. drivers/infiniband/hw/cxgb4/iw_cxgb4.h +0 −2
  39. drivers/infiniband/hw/cxgb4/mem.c +0 −84
  40. drivers/infiniband/hw/cxgb4/provider.c +6 −29
  41. drivers/infiniband/hw/cxgb4/qp.c +7 −1
  42. drivers/infiniband/hw/efa/efa_main.c +3 −31
  43. drivers/infiniband/hw/efa/efa_verbs.c +6 −0
  44. drivers/infiniband/hw/hfi1/qp.c +1 −0
  45. drivers/infiniband/hw/hfi1/sysfs.c +29 −33
  46. drivers/infiniband/hw/hfi1/tid_rdma.c +5 −0
  47. drivers/infiniband/hw/hns/hns_roce_ah.c +23 −32
  48. drivers/infiniband/hw/hns/hns_roce_alloc.c +76 −56
  49. drivers/infiniband/hw/hns/hns_roce_cmd.c +18 −19
  50. drivers/infiniband/hw/hns/hns_roce_cmd.h +3 −3
  51. drivers/infiniband/hw/hns/hns_roce_common.h +19 −7
  52. drivers/infiniband/hw/hns/hns_roce_cq.c +25 −21
  53. drivers/infiniband/hw/hns/hns_roce_db.c +4 −4
  54. drivers/infiniband/hw/hns/hns_roce_device.h +92 −86
  55. drivers/infiniband/hw/hns/hns_roce_hem.c +37 −22
  56. drivers/infiniband/hw/hns/hns_roce_hem.h +2 −1
  57. drivers/infiniband/hw/hns/hns_roce_hw_v1.c +18 −32
  58. drivers/infiniband/hw/hns/hns_roce_hw_v1.h +1 −1
  59. drivers/infiniband/hw/hns/hns_roce_hw_v2.c +313 −241
  60. drivers/infiniband/hw/hns/hns_roce_hw_v2.h +154 −111
  61. drivers/infiniband/hw/hns/hns_roce_main.c +37 −45
  62. drivers/infiniband/hw/hns/hns_roce_mr.c +33 −46
  63. drivers/infiniband/hw/hns/hns_roce_pd.c +7 −7
  64. drivers/infiniband/hw/hns/hns_roce_qp.c +196 −104
  65. drivers/infiniband/hw/hns/hns_roce_srq.c +31 −22
  66. drivers/infiniband/hw/i40iw/i40iw.h +0 −1
  67. drivers/infiniband/hw/i40iw/i40iw_cm.c +3 −3
  68. drivers/infiniband/hw/i40iw/i40iw_ctrl.c +11 −61
  69. drivers/infiniband/hw/i40iw/i40iw_d.h +12 −23
  70. drivers/infiniband/hw/i40iw/i40iw_status.h +0 −1
  71. drivers/infiniband/hw/i40iw/i40iw_type.h +10 −28
  72. drivers/infiniband/hw/i40iw/i40iw_uk.c +4 −37
  73. drivers/infiniband/hw/i40iw/i40iw_user.h +0 −8
  74. drivers/infiniband/hw/i40iw/i40iw_verbs.c +11 −110
  75. drivers/infiniband/hw/mlx4/mad.c +1 −0
  76. drivers/infiniband/hw/mlx4/main.c +7 −57
  77. drivers/infiniband/hw/mlx4/mcg.c +46 −36
  78. drivers/infiniband/hw/mlx4/mlx4_ib.h +4 −4
  79. drivers/infiniband/hw/mlx4/mr.c +8 −8
  80. drivers/infiniband/hw/mlx4/qp.c +12 −2
  81. drivers/infiniband/hw/mlx4/srq.c +4 −0
  82. drivers/infiniband/hw/mlx4/sysfs.c +28 −38
  83. drivers/infiniband/hw/mlx5/cq.c +49 −28
  84. drivers/infiniband/hw/mlx5/devx.c +43 −47
  85. drivers/infiniband/hw/mlx5/fs.c +2 −4
  86. drivers/infiniband/hw/mlx5/main.c +24 −81
  87. drivers/infiniband/hw/mlx5/mem.c +48 −144
  88. drivers/infiniband/hw/mlx5/mlx5_ib.h +81 −21
  89. drivers/infiniband/hw/mlx5/mr.c +571 −389
  90. drivers/infiniband/hw/mlx5/odp.c +29 −27
  91. drivers/infiniband/hw/mlx5/qp.c +89 −108
  92. drivers/infiniband/hw/mlx5/restrack.c +1 −1
  93. drivers/infiniband/hw/mlx5/srq.c +7 −27
  94. drivers/infiniband/hw/mlx5/srq.h +1 −0
  95. drivers/infiniband/hw/mlx5/srq_cmd.c +76 −4
  96. drivers/infiniband/hw/mthca/mthca_cq.c +1 −1
  97. drivers/infiniband/hw/mthca/mthca_dev.h +0 −1
  98. drivers/infiniband/hw/mthca/mthca_provider.c +21 −40
  99. drivers/infiniband/hw/mthca/mthca_qp.c +3 −0
  100. drivers/infiniband/hw/ocrdma/ocrdma_main.c +5 −37
  101. drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +10 −1
  102. drivers/infiniband/hw/qedr/main.c +5 −34
  103. drivers/infiniband/hw/qedr/verbs.c +13 −0
  104. drivers/infiniband/hw/qib/qib_pcie.c +2 −9
  105. drivers/infiniband/hw/qib/qib_sysfs.c +43 −53
  106. drivers/infiniband/hw/usnic/usnic_ib_main.c +0 −19
  107. drivers/infiniband/hw/usnic/usnic_ib_sysfs.c +40 −60
  108. drivers/infiniband/hw/usnic/usnic_ib_verbs.c +5 −2
  109. drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +3 −0
  110. drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +3 −31
  111. drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +4 −1
  112. drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c +1 −1
  113. drivers/infiniband/sw/rdmavt/Kconfig +0 −1
  114. drivers/infiniband/sw/rdmavt/ah.c +1 −2
  115. drivers/infiniband/sw/rdmavt/cq.c +1 −1
  116. drivers/infiniband/sw/rdmavt/mcast.c +6 −6
  117. drivers/infiniband/sw/rdmavt/mr.c +2 −4
  118. drivers/infiniband/sw/rdmavt/qp.c +11 −7
  119. drivers/infiniband/sw/rdmavt/vt.c +2 −34
  120. drivers/infiniband/sw/rxe/Kconfig +0 −1
  121. drivers/infiniband/sw/rxe/rxe_cq.c +0 −5
  122. drivers/infiniband/sw/rxe/rxe_mr.c +0 −1
  123. drivers/infiniband/sw/rxe/rxe_net.c +0 −18
  124. drivers/infiniband/sw/rxe/rxe_queue.h +60 −34
  125. drivers/infiniband/sw/rxe/rxe_req.c +2 −1
  126. drivers/infiniband/sw/rxe/rxe_resp.c +0 −5
  127. drivers/infiniband/sw/rxe/rxe_verbs.c +14 −53
  128. drivers/infiniband/sw/rxe/rxe_verbs.h +0 −2
  129. drivers/infiniband/sw/siw/Kconfig +0 −1
  130. drivers/infiniband/sw/siw/siw.h +0 −1
  131. drivers/infiniband/sw/siw/siw_cm.c +1 −1
  132. drivers/infiniband/sw/siw/siw_main.c +3 −49
  133. drivers/infiniband/sw/siw/siw_verbs.c +12 −0
  134. drivers/infiniband/ulp/ipoib/ipoib_cm.c +2 −2
  135. drivers/infiniband/ulp/ipoib/ipoib_ethtool.c +4 −0
  136. drivers/infiniband/ulp/ipoib/ipoib_main.c +4 −3
  137. drivers/infiniband/ulp/ipoib/ipoib_verbs.c +2 −2
  138. drivers/infiniband/ulp/ipoib/ipoib_vlan.c +1 −1
  139. drivers/infiniband/ulp/iser/iscsi_iser.c +6 −18
  140. drivers/infiniband/ulp/isert/ib_isert.c +27 −2
  141. drivers/infiniband/ulp/isert/ib_isert.h +6 −0
  142. drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h +1 −1
  143. drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c +1 −1
  144. drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c +31 −31
  145. drivers/infiniband/ulp/rtrs/rtrs-clt.c +36 −38
  146. drivers/infiniband/ulp/rtrs/rtrs-clt.h +1 −0
  147. drivers/infiniband/ulp/rtrs/rtrs-pri.h +1 −2
  148. drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c +9 −12
  149. drivers/infiniband/ulp/rtrs/rtrs-srv.c +46 −73
  150. drivers/infiniband/ulp/rtrs/rtrs-srv.h +1 −1
  151. drivers/infiniband/ulp/rtrs/rtrs.c +23 −38
  152. drivers/infiniband/ulp/srp/ib_srp.c +26 −22
  153. drivers/infiniband/ulp/srpt/ib_srpt.c +7 −7
  154. drivers/infiniband/ulp/srpt/ib_srpt.h +1 −1
  155. drivers/nvme/host/rdma.c +1 −1
  156. drivers/nvme/target/rdma.c +2 −1
  157. drivers/pci/p2pdma.c +1 −24
  158. include/linux/dma-mapping.h +0 −2
  159. include/rdma/ib_umem.h +42 −0
  160. include/rdma/ib_verbs.h +97 −101
  161. include/rdma/restrack.h +24 −0
  162. include/rdma/uverbs_ioctl.h +19 −6
  163. include/rdma/uverbs_types.h +7 −2
  164. include/uapi/rdma/hns-abi.h +10 −0
  165. include/uapi/rdma/ib_user_verbs.h +0 −14
  166. include/uapi/rdma/rdma_user_rxe.h +21 −0
  167. kernel/dma/Kconfig +0 −5
  168. kernel/dma/Makefile +0 −1
  169. kernel/dma/virt.c +0 −61
  170. net/rds/ib.c +0 −10
  171. net/rds/ib.h +0 −13
  172. net/rds/ib_cm.c +78 −50
  173. net/rds/ib_recv.c +15 −3
  174. net/rds/ib_send.c +8 −0
  175. tools/testing/scatterlist/main.c +41 −23
.mailmap+1−0
@@ -345,3 +345,4 @@ Wolfram Sang <wsa@kernel.org> <w.sang@pengutronix.de> · 344 unmodified lines
345345 Wolfram Sang <wsa@kernel.org> <wsa@the-dreams.de>
346346 Yakir Yang <kuankuan.y@gmail.com> <ykk@rock-chips.com>
347347 Yusuke Goda <goda.yusuke@renesas.com>
348+Zhu Yanjun <zyjzyj2000@gmail.com> <yanjunz@nvidia.com>
MAINTAINERS+1−1
@@ -16399,7 +16399,7 @@ F: drivers/infiniband/sw/siw/ · 16398 unmodified lines
1639916399 F: include/uapi/rdma/siw-abi.h
1640016400
1640116401 SOFT-ROCE DRIVER (rxe)
16402−M: Zhu Yanjun <yanjunz@nvidia.com>
16402+M: Zhu Yanjun <zyjzyj2000@gmail.com>
1640316403 L: linux-rdma@vger.kernel.org
1640416404 S: Supported
1640516405 F: drivers/infiniband/sw/rxe/
drivers/infiniband/core/cm.c+5−4
@@ -1251,7 +1251,8 @@ out: · 1250 unmodified lines
12511251 EXPORT_SYMBOL(ib_cm_listen);
12521252
12531253 /**
1254− * Create a new listening ib_cm_id and listen on the given service ID.
1254+ * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1255+ * the given service ID.
12551256 *
12561257 * If there's an existing ID listening on that same device and service ID,
12571258 * return it.
@@ -1765,7 +1766,7 @@ static u16 cm_get_bth_pkey(struct cm_work *work) · 507 unmodified lines
17651766 }
17661767
17671768 /**
1768− * Convert OPA SGID to IB SGID
1769+ * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
17691770 * ULPs (such as IPoIB) do not understand OPA GIDs and will
17701771 * reject them as the local_gid will not match the sgid. Therefore,
17711772 * change the pathrec's SGID to an IB SGID.
@@ -4273,8 +4274,8 @@ static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr, · 2501 unmodified lines
42734274 group = container_of(obj, struct cm_counter_group, obj);
42744275 cm_attr = container_of(attr, struct cm_counter_attribute, attr);
42754276
4276− return sprintf(buf, "%ld\n",
4277− atomic_long_read(&group->counter[cm_attr->index]));
4277+ return sysfs_emit(buf, "%ld\n",
4278+ atomic_long_read(&group->counter[cm_attr->index]));
42784279 }
42794280
42804281 static const struct sysfs_ops cm_counter_ops = {
drivers/infiniband/core/cma.c+117−78
@@ -477,6 +477,10 @@ static void cma_release_dev(struct rdma_id_private *id_priv) · 476 unmodified lines
477477 list_del(&id_priv->list);
478478 cma_dev_put(id_priv->cma_dev);
479479 id_priv->cma_dev = NULL;
480+ if (id_priv->id.route.addr.dev_addr.sgid_attr) {
481+ rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr);
482+ id_priv->id.route.addr.dev_addr.sgid_attr = NULL;
483+ }
480484 mutex_unlock(&lock);
481485 }
482486
@@ -1861,9 +1865,6 @@ static void _destroy_id(struct rdma_id_private *id_priv, · 1378 unmodified lines
18611865
18621866 kfree(id_priv->id.route.path_rec);
18631867
1864− if (id_priv->id.route.addr.dev_addr.sgid_attr)
1865− rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr);
1866−
18671868 put_net(id_priv->id.route.addr.dev_addr.net);
18681869 rdma_restrack_del(&id_priv->res);
18691870 kfree(id_priv);
@@ -2495,8 +2496,9 @@ static int cma_listen_handler(struct rdma_cm_id *id, · 625 unmodified lines
24952496 return id_priv->id.event_handler(id, event);
24962497 }
24972498
2498−static void cma_listen_on_dev(struct rdma_id_private *id_priv,
2499− struct cma_device *cma_dev)
2499+static int cma_listen_on_dev(struct rdma_id_private *id_priv,
2500+ struct cma_device *cma_dev,
2501+ struct rdma_id_private **to_destroy)
25002502 {
25012503 struct rdma_id_private *dev_id_priv;
25022504 struct net *net = id_priv->id.route.addr.dev_addr.net;
@@ -2504,21 +2506,21 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv, · 1 unmodified line
25042506
25052507 lockdep_assert_held(&lock);
25062508
2509+ *to_destroy = NULL;
25072510 if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))
2508− return;
2511+ return 0;
25092512
25102513 dev_id_priv =
25112514 __rdma_create_id(net, cma_listen_handler, id_priv,
25122515 id_priv->id.ps, id_priv->id.qp_type, id_priv);
25132516 if (IS_ERR(dev_id_priv))
2514− return;
2517+ return PTR_ERR(dev_id_priv);
25152518
25162519 dev_id_priv->state = RDMA_CM_ADDR_BOUND;
25172520 memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
25182521 rdma_addr_size(cma_src_addr(id_priv)));
25192522
25202523 _cma_attach_to_dev(dev_id_priv, cma_dev);
2521− list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
25222524 cma_id_get(id_priv);
25232525 dev_id_priv->internal_id = 1;
25242526 dev_id_priv->afonly = id_priv->afonly;
@@ -2527,19 +2529,42 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv, · 2 unmodified lines
25272529
25282530 ret = rdma_listen(&dev_id_priv->id, id_priv->backlog);
25292531 if (ret)
2530− dev_warn(&cma_dev->device->dev,
2531− "RDMA CMA: cma_listen_on_dev, error %d\n", ret);
2532+ goto err_listen;
2533+ list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
2534+ return 0;
2535+err_listen:
2536+ /* Caller must destroy this after releasing lock */
2537+ *to_destroy = dev_id_priv;
2538+ dev_warn(&cma_dev->device->dev, "RDMA CMA: %s, error %d\n", __func__, ret);
2539+ return ret;
25322540 }
25332541
2534−static void cma_listen_on_all(struct rdma_id_private *id_priv)
2542+static int cma_listen_on_all(struct rdma_id_private *id_priv)
25352543 {
2544+ struct rdma_id_private *to_destroy;
25362545 struct cma_device *cma_dev;
2546+ int ret;
25372547
25382548 mutex_lock(&lock);
25392549 list_add_tail(&id_priv->list, &listen_any_list);
2540− list_for_each_entry(cma_dev, &dev_list, list)
2541− cma_listen_on_dev(id_priv, cma_dev);
2550+ list_for_each_entry(cma_dev, &dev_list, list) {
2551+ ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
2552+ if (ret) {
2553+ /* Prevent racing with cma_process_remove() */
2554+ if (to_destroy)
2555+ list_del_init(&to_destroy->list);
2556+ goto err_listen;
2557+ }
2558+ }
25422559 mutex_unlock(&lock);
2560+ return 0;
2561+
2562+err_listen:
2563+ list_del(&id_priv->list);
2564+ mutex_unlock(&lock);
2565+ if (to_destroy)
2566+ rdma_destroy_id(&to_destroy->id);
2567+ return ret;
25432568 }
25442569
25452570 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
@@ -3692,8 +3717,11 @@ int rdma_listen(struct rdma_cm_id *id, int backlog) · 1146 unmodified lines
36923717 ret = -ENOSYS;
36933718 goto err;
36943719 }
3695− } else
3696− cma_listen_on_all(id_priv);
3720+ } else {
3721+ ret = cma_listen_on_all(id_priv);
3722+ if (ret)
3723+ goto err;
3724+ }
36973725
36983726 return 0;
36993727 err:
@@ -4773,69 +4801,6 @@ static struct notifier_block cma_nb = { · 1073 unmodified lines
47734801 .notifier_call = cma_netdev_callback
47744802 };
47754803
4776−static int cma_add_one(struct ib_device *device)
4777−{
4778− struct cma_device *cma_dev;
4779− struct rdma_id_private *id_priv;
4780− unsigned int i;
4781− unsigned long supported_gids = 0;
4782− int ret;
4783−
4784− cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
4785− if (!cma_dev)
4786− return -ENOMEM;
4787−
4788− cma_dev->device = device;
4789− cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
4790− sizeof(*cma_dev->default_gid_type),
4791− GFP_KERNEL);
4792− if (!cma_dev->default_gid_type) {
4793− ret = -ENOMEM;
4794− goto free_cma_dev;
4795− }
4796−
4797− cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
4798− sizeof(*cma_dev->default_roce_tos),
4799− GFP_KERNEL);
4800− if (!cma_dev->default_roce_tos) {
4801− ret = -ENOMEM;
4802− goto free_gid_type;
4803− }
4804−
4805− rdma_for_each_port (device, i) {
4806− supported_gids = roce_gid_type_mask_support(device, i);
4807− WARN_ON(!supported_gids);
4808− if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
4809− cma_dev->default_gid_type[i - rdma_start_port(device)] =
4810− CMA_PREFERRED_ROCE_GID_TYPE;
4811− else
4812− cma_dev->default_gid_type[i - rdma_start_port(device)] =
4813− find_first_bit(&supported_gids, BITS_PER_LONG);
4814− cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
4815− }
4816−
4817− init_completion(&cma_dev->comp);
4818− refcount_set(&cma_dev->refcount, 1);
4819− INIT_LIST_HEAD(&cma_dev->id_list);
4820− ib_set_client_data(device, &cma_client, cma_dev);
4821−
4822− mutex_lock(&lock);
4823− list_add_tail(&cma_dev->list, &dev_list);
4824− list_for_each_entry(id_priv, &listen_any_list, list)
4825− cma_listen_on_dev(id_priv, cma_dev);
4826− mutex_unlock(&lock);
4827−
4828− trace_cm_add_one(device);
4829− return 0;
4830−
4831−free_gid_type:
4832− kfree(cma_dev->default_gid_type);
4833−
4834−free_cma_dev:
4835− kfree(cma_dev);
4836− return ret;
4837−}
4838−
48394804 static void cma_send_device_removal_put(struct rdma_id_private *id_priv)
48404805 {
48414806 struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };
@@ -4898,6 +4863,80 @@ static void cma_process_remove(struct cma_device *cma_dev) · 56 unmodified lines
48984863 wait_for_completion(&cma_dev->comp);
48994864 }
49004865
4866+static int cma_add_one(struct ib_device *device)
4867+{
4868+ struct rdma_id_private *to_destroy;
4869+ struct cma_device *cma_dev;
4870+ struct rdma_id_private *id_priv;
4871+ unsigned int i;
4872+ unsigned long supported_gids = 0;
4873+ int ret;
4874+
4875+ cma_dev = kmalloc(sizeof(*cma_dev), GFP_KERNEL);
4876+ if (!cma_dev)
4877+ return -ENOMEM;
4878+
4879+ cma_dev->device = device;
4880+ cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
4881+ sizeof(*cma_dev->default_gid_type),
4882+ GFP_KERNEL);
4883+ if (!cma_dev->default_gid_type) {
4884+ ret = -ENOMEM;
4885+ goto free_cma_dev;
4886+ }
4887+
4888+ cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
4889+ sizeof(*cma_dev->default_roce_tos),
4890+ GFP_KERNEL);
4891+ if (!cma_dev->default_roce_tos) {
4892+ ret = -ENOMEM;
4893+ goto free_gid_type;
4894+ }
4895+
4896+ rdma_for_each_port (device, i) {
4897+ supported_gids = roce_gid_type_mask_support(device, i);
4898+ WARN_ON(!supported_gids);
4899+ if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
4900+ cma_dev->default_gid_type[i - rdma_start_port(device)] =
4901+ CMA_PREFERRED_ROCE_GID_TYPE;
4902+ else
4903+ cma_dev->default_gid_type[i - rdma_start_port(device)] =
4904+ find_first_bit(&supported_gids, BITS_PER_LONG);
4905+ cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
4906+ }
4907+
4908+ init_completion(&cma_dev->comp);
4909+ refcount_set(&cma_dev->refcount, 1);
4910+ INIT_LIST_HEAD(&cma_dev->id_list);
4911+ ib_set_client_data(device, &cma_client, cma_dev);
4912+
4913+ mutex_lock(&lock);
4914+ list_add_tail(&cma_dev->list, &dev_list);
4915+ list_for_each_entry(id_priv, &listen_any_list, list) {
4916+ ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
4917+ if (ret)
4918+ goto free_listen;
4919+ }
4920+ mutex_unlock(&lock);
4921+
4922+ trace_cm_add_one(device);
4923+ return 0;
4924+
4925+free_listen:
4926+ list_del(&cma_dev->list);
4927+ mutex_unlock(&lock);
4928+
4929+ /* cma_process_remove() will delete to_destroy */
4930+ cma_process_remove(cma_dev);
4931+ kfree(cma_dev->default_roce_tos);
4932+free_gid_type:
4933+ kfree(cma_dev->default_gid_type);
4934+
4935+free_cma_dev:
4936+ kfree(cma_dev);
4937+ return ret;
4938+}
4939+
49014940 static void cma_remove_one(struct ib_device *device, void *client_data)
49024941 {
49034942 struct cma_device *cma_dev = client_data;
drivers/infiniband/core/cma_configfs.c+2−2
@@ -115,7 +115,7 @@ static ssize_t default_roce_mode_show(struct config_item *item, · 114 unmodified lines
115115 if (gid_type < 0)
116116 return gid_type;
117117
118− return sprintf(buf, "%s\n", ib_cache_gid_type_str(gid_type));
118+ return sysfs_emit(buf, "%s\n", ib_cache_gid_type_str(gid_type));
119119 }
120120
121121 static ssize_t default_roce_mode_store(struct config_item *item,
@@ -157,7 +157,7 @@ static ssize_t default_roce_tos_show(struct config_item *item, char *buf) · 35 unmodified lines
157157 tos = cma_get_default_roce_tos(cma_dev, group->port_num);
158158 cma_configfs_params_put(cma_dev);
159159
160− return sprintf(buf, "%u\n", tos);
160+ return sysfs_emit(buf, "%u\n", tos);
161161 }
162162
163163 static ssize_t default_roce_tos_store(struct config_item *item,
drivers/infiniband/core/core_priv.h+9−19
@@ -318,15 +318,12 @@ struct ib_device *ib_device_get_by_index(const struct net *net, u32 index); · 317 unmodified lines
318318 void nldev_init(void);
319319 void nldev_exit(void);
320320
321−static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,
322− struct ib_pd *pd,
323− struct ib_qp_init_attr *attr,
324− struct ib_udata *udata,
325− struct ib_uqp_object *uobj)
321+static inline struct ib_qp *
322+_ib_create_qp(struct ib_device *dev, struct ib_pd *pd,
323+ struct ib_qp_init_attr *attr, struct ib_udata *udata,
324+ struct ib_uqp_object *uobj, const char *caller)
326325 {
327− enum ib_qp_type qp_type = attr->qp_type;
328326 struct ib_qp *qp;
329− bool is_xrc;
330327
331328 if (!dev->ops.create_qp)
332329 return ERR_PTR(-EOPNOTSUPP);
@@ -347,6 +344,7 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev, · 14 unmodified lines
347344 qp->srq = attr->srq;
348345 qp->rwq_ind_tbl = attr->rwq_ind_tbl;
349346 qp->event_handler = attr->event_handler;
347+ qp->port = attr->port_num;
350348
351349 atomic_set(&qp->usecnt, 0);
352350 spin_lock_init(&qp->mr_lock);
@@ -354,16 +352,9 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev, · 1 unmodified line
354352 INIT_LIST_HEAD(&qp->sig_mrs);
355353
356354 rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP);
357− /*
358− * We don't track XRC QPs for now, because they don't have PD
359− * and more importantly they are created internaly by driver,
360− * see mlx5 create_dev_resources() as an example.
361− */
362− is_xrc = qp_type == IB_QPT_XRC_INI || qp_type == IB_QPT_XRC_TGT;
363− if ((qp_type < IB_QPT_MAX && !is_xrc) || qp_type == IB_QPT_DRIVER) {
364− rdma_restrack_parent_name(&qp->res, &pd->res);
365− rdma_restrack_add(&qp->res);
366− }
355+ WARN_ONCE(!udata && !caller, "Missing kernel QP owner");
356+ rdma_restrack_set_name(&qp->res, udata ? NULL : caller);
357+ rdma_restrack_add(&qp->res);
367358 return qp;
368359 }
369360
@@ -411,7 +402,6 @@ void rdma_umap_priv_init(struct rdma_umap_priv *priv, · 41 unmodified lines
411402 struct vm_area_struct *vma,
412403 struct rdma_user_mmap_entry *entry);
413404
414−void ib_cq_pool_init(struct ib_device *dev);
415−void ib_cq_pool_destroy(struct ib_device *dev);
405+void ib_cq_pool_cleanup(struct ib_device *dev);
416406
417407 #endif /* _CORE_PRIV_H */
drivers/infiniband/core/counters.c+63−75
@@ -64,8 +64,40 @@ out: · 63 unmodified lines
6464 return ret;
6565 }
6666
67−static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port,
68− enum rdma_nl_counter_mode mode)
67+static void auto_mode_init_counter(struct rdma_counter *counter,
68+ const struct ib_qp *qp,
69+ enum rdma_nl_counter_mask new_mask)
70+{
71+ struct auto_mode_param *param = &counter->mode.param;
72+
73+ counter->mode.mode = RDMA_COUNTER_MODE_AUTO;
74+ counter->mode.mask = new_mask;
75+
76+ if (new_mask & RDMA_COUNTER_MASK_QP_TYPE)
77+ param->qp_type = qp->qp_type;
78+}
79+
80+static int __rdma_counter_bind_qp(struct rdma_counter *counter,
81+ struct ib_qp *qp)
82+{
83+ int ret;
84+
85+ if (qp->counter)
86+ return -EINVAL;
87+
88+ if (!qp->device->ops.counter_bind_qp)
89+ return -EOPNOTSUPP;
90+
91+ mutex_lock(&counter->lock);
92+ ret = qp->device->ops.counter_bind_qp(counter, qp);
93+ mutex_unlock(&counter->lock);
94+
95+ return ret;
96+}
97+
98+static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u8 port,
99+ struct ib_qp *qp,
100+ enum rdma_nl_counter_mode mode)
69101 {
70102 struct rdma_port_counter *port_counter;
71103 struct rdma_counter *counter;
@@ -88,11 +120,22 @@ static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port, · 16 unmodified lines
88120
89121 port_counter = &dev->port_data[port].port_counter;
90122 mutex_lock(&port_counter->lock);
91− if (mode == RDMA_COUNTER_MODE_MANUAL) {
123+ switch (mode) {
124+ case RDMA_COUNTER_MODE_MANUAL:
92125 ret = __counter_set_mode(&port_counter->mode,
93126 RDMA_COUNTER_MODE_MANUAL, 0);
94− if (ret)
127+ if (ret) {
128+ mutex_unlock(&port_counter->lock);
95129 goto err_mode;
130+ }
131+ break;
132+ case RDMA_COUNTER_MODE_AUTO:
133+ auto_mode_init_counter(counter, qp, port_counter->mode.mask);
134+ break;
135+ default:
136+ ret = -EOPNOTSUPP;
137+ mutex_unlock(&port_counter->lock);
138+ goto err_mode;
96139 }
97140
98141 port_counter->num_counters++;
@@ -102,10 +145,15 @@ static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port, · 3 unmodified lines
102145 kref_init(&counter->kref);
103146 mutex_init(&counter->lock);
104147
148+ ret = __rdma_counter_bind_qp(counter, qp);
149+ if (ret)
150+ goto err_mode;
151+
152+ rdma_restrack_parent_name(&counter->res, &qp->res);
153+ rdma_restrack_add(&counter->res);
105154 return counter;
106155
107156 err_mode:
108− mutex_unlock(&port_counter->lock);
109157 kfree(counter->stats);
110158 err_stats:
111159 rdma_restrack_put(&counter->res);
@@ -132,19 +180,6 @@ static void rdma_counter_free(struct rdma_counter *counter) · 20 unmodified lines
132180 kfree(counter);
133181 }
134182
135−static void auto_mode_init_counter(struct rdma_counter *counter,
136− const struct ib_qp *qp,
137− enum rdma_nl_counter_mask new_mask)
138−{
139− struct auto_mode_param *param = &counter->mode.param;
140−
141− counter->mode.mode = RDMA_COUNTER_MODE_AUTO;
142− counter->mode.mask = new_mask;
143−
144− if (new_mask & RDMA_COUNTER_MASK_QP_TYPE)
145− param->qp_type = qp->qp_type;
146−}
147−
148183 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,
149184 enum rdma_nl_counter_mask auto_mask)
150185 {
@@ -161,24 +196,6 @@ static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter, · 10 unmodified lines
161196 return match;
162197 }
163198
164−static int __rdma_counter_bind_qp(struct rdma_counter *counter,
165− struct ib_qp *qp)
166−{
167− int ret;
168−
169− if (qp->counter)
170− return -EINVAL;
171−
172− if (!qp->device->ops.counter_bind_qp)
173− return -EOPNOTSUPP;
174−
175− mutex_lock(&counter->lock);
176− ret = qp->device->ops.counter_bind_qp(counter, qp);
177− mutex_unlock(&counter->lock);
178−
179− return ret;
180−}
181−
182199 static int __rdma_counter_unbind_qp(struct ib_qp *qp)
183200 {
184201 struct rdma_counter *counter = qp->counter;
@@ -247,13 +264,6 @@ next: · 62 unmodified lines
247264 return counter;
248265 }
249266
250−static void rdma_counter_res_add(struct rdma_counter *counter,
251− struct ib_qp *qp)
252−{
253− rdma_restrack_parent_name(&counter->res, &qp->res);
254− rdma_restrack_add(&counter->res);
255−}
256−
257267 static void counter_release(struct kref *kref)
258268 {
259269 struct rdma_counter *counter;
@@ -275,7 +285,7 @@ int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port) · 15 unmodified lines
275285 struct rdma_counter *counter;
276286 int ret;
277287
278− if (!qp->res.valid || rdma_is_kernel_res(&qp->res))
288+ if (!rdma_restrack_is_tracked(&qp->res) || rdma_is_kernel_res(&qp->res))
279289 return 0;
280290
281291 if (!rdma_is_port_valid(dev, port))
@@ -293,19 +303,9 @@ int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port) · 11 unmodified lines
293303 return ret;
294304 }
295305 } else {
296− counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_AUTO);
306+ counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO);
297307 if (!counter)
298308 return -ENOMEM;
299−
300− auto_mode_init_counter(counter, qp, port_counter->mode.mask);
301−
302− ret = __rdma_counter_bind_qp(counter, qp);
303− if (ret) {
304− rdma_counter_free(counter);
305− return ret;
306− }
307−
308− rdma_counter_res_add(counter, qp);
309309 }
310310
311311 return 0;
@@ -419,15 +419,6 @@ err: · 107 unmodified lines
419419 return NULL;
420420 }
421421
422−static int rdma_counter_bind_qp_manual(struct rdma_counter *counter,
423− struct ib_qp *qp)
424−{
425− if ((counter->device != qp->device) || (counter->port != qp->port))
426− return -EINVAL;
427−
428− return __rdma_counter_bind_qp(counter, qp);
429−}
430−
431422 static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev,
432423 u32 counter_id)
433424 {
@@ -475,7 +466,12 @@ int rdma_counter_bind_qpn(struct ib_device *dev, u8 port, · 41 unmodified lines
475466 goto err_task;
476467 }
477468
478− ret = rdma_counter_bind_qp_manual(counter, qp);
469+ if ((counter->device != qp->device) || (counter->port != qp->port)) {
470+ ret = -EINVAL;
471+ goto err_task;
472+ }
473+
474+ ret = __rdma_counter_bind_qp(counter, qp);
479475 if (ret)
480476 goto err_task;
481477
@@ -520,26 +516,18 @@ int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u8 port, · 38 unmodified lines
520516 goto err;
521517 }
522518
523− counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_MANUAL);
519+ counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL);
524520 if (!counter) {
525521 ret = -ENOMEM;
526522 goto err;
527523 }
528524
529− ret = rdma_counter_bind_qp_manual(counter, qp);
530− if (ret)
531− goto err_bind;
532−
533525 if (counter_id)
534526 *counter_id = counter->id;
535527
536− rdma_counter_res_add(counter, qp);
537−
538528 rdma_restrack_put(&qp->res);
539− return ret;
529+ return 0;
540530
541−err_bind:
542− rdma_counter_free(counter);
543531 err:
544532 rdma_restrack_put(&qp->res);
545533 return ret;
drivers/infiniband/core/cq.c+4−12
@@ -123,7 +123,7 @@ static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs, · 122 unmodified lines
123123 }
124124
125125 /**
126− * ib_process_direct_cq - process a CQ in caller context
126+ * ib_process_cq_direct - process a CQ in caller context
127127 * @cq: CQ to process
128128 * @budget: number of CQEs to poll for
129129 *
@@ -197,7 +197,7 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private) · 67 unmodified lines
197197 }
198198
199199 /**
200− * __ib_alloc_cq allocate a completion queue
200+ * __ib_alloc_cq - allocate a completion queue
201201 * @dev: device to allocate the CQ for
202202 * @private: driver private data, accessible from cq->cq_context
203203 * @nr_cqe: number of CQEs to allocate
@@ -349,16 +349,7 @@ void ib_free_cq(struct ib_cq *cq) · 145 unmodified lines
349349 }
350350 EXPORT_SYMBOL(ib_free_cq);
351351
352−void ib_cq_pool_init(struct ib_device *dev)
353−{
354− unsigned int i;
355−
356− spin_lock_init(&dev->cq_pools_lock);
357− for (i = 0; i < ARRAY_SIZE(dev->cq_pools); i++)
358− INIT_LIST_HEAD(&dev->cq_pools[i]);
359−}
360−
361−void ib_cq_pool_destroy(struct ib_device *dev)
352+void ib_cq_pool_cleanup(struct ib_device *dev)
362353 {
363354 struct ib_cq *cq, *n;
364355 unsigned int i;
@@ -367,6 +358,7 @@ void ib_cq_pool_destroy(struct ib_device *dev) · 2 unmodified lines
367358 list_for_each_entry_safe(cq, n, &dev->cq_pools[i],
368359 pool_entry) {
369360 WARN_ON(cq->cqe_used);
361+ list_del(&cq->pool_entry);
370362 cq->shared = false;
371363 ib_free_cq(cq);
372364 }
drivers/infiniband/core/device.c+66−26
@@ -284,6 +284,7 @@ static void ib_device_check_mandatory(struct ib_device *device) · 283 unmodified lines
284284 IB_MANDATORY_FUNC(poll_cq),
285285 IB_MANDATORY_FUNC(req_notify_cq),
286286 IB_MANDATORY_FUNC(get_dma_mr),
287+ IB_MANDATORY_FUNC(reg_user_mr),
287288 IB_MANDATORY_FUNC(dereg_mr),
288289 IB_MANDATORY_FUNC(get_port_immutable)
289290 };
@@ -569,6 +570,7 @@ static void rdma_init_coredev(struct ib_core_device *coredev, · 279 unmodified lines
569570 struct ib_device *_ib_alloc_device(size_t size)
570571 {
571572 struct ib_device *device;
573+ unsigned int i;
572574
573575 if (WARN_ON(size < sizeof(struct ib_device)))
574576 return NULL;
@@ -600,6 +602,41 @@ struct ib_device *_ib_alloc_device(size_t size) · 25 unmodified lines
600602 init_completion(&device->unreg_completion);
601603 INIT_WORK(&device->unregistration_work, ib_unregister_work);
602604
605+ spin_lock_init(&device->cq_pools_lock);
606+ for (i = 0; i < ARRAY_SIZE(device->cq_pools); i++)
607+ INIT_LIST_HEAD(&device->cq_pools[i]);
608+
609+ device->uverbs_cmd_mask =
610+ BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) |
611+ BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) |
612+ BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) |
613+ BIT_ULL(IB_USER_VERBS_CMD_CLOSE_XRCD) |
614+ BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) |
615+ BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
616+ BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) |
617+ BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) |
618+ BIT_ULL(IB_USER_VERBS_CMD_CREATE_SRQ) |
619+ BIT_ULL(IB_USER_VERBS_CMD_CREATE_XSRQ) |
620+ BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) |
621+ BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) |
622+ BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) |
623+ BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) |
624+ BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) |
625+ BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |
626+ BIT_ULL(IB_USER_VERBS_CMD_DESTROY_SRQ) |
627+ BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST) |
628+ BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) |
629+ BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) |
630+ BIT_ULL(IB_USER_VERBS_CMD_MODIFY_SRQ) |
631+ BIT_ULL(IB_USER_VERBS_CMD_OPEN_QP) |
632+ BIT_ULL(IB_USER_VERBS_CMD_OPEN_XRCD) |
633+ BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) |
634+ BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) |
635+ BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |
636+ BIT_ULL(IB_USER_VERBS_CMD_QUERY_SRQ) |
637+ BIT_ULL(IB_USER_VERBS_CMD_REG_MR) |
638+ BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) |
639+ BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ);
603640 return device;
604641 }
605642 EXPORT_SYMBOL(_ib_alloc_device);
@@ -1177,25 +1214,6 @@ out: · 571 unmodified lines
11771214 return ret;
11781215 }
11791216
1180−static void setup_dma_device(struct ib_device *device,
1181− struct device *dma_device)
1182−{
1183− /*
1184− * If the caller does not provide a DMA capable device then the IB
1185− * device will be used. In this case the caller should fully setup the
1186− * ibdev for DMA. This usually means using dma_virt_ops.
1187− */
1188−#ifdef CONFIG_DMA_VIRT_OPS
1189− if (!dma_device) {
1190− device->dev.dma_ops = &dma_virt_ops;
1191− dma_device = &device->dev;
1192− }
1193−#endif
1194− WARN_ON(!dma_device);
1195− device->dma_device = dma_device;
1196− WARN_ON(!device->dma_device->dma_parms);
1197−}
1198−
11991217 /*
12001218 * setup_device() allocates memory and sets up data that requires calling the
12011219 * device ops, this is the only reason these actions are not done during
@@ -1249,7 +1267,7 @@ static void disable_device(struct ib_device *device) · 47 unmodified lines
12491267 remove_client_context(device, cid);
12501268 }
12511269
1252− ib_cq_pool_destroy(device);
1270+ ib_cq_pool_cleanup(device);
12531271
12541272 /* Pairs with refcount_set in enable_device */
12551273 ib_device_put(device);
@@ -1294,8 +1312,6 @@ static int enable_device_and_get(struct ib_device *device) · 38 unmodified lines
12941312 goto out;
12951313 }
12961314
1297− ib_cq_pool_init(device);
1298−
12991315 down_read(&clients_rwsem);
13001316 xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
13011317 ret = add_client_context(device, client);
@@ -1341,7 +1357,14 @@ int ib_register_device(struct ib_device *device, const char *name, · 39 unmodified lines
13411357 if (ret)
13421358 return ret;
13431359
1344− setup_dma_device(device, dma_device);
1360+ /*
1361+ * If the caller does not provide a DMA capable device then the IB core
1362+ * will set up ib_sge and scatterlist structures that stash the kernel
1363+ * virtual address into the address field.
1364+ */
1365+ WARN_ON(dma_device && !dma_device->dma_parms);
1366+ device->dma_device = dma_device;
1367+
13451368 ret = setup_device(device);
13461369 if (ret)
13471370 return ret;
@@ -1374,9 +1397,6 @@ int ib_register_device(struct ib_device *device, const char *name, · 26 unmodified lines
13741397 }
13751398
13761399 ret = enable_device_and_get(device);
1377− dev_set_uevent_suppress(&device->dev, false);
1378− /* Mark for userspace that device is ready */
1379− kobject_uevent(&device->dev.kobj, KOBJ_ADD);
13801400 if (ret) {
13811401 void (*dealloc_fn)(struct ib_device *);
13821402
@@ -1396,8 +1416,12 @@ int ib_register_device(struct ib_device *device, const char *name, · 13 unmodified lines
13961416 ib_device_put(device);
13971417 __ib_unregister_device(device);
13981418 device->ops.dealloc_driver = dealloc_fn;
1419+ dev_set_uevent_suppress(&device->dev, false);
13991420 return ret;
14001421 }
1422+ dev_set_uevent_suppress(&device->dev, false);
1423+ /* Mark for userspace that device is ready */
1424+ kobject_uevent(&device->dev.kobj, KOBJ_ADD);
14011425 ib_device_put(device);
14021426
14031427 return 0;
@@ -2576,6 +2600,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) · 1172 unmodified lines
25762600 SET_DEVICE_OP(dev_ops, create_qp);
25772601 SET_DEVICE_OP(dev_ops, create_rwq_ind_table);
25782602 SET_DEVICE_OP(dev_ops, create_srq);
2603+ SET_DEVICE_OP(dev_ops, create_user_ah);
25792604 SET_DEVICE_OP(dev_ops, create_wq);
25802605 SET_DEVICE_OP(dev_ops, dealloc_dm);
25812606 SET_DEVICE_OP(dev_ops, dealloc_driver);
@@ -2675,6 +2700,21 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) · 93 unmodified lines
26752700 }
26762701 EXPORT_SYMBOL(ib_set_device_ops);
26772702
2703+#ifdef CONFIG_INFINIBAND_VIRT_DMA
2704+int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents)
2705+{
2706+ struct scatterlist *s;
2707+ int i;
2708+
2709+ for_each_sg(sg, s, nents, i) {
2710+ sg_dma_address(s) = (uintptr_t)sg_virt(s);
2711+ sg_dma_len(s) = s->length;
2712+ }
2713+ return nents;
2714+}
2715+EXPORT_SYMBOL(ib_dma_virt_map_sg);
2716+#endif /* CONFIG_INFINIBAND_VIRT_DMA */
2717+
26782718 static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = {
26792719 [RDMA_NL_LS_OP_RESOLVE] = {
26802720 .doit = ib_nl_handle_resolve_resp,
drivers/infiniband/core/iwpm_util.h+1−1
@@ -141,7 +141,7 @@ int iwpm_wait_complete_req(struct iwpm_nlmsg_request *nlmsg_request); · 140 unmodified lines
141141 int iwpm_get_nlmsg_seq(void);
142142
143143 /**
144− * iwpm_add_reminfo - Add remote address info of the connecting peer
144+ * iwpm_add_remote_info - Add remote address info of the connecting peer
145145 * to the remote info hash table
146146 * @reminfo: The remote info to be added
147147 */
drivers/infiniband/core/rdma_core.c+68−33
@@ -137,15 +137,9 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj, · 136 unmodified lines
137137 } else if (uobj->object) {
138138 ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason,
139139 attrs);
140− if (ret) {
141− if (ib_is_destroy_retryable(ret, reason, uobj))
142− return ret;
143−
144− /* Nothing to be done, dangle the memory and move on */
145− WARN(true,
146− "ib_uverbs: failed to remove uobject id %d, driver err=%d",
147− uobj->id, ret);
148− }
140+ if (ret)
141+ /* Nothing to be done, wait till ucontext will clean it */
142+ return ret;
149143
150144 uobj->object = NULL;
151145 }
@@ -543,12 +537,7 @@ static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj, · 391 unmodified lines
543537 struct uverbs_obj_idr_type, type);
544538 int ret = idr_type->destroy_object(uobj, why, attrs);
545539
546− /*
547− * We can only fail gracefully if the user requested to destroy the
548− * object or when a retry may be called upon an error.
549− * In the rest of the cases, just remove whatever you can.
550− */
551− if (ib_is_destroy_retryable(ret, why, uobj))
540+ if (ret)
552541 return ret;
553542
554543 if (why == RDMA_REMOVE_ABORT)
@@ -581,11 +570,8 @@ static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj, · 26 unmodified lines
581570 {
582571 const struct uverbs_obj_fd_type *fd_type = container_of(
583572 uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type);
584− int ret = fd_type->destroy_object(uobj, why);
585−
586− if (ib_is_destroy_retryable(ret, why, uobj))
587− return ret;
588573
574+ fd_type->destroy_object(uobj, why);
589575 return 0;
590576 }
591577
@@ -609,6 +595,27 @@ static void alloc_commit_idr_uobject(struct ib_uobject *uobj) · 17 unmodified lines
609595 WARN_ON(old != NULL);
610596 }
611597
598+static void swap_idr_uobjects(struct ib_uobject *obj_old,
599+ struct ib_uobject *obj_new)
600+{
601+ struct ib_uverbs_file *ufile = obj_old->ufile;
602+ void *old;
603+
604+ /*
605+ * New must be an object that been allocated but not yet committed, this
606+ * moves the pre-committed state to obj_old, new still must be comitted.
607+ */
608+ old = xa_cmpxchg(&ufile->idr, obj_old->id, obj_old, XA_ZERO_ENTRY,
609+ GFP_KERNEL);
610+ if (WARN_ON(old != obj_old))
611+ return;
612+
613+ swap(obj_old->id, obj_new->id);
614+
615+ old = xa_cmpxchg(&ufile->idr, obj_old->id, NULL, obj_old, GFP_KERNEL);
616+ WARN_ON(old != NULL);
617+}
618+
612619 static void alloc_commit_fd_uobject(struct ib_uobject *uobj)
613620 {
614621 int fd = uobj->id;
@@ -654,6 +661,35 @@ void rdma_alloc_commit_uobject(struct ib_uobject *uobj, · 39 unmodified lines
654661 up_read(&ufile->hw_destroy_rwsem);
655662 }
656663
664+/*
665+ * new_uobj will be assigned to the handle currently used by to_uobj, and
666+ * to_uobj will be destroyed.
667+ *
668+ * Upon return the caller must do:
669+ * rdma_alloc_commit_uobject(new_uobj)
670+ * uobj_put_destroy(to_uobj)
671+ *
672+ * to_uobj must have a write get but the put mode switches to destroy once
673+ * this is called.
674+ */
675+void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj,
676+ struct uverbs_attr_bundle *attrs)
677+{
678+ assert_uverbs_usecnt(new_uobj, UVERBS_LOOKUP_WRITE);
679+
680+ if (WARN_ON(to_uobj->uapi_object != new_uobj->uapi_object ||
681+ !to_uobj->uapi_object->type_class->swap_uobjects))
682+ return;
683+
684+ to_uobj->uapi_object->type_class->swap_uobjects(to_uobj, new_uobj);
685+
686+ /*
687+ * If this fails then the uobject is still completely valid (though with
688+ * a new ID) and we leak it until context close.
689+ */
690+ uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs);
691+}
692+
657693 /*
658694 * This consumes the kref for uobj. It is up to the caller to unwind the HW
659695 * object and anything else connected to uobj before calling this.
@@ -761,6 +797,7 @@ const struct uverbs_obj_type_class uverbs_idr_class = { · 101 unmodified lines
761797 .lookup_put = lookup_put_idr_uobject,
762798 .destroy_hw = destroy_hw_idr_uobject,
763799 .remove_handle = remove_handle_idr_uobject,
800+ .swap_uobjects = swap_idr_uobjects,
764801 };
765802 EXPORT_SYMBOL(uverbs_idr_class);
766803
@@ -863,11 +900,18 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, · 96 unmodified lines
863900 * racing with a lookup_get.
864901 */
865902 WARN_ON(uverbs_try_lock_object(obj, UVERBS_LOOKUP_WRITE));
903+ if (reason == RDMA_REMOVE_DRIVER_FAILURE)
904+ obj->object = NULL;
866905 if (!uverbs_destroy_uobject(obj, reason, &attrs))
867906 ret = 0;
868907 else
869908 atomic_set(&obj->usecnt, 0);
870909 }
910+
911+ if (reason == RDMA_REMOVE_DRIVER_FAILURE) {
912+ WARN_ON(!list_empty(&ufile->uobjects));
913+ return 0;
914+ }
871915 return ret;
872916 }
873917
@@ -889,21 +933,12 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, · 15 unmodified lines
889933 if (!ufile->ucontext)
890934 goto done;
891935
892− ufile->ucontext->cleanup_retryable = true;
893− while (!list_empty(&ufile->uobjects))
894− if (__uverbs_cleanup_ufile(ufile, reason)) {
895− /*
896− * No entry was cleaned-up successfully during this
897− * iteration. It is a driver bug to fail destruction.
898− */
899− WARN_ON(!list_empty(&ufile->uobjects));
900− break;
901− }
902−
903− ufile->ucontext->cleanup_retryable = false;
904− if (!list_empty(&ufile->uobjects))
905− __uverbs_cleanup_ufile(ufile, reason);
936+ while (!list_empty(&ufile->uobjects) &&
937+ !__uverbs_cleanup_ufile(ufile, reason)) {
938+ }
906939
940+ if (WARN_ON(!list_empty(&ufile->uobjects)))
941+ __uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE);
907942 ufile_destroy_ucontext(ufile, reason);
908943
909944 done:
drivers/infiniband/core/restrack.c+19−4
@@ -221,19 +221,29 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) · 220 unmodified lines
221221 {
222222 struct ib_device *dev = res_to_dev(res);
223223 struct rdma_restrack_root *rt;
224− int ret;
224+ int ret = 0;
225225
226226 if (!dev)
227227 return;
228228
229+ if (res->no_track)
230+ goto out;
231+
229232 rt = &dev->res[res->type];
230233
231234 if (res->type == RDMA_RESTRACK_QP) {
232235 /* Special case to ensure that LQPN points to right QP */
233236 struct ib_qp *qp = container_of(res, struct ib_qp, res);
234237
235− ret = xa_insert(&rt->xa, qp->qp_num, res, GFP_KERNEL);
236− res->id = ret ? 0 : qp->qp_num;
238+ WARN_ONCE(qp->qp_num >> 24 || qp->port >> 8,
239+ "QP number 0x%0X and port 0x%0X", qp->qp_num,
240+ qp->port);
241+ res->id = qp->qp_num;
242+ if (qp->qp_type == IB_QPT_SMI || qp->qp_type == IB_QPT_GSI)
243+ res->id |= qp->port << 24;
244+ ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL);
245+ if (ret)
246+ res->id = 0;
237247 } else if (res->type == RDMA_RESTRACK_COUNTER) {
238248 /* Special case to ensure that cntn points to right counter */
239249 struct rdma_counter *counter;
@@ -246,6 +256,7 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) · 6 unmodified lines
246256 &rt->next_id, GFP_KERNEL);
247257 }
248258
259+out:
249260 if (!ret)
250261 res->valid = true;
251262 }
@@ -318,6 +329,9 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) · 66 unmodified lines
318329 return;
319330 }
320331
332+ if (res->no_track)
333+ goto out;
334+
321335 dev = res_to_dev(res);
322336 if (WARN_ON(!dev))
323337 return;
@@ -328,8 +342,9 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) · 4 unmodified lines
328342 if (res->type == RDMA_RESTRACK_MR || res->type == RDMA_RESTRACK_QP)
329343 return;
330344 WARN_ON(old != res);
331− res->valid = false;
332345
346+out:
347+ res->valid = false;
333348 rdma_restrack_put(res);
334349 wait_for_completion(&res->comp);
335350 }
drivers/infiniband/core/rw.c+4−1
@@ -285,8 +285,11 @@ static void rdma_rw_unmap_sg(struct ib_device *dev, struct scatterlist *sg, · 284 unmodified lines
285285 static int rdma_rw_map_sg(struct ib_device *dev, struct scatterlist *sg,
286286 u32 sg_cnt, enum dma_data_direction dir)
287287 {
288− if (is_pci_p2pdma_page(sg_page(sg)))
288+ if (is_pci_p2pdma_page(sg_page(sg))) {
289+ if (WARN_ON_ONCE(ib_uses_virt_dma(dev)))
290+ return 0;
289291 return pci_p2pdma_map_sg(dev->dma_device, sg, sg_cnt, dir);
292+ }
290293 return ib_dma_map_sg(dev, sg, sg_cnt, dir);
291294 }
292295
drivers/infiniband/core/sa_query.c+2−1
@@ -1435,7 +1435,8 @@ enum opa_pr_supported { · 1434 unmodified lines
14351435 };
14361436
14371437 /**
1438− * Check if current PR query can be an OPA query.
1438+ * opa_pr_query_possible - Check if current PR query can be an OPA query.
1439+ *
14391440 * Retuns PR_NOT_SUPPORTED if a path record query is not
14401441 * possible, PR_OPA_SUPPORTED if an OPA path record query
14411442 * is possible and PR_IB_SUPPORTED if an IB path record
drivers/infiniband/core/sysfs.c+97−69
@@ -165,9 +165,11 @@ static ssize_t state_show(struct ib_port *p, struct port_attribute *unused, · 164 unmodified lines
165165 if (ret)
166166 return ret;
167167
168− return sprintf(buf, "%d: %s\n", attr.state,
169− attr.state >= 0 && attr.state < ARRAY_SIZE(state_name) ?
170− state_name[attr.state] : "UNKNOWN");
168+ return sysfs_emit(buf, "%d: %s\n", attr.state,
169+ attr.state >= 0 &&
170+ attr.state < ARRAY_SIZE(state_name) ?
171+ state_name[attr.state] :
172+ "UNKNOWN");
171173 }
172174
173175 static ssize_t lid_show(struct ib_port *p, struct port_attribute *unused,
@@ -180,7 +182,7 @@ static ssize_t lid_show(struct ib_port *p, struct port_attribute *unused, · 6 unmodified lines
180182 if (ret)
181183 return ret;
182184
183− return sprintf(buf, "0x%x\n", attr.lid);
185+ return sysfs_emit(buf, "0x%x\n", attr.lid);
184186 }
185187
186188 static ssize_t lid_mask_count_show(struct ib_port *p,
@@ -194,7 +196,7 @@ static ssize_t lid_mask_count_show(struct ib_port *p, · 7 unmodified lines
194196 if (ret)
195197 return ret;
196198
197− return sprintf(buf, "%d\n", attr.lmc);
199+ return sysfs_emit(buf, "%d\n", attr.lmc);
198200 }
199201
200202 static ssize_t sm_lid_show(struct ib_port *p, struct port_attribute *unused,
@@ -207,7 +209,7 @@ static ssize_t sm_lid_show(struct ib_port *p, struct port_attribute *unused, · 6 unmodified lines
207209 if (ret)
208210 return ret;
209211
210− return sprintf(buf, "0x%x\n", attr.sm_lid);
212+ return sysfs_emit(buf, "0x%x\n", attr.sm_lid);
211213 }
212214
213215 static ssize_t sm_sl_show(struct ib_port *p, struct port_attribute *unused,
@@ -220,7 +222,7 @@ static ssize_t sm_sl_show(struct ib_port *p, struct port_attribute *unused, · 6 unmodified lines
220222 if (ret)
221223 return ret;
222224
223− return sprintf(buf, "%d\n", attr.sm_sl);
225+ return sysfs_emit(buf, "%d\n", attr.sm_sl);
224226 }
225227
226228 static ssize_t cap_mask_show(struct ib_port *p, struct port_attribute *unused,
@@ -233,7 +235,7 @@ static ssize_t cap_mask_show(struct ib_port *p, struct port_attribute *unused, · 6 unmodified lines
233235 if (ret)
234236 return ret;
235237
236− return sprintf(buf, "0x%08x\n", attr.port_cap_flags);
238+ return sysfs_emit(buf, "0x%08x\n", attr.port_cap_flags);
237239 }
238240
239241 static ssize_t rate_show(struct ib_port *p, struct port_attribute *unused,
@@ -273,6 +275,10 @@ static ssize_t rate_show(struct ib_port *p, struct port_attribute *unused, · 33 unmodified lines
273275 speed = " HDR";
274276 rate = 500;
275277 break;
278+ case IB_SPEED_NDR:
279+ speed = " NDR";
280+ rate = 1000;
281+ break;
276282 case IB_SPEED_SDR:
277283 default: /* default to SDR for invalid rates */
278284 speed = " SDR";
@@ -284,9 +290,9 @@ static ssize_t rate_show(struct ib_port *p, struct port_attribute *unused, · 5 unmodified lines
284290 if (rate < 0)
285291 return -EINVAL;
286292
287− return sprintf(buf, "%d%s Gb/sec (%dX%s)\n",
288− rate / 10, rate % 10 ? ".5" : "",
289− ib_width_enum_to_int(attr.active_width), speed);
293+ return sysfs_emit(buf, "%d%s Gb/sec (%dX%s)\n", rate / 10,
294+ rate % 10 ? ".5" : "",
295+ ib_width_enum_to_int(attr.active_width), speed);
290296 }
291297
292298 static const char *phys_state_to_str(enum ib_port_phys_state phys_state)
@@ -318,21 +324,28 @@ static ssize_t phys_state_show(struct ib_port *p, struct port_attribute *unused, · 25 unmodified lines
318324 if (ret)
319325 return ret;
320326
321− return sprintf(buf, "%d: %s\n", attr.phys_state,
322− phys_state_to_str(attr.phys_state));
327+ return sysfs_emit(buf, "%d: %s\n", attr.phys_state,
328+ phys_state_to_str(attr.phys_state));
323329 }
324330
325331 static ssize_t link_layer_show(struct ib_port *p, struct port_attribute *unused,
326332 char *buf)
327333 {
334+ const char *output;
335+
328336 switch (rdma_port_get_link_layer(p->ibdev, p->port_num)) {
329337 case IB_LINK_LAYER_INFINIBAND:
330− return sprintf(buf, "%s\n", "InfiniBand");
338+ output = "InfiniBand";
339+ break;
331340 case IB_LINK_LAYER_ETHERNET:
332− return sprintf(buf, "%s\n", "Ethernet");
341+ output = "Ethernet";
342+ break;
333343 default:
334− return sprintf(buf, "%s\n", "Unknown");
344+ output = "Unknown";
345+ break;
335346 }
347+
348+ return sysfs_emit(buf, "%s\n", output);
336349 }
337350
338351 static PORT_ATTR_RO(state);
@@ -358,27 +371,28 @@ static struct attribute *port_default_attrs[] = { · 19 unmodified lines
358371 NULL
359372 };
360373
361−static size_t print_ndev(const struct ib_gid_attr *gid_attr, char *buf)
374+static ssize_t print_ndev(const struct ib_gid_attr *gid_attr, char *buf)
362375 {
363376 struct net_device *ndev;
364− size_t ret = -EINVAL;
377+ int ret = -EINVAL;
365378
366379 rcu_read_lock();
367380 ndev = rcu_dereference(gid_attr->ndev);
368381 if (ndev)
369− ret = sprintf(buf, "%s\n", ndev->name);
382+ ret = sysfs_emit(buf, "%s\n", ndev->name);
370383 rcu_read_unlock();
371384 return ret;
372385 }
373386
374−static size_t print_gid_type(const struct ib_gid_attr *gid_attr, char *buf)
387+static ssize_t print_gid_type(const struct ib_gid_attr *gid_attr, char *buf)
375388 {
376− return sprintf(buf, "%s\n", ib_cache_gid_type_str(gid_attr->gid_type));
389+ return sysfs_emit(buf, "%s\n",
390+ ib_cache_gid_type_str(gid_attr->gid_type));
377391 }
378392
379393 static ssize_t _show_port_gid_attr(
380394 struct ib_port *p, struct port_attribute *attr, char *buf,
381− size_t (*print)(const struct ib_gid_attr *gid_attr, char *buf))
395+ ssize_t (*print)(const struct ib_gid_attr *gid_attr, char *buf))
382396 {
383397 struct port_table_attribute *tab_attr =
384398 container_of(attr, struct port_table_attribute, attr);
@@ -401,7 +415,7 @@ static ssize_t show_port_gid(struct ib_port *p, struct port_attribute *attr, · 16 unmodified lines
401415 struct port_table_attribute *tab_attr =
402416 container_of(attr, struct port_table_attribute, attr);
403417 const struct ib_gid_attr *gid_attr;
404− ssize_t ret;
418+ int len;
405419
406420 gid_attr = rdma_get_gid_attr(p->ibdev, p->port_num, tab_attr->index);
407421 if (IS_ERR(gid_attr)) {
@@ -416,12 +430,12 @@ static ssize_t show_port_gid(struct ib_port *p, struct port_attribute *attr, · 8 unmodified lines
416430 * space throwing such error on fail to read gid, return zero
417431 * GID as before. This maintains backward compatibility.
418432 */
419− return sprintf(buf, "%pI6\n", zgid.raw);
433+ return sysfs_emit(buf, "%pI6\n", zgid.raw);
420434 }
421435
422− ret = sprintf(buf, "%pI6\n", gid_attr->gid.raw);
436+ len = sysfs_emit(buf, "%pI6\n", gid_attr->gid.raw);
423437 rdma_put_gid_attr(gid_attr);
424− return ret;
438+ return len;
425439 }
426440
427441 static ssize_t show_port_gid_attr_ndev(struct ib_port *p,
@@ -443,13 +457,13 @@ static ssize_t show_port_pkey(struct ib_port *p, struct port_attribute *attr, · 15 unmodified lines
443457 struct port_table_attribute *tab_attr =
444458 container_of(attr, struct port_table_attribute, attr);
445459 u16 pkey;
446− ssize_t ret;
460+ int ret;
447461
448462 ret = ib_query_pkey(p->ibdev, p->port_num, tab_attr->index, &pkey);
449463 if (ret)
450464 return ret;
451465
452− return sprintf(buf, "0x%04x\n", pkey);
466+ return sysfs_emit(buf, "0x%04x\n", pkey);
453467 }
454468
455469 #define PORT_PMA_ATTR(_name, _counter, _width, _offset) \
@@ -521,8 +535,9 @@ static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr, · 65 unmodified lines
521535 container_of(attr, struct port_table_attribute, attr);
522536 int offset = tab_attr->index & 0xffff;
523537 int width = (tab_attr->index >> 16) & 0xff;
524− ssize_t ret;
538+ int ret;
525539 u8 data[8];
540+ int len;
526541
527542 ret = get_perf_mad(p->ibdev, p->port_num, tab_attr->attr_id, &data,
528543 40 + offset / 8, sizeof(data));
@@ -531,30 +546,27 @@ static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr, · 2 unmodified lines
531546
532547 switch (width) {
533548 case 4:
534− ret = sprintf(buf, "%u\n", (*data >>
535− (4 - (offset % 8))) & 0xf);
549+ len = sysfs_emit(buf, "%u\n",
550+ (*data >> (4 - (offset % 8))) & 0xf);
536551 break;
537552 case 8:
538− ret = sprintf(buf, "%u\n", *data);
553+ len = sysfs_emit(buf, "%u\n", *data);
539554 break;
540555 case 16:
541− ret = sprintf(buf, "%u\n",
542− be16_to_cpup((__be16 *)data));
556+ len = sysfs_emit(buf, "%u\n", be16_to_cpup((__be16 *)data));
543557 break;
544558 case 32:
545− ret = sprintf(buf, "%u\n",
546− be32_to_cpup((__be32 *)data));
559+ len = sysfs_emit(buf, "%u\n", be32_to_cpup((__be32 *)data));
547560 break;
548561 case 64:
549− ret = sprintf(buf, "%llu\n",
550− be64_to_cpup((__be64 *)data));
562+ len = sysfs_emit(buf, "%llu\n", be64_to_cpup((__be64 *)data));
551563 break;
552−
553564 default:
554− ret = 0;
565+ len = 0;
566+ break;
555567 }
556568
557− return ret;
569+ return len;
558570 }
559571
560572 static PORT_PMA_ATTR(symbol_error , 0, 16, 32);
@@ -815,12 +827,12 @@ static int update_hw_stats(struct ib_device *dev, struct rdma_hw_stats *stats, · 254 unmodified lines
815827 return 0;
816828 }
817829
818−static ssize_t print_hw_stat(struct ib_device *dev, int port_num,
819− struct rdma_hw_stats *stats, int index, char *buf)
830+static int print_hw_stat(struct ib_device *dev, int port_num,
831+ struct rdma_hw_stats *stats, int index, char *buf)
820832 {
821833 u64 v = rdma_counter_get_hwstat_value(dev, port_num, index);
822834
823− return sprintf(buf, "%llu\n", stats->value[index] + v);
835+ return sysfs_emit(buf, "%llu\n", stats->value[index] + v);
824836 }
825837
826838 static ssize_t show_hw_stats(struct kobject *kobj, struct attribute *attr,
@@ -877,7 +889,7 @@ static ssize_t show_stats_lifespan(struct kobject *kobj, · 50 unmodified lines
877889 msecs = jiffies_to_msecs(stats->lifespan);
878890 mutex_unlock(&stats->lock);
879891
880− return sprintf(buf, "%d\n", msecs);
892+ return sysfs_emit(buf, "%d\n", msecs);
881893 }
882894
883895 static ssize_t set_stats_lifespan(struct kobject *kobj,
@@ -1224,21 +1236,34 @@ err_put: · 340 unmodified lines
12241236 return ret;
12251237 }
12261238
1239+static const char *node_type_string(int node_type)
1240+{
1241+ switch (node_type) {
1242+ case RDMA_NODE_IB_CA:
1243+ return "CA";
1244+ case RDMA_NODE_IB_SWITCH:
1245+ return "switch";
1246+ case RDMA_NODE_IB_ROUTER:
1247+ return "router";
1248+ case RDMA_NODE_RNIC:
1249+ return "RNIC";
1250+ case RDMA_NODE_USNIC:
1251+ return "usNIC";
1252+ case RDMA_NODE_USNIC_UDP:
1253+ return "usNIC UDP";
1254+ case RDMA_NODE_UNSPECIFIED:
1255+ return "unspecified";
1256+ }
1257+ return "<unknown>";
1258+}
1259+
12271260 static ssize_t node_type_show(struct device *device,
12281261 struct device_attribute *attr, char *buf)
12291262 {
12301263 struct ib_device *dev = rdma_device_to_ibdev(device);
12311264
1232− switch (dev->node_type) {
1233− case RDMA_NODE_IB_CA: return sprintf(buf, "%d: CA\n", dev->node_type);
1234− case RDMA_NODE_RNIC: return sprintf(buf, "%d: RNIC\n", dev->node_type);
1235− case RDMA_NODE_USNIC: return sprintf(buf, "%d: usNIC\n", dev->node_type);
1236− case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
1237− case RDMA_NODE_UNSPECIFIED: return sprintf(buf, "%d: unspecified\n", dev->node_type);
1238− case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
1239− case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
1240− default: return sprintf(buf, "%d: <unknown>\n", dev->node_type);
1241− }
1265+ return sysfs_emit(buf, "%d: %s\n", dev->node_type,
1266+ node_type_string(dev->node_type));
12421267 }
12431268 static DEVICE_ATTR_RO(node_type);
12441269
@@ -1246,12 +1271,13 @@ static ssize_t sys_image_guid_show(struct device *device, · 1 unmodified line
12461271 struct device_attribute *dev_attr, char *buf)
12471272 {
12481273 struct ib_device *dev = rdma_device_to_ibdev(device);
1274+ __be16 *guid = (__be16 *)&dev->attrs.sys_image_guid;
12491275
1250− return sprintf(buf, "%04x:%04x:%04x:%04x\n",
1251− be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[0]),
1252− be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[1]),
1253− be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[2]),
1254− be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[3]));
1276+ return sysfs_emit(buf, "%04x:%04x:%04x:%04x\n",
1277+ be16_to_cpu(guid[0]),
1278+ be16_to_cpu(guid[1]),
1279+ be16_to_cpu(guid[2]),
1280+ be16_to_cpu(guid[3]));
12551281 }
12561282 static DEVICE_ATTR_RO(sys_image_guid);
12571283
@@ -1259,12 +1285,13 @@ static ssize_t node_guid_show(struct device *device, · 1 unmodified line
12591285 struct device_attribute *attr, char *buf)
12601286 {
12611287 struct ib_device *dev = rdma_device_to_ibdev(device);
1288+ __be16 *node_guid = (__be16 *)&dev->node_guid;
12621289
1263− return sprintf(buf, "%04x:%04x:%04x:%04x\n",
1264− be16_to_cpu(((__be16 *) &dev->node_guid)[0]),
1265− be16_to_cpu(((__be16 *) &dev->node_guid)[1]),
1266− be16_to_cpu(((__be16 *) &dev->node_guid)[2]),
1267− be16_to_cpu(((__be16 *) &dev->node_guid)[3]));
1290+ return sysfs_emit(buf, "%04x:%04x:%04x:%04x\n",
1291+ be16_to_cpu(node_guid[0]),
1292+ be16_to_cpu(node_guid[1]),
1293+ be16_to_cpu(node_guid[2]),
1294+ be16_to_cpu(node_guid[3]));
12681295 }
12691296 static DEVICE_ATTR_RO(node_guid);
12701297
@@ -1273,7 +1300,7 @@ static ssize_t node_desc_show(struct device *device, · 2 unmodified lines
12731300 {
12741301 struct ib_device *dev = rdma_device_to_ibdev(device);
12751302
1276− return sprintf(buf, "%.64s\n", dev->node_desc);
1303+ return sysfs_emit(buf, "%.64s\n", dev->node_desc);
12771304 }
12781305
12791306 static ssize_t node_desc_store(struct device *device,
@@ -1300,10 +1327,11 @@ static ssize_t fw_ver_show(struct device *device, struct device_attribute *attr, · 20 unmodified lines
13001327 char *buf)
13011328 {
13021329 struct ib_device *dev = rdma_device_to_ibdev(device);
1330+ char version[IB_FW_VERSION_NAME_MAX] = {};
1331+
1332+ ib_get_device_fw_str(dev, version);
13031333
1304− ib_get_device_fw_str(dev, buf);
1305− strlcat(buf, "\n", IB_FW_VERSION_NAME_MAX);
1306− return strlen(buf);
1334+ return sysfs_emit(buf, "%s\n", version);
13071335 }
13081336 static DEVICE_ATTR_RO(fw_ver);
13091337
drivers/infiniband/core/ucma.c+1−1
@@ -1825,7 +1825,7 @@ static ssize_t show_abi_version(struct device *dev, · 1824 unmodified lines
18251825 struct device_attribute *attr,
18261826 char *buf)
18271827 {
1828− return sprintf(buf, "%d\n", RDMA_USER_CM_ABI_VERSION);
1828+ return sysfs_emit(buf, "%d\n", RDMA_USER_CM_ABI_VERSION);
18291829 }
18301830 static DEVICE_ATTR(abi_version, S_IRUGO, show_abi_version, NULL);
18311831
drivers/infiniband/core/umem.c+13−4
@@ -84,6 +84,15 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, · 83 unmodified lines
8484 dma_addr_t mask;
8585 int i;
8686
87+ if (umem->is_odp) {
88+ unsigned int page_size = BIT(to_ib_umem_odp(umem)->page_shift);
89+
90+ /* ODP must always be self consistent. */
91+ if (!(pgsz_bitmap & page_size))
92+ return 0;
93+ return page_size;
94+ }
95+
8796 /* rdma_for_each_block() has a bug if the page size is smaller than the
8897 * page size used to build the umem. For now prevent smaller page sizes
8998 * from being returned.
@@ -220,10 +229,10 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, · 130 unmodified lines
220229
221230 cur_base += ret * PAGE_SIZE;
222231 npages -= ret;
223− sg = __sg_alloc_table_from_pages(
224− &umem->sg_head, page_list, ret, 0, ret << PAGE_SHIFT,
225− dma_get_max_seg_size(device->dma_device), sg, npages,
226− GFP_KERNEL);
232+ sg = __sg_alloc_table_from_pages(&umem->sg_head, page_list, ret,
233+ 0, ret << PAGE_SHIFT,
234+ ib_dma_max_seg_size(device), sg, npages,
235+ GFP_KERNEL);
227236 umem->sg_nents = umem->sg_head.nents;
228237 if (IS_ERR(sg)) {
229238 unpin_user_pages_dirty_lock(page_list, ret, 0);
drivers/infiniband/core/user_mad.c+3−3
@@ -1191,7 +1191,7 @@ static ssize_t ibdev_show(struct device *dev, struct device_attribute *attr, · 1190 unmodified lines
11911191 if (!port)
11921192 return -ENODEV;
11931193
1194− return sprintf(buf, "%s\n", dev_name(&port->ib_dev->dev));
1194+ return sysfs_emit(buf, "%s\n", dev_name(&port->ib_dev->dev));
11951195 }
11961196 static DEVICE_ATTR_RO(ibdev);
11971197
@@ -1203,7 +1203,7 @@ static ssize_t port_show(struct device *dev, struct device_attribute *attr, · 5 unmodified lines
12031203 if (!port)
12041204 return -ENODEV;
12051205
1206− return sprintf(buf, "%d\n", port->port_num);
1206+ return sysfs_emit(buf, "%d\n", port->port_num);
12071207 }
12081208 static DEVICE_ATTR_RO(port);
12091209
@@ -1222,7 +1222,7 @@ static char *umad_devnode(struct device *dev, umode_t *mode) · 12 unmodified lines
12221222 static ssize_t abi_version_show(struct class *class,
12231223 struct class_attribute *attr, char *buf)
12241224 {
1225− return sprintf(buf, "%d\n", IB_USER_MAD_ABI_VERSION);
1225+ return sysfs_emit(buf, "%d\n", IB_USER_MAD_ABI_VERSION);
12261226 }
12271227 static CLASS_ATTR_RO(abi_version);
12281228
drivers/infiniband/core/uverbs_cmd.c+91−58
@@ -681,8 +681,7 @@ int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd, · 680 unmodified lines
681681 return 0;
682682
683683 ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);
684−
685− if (ib_is_destroy_retryable(ret, why, uobject)) {
684+ if (ret) {
686685 atomic_inc(&xrcd->usecnt);
687686 return ret;
688687 }
@@ -690,7 +689,7 @@ int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd, · 1 unmodified line
690689 if (inode)
691690 xrcd_table_delete(dev, inode);
692691
693− return ret;
692+ return 0;
694693 }
695694
696695 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
@@ -710,29 +709,20 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) · 13 unmodified lines
710709 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
711710 return -EINVAL;
712711
713− ret = ib_check_mr_access(cmd.access_flags);
714− if (ret)
715− return ret;
716−
717712 uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
718713 if (IS_ERR(uobj))
719714 return PTR_ERR(uobj);
720715
716+ ret = ib_check_mr_access(ib_dev, cmd.access_flags);
717+ if (ret)
718+ goto err_free;
719+
721720 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
722721 if (!pd) {
723722 ret = -EINVAL;
724723 goto err_free;
725724 }
726725
727− if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
728− if (!(pd->device->attrs.device_cap_flags &
729− IB_DEVICE_ON_DEMAND_PAGING)) {
730− pr_debug("ODP support not available\n");
731− ret = -EINVAL;
732− goto err_put;
733− }
734− }
735−
736726 mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
737727 cmd.access_flags,
738728 &attrs->driver_udata);
@@ -774,23 +764,28 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) · 35 unmodified lines
774764 {
775765 struct ib_uverbs_rereg_mr cmd;
776766 struct ib_uverbs_rereg_mr_resp resp;
777− struct ib_pd *pd = NULL;
778767 struct ib_mr *mr;
779− struct ib_pd *old_pd;
780768 int ret;
781769 struct ib_uobject *uobj;
770+ struct ib_uobject *new_uobj;
771+ struct ib_device *ib_dev;
772+ struct ib_pd *orig_pd;
773+ struct ib_pd *new_pd;
774+ struct ib_mr *new_mr;
782775
783776 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
784777 if (ret)
785778 return ret;
786779
787− if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
780+ if (!cmd.flags)
788781 return -EINVAL;
789782
783+ if (cmd.flags & ~IB_MR_REREG_SUPPORTED)
784+ return -EOPNOTSUPP;
785+
790786 if ((cmd.flags & IB_MR_REREG_TRANS) &&
791− (!cmd.start || !cmd.hca_va || 0 >= cmd.length ||
792− (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
793− return -EINVAL;
787+ (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
788+ return -EINVAL;
794789
795790 uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
796791 if (IS_ERR(uobj))
@@ -804,36 +799,74 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) · 7 unmodified lines
804799 }
805800
806801 if (cmd.flags & IB_MR_REREG_ACCESS) {
807− ret = ib_check_mr_access(cmd.access_flags);
802+ ret = ib_check_mr_access(mr->device, cmd.access_flags);
808803 if (ret)
809804 goto put_uobjs;
810805 }
811806
807+ orig_pd = mr->pd;
812808 if (cmd.flags & IB_MR_REREG_PD) {
813− pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
814− attrs);
815− if (!pd) {
809+ new_pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
810+ attrs);
811+ if (!new_pd) {
816812 ret = -EINVAL;
817813 goto put_uobjs;
818814 }
815+ } else {
816+ new_pd = mr->pd;
819817 }
820818
821− old_pd = mr->pd;
822− ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start,
823− cmd.length, cmd.hca_va,
824− cmd.access_flags, pd,
825− &attrs->driver_udata);
826− if (ret)
819+ /*
820+ * The driver might create a new HW object as part of the rereg, we need
821+ * to have a uobject ready to hold it.
822+ */
823+ new_uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
824+ if (IS_ERR(new_uobj)) {
825+ ret = PTR_ERR(new_uobj);
827826 goto put_uobj_pd;
828−
829− if (cmd.flags & IB_MR_REREG_PD) {
830− atomic_inc(&pd->usecnt);
831− mr->pd = pd;
832− atomic_dec(&old_pd->usecnt);
833827 }
834828
835− if (cmd.flags & IB_MR_REREG_TRANS)
836− mr->iova = cmd.hca_va;
829+ new_mr = ib_dev->ops.rereg_user_mr(mr, cmd.flags, cmd.start, cmd.length,
830+ cmd.hca_va, cmd.access_flags, new_pd,
831+ &attrs->driver_udata);
832+ if (IS_ERR(new_mr)) {
833+ ret = PTR_ERR(new_mr);
834+ goto put_new_uobj;
835+ }
836+ if (new_mr) {
837+ new_mr->device = new_pd->device;
838+ new_mr->pd = new_pd;
839+ new_mr->type = IB_MR_TYPE_USER;
840+ new_mr->dm = NULL;
841+ new_mr->sig_attrs = NULL;
842+ new_mr->uobject = uobj;
843+ atomic_inc(&new_pd->usecnt);
844+ new_mr->iova = cmd.hca_va;
845+ new_uobj->object = new_mr;
846+
847+ rdma_restrack_new(&new_mr->res, RDMA_RESTRACK_MR);
848+ rdma_restrack_set_name(&new_mr->res, NULL);
849+ rdma_restrack_add(&new_mr->res);
850+
851+ /*
852+ * The new uobj for the new HW object is put into the same spot
853+ * in the IDR and the old uobj & HW object is deleted.
854+ */
855+ rdma_assign_uobject(uobj, new_uobj, attrs);
856+ rdma_alloc_commit_uobject(new_uobj, attrs);
857+ uobj_put_destroy(uobj);
858+ new_uobj = NULL;
859+ uobj = NULL;
860+ mr = new_mr;
861+ } else {
862+ if (cmd.flags & IB_MR_REREG_PD) {
863+ atomic_dec(&orig_pd->usecnt);
864+ mr->pd = new_pd;
865+ atomic_inc(&new_pd->usecnt);
866+ }
867+ if (cmd.flags & IB_MR_REREG_TRANS)
868+ mr->iova = cmd.hca_va;
869+ }
837870
838871 memset(&resp, 0, sizeof(resp));
839872 resp.lkey = mr->lkey;
@@ -841,12 +874,16 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) · 1 unmodified line
841874
842875 ret = uverbs_response(attrs, &resp, sizeof(resp));
843876
877+put_new_uobj:
878+ if (new_uobj)
879+ uobj_alloc_abort(new_uobj, attrs);
844880 put_uobj_pd:
845881 if (cmd.flags & IB_MR_REREG_PD)
846− uobj_put_obj_read(pd);
882+ uobj_put_obj_read(new_pd);
847883
848884 put_uobjs:
849− uobj_put_write(uobj);
885+ if (uobj)
886+ uobj_put_write(uobj);
850887
851888 return ret;
852889 }
@@ -1401,8 +1438,8 @@ static int create_qp(struct uverbs_attr_bundle *attrs, · 548 unmodified lines
14011438 if (cmd->qp_type == IB_QPT_XRC_TGT)
14021439 qp = ib_create_qp(pd, &attr);
14031440 else
1404− qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
1405− obj);
1441+ qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, obj,
1442+ NULL);
14061443
14071444 if (IS_ERR(qp)) {
14081445 ret = PTR_ERR(qp);
@@ -1906,8 +1943,7 @@ static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs) · 497 unmodified lines
19061943 if (ret)
19071944 return ret;
19081945
1909− if (cmd.base.attr_mask &
1910− ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
1946+ if (cmd.base.attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
19111947 return -EOPNOTSUPP;
19121948
19131949 return modify_qp(attrs, &cmd);
@@ -1929,10 +1965,7 @@ static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs) · 15 unmodified lines
19291965 * Last bit is reserved for extending the attr_mask by
19301966 * using another field.
19311967 */
1932− BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1ULL << 31));
1933−
1934− if (cmd.base.attr_mask &
1935− ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
1968+ if (cmd.base.attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
19361969 return -EOPNOTSUPP;
19371970
19381971 ret = modify_qp(attrs, &cmd);
@@ -3693,13 +3726,13 @@ const struct uapi_definition uverbs_def_write_intf[] = { · 1754 unmodified lines
36933726 ib_uverbs_create_ah,
36943727 UAPI_DEF_WRITE_UDATA_IO(
36953728 struct ib_uverbs_create_ah,
3696− struct ib_uverbs_create_ah_resp),
3697− UAPI_DEF_METHOD_NEEDS_FN(create_ah)),
3729+ struct ib_uverbs_create_ah_resp)),
36983730 DECLARE_UVERBS_WRITE(
36993731 IB_USER_VERBS_CMD_DESTROY_AH,
37003732 ib_uverbs_destroy_ah,
3701− UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah),
3702− UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))),
3733+ UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah)),
3734+ UAPI_DEF_OBJ_NEEDS_FN(create_user_ah),
3735+ UAPI_DEF_OBJ_NEEDS_FN(destroy_ah)),
37033736
37043737 DECLARE_UVERBS_OBJECT(
37053738 UVERBS_OBJECT_COMP_CHANNEL,
@@ -3753,7 +3786,7 @@ const struct uapi_definition uverbs_def_write_intf[] = { · 47 unmodified lines
37533786 IB_USER_VERBS_EX_CMD_MODIFY_CQ,
37543787 ib_uverbs_ex_modify_cq,
37553788 UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
3756− UAPI_DEF_METHOD_NEEDS_FN(create_cq))),
3789+ UAPI_DEF_METHOD_NEEDS_FN(modify_cq))),
37573790
37583791 DECLARE_UVERBS_OBJECT(
37593792 UVERBS_OBJECT_DEVICE,
@@ -3999,8 +4032,7 @@ const struct uapi_definition uverbs_def_write_intf[] = { · 239 unmodified lines
39994032 DECLARE_UVERBS_WRITE(
40004033 IB_USER_VERBS_CMD_CLOSE_XRCD,
40014034 ib_uverbs_close_xrcd,
4002− UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd),
4003− UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)),
4035+ UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd)),
40044036 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
40054037 ib_uverbs_open_qp,
40064038 UAPI_DEF_WRITE_UDATA_IO(
@@ -4010,8 +4042,9 @@ const struct uapi_definition uverbs_def_write_intf[] = { · 3 unmodified lines
40104042 ib_uverbs_open_xrcd,
40114043 UAPI_DEF_WRITE_UDATA_IO(
40124044 struct ib_uverbs_open_xrcd,
4013− struct ib_uverbs_open_xrcd_resp),
4014− UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))),
4045+ struct ib_uverbs_open_xrcd_resp)),
4046+ UAPI_DEF_OBJ_NEEDS_FN(alloc_xrcd),
4047+ UAPI_DEF_OBJ_NEEDS_FN(dealloc_xrcd)),
40154048
40164049 {},
40174050 };
drivers/infiniband/core/uverbs_main.c+2−2
@@ -1046,7 +1046,7 @@ static ssize_t ibdev_show(struct device *device, struct device_attribute *attr, · 1045 unmodified lines
10461046 srcu_key = srcu_read_lock(&dev->disassociate_srcu);
10471047 ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
10481048 if (ib_dev)
1049− ret = sprintf(buf, "%s\n", dev_name(&ib_dev->dev));
1049+ ret = sysfs_emit(buf, "%s\n", dev_name(&ib_dev->dev));
10501050 srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
10511051
10521052 return ret;
@@ -1065,7 +1065,7 @@ static ssize_t abi_version_show(struct device *device, · 12 unmodified lines
10651065 srcu_key = srcu_read_lock(&dev->disassociate_srcu);
10661066 ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
10671067 if (ib_dev)
1068− ret = sprintf(buf, "%u\n", ib_dev->ops.uverbs_abi_ver);
1068+ ret = sysfs_emit(buf, "%u\n", ib_dev->ops.uverbs_abi_ver);
10691069 srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
10701070
10711071 return ret;
drivers/infiniband/core/uverbs_std_types.c+7−11
@@ -88,7 +88,7 @@ static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject, · 87 unmodified lines
8888 return -EBUSY;
8989
9090 ret = rwq_ind_tbl->device->ops.destroy_rwq_ind_table(rwq_ind_tbl);
91− if (ib_is_destroy_retryable(ret, why, uobject))
91+ if (ret)
9292 return ret;
9393
9494 for (i = 0; i < table_size; i++)
@@ -96,7 +96,7 @@ static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject, · 1 unmodified line
9696
9797 kfree(rwq_ind_tbl);
9898 kfree(ind_tbl);
99− return ret;
99+ return 0;
100100 }
101101
102102 static int uverbs_free_xrcd(struct ib_uobject *uobject,
@@ -108,9 +108,8 @@ static int uverbs_free_xrcd(struct ib_uobject *uobject, · 5 unmodified lines
108108 container_of(uobject, struct ib_uxrcd_object, uobject);
109109 int ret;
110110
111− ret = ib_destroy_usecnt(&uxrcd->refcnt, why, uobject);
112− if (ret)
113− return ret;
111+ if (atomic_read(&uxrcd->refcnt))
112+ return -EBUSY;
114113
115114 mutex_lock(&attrs->ufile->device->xrcd_tree_mutex);
116115 ret = ib_uverbs_dealloc_xrcd(uobject, xrcd, why, attrs);
@@ -124,11 +123,9 @@ static int uverbs_free_pd(struct ib_uobject *uobject, · 7 unmodified lines
124123 struct uverbs_attr_bundle *attrs)
125124 {
126125 struct ib_pd *pd = uobject->object;
127− int ret;
128126
129− ret = ib_destroy_usecnt(&pd->usecnt, why, uobject);
130− if (ret)
131− return ret;
127+ if (atomic_read(&pd->usecnt))
128+ return -EBUSY;
132129
133130 return ib_dealloc_pd_user(pd, &attrs->driver_udata);
134131 }
@@ -157,7 +154,7 @@ void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue) · 22 unmodified lines
157154 spin_unlock_irq(&event_queue->lock);
158155 }
159156
160−static int
157+static void
161158 uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj,
162159 enum rdma_remove_reason why)
163160 {
@@ -166,7 +163,6 @@ uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj, · 2 unmodified lines
166163 uobj);
167164
168165 ib_uverbs_free_event_queue(&file->ev_queue);
169− return 0;
170166 }
171167
172168 int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs)
drivers/infiniband/core/uverbs_std_types_async_fd.c+2−3
@@ -19,8 +19,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_ASYNC_EVENT_ALLOC)( · 18 unmodified lines
1919 return 0;
2020 }
2121
22−static int uverbs_async_event_destroy_uobj(struct ib_uobject *uobj,
23− enum rdma_remove_reason why)
22+static void uverbs_async_event_destroy_uobj(struct ib_uobject *uobj,
23+ enum rdma_remove_reason why)
2424 {
2525 struct ib_uverbs_async_event_file *event_file =
2626 container_of(uobj, struct ib_uverbs_async_event_file, uobj);
@@ -30,7 +30,6 @@ static int uverbs_async_event_destroy_uobj(struct ib_uobject *uobj, · 3 unmodified lines
3030 if (why == RDMA_REMOVE_DRIVER_REMOVE)
3131 ib_uverbs_async_handler(event_file, 0, IB_EVENT_DEVICE_FATAL,
3232 NULL, NULL);
33− return 0;
3433 }
3534
3635 int uverbs_async_event_release(struct inode *inode, struct file *filp)
drivers/infiniband/core/uverbs_std_types_counters.c+2−3
@@ -42,9 +42,8 @@ static int uverbs_free_counters(struct ib_uobject *uobject, · 41 unmodified lines
4242 struct ib_counters *counters = uobject->object;
4343 int ret;
4444
45− ret = ib_destroy_usecnt(&counters->usecnt, why, uobject);
46− if (ret)
47− return ret;
45+ if (atomic_read(&counters->usecnt))
46+ return -EBUSY;
4847
4948 ret = counters->device->ops.destroy_counters(counters);
5049 if (ret)
drivers/infiniband/core/uverbs_std_types_cq.c+2−2
@@ -46,7 +46,7 @@ static int uverbs_free_cq(struct ib_uobject *uobject, · 45 unmodified lines
4646 int ret;
4747
4848 ret = ib_destroy_cq_user(cq, &attrs->driver_udata);
49− if (ib_is_destroy_retryable(ret, why, uobject))
49+ if (ret)
5050 return ret;
5151
5252 ib_uverbs_release_ucq(
@@ -55,7 +55,7 @@ static int uverbs_free_cq(struct ib_uobject *uobject, · 2 unmodified lines
5555 ev_queue) :
5656 NULL,
5757 ucq);
58− return ret;
58+ return 0;
5959 }
6060
6161 static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
drivers/infiniband/core/uverbs_std_types_device.c+5−9
@@ -317,8 +317,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_GID_TABLE)( · 316 unmodified lines
317317 struct ib_device *ib_dev;
318318 size_t user_entry_size;
319319 ssize_t num_entries;
320− size_t max_entries;
321− size_t num_bytes;
320+ int max_entries;
322321 u32 flags;
323322 int ret;
324323
@@ -336,19 +335,16 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_GID_TABLE)( · 11 unmodified lines
336335 attrs, UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES,
337336 user_entry_size);
338337 if (max_entries <= 0)
339− return -EINVAL;
338+ return max_entries ?: -EINVAL;
340339
341340 ucontext = ib_uverbs_get_ucontext(attrs);
342341 if (IS_ERR(ucontext))
343342 return PTR_ERR(ucontext);
344343 ib_dev = ucontext->device;
345344
346− if (check_mul_overflow(max_entries, sizeof(*entries), &num_bytes))
347− return -EINVAL;
348−
349− entries = uverbs_zalloc(attrs, num_bytes);
350− if (!entries)
351− return -ENOMEM;
345+ entries = uverbs_kcalloc(attrs, max_entries, sizeof(*entries));
346+ if (IS_ERR(entries))
347+ return PTR_ERR(entries);
352348
353349 num_entries = rdma_query_gid_table(ib_dev, entries, max_entries);
354350 if (num_entries < 0)
drivers/infiniband/core/uverbs_std_types_dm.c+2−4
@@ -39,11 +39,9 @@ static int uverbs_free_dm(struct ib_uobject *uobject, · 38 unmodified lines
3939 struct uverbs_attr_bundle *attrs)
4040 {
4141 struct ib_dm *dm = uobject->object;
42− int ret;
4342
44− ret = ib_destroy_usecnt(&dm->usecnt, why, uobject);
45− if (ret)
46− return ret;
43+ if (atomic_read(&dm->usecnt))
44+ return -EBUSY;
4745
4846 return dm->device->ops.dealloc_dm(dm, attrs);
4947 }
drivers/infiniband/core/uverbs_std_types_flow_action.c+2−4
@@ -39,11 +39,9 @@ static int uverbs_free_flow_action(struct ib_uobject *uobject, · 38 unmodified lines
3939 struct uverbs_attr_bundle *attrs)
4040 {
4141 struct ib_flow_action *action = uobject->object;
42− int ret;
4342
44− ret = ib_destroy_usecnt(&action->usecnt, why, uobject);
45− if (ret)
46− return ret;
43+ if (atomic_read(&action->usecnt))
44+ return -EBUSY;
4745
4846 return action->device->ops.destroy_flow_action(action);
4947 }
drivers/infiniband/core/uverbs_std_types_mr.c+5−1
@@ -33,6 +33,7 @@ · 32 unmodified lines
3333 #include "rdma_core.h"
3434 #include "uverbs.h"
3535 #include <rdma/uverbs_std_types.h>
36+#include "restrack.h"
3637
3738 static int uverbs_free_mr(struct ib_uobject *uobject,
3839 enum rdma_remove_reason why,
@@ -114,7 +115,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)( · 75 unmodified lines
114115 if (!(attr.access_flags & IB_ZERO_BASED))
115116 return -EINVAL;
116117
117− ret = ib_check_mr_access(attr.access_flags);
118+ ret = ib_check_mr_access(ib_dev, attr.access_flags);
118119 if (ret)
119120 return ret;
120121
@@ -134,6 +135,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)( · 13 unmodified lines
134135 atomic_inc(&pd->usecnt);
135136 atomic_inc(&dm->usecnt);
136137
138+ rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
139+ rdma_restrack_set_name(&mr->res, NULL);
140+ rdma_restrack_add(&mr->res);
137141 uobj->object = mr;
138142
139143 uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_REG_DM_MR_HANDLE);
drivers/infiniband/core/uverbs_std_types_qp.c+4−4
@@ -32,14 +32,14 @@ static int uverbs_free_qp(struct ib_uobject *uobject, · 31 unmodified lines
3232 }
3333
3434 ret = ib_destroy_qp_user(qp, &attrs->driver_udata);
35− if (ib_is_destroy_retryable(ret, why, uobject))
35+ if (ret)
3636 return ret;
3737
3838 if (uqp->uxrcd)
3939 atomic_dec(&uqp->uxrcd->refcnt);
4040
4141 ib_uverbs_release_uevent(&uqp->uevent);
42− return ret;
42+ return 0;
4343 }
4444
4545 static int check_creation_flags(enum ib_qp_type qp_type,
@@ -251,8 +251,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( · 205 unmodified lines
251251 if (attr.qp_type == IB_QPT_XRC_TGT)
252252 qp = ib_create_qp(pd, &attr);
253253 else
254− qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
255− obj);
254+ qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, obj,
255+ NULL);
256256
257257 if (IS_ERR(qp)) {
258258 ret = PTR_ERR(qp);
drivers/infiniband/core/uverbs_std_types_srq.c+2−2
@@ -18,7 +18,7 @@ static int uverbs_free_srq(struct ib_uobject *uobject, · 17 unmodified lines
1818 int ret;
1919
2020 ret = ib_destroy_srq_user(srq, &attrs->driver_udata);
21− if (ib_is_destroy_retryable(ret, why, uobject))
21+ if (ret)
2222 return ret;
2323
2424 if (srq_type == IB_SRQT_XRC) {
@@ -30,7 +30,7 @@ static int uverbs_free_srq(struct ib_uobject *uobject, · 5 unmodified lines
3030 }
3131
3232 ib_uverbs_release_uevent(uevent);
33− return ret;
33+ return 0;
3434 }
3535
3636 static int UVERBS_HANDLER(UVERBS_METHOD_SRQ_CREATE)(
drivers/infiniband/core/uverbs_std_types_wq.c+2−2
@@ -17,11 +17,11 @@ static int uverbs_free_wq(struct ib_uobject *uobject, · 16 unmodified lines
1717 int ret;
1818
1919 ret = ib_destroy_wq_user(wq, &attrs->driver_udata);
20− if (ib_is_destroy_retryable(ret, why, uobject))
20+ if (ret)
2121 return ret;
2222
2323 ib_uverbs_release_uevent(&uwq->uevent);
24− return ret;
24+ return 0;
2525 }
2626
2727 static int UVERBS_HANDLER(UVERBS_METHOD_WQ_CREATE)(
drivers/infiniband/core/uverbs_uapi.c+1−4
@@ -79,10 +79,7 @@ static int uapi_create_write(struct uverbs_api *uapi, · 78 unmodified lines
7979
8080 method_elm->is_ex = def->write.is_ex;
8181 method_elm->handler = def->func_write;
82− if (def->write.is_ex)
83− method_elm->disabled = !(ibdev->uverbs_ex_cmd_mask &
84− BIT_ULL(def->write.command_num));
85− else
82+ if (!def->write.is_ex)
8683 method_elm->disabled = !(ibdev->uverbs_cmd_mask &
8784 BIT_ULL(def->write.command_num));
8885
drivers/infiniband/core/verbs.c+17−10
@@ -244,7 +244,7 @@ EXPORT_SYMBOL(rdma_port_get_link_layer); · 243 unmodified lines
244244 /* Protection domains */
245245
246246 /**
247− * ib_alloc_pd - Allocates an unused protection domain.
247+ * __ib_alloc_pd - Allocates an unused protection domain.
248248 * @device: The device on which to allocate the protection domain.
249249 * @flags: protection domain flags
250250 * @caller: caller's build-time module name
@@ -516,7 +516,7 @@ static struct ib_ah *_rdma_create_ah(struct ib_pd *pd, · 265 unmodified lines
516516
517517 might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
518518
519− if (!device->ops.create_ah)
519+ if (!udata && !device->ops.create_ah)
520520 return ERR_PTR(-EOPNOTSUPP);
521521
522522 ah = rdma_zalloc_drv_obj_gfp(
@@ -533,7 +533,10 @@ static struct ib_ah *_rdma_create_ah(struct ib_pd *pd, · 10 unmodified lines
533533 init_attr.flags = flags;
534534 init_attr.xmit_slave = xmit_slave;
535535
536− ret = device->ops.create_ah(ah, &init_attr, udata);
536+ if (udata)
537+ ret = device->ops.create_user_ah(ah, &init_attr, udata);
538+ else
539+ ret = device->ops.create_ah(ah, &init_attr, NULL);
537540 if (ret) {
538541 kfree(ah);
539542 return ERR_PTR(ret);
@@ -1188,17 +1191,19 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp, · 648 unmodified lines
11881191 }
11891192
11901193 /**
1191− * ib_create_qp - Creates a kernel QP associated with the specified protection
1194+ * ib_create_named_qp - Creates a kernel QP associated with the specified protection
11921195 * domain.
11931196 * @pd: The protection domain associated with the QP.
11941197 * @qp_init_attr: A list of initial attributes required to create the
11951198 * QP. If QP creation succeeds, then the attributes are updated to
11961199 * the actual capabilities of the created QP.
1200+ * @caller: caller's build-time module name
11971201 *
11981202 * NOTE: for user qp use ib_create_qp_user with valid udata!
11991203 */
1200−struct ib_qp *ib_create_qp(struct ib_pd *pd,
1201− struct ib_qp_init_attr *qp_init_attr)
1204+struct ib_qp *ib_create_named_qp(struct ib_pd *pd,
1205+ struct ib_qp_init_attr *qp_init_attr,
1206+ const char *caller)
12021207 {
12031208 struct ib_device *device = pd ? pd->device : qp_init_attr->xrcd->device;
12041209 struct ib_qp *qp;
@@ -1223,7 +1228,7 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd, · 18 unmodified lines
12231228 if (qp_init_attr->cap.max_rdma_ctxs)
12241229 rdma_rw_init_qp(device, qp_init_attr);
12251230
1226− qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL);
1231+ qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL, caller);
12271232 if (IS_ERR(qp))
12281233 return qp;
12291234
@@ -1289,7 +1294,7 @@ err: · 59 unmodified lines
12891294 return ERR_PTR(ret);
12901295
12911296 }
1292−EXPORT_SYMBOL(ib_create_qp);
1297+EXPORT_SYMBOL(ib_create_named_qp);
12931298
12941299 static const struct {
12951300 int valid;
@@ -1662,7 +1667,7 @@ static bool is_qp_type_connected(const struct ib_qp *qp) · 366 unmodified lines
16621667 qp->qp_type == IB_QPT_XRC_TGT);
16631668 }
16641669
1665−/**
1670+/*
16661671 * IB core internal function to perform QP attributes modification.
16671672 */
16681673 static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
@@ -1698,8 +1703,10 @@ static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, · 29 unmodified lines
16981703 slave = rdma_lag_get_ah_roce_slave(qp->device,
16991704 &attr->ah_attr,
17001705 GFP_KERNEL);
1701− if (IS_ERR(slave))
1706+ if (IS_ERR(slave)) {
1707+ ret = PTR_ERR(slave);
17021708 goto out_av;
1709+ }
17031710 attr->xmit_slave = slave;
17041711 }
17051712 }
drivers/infiniband/hw/bnxt_re/ib_verbs.c+12−3
@@ -1271,10 +1271,12 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, · 1270 unmodified lines
12711271 }
12721272 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
12731273 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1274− if (init_attr->create_flags)
1274+ if (init_attr->create_flags) {
12751275 ibdev_dbg(&rdev->ibdev,
12761276 "QP create flags 0x%x not supported",
12771277 init_attr->create_flags);
1278+ return -EOPNOTSUPP;
1279+ }
12781280
12791281 /* Setup CQs */
12801282 if (init_attr->send_cq) {
@@ -1657,8 +1659,8 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, · 376 unmodified lines
16571659 srq->qplib_srq.max_wqe = entries;
16581660
16591661 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1660− srq->qplib_srq.wqe_size =
1661− bnxt_re_get_rwqe_size(srq->qplib_srq.max_sge);
1662+ /* 128 byte wqe size for SRQ . So use max sges */
1663+ srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
16621664 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
16631665 srq->srq_limit = srq_init_attr->attr.srq_limit;
16641666 srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
@@ -1829,6 +1831,9 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, · 164 unmodified lines
18291831 unsigned int flags;
18301832 u8 nw_type;
18311833
1834+ if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1835+ return -EOPNOTSUPP;
1836+
18321837 qp->qplib_qp.modify_flags = 0;
18331838 if (qp_attr_mask & IB_QP_STATE) {
18341839 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
@@ -2078,6 +2083,7 @@ int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, · 243 unmodified lines
20782083 goto out;
20792084 }
20802085 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2086+ qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
20812087 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
20822088 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
20832089 qp_attr->pkey_index = qplib_qp->pkey_index;
@@ -2827,6 +2833,9 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 743 unmodified lines
28272833 struct bnxt_qplib_nq *nq = NULL;
28282834 unsigned int nq_alloc_cnt;
28292835
2836+ if (attr->flags)
2837+ return -EOPNOTSUPP;
2838+
28302839 /* Validate CQ fields */
28312840 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
28322841 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
drivers/infiniband/hw/bnxt_re/main.c+3−31
@@ -608,7 +608,7 @@ static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr, · 607 unmodified lines
608608 struct bnxt_re_dev *rdev =
609609 rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);
610610
611− return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
611+ return sysfs_emit(buf, "0x%x\n", rdev->en_dev->pdev->vendor);
612612 }
613613 static DEVICE_ATTR_RO(hw_rev);
614614
@@ -618,7 +618,7 @@ static ssize_t hca_type_show(struct device *device, · 3 unmodified lines
618618 struct bnxt_re_dev *rdev =
619619 rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);
620620
621− return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc);
621+ return sysfs_emit(buf, "%s\n", rdev->ibdev.node_desc);
622622 }
623623 static DEVICE_ATTR_RO(hca_type);
624624
@@ -646,6 +646,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = { · 21 unmodified lines
646646 .create_cq = bnxt_re_create_cq,
647647 .create_qp = bnxt_re_create_qp,
648648 .create_srq = bnxt_re_create_srq,
649+ .create_user_ah = bnxt_re_create_ah,
649650 .dealloc_driver = bnxt_re_dealloc_driver,
650651 .dealloc_pd = bnxt_re_dealloc_pd,
651652 .dealloc_ucontext = bnxt_re_dealloc_ucontext,
@@ -701,35 +702,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) · 49 unmodified lines
701702 ibdev->dev.parent = &rdev->en_dev->pdev->dev;
702703 ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
703704
704− /* User space */
705− ibdev->uverbs_cmd_mask =
706− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
707− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
708− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
709− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
710− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
711− (1ull << IB_USER_VERBS_CMD_REG_MR) |
712− (1ull << IB_USER_VERBS_CMD_REREG_MR) |
713− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
714− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
715− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
716− (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
717− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
718− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
719− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
720− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
721− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
722− (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
723− (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
724− (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
725− (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
726− (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
727− (1ull << IB_USER_VERBS_CMD_MODIFY_AH) |
728− (1ull << IB_USER_VERBS_CMD_QUERY_AH) |
729− (1ull << IB_USER_VERBS_CMD_DESTROY_AH);
730− /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */
731−
732−
733705 rdma_set_device_sysfs_group(ibdev, &bnxt_re_dev_attr_group);
734706 ib_set_device_ops(ibdev, &bnxt_re_dev_ops);
735707 ret = ib_device_set_netdev(&rdev->ibdev, rdev->netdev, 1);
drivers/infiniband/hw/bnxt_re/qplib_sp.c+1−1
@@ -118,7 +118,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, · 117 unmodified lines
118118 * 128 WQEs needs to be reserved for the HW (8916). Prevent
119119 * reporting the max number
120120 */
121− attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS;
121+ attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS + 1;
122122 attr->max_qp_sges = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx) ?
123123 6 : sb->max_sge;
124124 attr->max_cq = le32_to_cpu(sb->max_cq);
drivers/infiniband/hw/cxgb4/cq.c+3−0
@@ -1006,6 +1006,9 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 1005 unmodified lines
10061006
10071007 pr_debug("ib_dev %p entries %d\n", ibdev, entries);
10081008 if (attr->flags)
1009+ return -EOPNOTSUPP;
1010+
1011+ if (entries < 1 || entries > ibdev->attrs.max_cqe)
10091012 return -EINVAL;
10101013
10111014 if (vector >= rhp->rdev.lldi.nciq)
drivers/infiniband/hw/cxgb4/iw_cxgb4.h+0−2
@@ -983,9 +983,7 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, · 982 unmodified lines
983983 u32 max_num_sg);
984984 int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
985985 unsigned int *sg_offset);
986−int c4iw_dealloc_mw(struct ib_mw *mw);
987986 void c4iw_dealloc(struct uld_ctx *ctx);
988−int c4iw_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);
989987 struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
990988 u64 length, u64 virt, int acc,
991989 struct ib_udata *udata);
drivers/infiniband/hw/cxgb4/mem.c+0−84
@@ -365,22 +365,6 @@ static int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size, · 364 unmodified lines
365365 pbl_size, pbl_addr, skb, wr_waitp);
366366 }
367367
368−static int allocate_window(struct c4iw_rdev *rdev, u32 *stag, u32 pdid,
369− struct c4iw_wr_wait *wr_waitp)
370−{
371− *stag = T4_STAG_UNSET;
372− return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0,
373− 0UL, 0, 0, 0, 0, NULL, wr_waitp);
374−}
375−
376−static int deallocate_window(struct c4iw_rdev *rdev, u32 stag,
377− struct sk_buff *skb,
378− struct c4iw_wr_wait *wr_waitp)
379−{
380− return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0,
381− 0, skb, wr_waitp);
382−}
383−
384368 static int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid,
385369 u32 pbl_size, u32 pbl_addr,
386370 struct c4iw_wr_wait *wr_waitp)
@@ -611,74 +595,6 @@ err_free_mhp: · 224 unmodified lines
611595 return ERR_PTR(err);
612596 }
613597
614−int c4iw_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
615−{
616− struct c4iw_mw *mhp = to_c4iw_mw(ibmw);
617− struct c4iw_dev *rhp;
618− struct c4iw_pd *php;
619− u32 mmid;
620− u32 stag = 0;
621− int ret;
622−
623− if (ibmw->type != IB_MW_TYPE_1)
624− return -EINVAL;
625−
626− php = to_c4iw_pd(ibmw->pd);
627− rhp = php->rhp;
628− mhp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);
629− if (!mhp->wr_waitp)
630− return -ENOMEM;
631−
632− mhp->dereg_skb = alloc_skb(SGE_MAX_WR_LEN, GFP_KERNEL);
633− if (!mhp->dereg_skb) {
634− ret = -ENOMEM;
635− goto free_wr_wait;
636− }
637−
638− ret = allocate_window(&rhp->rdev, &stag, php->pdid, mhp->wr_waitp);
639− if (ret)
640− goto free_skb;
641−
642− mhp->rhp = rhp;
643− mhp->attr.pdid = php->pdid;
644− mhp->attr.type = FW_RI_STAG_MW;
645− mhp->attr.stag = stag;
646− mmid = (stag) >> 8;
647− ibmw->rkey = stag;
648− if (xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL)) {
649− ret = -ENOMEM;
650− goto dealloc_win;
651− }
652− pr_debug("mmid 0x%x mhp %p stag 0x%x\n", mmid, mhp, stag);
653− return 0;
654−
655−dealloc_win:
656− deallocate_window(&rhp->rdev, mhp->attr.stag, mhp->dereg_skb,
657− mhp->wr_waitp);
658−free_skb:
659− kfree_skb(mhp->dereg_skb);
660−free_wr_wait:
661− c4iw_put_wr_wait(mhp->wr_waitp);
662− return ret;
663−}
664−
665−int c4iw_dealloc_mw(struct ib_mw *mw)
666−{
667− struct c4iw_dev *rhp;
668− struct c4iw_mw *mhp;
669− u32 mmid;
670−
671− mhp = to_c4iw_mw(mw);
672− rhp = mhp->rhp;
673− mmid = (mw->rkey) >> 8;
674− xa_erase_irq(&rhp->mrs, mmid);
675− deallocate_window(&rhp->rdev, mhp->attr.stag, mhp->dereg_skb,
676− mhp->wr_waitp);
677− kfree_skb(mhp->dereg_skb);
678− c4iw_put_wr_wait(mhp->wr_waitp);
679− return 0;
680−}
681−
682598 struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
683599 u32 max_num_sg)
684600 {
drivers/infiniband/hw/cxgb4/provider.c+6−29
@@ -322,8 +322,9 @@ static ssize_t hw_rev_show(struct device *dev, · 321 unmodified lines
322322 rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);
323323
324324 pr_debug("dev 0x%p\n", dev);
325− return sprintf(buf, "%d\n",
326− CHELSIO_CHIP_RELEASE(c4iw_dev->rdev.lldi.adapter_type));
325+ return sysfs_emit(
326+ buf, "%d\n",
327+ CHELSIO_CHIP_RELEASE(c4iw_dev->rdev.lldi.adapter_type));
327328 }
328329 static DEVICE_ATTR_RO(hw_rev);
329330
@@ -337,7 +338,7 @@ static ssize_t hca_type_show(struct device *dev, · 7 unmodified lines
337338
338339 pr_debug("dev 0x%p\n", dev);
339340 lldev->ethtool_ops->get_drvinfo(lldev, &info);
340− return sprintf(buf, "%s\n", info.driver);
341+ return sysfs_emit(buf, "%s\n", info.driver);
341342 }
342343 static DEVICE_ATTR_RO(hca_type);
343344
@@ -348,8 +349,8 @@ static ssize_t board_id_show(struct device *dev, struct device_attribute *attr, · 4 unmodified lines
348349 rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);
349350
350351 pr_debug("dev 0x%p\n", dev);
351− return sprintf(buf, "%x.%x\n", c4iw_dev->rdev.lldi.pdev->vendor,
352− c4iw_dev->rdev.lldi.pdev->device);
352+ return sysfs_emit(buf, "%x.%x\n", c4iw_dev->rdev.lldi.pdev->vendor,
353+ c4iw_dev->rdev.lldi.pdev->device);
353354 }
354355 static DEVICE_ATTR_RO(board_id);
355356
@@ -456,13 +457,11 @@ static const struct ib_device_ops c4iw_dev_ops = { · 100 unmodified lines
456457
457458 .alloc_hw_stats = c4iw_alloc_stats,
458459 .alloc_mr = c4iw_alloc_mr,
459− .alloc_mw = c4iw_alloc_mw,
460460 .alloc_pd = c4iw_allocate_pd,
461461 .alloc_ucontext = c4iw_alloc_ucontext,
462462 .create_cq = c4iw_create_cq,
463463 .create_qp = c4iw_create_qp,
464464 .create_srq = c4iw_create_srq,
465− .dealloc_mw = c4iw_dealloc_mw,
466465 .dealloc_pd = c4iw_deallocate_pd,
467466 .dealloc_ucontext = c4iw_dealloc_ucontext,
468467 .dereg_mr = c4iw_dereg_mr,
@@ -533,28 +532,6 @@ void c4iw_register_device(struct work_struct *work) · 64 unmodified lines
533532 if (fastreg_support)
534533 dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
535534 dev->ibdev.local_dma_lkey = 0;
536− dev->ibdev.uverbs_cmd_mask =
537− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
538− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
539− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
540− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
541− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
542− (1ull << IB_USER_VERBS_CMD_REG_MR) |
543− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
544− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
545− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
546− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
547− (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
548− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
549− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
550− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
551− (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
552− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
553− (1ull << IB_USER_VERBS_CMD_POST_SEND) |
554− (1ull << IB_USER_VERBS_CMD_POST_RECV) |
555− (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
556− (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
557− (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
558535 dev->ibdev.node_type = RDMA_NODE_RNIC;
559536 BUILD_BUG_ON(sizeof(C4IW_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
560537 memcpy(dev->ibdev.node_desc, C4IW_NODE_DESC, sizeof(C4IW_NODE_DESC));
drivers/infiniband/hw/cxgb4/qp.c+7−1
@@ -2126,7 +2126,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, · 2125 unmodified lines
21262126
21272127 pr_debug("ib_pd %p\n", pd);
21282128
2129− if (attrs->qp_type != IB_QPT_RC)
2129+ if (attrs->qp_type != IB_QPT_RC || attrs->create_flags)
21302130 return ERR_PTR(-EOPNOTSUPP);
21312131
21322132 php = to_c4iw_pd(pd);
@@ -2374,6 +2374,9 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 241 unmodified lines
23742374
23752375 pr_debug("ib_qp %p\n", ibqp);
23762376
2377+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2378+ return -EOPNOTSUPP;
2379+
23772380 /* iwarp does not support the RTR state */
23782381 if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
23792382 attr_mask &= ~IB_QP_STATE;
@@ -2680,6 +2683,9 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs, · 300 unmodified lines
26802683 int ret;
26812684 int wr_len;
26822685
2686+ if (attrs->srq_type != IB_SRQT_BASIC)
2687+ return -EOPNOTSUPP;
2688+
26832689 pr_debug("%s ib_pd %p\n", __func__, pd);
26842690
26852691 php = to_c4iw_pd(pd);
drivers/infiniband/hw/efa/efa_main.c+3−31
@@ -245,9 +245,9 @@ static const struct ib_device_ops efa_dev_ops = { · 244 unmodified lines
245245 .alloc_hw_stats = efa_alloc_hw_stats,
246246 .alloc_pd = efa_alloc_pd,
247247 .alloc_ucontext = efa_alloc_ucontext,
248− .create_ah = efa_create_ah,
249248 .create_cq = efa_create_cq,
250249 .create_qp = efa_create_qp,
250+ .create_user_ah = efa_create_ah,
251251 .dealloc_pd = efa_dealloc_pd,
252252 .dealloc_ucontext = efa_dealloc_ucontext,
253253 .dereg_mr = efa_dereg_mr,
@@ -308,27 +308,6 @@ static int efa_ib_device_add(struct efa_dev *dev) · 54 unmodified lines
308308 dev->ibdev.num_comp_vectors = 1;
309309 dev->ibdev.dev.parent = &pdev->dev;
310310
311− dev->ibdev.uverbs_cmd_mask =
312− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
313− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
314− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
315− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
316− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
317− (1ull << IB_USER_VERBS_CMD_REG_MR) |
318− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
319− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
320− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
321− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
322− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
323− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
324− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
325− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
326− (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
327− (1ull << IB_USER_VERBS_CMD_DESTROY_AH);
328−
329− dev->ibdev.uverbs_ex_cmd_mask =
330− (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE);
331−
332311 ib_set_device_ops(&dev->ibdev, &efa_dev_ops);
333312
334313 err = ib_register_device(&dev->ibdev, "efa_%d", &pdev->dev);
@@ -405,19 +384,12 @@ static int efa_device_init(struct efa_com_dev *edev, struct pci_dev *pdev) · 70 unmodified lines
405384 return err;
406385 }
407386
408− err = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_width));
387+ err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_width));
409388 if (err) {
410− dev_err(&pdev->dev, "pci_set_dma_mask failed %d\n", err);
389+ dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", err);
411390 return err;
412391 }
413392
414− err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_width));
415− if (err) {
416− dev_err(&pdev->dev,
417− "err_pci_set_consistent_dma_mask failed %d\n",
418− err);
419− return err;
420− }
421393 dma_set_max_seg_size(&pdev->dev, UINT_MAX);
422394 return 0;
423395 }
drivers/infiniband/hw/efa/efa_verbs.c+6−0
@@ -917,6 +917,9 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, · 916 unmodified lines
917917 enum ib_qp_state new_state;
918918 int err;
919919
920+ if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
921+ return -EOPNOTSUPP;
922+
920923 if (udata->inlen &&
921924 !ib_is_udata_cleared(udata, 0, udata->inlen)) {
922925 ibdev_dbg(&dev->ibdev,
@@ -1029,6 +1032,9 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 106 unmodified lines
10291032
10301033 ibdev_dbg(ibdev, "create_cq entries %d\n", entries);
10311034
1035+ if (attr->flags)
1036+ return -EOPNOTSUPP;
1037+
10321038 if (entries < 1 || entries > dev->dev_attr.max_cq_depth) {
10331039 ibdev_dbg(ibdev,
10341040 "cq: requested entries[%u] non-positive or greater than max[%u]\n",
drivers/infiniband/hw/hfi1/qp.c+1−0
@@ -339,6 +339,7 @@ int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) · 338 unmodified lines
339339 return -EINVAL;
340340 if (ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)] == 0xf)
341341 return -EINVAL;
342+ break;
342343 default:
343344 break;
344345 }
drivers/infiniband/hw/hfi1/sysfs.c+29−33
@@ -151,7 +151,7 @@ struct hfi1_port_attr { · 150 unmodified lines
151151
152152 static ssize_t cc_prescan_show(struct hfi1_pportdata *ppd, char *buf)
153153 {
154− return sprintf(buf, "%s\n", ppd->cc_prescan ? "on" : "off");
154+ return sysfs_emit(buf, "%s\n", ppd->cc_prescan ? "on" : "off");
155155 }
156156
157157 static ssize_t cc_prescan_store(struct hfi1_pportdata *ppd, const char *buf,
@@ -296,7 +296,7 @@ static ssize_t sc2vl_attr_show(struct kobject *kobj, struct attribute *attr, · 138 unmodified lines
296296 container_of(kobj, struct hfi1_pportdata, sc2vl_kobj);
297297 struct hfi1_devdata *dd = ppd->dd;
298298
299− return sprintf(buf, "%u\n", *((u8 *)dd->sc2vl + sattr->sc));
299+ return sysfs_emit(buf, "%u\n", *((u8 *)dd->sc2vl + sattr->sc));
300300 }
301301
302302 static const struct sysfs_ops hfi1_sc2vl_ops = {
@@ -401,7 +401,7 @@ static ssize_t sl2sc_attr_show(struct kobject *kobj, struct attribute *attr, · 98 unmodified lines
401401 container_of(kobj, struct hfi1_pportdata, sl2sc_kobj);
402402 struct hfi1_ibport *ibp = &ppd->ibport_data;
403403
404− return sprintf(buf, "%u\n", ibp->sl_to_sc[sattr->sl]);
404+ return sysfs_emit(buf, "%u\n", ibp->sl_to_sc[sattr->sl]);
405405 }
406406
407407 static const struct sysfs_ops hfi1_sl2sc_ops = {
@@ -475,7 +475,7 @@ static ssize_t vl2mtu_attr_show(struct kobject *kobj, struct attribute *attr, · 67 unmodified lines
475475 container_of(kobj, struct hfi1_pportdata, vl2mtu_kobj);
476476 struct hfi1_devdata *dd = ppd->dd;
477477
478− return sprintf(buf, "%u\n", dd->vld[vlattr->vl].mtu);
478+ return sysfs_emit(buf, "%u\n", dd->vld[vlattr->vl].mtu);
479479 }
480480
481481 static const struct sysfs_ops hfi1_vl2mtu_ops = {
@@ -500,7 +500,7 @@ static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr, · 18 unmodified lines
500500 struct hfi1_ibdev *dev =
501501 rdma_device_to_drv_device(device, struct hfi1_ibdev, rdi.ibdev);
502502
503− return sprintf(buf, "%x\n", dd_from_dev(dev)->minrev);
503+ return sysfs_emit(buf, "%x\n", dd_from_dev(dev)->minrev);
504504 }
505505 static DEVICE_ATTR_RO(hw_rev);
506506
@@ -510,13 +510,11 @@ static ssize_t board_id_show(struct device *device, · 3 unmodified lines
510510 struct hfi1_ibdev *dev =
511511 rdma_device_to_drv_device(device, struct hfi1_ibdev, rdi.ibdev);
512512 struct hfi1_devdata *dd = dd_from_dev(dev);
513− int ret;
514513
515514 if (!dd->boardname)
516− ret = -EINVAL;
517− else
518− ret = scnprintf(buf, PAGE_SIZE, "%s\n", dd->boardname);
519− return ret;
515+ return -EINVAL;
516+
517+ return sysfs_emit(buf, "%s\n", dd->boardname);
520518 }
521519 static DEVICE_ATTR_RO(board_id);
522520
@@ -528,7 +526,7 @@ static ssize_t boardversion_show(struct device *device, · 5 unmodified lines
528526 struct hfi1_devdata *dd = dd_from_dev(dev);
529527
530528 /* The string printed here is already newline-terminated. */
531− return scnprintf(buf, PAGE_SIZE, "%s", dd->boardversion);
529+ return sysfs_emit(buf, "%s", dd->boardversion);
532530 }
533531 static DEVICE_ATTR_RO(boardversion);
534532
@@ -545,9 +543,9 @@ static ssize_t nctxts_show(struct device *device, · 10 unmodified lines
545543 * and a receive context, so returning the smaller of the two counts
546544 * give a more accurate picture of total contexts available.
547545 */
548− return scnprintf(buf, PAGE_SIZE, "%u\n",
549− min(dd->num_user_contexts,
550− (u32)dd->sc_sizes[SC_USER].count));
546+ return sysfs_emit(buf, "%u\n",
547+ min(dd->num_user_contexts,
548+ (u32)dd->sc_sizes[SC_USER].count));
551549 }
552550 static DEVICE_ATTR_RO(nctxts);
553551
@@ -559,7 +557,7 @@ static ssize_t nfreectxts_show(struct device *device, · 5 unmodified lines
559557 struct hfi1_devdata *dd = dd_from_dev(dev);
560558
561559 /* Return the number of free user ports (contexts) available. */
562− return scnprintf(buf, PAGE_SIZE, "%u\n", dd->freectxts);
560+ return sysfs_emit(buf, "%u\n", dd->freectxts);
563561 }
564562 static DEVICE_ATTR_RO(nfreectxts);
565563
@@ -570,7 +568,8 @@ static ssize_t serial_show(struct device *device, · 4 unmodified lines
570568 rdma_device_to_drv_device(device, struct hfi1_ibdev, rdi.ibdev);
571569 struct hfi1_devdata *dd = dd_from_dev(dev);
572570
573− return scnprintf(buf, PAGE_SIZE, "%s", dd->serial);
571+ /* dd->serial is already newline terminated in chip.c */
572+ return sysfs_emit(buf, "%s", dd->serial);
574573 }
575574 static DEVICE_ATTR_RO(serial);
576575
@@ -598,9 +597,8 @@ static DEVICE_ATTR_WO(chip_reset); · 21 unmodified lines
598597 * Convert the reported temperature from an integer (reported in
599598 * units of 0.25C) to a floating point number.
600599 */
601−#define temp2str(temp, buf, size, idx) \
602− scnprintf((buf) + (idx), (size) - (idx), "%u.%02u ", \
603− ((temp) >> 2), ((temp) & 0x3) * 25)
600+#define temp_d(t) ((t) >> 2)
601+#define temp_f(t) (((t)&0x3) * 25u)
604602
605603 /*
606604 * Dump tempsense values, in decimal, to ease shell-scripts.
@@ -615,19 +613,17 @@ static ssize_t tempsense_show(struct device *device, · 8 unmodified lines
615613 int ret;
616614
617615 ret = hfi1_tempsense_rd(dd, &temp);
618− if (!ret) {
619− int idx = 0;
620−
621− idx += temp2str(temp.curr, buf, PAGE_SIZE, idx);
622− idx += temp2str(temp.lo_lim, buf, PAGE_SIZE, idx);
623− idx += temp2str(temp.hi_lim, buf, PAGE_SIZE, idx);
624− idx += temp2str(temp.crit_lim, buf, PAGE_SIZE, idx);
625− idx += scnprintf(buf + idx, PAGE_SIZE - idx,
626− "%u %u %u\n", temp.triggers & 0x1,
627− temp.triggers & 0x2, temp.triggers & 0x4);
628− ret = idx;
629− }
630− return ret;
616+ if (ret)
617+ return ret;
618+
619+ return sysfs_emit(buf, "%u.%02u %u.%02u %u.%02u %u.%02u %u %u %u\n",
620+ temp_d(temp.curr), temp_f(temp.curr),
621+ temp_d(temp.lo_lim), temp_f(temp.lo_lim),
622+ temp_d(temp.hi_lim), temp_f(temp.hi_lim),
623+ temp_d(temp.crit_lim), temp_f(temp.crit_lim),
624+ temp.triggers & 0x1,
625+ temp.triggers & 0x2,
626+ temp.triggers & 0x4);
631627 }
632628 static DEVICE_ATTR_RO(tempsense);
633629
@@ -817,7 +813,7 @@ static ssize_t sde_show_vl(struct sdma_engine *sde, char *buf) · 183 unmodified lines
817813 if (vl < 0)
818814 return vl;
819815
820− return snprintf(buf, PAGE_SIZE, "%d\n", vl);
816+ return sysfs_emit(buf, "%d\n", vl);
821817 }
822818
823819 static SDE_ATTR(cpu_list, S_IWUSR | S_IRUGO,
drivers/infiniband/hw/hfi1/tid_rdma.c+5−0
@@ -2826,6 +2826,7 @@ static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd, · 2825 unmodified lines
28262826 default:
28272827 break;
28282828 }
2829+ break;
28292830 default:
28302831 break;
28312832 }
@@ -3005,6 +3006,7 @@ bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd, · 173 unmodified lines
30053006 default:
30063007 break;
30073008 }
3009+ break;
30083010 default:
30093011 break;
30103012 }
@@ -3221,6 +3223,7 @@ bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) · 210 unmodified lines
32213223 req = wqe_to_tid_req(prev);
32223224 if (req->ack_seg != req->total_segs)
32233225 goto interlock;
3226+ break;
32243227 default:
32253228 break;
32263229 }
@@ -3239,9 +3242,11 @@ bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) · 12 unmodified lines
32393242 req = wqe_to_tid_req(prev);
32403243 if (req->ack_seg != req->total_segs)
32413244 goto interlock;
3245+ break;
32423246 default:
32433247 break;
32443248 }
3249+ break;
32453250 default:
32463251 break;
32473252 }
drivers/infiniband/hw/hns/hns_roce_ah.c+23−32
@@ -31,14 +31,11 @@ · 30 unmodified lines
3131 */
3232
3333 #include <linux/platform_device.h>
34+#include <linux/pci.h>
3435 #include <rdma/ib_addr.h>
3536 #include <rdma/ib_cache.h>
3637 #include "hns_roce_device.h"
3738
38−#define HNS_ROCE_PORT_NUM_SHIFT 24
39−#define HNS_ROCE_VLAN_SL_BIT_MASK 7
40−#define HNS_ROCE_VLAN_SL_SHIFT 13
41−
4239 static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr)
4340 {
4441 u32 fl = ah_attr->grh.flow_label;
@@ -58,47 +55,41 @@ static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr) · 13 unmodified lines
5855 int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
5956 struct ib_udata *udata)
6057 {
61− struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device);
62− const struct ib_gid_attr *gid_attr;
63− struct device *dev = hr_dev->dev;
64− struct hns_roce_ah *ah = to_hr_ah(ibah);
6558 struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
6659 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
67− u16 vlan_id = 0xffff;
68− bool vlan_en = false;
69− int ret;
70−
71− gid_attr = ah_attr->grh.sgid_attr;
72− ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
73− if (ret)
74− return ret;
75−
76− /* Get mac address */
77− memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN);
60+ struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device);
61+ struct hns_roce_ah *ah = to_hr_ah(ibah);
62+ int ret = 0;
7863
79− if (vlan_id < VLAN_N_VID) {
80− vlan_en = true;
81− vlan_id |= (rdma_ah_get_sl(ah_attr) &
82− HNS_ROCE_VLAN_SL_BIT_MASK) <<
83− HNS_ROCE_VLAN_SL_SHIFT;
84− }
64+ if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && udata)
65+ return -EOPNOTSUPP;
8566
8667 ah->av.port = rdma_ah_get_port_num(ah_attr);
8768 ah->av.gid_index = grh->sgid_index;
88− ah->av.vlan_id = vlan_id;
89− ah->av.vlan_en = vlan_en;
90− dev_dbg(dev, "gid_index = 0x%x,vlan_id = 0x%x\n", ah->av.gid_index,
91− ah->av.vlan_id);
9269
9370 if (rdma_ah_get_static_rate(ah_attr))
9471 ah->av.stat_rate = IB_RATE_10_GBPS;
9572
96− memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE);
97− ah->av.sl = rdma_ah_get_sl(ah_attr);
73+ ah->av.hop_limit = grh->hop_limit;
9874 ah->av.flowlabel = grh->flow_label;
9975 ah->av.udp_sport = get_ah_udp_sport(ah_attr);
76+ ah->av.sl = rdma_ah_get_sl(ah_attr);
77+ ah->av.tclass = get_tclass(grh);
10078
101− return 0;
79+ memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE);
80+ memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN);
81+
82+ /* HIP08 needs to record vlan info in Address Vector */
83+ if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) {
84+ ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr,
85+ &ah->av.vlan_id, NULL);
86+ if (ret)
87+ return ret;
88+
89+ ah->av.vlan_en = ah->av.vlan_id < VLAN_N_VID;
90+ }
91+
92+ return ret;
10293 }
10394
10495 int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
drivers/infiniband/hw/hns/hns_roce_alloc.c+76−56
@@ -159,76 +159,96 @@ void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap) · 158 unmodified lines
159159
160160 void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf)
161161 {
162− struct device *dev = hr_dev->dev;
163− u32 size = buf->size;
164− int i;
162+ struct hns_roce_buf_list *trunks;
163+ u32 i;
165164
166− if (size == 0)
165+ if (!buf)
167166 return;
168167
169− buf->size = 0;
168+ trunks = buf->trunk_list;
169+ if (trunks) {
170+ buf->trunk_list = NULL;
171+ for (i = 0; i < buf->ntrunks; i++)
172+ dma_free_coherent(hr_dev->dev, 1 << buf->trunk_shift,
173+ trunks[i].buf, trunks[i].map);
170174
171− if (hns_roce_buf_is_direct(buf)) {
172− dma_free_coherent(dev, size, buf->direct.buf, buf->direct.map);
173− } else {
174− for (i = 0; i < buf->npages; ++i)
175− if (buf->page_list[i].buf)
176− dma_free_coherent(dev, 1 << buf->page_shift,
177− buf->page_list[i].buf,
178− buf->page_list[i].map);
179− kfree(buf->page_list);
180− buf->page_list = NULL;
175+ kfree(trunks);
181176 }
177+
178+ kfree(buf);
182179 }
183180
184−int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
185− struct hns_roce_buf *buf, u32 page_shift)
181+/*
182+ * Allocate the dma buffer for storing ROCEE table entries
183+ *
184+ * @size: required size
185+ * @page_shift: the unit size in a continuous dma address range
186+ * @flags: HNS_ROCE_BUF_ flags to control the allocation flow.
187+ */
188+struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size,
189+ u32 page_shift, u32 flags)
186190 {
187− struct hns_roce_buf_list *buf_list;
188− struct device *dev = hr_dev->dev;
189− u32 page_size;
190− int i;
191+ u32 trunk_size, page_size, alloced_size;
192+ struct hns_roce_buf_list *trunks;
193+ struct hns_roce_buf *buf;
194+ gfp_t gfp_flags;
195+ u32 ntrunk, i;
191196
192197 /* The minimum shift of the page accessed by hw is HNS_HW_PAGE_SHIFT */
193− buf->page_shift = max_t(int, HNS_HW_PAGE_SHIFT, page_shift);
198+ if (WARN_ON(page_shift < HNS_HW_PAGE_SHIFT))
199+ return ERR_PTR(-EINVAL);
200+
201+ gfp_flags = (flags & HNS_ROCE_BUF_NOSLEEP) ? GFP_ATOMIC : GFP_KERNEL;
202+ buf = kzalloc(sizeof(*buf), gfp_flags);
203+ if (!buf)
204+ return ERR_PTR(-ENOMEM);
194205
206+ buf->page_shift = page_shift;
195207 page_size = 1 << buf->page_shift;
196− buf->npages = DIV_ROUND_UP(size, page_size);
197−
198− /* required size is not bigger than one trunk size */
199− if (size <= max_direct) {
200− buf->page_list = NULL;
201− buf->direct.buf = dma_alloc_coherent(dev, size,
202− &buf->direct.map,
203− GFP_KERNEL);
204− if (!buf->direct.buf)
205− return -ENOMEM;
208+
209+ /* Calc the trunk size and num by required size and page_shift */
210+ if (flags & HNS_ROCE_BUF_DIRECT) {
211+ buf->trunk_shift = ilog2(ALIGN(size, PAGE_SIZE));
212+ ntrunk = 1;
206213 } else {
207− buf_list = kcalloc(buf->npages, sizeof(*buf_list), GFP_KERNEL);
208− if (!buf_list)
209− return -ENOMEM;
210−
211− for (i = 0; i < buf->npages; i++) {
212− buf_list[i].buf = dma_alloc_coherent(dev, page_size,
213− &buf_list[i].map,
214− GFP_KERNEL);
215− if (!buf_list[i].buf)
216− break;
217− }
214+ buf->trunk_shift = ilog2(ALIGN(page_size, PAGE_SIZE));
215+ ntrunk = DIV_ROUND_UP(size, 1 << buf->trunk_shift);
216+ }
218217
219− if (i != buf->npages && i > 0) {
220− while (i-- > 0)
221− dma_free_coherent(dev, page_size,
222− buf_list[i].buf,
223− buf_list[i].map);
224− kfree(buf_list);
225− return -ENOMEM;
226− }
227− buf->page_list = buf_list;
218+ trunks = kcalloc(ntrunk, sizeof(*trunks), gfp_flags);
219+ if (!trunks) {
220+ kfree(buf);
221+ return ERR_PTR(-ENOMEM);
228222 }
229− buf->size = size;
230223
231− return 0;
224+ trunk_size = 1 << buf->trunk_shift;
225+ alloced_size = 0;
226+ for (i = 0; i < ntrunk; i++) {
227+ trunks[i].buf = dma_alloc_coherent(hr_dev->dev, trunk_size,
228+ &trunks[i].map, gfp_flags);
229+ if (!trunks[i].buf)
230+ break;
231+
232+ alloced_size += trunk_size;
233+ }
234+
235+ buf->ntrunks = i;
236+
237+ /* In nofail mode, it's only failed when the alloced size is 0 */
238+ if ((flags & HNS_ROCE_BUF_NOFAIL) ? i == 0 : i != ntrunk) {
239+ for (i = 0; i < buf->ntrunks; i++)
240+ dma_free_coherent(hr_dev->dev, trunk_size,
241+ trunks[i].buf, trunks[i].map);
242+
243+ kfree(trunks);
244+ kfree(buf);
245+ return ERR_PTR(-ENOMEM);
246+ }
247+
248+ buf->npages = DIV_ROUND_UP(alloced_size, page_size);
249+ buf->trunk_list = trunks;
250+
251+ return buf;
232252 }
233253
234254 int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
@@ -240,7 +260,7 @@ int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, · 5 unmodified lines
240260 end = start + buf_cnt;
241261 if (end > buf->npages) {
242262 dev_err(hr_dev->dev,
243− "Failed to check kmem bufs, end %d + %d total %d!\n",
263+ "failed to check kmem bufs, end %d + %d total %u!\n",
244264 start, buf_cnt, buf->npages);
245265 return -EINVAL;
246266 }
@@ -262,7 +282,7 @@ int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, · 15 unmodified lines
262282 u64 addr;
263283
264284 if (page_shift < HNS_HW_PAGE_SHIFT) {
265− dev_err(hr_dev->dev, "Failed to check umem page shift %d!\n",
285+ dev_err(hr_dev->dev, "failed to check umem page shift %u!\n",
266286 page_shift);
267287 return -EINVAL;
268288 }
drivers/infiniband/hw/hns/hns_roce_cmd.c+18−19
@@ -36,9 +36,9 @@ · 35 unmodified lines
3636 #include "hns_roce_device.h"
3737 #include "hns_roce_cmd.h"
3838
39−#define CMD_POLL_TOKEN 0xffff
40−#define CMD_MAX_NUM 32
41−#define CMD_TOKEN_MASK 0x1f
39+#define CMD_POLL_TOKEN 0xffff
40+#define CMD_MAX_NUM 32
41+#define CMD_TOKEN_MASK 0x1f
4242
4343 static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param,
4444 u64 out_param, u32 in_modifier,
@@ -60,7 +60,7 @@ static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param, · 15 unmodified lines
6060 static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,
6161 u64 out_param, unsigned long in_modifier,
6262 u8 op_modifier, u16 op,
63− unsigned long timeout)
63+ unsigned int timeout)
6464 {
6565 struct device *dev = hr_dev->dev;
6666 int ret;
@@ -78,7 +78,7 @@ static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, · 11 unmodified lines
7878
7979 static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,
8080 u64 out_param, unsigned long in_modifier,
81− u8 op_modifier, u16 op, unsigned long timeout)
81+ u8 op_modifier, u16 op, unsigned int timeout)
8282 {
8383 int ret;
8484
@@ -93,8 +93,8 @@ static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, · 8 unmodified lines
9393 void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
9494 u64 out_param)
9595 {
96− struct hns_roce_cmd_context
97− *context = &hr_dev->cmd.context[token & hr_dev->cmd.token_mask];
96+ struct hns_roce_cmd_context *context =
97+ &hr_dev->cmd.context[token % hr_dev->cmd.max_cmds];
9898
9999 if (token != context->token)
100100 return;
@@ -108,7 +108,7 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, · 7 unmodified lines
108108 static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
109109 u64 out_param, unsigned long in_modifier,
110110 u8 op_modifier, u16 op,
111− unsigned long timeout)
111+ unsigned int timeout)
112112 {
113113 struct hns_roce_cmdq *cmd = &hr_dev->cmd;
114114 struct hns_roce_cmd_context *context;
@@ -159,13 +159,13 @@ out: · 44 unmodified lines
159159
160160 static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
161161 u64 out_param, unsigned long in_modifier,
162− u8 op_modifier, u16 op, unsigned long timeout)
162+ u8 op_modifier, u16 op, unsigned int timeout)
163163 {
164164 int ret;
165165
166166 down(&hr_dev->cmd.event_sem);
167− ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param,
168− in_modifier, op_modifier, op, timeout);
167+ ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, in_modifier,
168+ op_modifier, op, timeout);
169169 up(&hr_dev->cmd.event_sem);
170170
171171 return ret;
@@ -173,7 +173,7 @@ static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, · 1 unmodified line
173173
174174 int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
175175 unsigned long in_modifier, u8 op_modifier, u16 op,
176− unsigned long timeout)
176+ unsigned int timeout)
177177 {
178178 int ret;
179179
@@ -231,9 +231,8 @@ int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev) · 51 unmodified lines
231231 struct hns_roce_cmdq *hr_cmd = &hr_dev->cmd;
232232 int i;
233233
234− hr_cmd->context = kmalloc_array(hr_cmd->max_cmds,
235− sizeof(*hr_cmd->context),
236− GFP_KERNEL);
234+ hr_cmd->context =
235+ kcalloc(hr_cmd->max_cmds, sizeof(*hr_cmd->context), GFP_KERNEL);
237236 if (!hr_cmd->context)
238237 return -ENOMEM;
239238
@@ -262,8 +261,8 @@ void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev) · 22 unmodified lines
262261 hr_cmd->use_events = 0;
263262 }
264263
265−struct hns_roce_cmd_mailbox
266− *hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev)
264+struct hns_roce_cmd_mailbox *
265+hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev)
267266 {
268267 struct hns_roce_cmd_mailbox *mailbox;
269268
@@ -271,8 +270,8 @@ struct hns_roce_cmd_mailbox · 1 unmodified line
271270 if (!mailbox)
272271 return ERR_PTR(-ENOMEM);
273272
274− mailbox->buf = dma_pool_alloc(hr_dev->cmd.pool, GFP_KERNEL,
275− &mailbox->dma);
273+ mailbox->buf =
274+ dma_pool_alloc(hr_dev->cmd.pool, GFP_KERNEL, &mailbox->dma);
276275 if (!mailbox->buf) {
277276 kfree(mailbox);
278277 return ERR_PTR(-ENOMEM);
drivers/infiniband/hw/hns/hns_roce_cmd.h+3−3
@@ -141,10 +141,10 @@ enum { · 140 unmodified lines
141141
142142 int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
143143 unsigned long in_modifier, u8 op_modifier, u16 op,
144− unsigned long timeout);
144+ unsigned int timeout);
145145
146−struct hns_roce_cmd_mailbox
147− *hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev);
146+struct hns_roce_cmd_mailbox *
147+hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev);
148148 void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,
149149 struct hns_roce_cmd_mailbox *mailbox);
150150
drivers/infiniband/hw/hns/hns_roce_common.h+19−7
@@ -38,21 +38,33 @@ · 37 unmodified lines
3838 #define roce_raw_write(value, addr) \
3939 __raw_writel((__force u32)cpu_to_le32(value), (addr))
4040
41−#define roce_get_field(origin, mask, shift) \
42− (((le32_to_cpu(origin)) & (mask)) >> (shift))
41+#define roce_get_field(origin, mask, shift) \
42+ ((le32_to_cpu(origin) & (mask)) >> (u32)(shift))
4343
4444 #define roce_get_bit(origin, shift) \
4545 roce_get_field((origin), (1ul << (shift)), (shift))
4646
47−#define roce_set_field(origin, mask, shift, val) \
48− do { \
49− (origin) &= ~cpu_to_le32(mask); \
50− (origin) |= cpu_to_le32(((u32)(val) << (shift)) & (mask)); \
47+#define roce_set_field(origin, mask, shift, val) \
48+ do { \
49+ (origin) &= ~cpu_to_le32(mask); \
50+ (origin) |= cpu_to_le32(((u32)(val) << (u32)(shift)) & (mask)); \
5151 } while (0)
5252
53−#define roce_set_bit(origin, shift, val) \
53+#define roce_set_bit(origin, shift, val) \
5454 roce_set_field((origin), (1ul << (shift)), (shift), (val))
5555
56+#define FIELD_LOC(field_type, field_h, field_l) field_type, field_h, field_l
57+
58+#define _hr_reg_enable(ptr, field_type, field_h, field_l) \
59+ ({ \
60+ const field_type *_ptr = ptr; \
61+ *((__le32 *)_ptr + (field_h) / 32) |= \
62+ cpu_to_le32(BIT((field_l) % 32)) + \
63+ BUILD_BUG_ON_ZERO((field_h) != (field_l)); \
64+ })
65+
66+#define hr_reg_enable(ptr, field) _hr_reg_enable(ptr, field)
67+
5668 #define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3
5769 #define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4
5870
drivers/infiniband/hw/hns/hns_roce_cq.c+25−21
@@ -36,43 +36,42 @@ · 35 unmodified lines
3636 #include "hns_roce_device.h"
3737 #include "hns_roce_cmd.h"
3838 #include "hns_roce_hem.h"
39−#include <rdma/hns-abi.h>
4039 #include "hns_roce_common.h"
4140
4241 static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
4342 {
43+ struct ib_device *ibdev = &hr_dev->ib_dev;
4444 struct hns_roce_cmd_mailbox *mailbox;
4545 struct hns_roce_cq_table *cq_table;
46− struct ib_device *ibdev = &hr_dev->ib_dev;
4746 u64 mtts[MTT_MIN_COUNT] = { 0 };
4847 dma_addr_t dma_handle;
4948 int ret;
5049
5150 ret = hns_roce_mtr_find(hr_dev, &hr_cq->mtr, 0, mtts, ARRAY_SIZE(mtts),
5251 &dma_handle);
53− if (ret < 1) {
54− ibdev_err(ibdev, "Failed to find CQ mtr\n");
52+ if (!ret) {
53+ ibdev_err(ibdev, "failed to find CQ mtr, ret = %d.\n", ret);
5554 return -EINVAL;
5655 }
5756
5857 cq_table = &hr_dev->cq_table;
5958 ret = hns_roce_bitmap_alloc(&cq_table->bitmap, &hr_cq->cqn);
6059 if (ret) {
61− ibdev_err(ibdev, "Failed to alloc CQ bitmap, err %d\n", ret);
60+ ibdev_err(ibdev, "failed to alloc CQ bitmap, ret = %d.\n", ret);
6261 return ret;
6362 }
6463
6564 /* Get CQC memory HEM(Hardware Entry Memory) table */
6665 ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn);
6766 if (ret) {
68− ibdev_err(ibdev, "Failed to get CQ(0x%lx) context, err %d\n",
67+ ibdev_err(ibdev, "failed to get CQ(0x%lx) context, ret = %d.\n",
6968 hr_cq->cqn, ret);
7069 goto err_out;
7170 }
7271
7372 ret = xa_err(xa_store(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL));
7473 if (ret) {
75− ibdev_err(ibdev, "Failed to xa_store CQ\n");
74+ ibdev_err(ibdev, "failed to xa_store CQ, ret = %d.\n", ret);
7675 goto err_put;
7776 }
7877
@@ -91,7 +90,7 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) · 12 unmodified lines
9190 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
9291 if (ret) {
9392 ibdev_err(ibdev,
94− "Failed to send create cmd for CQ(0x%lx), err %d\n",
93+ "failed to send create cmd for CQ(0x%lx), ret = %d.\n",
9594 hr_cq->cqn, ret);
9695 goto err_xa;
9796 }
@@ -147,7 +146,7 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, · 49 unmodified lines
147146 {
148147 struct ib_device *ibdev = &hr_dev->ib_dev;
149148 struct hns_roce_buf_attr buf_attr = {};
150− int err;
149+ int ret;
151150
152151 buf_attr.page_shift = hr_dev->caps.cqe_buf_pg_sz + HNS_HW_PAGE_SHIFT;
153152 buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size;
@@ -155,13 +154,13 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, · 1 unmodified line
155154 buf_attr.region_count = 1;
156155 buf_attr.fixed_page = true;
157156
158− err = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,
157+ ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,
159158 hr_dev->caps.cqe_ba_pg_sz + HNS_HW_PAGE_SHIFT,
160159 udata, addr);
161− if (err)
162− ibdev_err(ibdev, "Failed to alloc CQ mtr, err %d\n", err);
160+ if (ret)
161+ ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret);
163162
164− return err;
163+ return ret;
165164 }
166165
167166 static void free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
@@ -251,14 +250,17 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, · 83 unmodified lines
251250 u32 cq_entries = attr->cqe;
252251 int ret;
253252
253+ if (attr->flags)
254+ return -EOPNOTSUPP;
255+
254256 if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) {
255− ibdev_err(ibdev, "Failed to check CQ count %d max=%d\n",
257+ ibdev_err(ibdev, "failed to check CQ count %u, max = %u.\n",
256258 cq_entries, hr_dev->caps.max_cqes);
257259 return -EINVAL;
258260 }
259261
260262 if (vector >= hr_dev->caps.num_comp_vectors) {
261− ibdev_err(ibdev, "Failed to check CQ vector=%d max=%d\n",
263+ ibdev_err(ibdev, "failed to check CQ vector = %d, max = %d.\n",
262264 vector, hr_dev->caps.num_comp_vectors);
263265 return -EINVAL;
264266 }
@@ -274,9 +276,9 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, · 9 unmodified lines
274276
275277 if (udata) {
276278 ret = ib_copy_from_udata(&ucmd, udata,
277− min(sizeof(ucmd), udata->inlen));
279+ min(udata->inlen, sizeof(ucmd)));
278280 if (ret) {
279− ibdev_err(ibdev, "Failed to copy CQ udata, err %d\n",
281+ ibdev_err(ibdev, "failed to copy CQ udata, ret = %d.\n",
280282 ret);
281283 return ret;
282284 }
@@ -286,19 +288,20 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, · 3 unmodified lines
286288
287289 ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr);
288290 if (ret) {
289− ibdev_err(ibdev, "Failed to alloc CQ buf, err %d\n", ret);
291+ ibdev_err(ibdev, "failed to alloc CQ buf, ret = %d.\n", ret);
290292 return ret;
291293 }
292294
293295 ret = alloc_cq_db(hr_dev, hr_cq, udata, ucmd.db_addr, &resp);
294296 if (ret) {
295− ibdev_err(ibdev, "Failed to alloc CQ db, err %d\n", ret);
297+ ibdev_err(ibdev, "failed to alloc CQ db, ret = %d.\n", ret);
296298 goto err_cq_buf;
297299 }
298300
299301 ret = alloc_cqc(hr_dev, hr_cq);
300302 if (ret) {
301− ibdev_err(ibdev, "Failed to alloc CQ context, err %d\n", ret);
303+ ibdev_err(ibdev,
304+ "failed to alloc CQ context, ret = %d.\n", ret);
302305 goto err_cq_db;
303306 }
304307
@@ -313,7 +316,8 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, · 8 unmodified lines
313316
314317 if (udata) {
315318 resp.cqn = hr_cq->cqn;
316− ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
319+ ret = ib_copy_to_udata(udata, &resp,
320+ min(udata->outlen, sizeof(resp)));
317321 if (ret)
318322 goto err_cqc;
319323 }
drivers/infiniband/hw/hns/hns_roce_db.c+4−4
@@ -95,8 +95,8 @@ static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( · 94 unmodified lines
9595 static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir,
9696 struct hns_roce_db *db, int order)
9797 {
98− int o;
99− int i;
98+ unsigned long o;
99+ unsigned long i;
100100
101101 for (o = order; o <= 1; ++o) {
102102 i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o);
@@ -154,8 +154,8 @@ out: · 51 unmodified lines
154154
155155 void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
156156 {
157− int o;
158− int i;
157+ unsigned long o;
158+ unsigned long i;
159159
160160 mutex_lock(&hr_dev->pgdir_mutex);
161161
drivers/infiniband/hw/hns/hns_roce_device.h+92−86
@@ -34,6 +34,7 @@ · 33 unmodified lines
3434 #define _HNS_ROCE_DEVICE_H
3535
3636 #include <rdma/ib_verbs.h>
37+#include <rdma/hns-abi.h>
3738
3839 #define DRV_NAME "hns_roce"
3940
@@ -117,6 +118,8 @@ · 77 unmodified lines
117118 #define HNS_ROCE_IDX_QUE_ENTRY_SZ 4
118119 #define SRQ_DB_REG 0x230
119120
121+#define HNS_ROCE_QP_BANK_NUM 8
122+
120123 /* The chip implementation of the consumer index is calculated
121124 * according to twice the actual EQ depth
122125 */
@@ -129,15 +132,6 @@ enum { · 6 unmodified lines
129132 SERV_TYPE_UD,
130133 };
131134
132−enum {
133− HNS_ROCE_QP_CAP_RQ_RECORD_DB = BIT(0),
134− HNS_ROCE_QP_CAP_SQ_RECORD_DB = BIT(1),
135−};
136−
137−enum hns_roce_cq_flags {
138− HNS_ROCE_CQ_FLAG_RECORD_DB = BIT(0),
139−};
140−
141135 enum hns_roce_qp_state {
142136 HNS_ROCE_QP_STATE_RST,
143137 HNS_ROCE_QP_STATE_INIT,
@@ -166,7 +160,6 @@ enum hns_roce_event { · 22 unmodified lines
166160 /* 0x10 and 0x11 is unused in currently application case */
167161 HNS_ROCE_EVENT_TYPE_DB_OVERFLOW = 0x12,
168162 HNS_ROCE_EVENT_TYPE_MB = 0x13,
169− HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW = 0x14,
170163 HNS_ROCE_EVENT_TYPE_FLR = 0x15,
171164 };
172165
@@ -221,6 +214,8 @@ enum { · 48 unmodified lines
221214 HNS_ROCE_CAP_FLAG_FRMR = BIT(8),
222215 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),
223216 HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10),
217+ HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14),
218+ HNS_ROCE_CAP_FLAG_STASH = BIT(17),
224219 };
225220
226221 #define HNS_ROCE_DB_TYPE_COUNT 2
@@ -265,9 +260,6 @@ enum { · 38 unmodified lines
265260 #define HNS_HW_PAGE_SHIFT 12
266261 #define HNS_HW_PAGE_SIZE (1 << HNS_HW_PAGE_SHIFT)
267262
268−/* The minimum page count for hardware access page directly. */
269−#define HNS_HW_DIRECT_PAGE_COUNT 2
270−
271263 struct hns_roce_uar {
272264 u64 pfn;
273265 unsigned long index;
@@ -318,7 +310,7 @@ struct hns_roce_hem_table { · 44 unmodified lines
318310 };
319311
320312 struct hns_roce_buf_region {
321− int offset; /* page offset */
313+ u32 offset; /* page offset */
322314 u32 count; /* page count */
323315 int hopnum; /* addressing hop num */
324316 };
@@ -338,10 +330,10 @@ struct hns_roce_buf_attr { · 13 unmodified lines
338330 size_t size; /* region size */
339331 int hopnum; /* multi-hop addressing hop num */
340332 } region[HNS_ROCE_MAX_BT_REGION];
341− int region_count; /* valid region count */
333+ unsigned int region_count; /* valid region count */
342334 unsigned int page_shift; /* buffer page shift */
343335 bool fixed_page; /* decide page shift is fixed-size or maximum size */
344− int user_access; /* umem access flag */
336+ unsigned int user_access; /* umem access flag */
345337 bool mtt_only; /* only alloc buffer-required MTT memory */
346338 };
347339
@@ -352,7 +344,7 @@ struct hns_roce_hem_cfg { · 4 unmodified lines
352344 unsigned int buf_pg_shift; /* buffer page shift */
353345 unsigned int buf_pg_count; /* buffer page count */
354346 struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
355− int region_count;
347+ unsigned int region_count;
356348 };
357349
358350 /* memory translate region */
@@ -400,7 +392,7 @@ struct hns_roce_wq { · 41 unmodified lines
400392 u64 *wrid; /* Work request ID */
401393 spinlock_t lock;
402394 u32 wqe_cnt; /* WQE num */
403− int max_gs;
395+ u32 max_gs;
404396 int offset;
405397 int wqe_shift; /* WQE size */
406398 u32 head;
@@ -419,11 +411,26 @@ struct hns_roce_buf_list { · 12 unmodified lines
419411 dma_addr_t map;
420412 };
421413
414+/*
415+ * %HNS_ROCE_BUF_DIRECT indicates that the all memory must be in a continuous
416+ * dma address range.
417+ *
418+ * %HNS_ROCE_BUF_NOSLEEP indicates that the caller cannot sleep.
419+ *
420+ * %HNS_ROCE_BUF_NOFAIL allocation only failed when allocated size is zero, even
421+ * the allocated size is smaller than the required size.
422+ */
423+enum {
424+ HNS_ROCE_BUF_DIRECT = BIT(0),
425+ HNS_ROCE_BUF_NOSLEEP = BIT(1),
426+ HNS_ROCE_BUF_NOFAIL = BIT(2),
427+};
428+
422429 struct hns_roce_buf {
423− struct hns_roce_buf_list direct;
424− struct hns_roce_buf_list *page_list;
430+ struct hns_roce_buf_list *trunk_list;
431+ u32 ntrunks;
425432 u32 npages;
426− u32 size;
433+ unsigned int trunk_shift;
427434 unsigned int page_shift;
428435 };
429436
@@ -451,8 +458,8 @@ struct hns_roce_db { · 21 unmodified lines
451458 } u;
452459 dma_addr_t dma;
453460 void *virt_addr;
454− int index;
455− int order;
461+ unsigned long index;
462+ unsigned long order;
456463 };
457464
458465 struct hns_roce_cq {
@@ -500,8 +507,8 @@ struct hns_roce_srq { · 41 unmodified lines
500507 u64 *wrid;
501508 struct hns_roce_idx_que idx_que;
502509 spinlock_t lock;
503− int head;
504− int tail;
510+ u16 head;
511+ u16 tail;
505512 struct mutex mutex;
506513 void (*event)(struct hns_roce_srq *srq, enum hns_roce_event event);
507514 };
@@ -510,13 +517,22 @@ struct hns_roce_uar_table { · 2 unmodified lines
510517 struct hns_roce_bitmap bitmap;
511518 };
512519
520+struct hns_roce_bank {
521+ struct ida ida;
522+ u32 inuse; /* Number of IDs allocated */
523+ u32 min; /* Lowest ID to allocate. */
524+ u32 max; /* Highest ID to allocate. */
525+ u32 next; /* Next ID to allocate. */
526+};
527+
513528 struct hns_roce_qp_table {
514− struct hns_roce_bitmap bitmap;
515529 struct hns_roce_hem_table qp_table;
516530 struct hns_roce_hem_table irrl_table;
517531 struct hns_roce_hem_table trrl_table;
518532 struct hns_roce_hem_table sccc_table;
519533 struct mutex scc_mutex;
534+ struct hns_roce_bank bank[HNS_ROCE_QP_BANK_NUM];
535+ spinlock_t bank_lock;
520536 };
521537
522538 struct hns_roce_cq_table {
@@ -547,7 +563,7 @@ struct hns_roce_av { · 24 unmodified lines
547563 u8 dgid[HNS_ROCE_GID_SIZE];
548564 u8 mac[ETH_ALEN];
549565 u16 vlan_id;
550− bool vlan_en;
566+ u8 vlan_en;
551567 };
552568
553569 struct hns_roce_ah {
@@ -619,10 +635,9 @@ enum { · 65 unmodified lines
619635 struct hns_roce_work {
620636 struct hns_roce_dev *hr_dev;
621637 struct work_struct work;
622− u32 qpn;
623− u32 cqn;
624638 int event_type;
625639 int sub_type;
640+ u32 queue_num;
626641 };
627642
628643 struct hns_roce_qp {
@@ -690,28 +705,10 @@ struct hns_roce_aeqe { · 61 unmodified lines
690705 __le32 asyn;
691706 union {
692707 struct {
693− __le32 qp;
694− u32 rsv0;
695− u32 rsv1;
696− } qp_event;
697−
698− struct {
699− __le32 srq;
700− u32 rsv0;
701− u32 rsv1;
702− } srq_event;
703−
704− struct {
705− __le32 cq;
706− u32 rsv0;
707− u32 rsv1;
708− } cq_event;
709−
710− struct {
711− __le32 ceqe;
708+ __le32 num;
712709 u32 rsv0;
713710 u32 rsv1;
714− } ce_event;
711+ } queue_event;
715712
716713 struct {
717714 __le64 out_param;
@@ -730,11 +727,11 @@ struct hns_roce_eq { · 12 unmodified lines
730727 int type_flag; /* Aeq:1 ceq:0 */
731728 int eqn;
732729 u32 entries;
733− int log_entries;
730+ u32 log_entries;
734731 int eqe_size;
735732 int irq;
736733 int log_page_size;
737− int cons_index;
734+ u32 cons_index;
738735 struct hns_roce_buf_list *buf_list;
739736 int over_ignore;
740737 int coalesce;
@@ -742,7 +739,7 @@ struct hns_roce_eq { · 1 unmodified line
742739 int hop_num;
743740 struct hns_roce_mtr mtr;
744741 u16 eq_max_cnt;
745− int eq_period;
742+ u32 eq_period;
746743 int shift;
747744 int event_type;
748745 int sub_type;
@@ -765,8 +762,8 @@ struct hns_roce_caps { · 16 unmodified lines
765762 u32 max_sq_inline;
766763 u32 max_rq_sg;
767764 u32 max_extend_sg;
768− int num_qps;
769− int reserved_qps;
765+ u32 num_qps;
766+ u32 reserved_qps;
770767 int num_qpc_timer;
771768 int num_cqc_timer;
772769 int num_srqs;
@@ -778,7 +775,7 @@ struct hns_roce_caps { · 5 unmodified lines
778775 u32 max_srq_desc_sz;
779776 int max_qp_init_rdma;
780777 int max_qp_dest_rdma;
781− int num_cqs;
778+ u32 num_cqs;
782779 u32 max_cqes;
783780 u32 min_cqes;
784781 u32 min_wqes;
@@ -787,7 +784,7 @@ struct hns_roce_caps { · 2 unmodified lines
787784 int num_aeq_vectors;
788785 int num_comp_vectors;
789786 int num_other_vectors;
790− int num_mtpts;
787+ u32 num_mtpts;
791788 u32 num_mtt_segs;
792789 u32 num_cqe_segs;
793790 u32 num_srqwqe_segs;
@@ -825,6 +822,7 @@ struct hns_roce_caps { · 31 unmodified lines
825822 u32 cqc_timer_bt_num;
826823 u32 mpt_bt_num;
827824 u32 sccc_bt_num;
825+ u32 gmv_bt_num;
828826 u32 qpc_ba_pg_sz;
829827 u32 qpc_buf_pg_sz;
830828 u32 qpc_hop_num;
@@ -864,6 +862,11 @@ struct hns_roce_caps { · 33 unmodified lines
864862 u32 eqe_ba_pg_sz;
865863 u32 eqe_buf_pg_sz;
866864 u32 eqe_hop_num;
865+ u32 gmv_entry_num;
866+ u32 gmv_entry_sz;
867+ u32 gmv_ba_pg_sz;
868+ u32 gmv_buf_pg_sz;
869+ u32 gmv_hop_num;
867870 u32 sl_num;
868871 u32 tsq_buf_pg_sz;
869872 u32 tpq_buf_pg_sz;
@@ -898,7 +901,7 @@ struct hns_roce_hw { · 28 unmodified lines
898901 int (*post_mbox)(struct hns_roce_dev *hr_dev, u64 in_param,
899902 u64 out_param, u32 in_modifier, u8 op_modifier, u16 op,
900903 u16 token, int event);
901− int (*chk_mbox)(struct hns_roce_dev *hr_dev, unsigned long timeout);
904+ int (*chk_mbox)(struct hns_roce_dev *hr_dev, unsigned int timeout);
902905 int (*rst_prc_mbox)(struct hns_roce_dev *hr_dev);
903906 int (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
904907 const union ib_gid *gid, const struct ib_gid_attr *attr);
@@ -999,6 +1002,10 @@ struct hns_roce_dev { · 94 unmodified lines
9991002 struct hns_roce_eq_table eq_table;
10001003 struct hns_roce_hem_table qpc_timer_table;
10011004 struct hns_roce_hem_table cqc_timer_table;
1005+ /* GMV is the memory area that the driver allocates for the hardware
1006+ * to store SGID, SMAC and VLAN information.
1007+ */
1008+ struct hns_roce_hem_table gmv_table;
10021009
10031010 int cmd_mod;
10041011 int loop_idc;
@@ -1069,29 +1076,19 @@ static inline struct hns_roce_qp · 64 unmodified lines
10691076 return xa_load(&hr_dev->qp_table_xa, qpn & (hr_dev->caps.num_qps - 1));
10701077 }
10711078
1072−static inline bool hns_roce_buf_is_direct(struct hns_roce_buf *buf)
1079+static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf,
1080+ unsigned int offset)
10731081 {
1074− if (buf->page_list)
1075− return false;
1076−
1077− return true;
1082+ return (char *)(buf->trunk_list[offset >> buf->trunk_shift].buf) +
1083+ (offset & ((1 << buf->trunk_shift) - 1));
10781084 }
10791085
1080−static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset)
1086+static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, u32 idx)
10811087 {
1082− if (hns_roce_buf_is_direct(buf))
1083− return (char *)(buf->direct.buf) + (offset & (buf->size - 1));
1084−
1085− return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
1086− (offset & ((1 << buf->page_shift) - 1));
1087−}
1088+ unsigned int offset = idx << buf->page_shift;
10881089
1089−static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, int idx)
1090−{
1091− if (hns_roce_buf_is_direct(buf))
1092− return buf->direct.map + ((dma_addr_t)idx << buf->page_shift);
1093− else
1094− return buf->page_list[idx].map;
1090+ return buf->trunk_list[offset >> buf->trunk_shift].map +
1091+ (offset & ((1 << buf->trunk_shift) - 1));
10951092 }
10961093
10971094 #define hr_hw_page_align(x) ALIGN(x, 1 << HNS_HW_PAGE_SHIFT)
@@ -1132,6 +1129,14 @@ static inline u32 to_hr_hem_entries_shift(u32 count, u32 buf_shift) · 34 unmodified lines
11321129 return ilog2(to_hr_hem_entries_count(count, buf_shift));
11331130 }
11341131
1132+#define DSCP_SHIFT 2
1133+
1134+static inline u8 get_tclass(const struct ib_global_route *grh)
1135+{
1136+ return grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP ?
1137+ grh->traffic_class >> DSCP_SHIFT : grh->traffic_class;
1138+}
1139+
11351140 int hns_roce_init_uar_table(struct hns_roce_dev *dev);
11361141 int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
11371142 void hns_roce_uar_free(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
@@ -1155,7 +1160,7 @@ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 17 unmodified lines
11551160 void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
11561161 struct hns_roce_mtr *mtr);
11571162 int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
1158− dma_addr_t *pages, int page_cnt);
1163+ dma_addr_t *pages, unsigned int page_cnt);
11591164
11601165 int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);
11611166 int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
@@ -1198,9 +1203,10 @@ struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc); · 36 unmodified lines
11981203 struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
11991204 u64 virt_addr, int access_flags,
12001205 struct ib_udata *udata);
1201−int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length,
1202− u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
1203− struct ib_udata *udata);
1206+struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,
1207+ u64 length, u64 virt_addr,
1208+ int mr_access_flags, struct ib_pd *pd,
1209+ struct ib_udata *udata);
12041210 struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
12051211 u32 max_num_sg);
12061212 int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
@@ -1215,8 +1221,8 @@ int hns_roce_alloc_mw(struct ib_mw *mw, struct ib_udata *udata); · 8 unmodified lines
12151221 int hns_roce_dealloc_mw(struct ib_mw *ibmw);
12161222
12171223 void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf);
1218−int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
1219− struct hns_roce_buf *buf, u32 page_shift);
1224+struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size,
1225+ u32 page_shift, u32 flags);
12201226
12211227 int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
12221228 int buf_cnt, int start, struct hns_roce_buf *buf);
@@ -1238,10 +1244,10 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd, · 15 unmodified lines
12381244 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
12391245 int attr_mask, struct ib_udata *udata);
12401246 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
1241−void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, int n);
1242−void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, int n);
1243−void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, int n);
1244−bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
1247+void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n);
1248+void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n);
1249+void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n);
1250+bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,
12451251 struct ib_cq *ib_cq);
12461252 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state);
12471253 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq,
@@ -1271,7 +1277,7 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn); · 23 unmodified lines
12711277 void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
12721278 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);
12731279 void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type);
1274−int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
1280+u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
12751281 void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
12761282 int hns_roce_init(struct hns_roce_dev *hr_dev);
12771283 void hns_roce_exit(struct hns_roce_dev *hr_dev);
drivers/infiniband/hw/hns/hns_roce_hem.c+37−22
@@ -75,6 +75,9 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) · 74 unmodified lines
7575 case HEM_TYPE_CQC_TIMER:
7676 hop_num = hr_dev->caps.cqc_timer_hop_num;
7777 break;
78+ case HEM_TYPE_GMV:
79+ hop_num = hr_dev->caps.gmv_hop_num;
80+ break;
7881 default:
7982 return false;
8083 }
@@ -183,8 +186,16 @@ static int get_hem_table_config(struct hns_roce_dev *hr_dev, · 102 unmodified lines
183186 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
184187 mhop->hop_num = hr_dev->caps.srqc_hop_num;
185188 break;
189+ case HEM_TYPE_GMV:
190+ mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz +
191+ PAGE_SHIFT);
192+ mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz +
193+ PAGE_SHIFT);
194+ mhop->ba_l0_num = hr_dev->caps.gmv_bt_num;
195+ mhop->hop_num = hr_dev->caps.gmv_hop_num;
196+ break;
186197 default:
187− dev_err(dev, "Table %d not support multi-hop addressing!\n",
198+ dev_err(dev, "table %u not support multi-hop addressing!\n",
188199 type);
189200 return -EINVAL;
190201 }
@@ -198,9 +209,9 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, · 7 unmodified lines
198209 {
199210 struct device *dev = hr_dev->dev;
200211 u32 chunk_ba_num;
212+ u32 chunk_size;
201213 u32 table_idx;
202214 u32 bt_num;
203− u32 chunk_size;
204215
205216 if (get_hem_table_config(hr_dev, mhop, table->type))
206217 return -EINVAL;
@@ -232,8 +243,8 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, · 25 unmodified lines
232243 mhop->l0_idx = table_idx;
233244 break;
234245 default:
235− dev_err(dev, "Table %d not support hop_num = %d!\n",
236− table->type, mhop->hop_num);
246+ dev_err(dev, "table %u not support hop_num = %u!\n",
247+ table->type, mhop->hop_num);
237248 return -EINVAL;
238249 }
239250 if (mhop->l0_idx >= mhop->ba_l0_num)
@@ -332,15 +343,15 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, · 92 unmodified lines
332343 {
333344 spinlock_t *lock = &hr_dev->bt_cmd_lock;
334345 struct device *dev = hr_dev->dev;
335− long end;
336− unsigned long flags;
337346 struct hns_roce_hem_iter iter;
338347 void __iomem *bt_cmd;
339348 __le32 bt_cmd_val[2];
340349 __le32 bt_cmd_h = 0;
350+ unsigned long flags;
341351 __le32 bt_cmd_l;
342− u64 bt_ba;
343352 int ret = 0;
353+ u64 bt_ba;
354+ long end;
344355
345356 /* Find the HEM(Hardware Entry Memory) entry */
346357 unsigned long i = (obj & (table->num_obj - 1)) /
@@ -438,13 +449,13 @@ static int calc_hem_config(struct hns_roce_dev *hr_dev, · 91 unmodified lines
438449 index->buf = l0_idx;
439450 break;
440451 default:
441− ibdev_err(ibdev, "Table %d not support mhop.hop_num = %d!\n",
452+ ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
442453 table->type, mhop->hop_num);
443454 return -EINVAL;
444455 }
445456
446457 if (unlikely(index->buf >= table->num_hem)) {
447− ibdev_err(ibdev, "Table %d exceed hem limt idx %llu,max %lu!\n",
458+ ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
448459 table->type, index->buf, table->num_hem);
449460 return -EINVAL;
450461 }
@@ -640,8 +651,8 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev, · 189 unmodified lines
640651 struct hns_roce_hem_table *table, unsigned long obj)
641652 {
642653 struct device *dev = hr_dev->dev;
643− int ret = 0;
644654 unsigned long i;
655+ int ret = 0;
645656
646657 if (hns_roce_check_whether_mhop(hr_dev, table->type))
647658 return hns_roce_table_mhop_get(hr_dev, table, obj);
@@ -714,15 +725,15 @@ static void clear_mhop_hem(struct hns_roce_dev *hr_dev, · 66 unmodified lines
714725 step_idx = hop_num;
715726
716727 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
717− ibdev_warn(ibdev, "Clear hop%d HEM failed.\n", hop_num);
728+ ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num);
718729
719730 if (index->inited & HEM_INDEX_L1)
720731 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
721− ibdev_warn(ibdev, "Clear HEM step 1 failed.\n");
732+ ibdev_warn(ibdev, "failed to clear HEM step 1.\n");
722733
723734 if (index->inited & HEM_INDEX_L0)
724735 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
725− ibdev_warn(ibdev, "Clear HEM step 0 failed.\n");
736+ ibdev_warn(ibdev, "failed to clear HEM step 0.\n");
726737 }
727738 }
728739
@@ -789,14 +800,14 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev, · 60 unmodified lines
789800 struct hns_roce_hem_chunk *chunk;
790801 struct hns_roce_hem_mhop mhop;
791802 struct hns_roce_hem *hem;
792− void *addr = NULL;
793803 unsigned long mhop_obj = obj;
794804 unsigned long obj_per_chunk;
795805 unsigned long idx_offset;
796806 int offset, dma_offset;
807+ void *addr = NULL;
808+ u32 hem_idx = 0;
797809 int length;
798810 int i, j;
799− u32 hem_idx = 0;
800811
801812 if (!table->lowmem)
802813 return NULL;
@@ -876,7 +887,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, · 73 unmodified lines
876887 unsigned long buf_chunk_size;
877888 unsigned long bt_chunk_size;
878889 unsigned long bt_chunk_num;
879− unsigned long num_bt_l0 = 0;
890+ unsigned long num_bt_l0;
880891 u32 hop_num;
881892
882893 if (get_hem_table_config(hr_dev, &mhop, type))
@@ -966,8 +977,8 @@ static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, · 83 unmodified lines
966977 {
967978 struct hns_roce_hem_mhop mhop;
968979 u32 buf_chunk_size;
969− int i;
970980 u64 obj;
981+ int i;
971982
972983 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
973984 return;
@@ -1017,7 +1028,7 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, · 43 unmodified lines
10171028
10181029 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
10191030 {
1020− if (hr_dev->caps.srqc_entry_sz)
1031+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
10211032 hns_roce_cleanup_hem_table(hr_dev,
10221033 &hr_dev->srq_table.table);
10231034 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
@@ -1027,12 +1038,16 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) · 3 unmodified lines
10271038 if (hr_dev->caps.cqc_timer_entry_sz)
10281039 hns_roce_cleanup_hem_table(hr_dev,
10291040 &hr_dev->cqc_timer_table);
1030− if (hr_dev->caps.sccc_sz)
1041+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
10311042 hns_roce_cleanup_hem_table(hr_dev,
10321043 &hr_dev->qp_table.sccc_table);
10331044 if (hr_dev->caps.trrl_entry_sz)
10341045 hns_roce_cleanup_hem_table(hr_dev,
10351046 &hr_dev->qp_table.trrl_table);
1047+
1048+ if (hr_dev->caps.gmv_entry_sz)
1049+ hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table);
1050+
10361051 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
10371052 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
10381053 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
@@ -1234,7 +1249,7 @@ static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, · 195 unmodified lines
12341249 }
12351250
12361251 if (offset < r->offset) {
1237− dev_err(hr_dev->dev, "invalid offset %d,min %d!\n",
1252+ dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
12381253 offset, r->offset);
12391254 return -EINVAL;
12401255 }
@@ -1298,8 +1313,8 @@ static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, · 57 unmodified lines
12981313 const struct hns_roce_buf_region *regions,
12991314 int region_cnt)
13001315 {
1301− struct roce_hem_item *hem, *temp_hem, *root_hem;
13021316 struct list_head temp_list[HNS_ROCE_MAX_BT_REGION];
1317+ struct roce_hem_item *hem, *temp_hem, *root_hem;
13031318 const struct hns_roce_buf_region *r;
13041319 struct list_head temp_root;
13051320 struct list_head temp_btm;
@@ -1404,8 +1419,8 @@ int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, · 98 unmodified lines
14041419 {
14051420 const struct hns_roce_buf_region *r;
14061421 int ofs, end;
1407− int ret;
14081422 int unit;
1423+ int ret;
14091424 int i;
14101425
14111426 if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
drivers/infiniband/hw/hns/hns_roce_hem.h+2−1
@@ -47,6 +47,7 @@ enum { · 46 unmodified lines
4747 HEM_TYPE_SCCC,
4848 HEM_TYPE_QPC_TIMER,
4949 HEM_TYPE_CQC_TIMER,
50+ HEM_TYPE_GMV,
5051
5152 /* UNMAP HEM */
5253 HEM_TYPE_MTT,
@@ -174,4 +175,4 @@ static inline dma_addr_t hns_roce_hem_addr(struct hns_roce_hem_iter *iter) · 121 unmodified lines
174175 return sg_dma_address(&iter->chunk->mem[iter->page_idx]);
175176 }
176177
177−#endif /*_HNS_ROCE_HEM_H*/
178+#endif /* _HNS_ROCE_HEM_H */
drivers/infiniband/hw/hns/hns_roce_hw_v1.c+18−32
@@ -239,7 +239,7 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, · 238 unmodified lines
239239 break;
240240 }
241241
242− /*Ctrl field, ctrl set type: sig, solic, imm, fence */
242+ /* Ctrl field, ctrl set type: sig, solic, imm, fence */
243243 /* SO wait for conforming application scenarios */
244244 ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ?
245245 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
@@ -288,7 +288,7 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, · 42 unmodified lines
288288 ret = -EINVAL;
289289 *bad_wr = wr;
290290 dev_err(dev, "inline len(1-%d)=%d, illegal",
291− ctrl->msg_length,
291+ le32_to_cpu(ctrl->msg_length),
292292 hr_dev->caps.max_sq_inline);
293293 goto out;
294294 }
@@ -300,7 +300,7 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, · 5 unmodified lines
300300 }
301301 ctrl->flag |= cpu_to_le32(HNS_ROCE_WQE_INLINE);
302302 } else {
303− /*sqe num is two */
303+ /* sqe num is two */
304304 for (i = 0; i < wr->num_sge; i++)
305305 set_data_seg(dseg + i, wr->sg_list + i);
306306
@@ -353,8 +353,8 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp, · 46 unmodified lines
353353 unsigned long flags = 0;
354354 unsigned int wqe_idx;
355355 int ret = 0;
356− int nreq = 0;
357− int i = 0;
356+ int nreq;
357+ int i;
358358 u32 reg_val;
359359
360360 spin_lock_irqsave(&hr_qp->rq.lock, flags);
@@ -1165,7 +1165,7 @@ static int hns_roce_raq_init(struct hns_roce_dev *hr_dev) · 804 unmodified lines
11651165 }
11661166 raq->e_raq_buf->map = addr;
11671167
1168− /* Configure raq extended address. 48bit 4K align*/
1168+ /* Configure raq extended address. 48bit 4K align */
11691169 roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12);
11701170
11711171 /* Configure raq_shift */
@@ -1639,7 +1639,7 @@ static int hns_roce_v1_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, · 467 unmodified lines
16391639 }
16401640
16411641 static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev,
1642− unsigned long timeout)
1642+ unsigned int timeout)
16431643 {
16441644 u8 __iomem *hcr = hr_dev->reg_base + ROCEE_MB1_REG;
16451645 unsigned long end;
@@ -2062,11 +2062,6 @@ static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, · 416 unmodified lines
20622062 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
20632063 }
20642064
2065−static int hns_roce_v1_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
2066−{
2067− return -EOPNOTSUPP;
2068−}
2069−
20702065 static int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq,
20712066 enum ib_cq_notify_flags flags)
20722067 {
@@ -2305,7 +2300,7 @@ int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) · 232 unmodified lines
23052300 struct hns_roce_qp *cur_qp = NULL;
23062301 unsigned long flags;
23072302 int npolled;
2308− int ret = 0;
2303+ int ret;
23092304
23102305 spin_lock_irqsave(&hr_cq->lock, flags);
23112306
@@ -2765,7 +2760,6 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, · 453 unmodified lines
27652760 roce_set_field(context->qpc_bytes_16,
27662761 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
27672762 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2768−
27692763 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
27702764 roce_set_field(context->qpc_bytes_4,
27712765 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
@@ -3261,6 +3255,8 @@ static int hns_roce_v1_modify_qp(struct ib_qp *ibqp, · 489 unmodified lines
32613255 enum ib_qp_state cur_state,
32623256 enum ib_qp_state new_state)
32633257 {
3258+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
3259+ return -EOPNOTSUPP;
32643260
32653261 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
32663262 return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state,
@@ -3604,10 +3600,10 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) · 337 unmodified lines
36043600 return 0;
36053601 }
36063602
3607−static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not)
3603+static void set_eq_cons_index_v1(struct hns_roce_eq *eq, u32 req_not)
36083604 {
36093605 roce_raw_write((eq->cons_index & HNS_ROCE_V1_CONS_IDX_M) |
3610− (req_not << eq->log_entries), eq->doorbell);
3606+ (req_not << eq->log_entries), eq->doorbell);
36113607 }
36123608
36133609 static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
@@ -3687,10 +3683,10 @@ static void hns_roce_v1_qp_err_handle(struct hns_roce_dev *hr_dev, · 73 unmodified lines
36873683 int phy_port;
36883684 int qpn;
36893685
3690− qpn = roce_get_field(aeqe->event.qp_event.qp,
3686+ qpn = roce_get_field(aeqe->event.queue_event.num,
36913687 HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
36923688 HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);
3693− phy_port = roce_get_field(aeqe->event.qp_event.qp,
3689+ phy_port = roce_get_field(aeqe->event.queue_event.num,
36943690 HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M,
36953691 HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S);
36963692 if (qpn <= 1)
@@ -3721,9 +3717,9 @@ static void hns_roce_v1_cq_err_handle(struct hns_roce_dev *hr_dev, · 24 unmodified lines
37213717 struct device *dev = &hr_dev->pdev->dev;
37223718 u32 cqn;
37233719
3724− cqn = roce_get_field(aeqe->event.cq_event.cq,
3725− HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
3726− HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S);
3720+ cqn = roce_get_field(aeqe->event.queue_event.num,
3721+ HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
3722+ HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S);
37273723
37283724 switch (event_type) {
37293725 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
@@ -3798,7 +3794,6 @@ static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, · 68 unmodified lines
37983794 int event_type;
37993795
38003796 while ((aeqe = next_aeqe_sw_v1(eq))) {
3801−
38023797 /* Make sure we read the AEQ entry after we have checked the
38033798 * ownership bit
38043799 */
@@ -3853,12 +3848,6 @@ static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, · 48 unmodified lines
38533848 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
38543849 hns_roce_v1_db_overflow_handle(hr_dev, aeqe);
38553850 break;
3856− case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
3857− dev_warn(dev, "CEQ 0x%lx overflow.\n",
3858− roce_get_field(aeqe->event.ce_event.ceqe,
3859− HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M,
3860− HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S));
3861− break;
38623851 default:
38633852 dev_warn(dev, "Unhandled event %d on EQ %d at idx %u.\n",
38643853 event_type, eq->eqn, eq->cons_index);
@@ -3903,7 +3892,6 @@ static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev, · 38 unmodified lines
39033892 u32 cqn;
39043893
39053894 while ((ceqe = next_ceqe_sw_v1(eq))) {
3906−
39073895 /* Make sure we read CEQ entry after we have checked the
39083896 * ownership bit
39093897 */
@@ -4129,7 +4117,7 @@ static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, · 219 unmodified lines
41294117 void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn];
41304118 struct device *dev = &hr_dev->pdev->dev;
41314119 dma_addr_t tmp_dma_addr;
4132− u32 eqcuridx_val = 0;
4120+ u32 eqcuridx_val;
41334121 u32 eqconsindx_val;
41344122 u32 eqshift_val;
41354123 __le32 tmp2 = 0;
@@ -4347,7 +4335,6 @@ static void hns_roce_v1_cleanup_eq_table(struct hns_roce_dev *hr_dev) · 211 unmodified lines
43474335
43484336 static const struct ib_device_ops hns_roce_v1_dev_ops = {
43494337 .destroy_qp = hns_roce_v1_destroy_qp,
4350− .modify_cq = hns_roce_v1_modify_cq,
43514338 .poll_cq = hns_roce_v1_poll_cq,
43524339 .post_recv = hns_roce_v1_post_recv,
43534340 .post_send = hns_roce_v1_post_send,
@@ -4367,7 +4354,6 @@ static const struct hns_roce_hw hns_roce_hw_v1 = { · 13 unmodified lines
43674354 .set_mtu = hns_roce_v1_set_mtu,
43684355 .write_mtpt = hns_roce_v1_write_mtpt,
43694356 .write_cqc = hns_roce_v1_write_cqc,
4370− .modify_cq = hns_roce_v1_modify_cq,
43714357 .clear_hem = hns_roce_v1_clear_hem,
43724358 .modify_qp = hns_roce_v1_modify_qp,
43734359 .query_qp = hns_roce_v1_query_qp,
drivers/infiniband/hw/hns/hns_roce_hw_v1.h+1−1
@@ -419,7 +419,7 @@ struct hns_roce_wqe_data_seg { · 418 unmodified lines
419419
420420 struct hns_roce_wqe_raddr_seg {
421421 __le32 rkey;
422− __le32 len;/* reserved */
422+ __le32 len; /* reserved */
423423 __le64 raddr;
424424 };
425425
drivers/infiniband/hw/hns/hns_roce_hw_v2.c+313−241
@@ -214,25 +214,20 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, · 213 unmodified lines
214214 return 0;
215215 }
216216
217−static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
218− unsigned int *sge_ind, unsigned int valid_num_sge)
217+static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge,
218+ unsigned int *sge_ind, unsigned int cnt)
219219 {
220220 struct hns_roce_v2_wqe_data_seg *dseg;
221− unsigned int cnt = valid_num_sge;
222− struct ib_sge *sge = wr->sg_list;
223221 unsigned int idx = *sge_ind;
224222
225− if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
226− cnt -= HNS_ROCE_SGE_IN_WQE;
227− sge += HNS_ROCE_SGE_IN_WQE;
228− }
229−
230223 while (cnt > 0) {
231224 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
232− set_data_seg_v2(dseg, sge);
233− idx++;
225+ if (likely(sge->length)) {
226+ set_data_seg_v2(dseg, sge);
227+ idx++;
228+ cnt--;
229+ }
234230 sge++;
235− cnt--;
236231 }
237232
238233 *sge_ind = idx;
@@ -340,7 +335,8 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, · 101 unmodified lines
340335 }
341336 }
342337
343− set_extend_sge(qp, wr, sge_ind, valid_num_sge);
338+ set_extend_sge(qp, wr->sg_list + i, sge_ind,
339+ valid_num_sge - HNS_ROCE_SGE_IN_WQE);
344340 }
345341
346342 roce_set_field(rc_sq_wqe->byte_16,
@@ -365,7 +361,7 @@ static int check_send_valid(struct hns_roce_dev *hr_dev, · 18 unmodified lines
365361 } else if (unlikely(hr_qp->state == IB_QPS_RESET ||
366362 hr_qp->state == IB_QPS_INIT ||
367363 hr_qp->state == IB_QPS_RTR)) {
368− ibdev_err(ibdev, "failed to post WQE, QP state %d!\n",
364+ ibdev_err(ibdev, "failed to post WQE, QP state %hhu!\n",
369365 hr_qp->state);
370366 return -EINVAL;
371367 } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {
@@ -422,19 +418,54 @@ static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, · 50 unmodified lines
422418 return 0;
423419 }
424420
421+static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
422+ struct hns_roce_ah *ah)
423+{
424+ struct ib_device *ib_dev = ah->ibah.device;
425+ struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
426+
427+ roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
428+ V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
429+
430+ roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
431+ V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
432+ roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
433+ V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
434+ roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
435+ V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
436+
437+ if (WARN_ON(ah->av.sl > MAX_SERVICE_LEVEL))
438+ return -EINVAL;
439+
440+ roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
441+ V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
442+
443+ ud_sq_wqe->sgid_index = ah->av.gid_index;
444+
445+ memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN);
446+ memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2);
447+
448+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
449+ return 0;
450+
451+ roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
452+ ah->av.vlan_en);
453+ roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M,
454+ V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
455+
456+ return 0;
457+}
458+
425459 static inline int set_ud_wqe(struct hns_roce_qp *qp,
426460 const struct ib_send_wr *wr,
427461 void *wqe, unsigned int *sge_idx,
428462 unsigned int owner_bit)
429463 {
430− struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
431464 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
432465 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
433466 unsigned int curr_idx = *sge_idx;
434− int valid_num_sge;
467+ unsigned int valid_num_sge;
435468 u32 msg_len = 0;
436− bool loopback;
437− u8 *smac;
438469 int ret;
439470
440471 valid_num_sge = calc_wr_sge_num(wr, &msg_len);
@@ -444,38 +475,13 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp, · 3 unmodified lines
444475 if (WARN_ON(ret))
445476 return ret;
446477
447− roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
448− V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
449− roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
450− V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
451− roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
452− V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
453− roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
454− V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
455− roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
456− V2_UD_SEND_WQE_BYTE_48_DMAC_4_S, ah->av.mac[4]);
457− roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
458− V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, ah->av.mac[5]);
459−
460− /* MAC loopback */
461− smac = (u8 *)hr_dev->dev_addr[qp->port];
462− loopback = ether_addr_equal_unaligned(ah->av.mac, smac) ? 1 : 0;
463−
464− roce_set_bit(ud_sq_wqe->byte_40,
465− V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
466−
467478 ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
468479
469− /* Set sig attr */
470480 roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S,
471− (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
481+ !!(wr->send_flags & IB_SEND_SIGNALED));
472482
473− /* Set se attr */
474483 roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S,
475− (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
476−
477− roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S,
478− owner_bit);
484+ !!(wr->send_flags & IB_SEND_SOLICITED));
479485
480486 roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M,
481487 V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn);
@@ -488,36 +494,29 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp, · 6 unmodified lines
488494 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
489495 curr_idx & (qp->sge.sge_cnt - 1));
490496
491− roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
492− V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
493497 ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
494498 qp->qkey : ud_wr(wr)->remote_qkey);
495499 roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
496500 V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn);
497501
498− roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M,
499− V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
500− roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
501− V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
502− roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
503− V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
504− roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
505− V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
506− roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
507− V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
508− roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M,
509− V2_UD_SEND_WQE_BYTE_40_PORTN_S, qp->port);
510−
511− roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
512− ah->av.vlan_en ? 1 : 0);
513− roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
514− V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index);
502+ ret = fill_ud_av(ud_sq_wqe, ah);
503+ if (ret)
504+ return ret;
515505
516− memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2);
506+ set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
517507
518− set_extend_sge(qp, wr, &curr_idx, valid_num_sge);
508+ /*
509+ * The pipeline can sequentially post all valid WQEs into WQ buffer,
510+ * including new WQEs waiting for the doorbell to update the PI again.
511+ * Therefore, the owner bit of WQE MUST be updated after all fields
512+ * and extSGEs have been written into DDR instead of cache.
513+ */
514+ if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
515+ dma_wmb();
519516
520517 *sge_idx = curr_idx;
518+ roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S,
519+ owner_bit);
521520
522521 return 0;
523522 }
@@ -591,9 +590,6 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, · 67 unmodified lines
591590 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S,
592591 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
593592
594− roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
595− owner_bit);
596−
597593 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
598594 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
599595 set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
@@ -601,7 +597,18 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, · 1 unmodified line
601597 ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
602598 &curr_idx, valid_num_sge);
603599
600+ /*
601+ * The pipeline can sequentially post all valid WQEs into WQ buffer,
602+ * including new WQEs waiting for the doorbell to update the PI again.
603+ * Therefore, the owner bit of WQE MUST be updated after all fields
604+ * and extSGEs have been written into DDR instead of cache.
605+ */
606+ if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
607+ dma_wmb();
608+
604609 *sge_idx = curr_idx;
610+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
611+ owner_bit);
605612
606613 return ret;
607614 }
@@ -649,7 +656,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, · 41 unmodified lines
649656 unsigned int sge_idx;
650657 unsigned int wqe_idx;
651658 void *wqe = NULL;
652− int nreq;
659+ u32 nreq;
653660 int ret;
654661
655662 spin_lock_irqsave(&qp->sq.lock, flags);
@@ -673,7 +680,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, · 17 unmodified lines
673680 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
674681
675682 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
676− ibdev_err(ibdev, "num_sge=%d > qp->sq.max_gs=%d\n",
683+ ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n",
677684 wr->num_sge, qp->sq.max_gs);
678685 ret = -EINVAL;
679686 *bad_wr = wr;
@@ -686,7 +693,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, · 6 unmodified lines
686693 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
687694
688695 /* Corresponding to the QP type, wqe process separately */
689− if (ibqp->qp_type == IB_QPT_GSI)
696+ if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
690697 ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
691698 else if (ibqp->qp_type == IB_QPT_RC)
692699 ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
@@ -758,7 +765,7 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, · 65 unmodified lines
758765 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
759766
760767 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
761− ibdev_err(ibdev, "rq:num_sge=%d >= qp->sq.max_gs=%d\n",
768+ ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n",
762769 wr->num_sge, hr_qp->rq.max_gs);
763770 ret = -EINVAL;
764771 *bad_wr = wr;
@@ -827,7 +834,7 @@ static void *get_srq_wqe(struct hns_roce_srq *srq, int n) · 62 unmodified lines
827834 return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
828835 }
829836
830−static void *get_idx_buf(struct hns_roce_idx_que *idx_que, int n)
837+static void *get_idx_buf(struct hns_roce_idx_que *idx_que, unsigned int n)
831838 {
832839 return hns_roce_buf_offset(idx_que->mtr.kmem,
833840 n << idx_que->entry_shift);
@@ -868,12 +875,12 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, · 34 unmodified lines
868875 struct hns_roce_v2_wqe_data_seg *dseg;
869876 struct hns_roce_v2_db srq_db;
870877 unsigned long flags;
878+ unsigned int ind;
871879 __le32 *srq_idx;
872880 int ret = 0;
873881 int wqe_idx;
874882 void *wqe;
875883 int nreq;
876− int ind;
877884 int i;
878885
879886 spin_lock_irqsave(&srq->lock, flags);
@@ -1018,8 +1025,8 @@ static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev) · 138 unmodified lines
10181025 struct hns_roce_v2_priv *priv = hr_dev->priv;
10191026 struct hnae3_handle *handle = priv->handle;
10201027 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1021− unsigned long instance_stage; /* the current instance stage */
1022− unsigned long reset_stage; /* the current reset stage */
1028+ unsigned long instance_stage; /* the current instance stage */
1029+ unsigned long reset_stage; /* the current reset stage */
10231030 unsigned long reset_cnt;
10241031 bool sw_resetting;
10251032 bool hw_resetting;
@@ -1118,7 +1125,7 @@ static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type) · 92 unmodified lines
11181125 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
11191126 upper_32_bits(dma));
11201127 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
1121− ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
1128+ (u32)ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
11221129 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
11231130 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
11241131 } else {
@@ -1126,7 +1133,7 @@ static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type) · 1 unmodified line
11261133 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
11271134 upper_32_bits(dma));
11281135 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
1129− ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
1136+ (u32)ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
11301137 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
11311138 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
11321139 }
@@ -1573,6 +1580,10 @@ static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) · 440 unmodified lines
15731580 PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
15741581 PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
15751582
1583+ hr_dev->caps.gmv_bt_num = roce_get_field(req_b->gmv_idx_num,
1584+ PF_RES_DATA_5_PF_GMV_BT_NUM_M,
1585+ PF_RES_DATA_5_PF_GMV_BT_NUM_S);
1586+
15761587 return 0;
15771588 }
15781589
@@ -1896,11 +1907,20 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) · 317 unmodified lines
18961907 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
18971908 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
18981909 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
1910+ caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
1911+ caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;
1912+ caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE /
1913+ caps->gmv_entry_sz);
1914+ caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;
1915+ caps->gmv_ba_pg_sz = 0;
1916+ caps->gmv_buf_pg_sz = 0;
1917+ caps->gid_table_len[0] = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /
1918+ caps->gmv_entry_sz);
18991919 }
19001920 }
19011921
1902−static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num,
1903− int *buf_page_size, int *bt_page_size, u32 hem_type)
1922+static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num,
1923+ u32 *buf_page_size, u32 *bt_page_size, u32 hem_type)
19041924 {
19051925 u64 obj_per_chunk;
19061926 u64 bt_chunk_size = PAGE_SIZE;
@@ -1930,8 +1950,8 @@ static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num, · 23 unmodified lines
19301950 obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
19311951 break;
19321952 default:
1933− pr_err("Table %d not support hop_num = %d!\n", hem_type,
1934− hop_num);
1953+ pr_err("table %u not support hop_num = %u!\n", hem_type,
1954+ hop_num);
19351955 return;
19361956 }
19371957
@@ -2122,6 +2142,14 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) · 184 unmodified lines
21222142 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
21232143 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
21242144 caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
2145+ caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;
2146+ caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE /
2147+ caps->gmv_entry_sz);
2148+ caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;
2149+ caps->gmv_ba_pg_sz = 0;
2150+ caps->gmv_buf_pg_sz = 0;
2151+ caps->gid_table_len[0] = caps->gmv_bt_num *
2152+ (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz);
21252153 }
21262154
21272155 calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,
@@ -2371,10 +2399,10 @@ static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev, · 243 unmodified lines
23712399 u32 buf_chk_sz;
23722400 dma_addr_t t;
23732401 int func_num = 1;
2374− int pg_num_a;
2375− int pg_num_b;
2376− int pg_num;
2377− int size;
2402+ u32 pg_num_a;
2403+ u32 pg_num_b;
2404+ u32 pg_num;
2405+ u32 size;
23782406 int i;
23792407
23802408 switch (type) {
@@ -2423,7 +2451,6 @@ static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev, · 42 unmodified lines
24232451 if (i < (pg_num - 1))
24242452 entry[i].blk_ba1_nxt_ptr |=
24252453 (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S;
2426−
24272454 }
24282455 link_tbl->npages = pg_num;
24292456 link_tbl->pg_sz = buf_chk_sz;
@@ -2465,24 +2492,13 @@ static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev, · 35 unmodified lines
24652492 link_tbl->table.map);
24662493 }
24672494
2468−static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
2495+static int get_hem_table(struct hns_roce_dev *hr_dev)
24692496 {
2470− struct hns_roce_v2_priv *priv = hr_dev->priv;
2471− int qpc_count, cqc_count;
2472− int ret, i;
2473−
2474− /* TSQ includes SQ doorbell and ack doorbell */
2475− ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
2476− if (ret) {
2477− dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);
2478− return ret;
2479− }
2480−
2481− ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
2482− if (ret) {
2483− dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);
2484− goto err_tpq_init_failed;
2485− }
2497+ unsigned int qpc_count;
2498+ unsigned int cqc_count;
2499+ unsigned int gmv_count;
2500+ int ret;
2501+ int i;
24862502
24872503 /* Alloc memory for QPC Timer buffer space chunk */
24882504 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
@@ -2506,8 +2522,23 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) · 17 unmodified lines
25062522 }
25072523 }
25082524
2525+ /* Alloc memory for GMV(GID/MAC/VLAN) table buffer space chunk */
2526+ for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num;
2527+ gmv_count++) {
2528+ ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count);
2529+ if (ret) {
2530+ dev_err(hr_dev->dev,
2531+ "failed to get gmv table, ret = %d.\n", ret);
2532+ goto err_gmv_failed;
2533+ }
2534+ }
2535+
25092536 return 0;
25102537
2538+err_gmv_failed:
2539+ for (i = 0; i < gmv_count; i++)
2540+ hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);
2541+
25112542 err_cqc_timer_failed:
25122543 for (i = 0; i < cqc_count; i++)
25132544 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
@@ -2516,6 +2547,34 @@ err_qpc_timer_failed: · 2 unmodified lines
25162547 for (i = 0; i < qpc_count; i++)
25172548 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
25182549
2550+ return ret;
2551+}
2552+
2553+static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
2554+{
2555+ struct hns_roce_v2_priv *priv = hr_dev->priv;
2556+ int ret;
2557+
2558+ /* TSQ includes SQ doorbell and ack doorbell */
2559+ ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
2560+ if (ret) {
2561+ dev_err(hr_dev->dev, "failed to init TSQ, ret = %d.\n", ret);
2562+ return ret;
2563+ }
2564+
2565+ ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
2566+ if (ret) {
2567+ dev_err(hr_dev->dev, "failed to init TPQ, ret = %d.\n", ret);
2568+ goto err_tpq_init_failed;
2569+ }
2570+
2571+ ret = get_hem_table(hr_dev);
2572+ if (ret)
2573+ goto err_get_hem_table_failed;
2574+
2575+ return 0;
2576+
2577+err_get_hem_table_failed:
25192578 hns_roce_free_link_table(hr_dev, &priv->tpq);
25202579
25212580 err_tpq_init_failed:
@@ -2539,7 +2598,7 @@ static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev) · 17 unmodified lines
25392598 struct hns_roce_cmq_desc desc;
25402599 struct hns_roce_mbox_status *mb_st =
25412600 (struct hns_roce_mbox_status *)desc.data;
2542− enum hns_roce_cmd_return_status status;
2601+ int status;
25432602
25442603 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
25452604
@@ -2610,7 +2669,7 @@ static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, · 64 unmodified lines
26102669 }
26112670
26122671 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
2613− unsigned long timeout)
2672+ unsigned int timeout)
26142673 {
26152674 struct device *dev = hr_dev->dev;
26162675 unsigned long end;
@@ -2637,14 +2696,27 @@ static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, · 20 unmodified lines
26372696 return 0;
26382697 }
26392698
2640−static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,
2641− int gid_index, const union ib_gid *gid,
2642− enum hns_roce_sgid_type sgid_type)
2699+static void copy_gid(void *dest, const union ib_gid *gid)
2700+{
2701+#define GID_SIZE 4
2702+ const union ib_gid *src = gid;
2703+ __le32 (*p)[GID_SIZE] = dest;
2704+ int i;
2705+
2706+ if (!gid)
2707+ src = &zgid;
2708+
2709+ for (i = 0; i < GID_SIZE; i++)
2710+ (*p)[i] = cpu_to_le32(*(u32 *)&src->raw[i * sizeof(u32)]);
2711+}
2712+
2713+static int config_sgid_table(struct hns_roce_dev *hr_dev,
2714+ int gid_index, const union ib_gid *gid,
2715+ enum hns_roce_sgid_type sgid_type)
26432716 {
26442717 struct hns_roce_cmq_desc desc;
26452718 struct hns_roce_cfg_sgid_tb *sgid_tb =
26462719 (struct hns_roce_cfg_sgid_tb *)desc.data;
2647− u32 *p;
26482720
26492721 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
26502722
@@ -2653,19 +2725,54 @@ static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev, · 2 unmodified lines
26532725 roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M,
26542726 CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
26552727
2656− p = (u32 *)&gid->raw[0];
2657− sgid_tb->vf_sgid_l = cpu_to_le32(*p);
2728+ copy_gid(&sgid_tb->vf_sgid_l, gid);
26582729
2659− p = (u32 *)&gid->raw[4];
2660− sgid_tb->vf_sgid_ml = cpu_to_le32(*p);
2730+ return hns_roce_cmq_send(hr_dev, &desc, 1);
2731+}
26612732
2662− p = (u32 *)&gid->raw[8];
2663− sgid_tb->vf_sgid_mh = cpu_to_le32(*p);
2733+static int config_gmv_table(struct hns_roce_dev *hr_dev,
2734+ int gid_index, const union ib_gid *gid,
2735+ enum hns_roce_sgid_type sgid_type,
2736+ const struct ib_gid_attr *attr)
2737+{
2738+ struct hns_roce_cmq_desc desc[2];
2739+ struct hns_roce_cfg_gmv_tb_a *tb_a =
2740+ (struct hns_roce_cfg_gmv_tb_a *)desc[0].data;
2741+ struct hns_roce_cfg_gmv_tb_b *tb_b =
2742+ (struct hns_roce_cfg_gmv_tb_b *)desc[1].data;
26642743
2665− p = (u32 *)&gid->raw[0xc];
2666− sgid_tb->vf_sgid_h = cpu_to_le32(*p);
2744+ u16 vlan_id = VLAN_CFI_MASK;
2745+ u8 mac[ETH_ALEN] = {};
2746+ int ret;
26672747
2668− return hns_roce_cmq_send(hr_dev, &desc, 1);
2748+ if (gid) {
2749+ ret = rdma_read_gid_l2_fields(attr, &vlan_id, mac);
2750+ if (ret)
2751+ return ret;
2752+ }
2753+
2754+ hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false);
2755+ desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2756+
2757+ hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false);
2758+
2759+ copy_gid(&tb_a->vf_sgid_l, gid);
2760+
2761+ roce_set_field(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_SGID_TYPE_M,
2762+ CFG_GMV_TB_VF_SGID_TYPE_S, sgid_type);
2763+ roce_set_bit(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_VLAN_EN_S,
2764+ vlan_id < VLAN_CFI_MASK);
2765+ roce_set_field(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_VLAN_ID_M,
2766+ CFG_GMV_TB_VF_VLAN_ID_S, vlan_id);
2767+
2768+ tb_b->vf_smac_l = cpu_to_le32(*(u32 *)mac);
2769+ roce_set_field(tb_b->vf_smac_h, CFG_GMV_TB_SMAC_H_M,
2770+ CFG_GMV_TB_SMAC_H_S, *(u16 *)&mac[4]);
2771+
2772+ roce_set_field(tb_b->table_idx_rsv, CFG_GMV_TB_SGID_IDX_M,
2773+ CFG_GMV_TB_SGID_IDX_S, gid_index);
2774+
2775+ return hns_roce_cmq_send(hr_dev, desc, 2);
26692776 }
26702777
26712778 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
@@ -2675,23 +2782,24 @@ static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port, · 3 unmodified lines
26752782 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
26762783 int ret;
26772784
2678− if (!gid || !attr)
2679− return -EINVAL;
2680−
2681− if (attr->gid_type == IB_GID_TYPE_ROCE)
2682− sgid_type = GID_TYPE_FLAG_ROCE_V1;
2683−
2684− if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2685− if (ipv6_addr_v4mapped((void *)gid))
2686− sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2687− else
2688− sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2785+ if (gid) {
2786+ if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2787+ if (ipv6_addr_v4mapped((void *)gid))
2788+ sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2789+ else
2790+ sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2791+ } else if (attr->gid_type == IB_GID_TYPE_ROCE) {
2792+ sgid_type = GID_TYPE_FLAG_ROCE_V1;
2793+ }
26892794 }
26902795
2691− ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2796+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
2797+ ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr);
2798+ else
2799+ ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2800+
26922801 if (ret)
2693− ibdev_err(&hr_dev->ib_dev,
2694− "failed to configure sgid table, ret = %d!\n",
2802+ ibdev_err(&hr_dev->ib_dev, "failed to set gid, ret = %d!\n",
26952803 ret);
26962804
26972805 return ret;
@@ -2959,7 +3067,7 @@ static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) · 261 unmodified lines
29593067 return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);
29603068 }
29613069
2962−static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
3070+static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n)
29633071 {
29643072 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
29653073
@@ -3060,6 +3168,9 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, · 94 unmodified lines
30603168 V2_CQC_BYTE_8_CQE_SIZE_S, hr_cq->cqe_size ==
30613169 HNS_ROCE_V3_CQE_SIZE ? 1 : 0);
30623170
3171+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
3172+ hr_reg_enable(cq_context, CQC_STASH);
3173+
30633174 cq_context->cqe_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
30643175
30653176 roce_set_field(cq_context->byte_16_hop_addr,
@@ -3303,7 +3414,7 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 237 unmodified lines
33033414 int is_send;
33043415 u16 wqe_ctr;
33053416 u32 opcode;
3306− int qpn;
3417+ u32 qpn;
33073418 int ret;
33083419
33093420 /* Find cqe according to consumer index */
@@ -3572,7 +3683,7 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, · 262 unmodified lines
35723683 break;
35733684 default:
35743685 dev_warn(hr_dev->dev,
3575− "Table %d not to be written by mailbox!\n", type);
3686+ "table %u not to be written by mailbox!\n", type);
35763687 return -EINVAL;
35773688 }
35783689
@@ -3583,9 +3694,25 @@ static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba, · 4 unmodified lines
35833694 u32 hem_type, int step_idx)
35843695 {
35853696 struct hns_roce_cmd_mailbox *mailbox;
3697+ struct hns_roce_cmq_desc desc;
3698+ struct hns_roce_cfg_gmv_bt *gmv_bt =
3699+ (struct hns_roce_cfg_gmv_bt *)desc.data;
35863700 int ret;
35873701 int op;
35883702
3703+ if (hem_type == HEM_TYPE_GMV) {
3704+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT,
3705+ false);
3706+
3707+ gmv_bt->gmv_ba_l = cpu_to_le32(bt_ba >> HNS_HW_PAGE_SHIFT);
3708+ gmv_bt->gmv_ba_h = cpu_to_le32(bt_ba >> (HNS_HW_PAGE_SHIFT +
3709+ 32));
3710+ gmv_bt->gmv_bt_idx = cpu_to_le32(obj /
3711+ (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz));
3712+
3713+ return hns_roce_cmq_send(hr_dev, &desc, 1);
3714+ }
3715+
35893716 op = get_op_for_set_hem(hr_dev, hem_type, step_idx);
35903717 if (op < 0)
35913718 return 0;
@@ -3683,24 +3810,20 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, · 91 unmodified lines
36833810 case HEM_TYPE_CQC:
36843811 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
36853812 break;
3686− case HEM_TYPE_SCCC:
3687− case HEM_TYPE_QPC_TIMER:
3688− case HEM_TYPE_CQC_TIMER:
3689− break;
36903813 case HEM_TYPE_SRQC:
36913814 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
36923815 break;
3816+ case HEM_TYPE_SCCC:
3817+ case HEM_TYPE_QPC_TIMER:
3818+ case HEM_TYPE_CQC_TIMER:
3819+ case HEM_TYPE_GMV:
3820+ return 0;
36933821 default:
3694− dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
3822+ dev_warn(dev, "table %u not to be destroyed by mailbox!\n",
36953823 table->type);
36963824 return 0;
36973825 }
36983826
3699− if (table->type == HEM_TYPE_SCCC ||
3700− table->type == HEM_TYPE_QPC_TIMER ||
3701− table->type == HEM_TYPE_CQC_TIMER)
3702− return 0;
3703−
37043827 op += step_idx;
37053828
37063829 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
@@ -3851,9 +3974,14 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, · 144 unmodified lines
38513974
38523975 roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
38533976
3854− hr_qp->access_flags = attr->qp_access_flags;
38553977 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
38563978 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3979+
3980+ if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ)
3981+ return;
3982+
3983+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
3984+ hr_reg_enable(&context->ext, QPCEX_STASH);
38573985 }
38583986
38593987 static void modify_qp_init_to_init(struct ib_qp *ibqp,
@@ -3874,51 +4002,6 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, · 14 unmodified lines
38744002 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
38754003 V2_QPC_BYTE_4_TST_S, 0);
38764004
3877− if (attr_mask & IB_QP_ACCESS_FLAGS) {
3878− roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3879− !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
3880− roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3881− 0);
3882−
3883− roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3884− !!(attr->qp_access_flags &
3885− IB_ACCESS_REMOTE_WRITE));
3886− roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3887− 0);
3888−
3889− roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3890− !!(attr->qp_access_flags &
3891− IB_ACCESS_REMOTE_ATOMIC));
3892− roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3893− 0);
3894− roce_set_bit(context->byte_76_srqn_op_en,
3895− V2_QPC_BYTE_76_EXT_ATE_S,
3896− !!(attr->qp_access_flags &
3897− IB_ACCESS_REMOTE_ATOMIC));
3898− roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3899− V2_QPC_BYTE_76_EXT_ATE_S, 0);
3900− } else {
3901− roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3902− !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
3903− roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3904− 0);
3905−
3906− roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3907− !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
3908− roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3909− 0);
3910−
3911− roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3912− !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3913− roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3914− 0);
3915− roce_set_bit(context->byte_76_srqn_op_en,
3916− V2_QPC_BYTE_76_EXT_ATE_S,
3917− !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3918− roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3919− V2_QPC_BYTE_76_EXT_ATE_S, 0);
3920− }
3921−
39224005 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
39234006 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
39244007 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
@@ -4328,7 +4411,7 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, · 403 unmodified lines
43284411
43294412 ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);
43304413 if (ret) {
4331− ibdev_err(ibdev, "failed to config sq buf, ret %d\n", ret);
4414+ ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);
43324415 return ret;
43334416 }
43344417
@@ -4421,7 +4504,9 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, · 86 unmodified lines
44214504 IB_GID_TYPE_ROCE_UDP_ENCAP);
44224505 }
44234506
4424− if (vlan_id < VLAN_N_VID) {
4507+ /* Only HIP08 needs to set the vlan_en bits in QPC */
4508+ if (vlan_id < VLAN_N_VID &&
4509+ hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
44254510 roce_set_bit(context->byte_76_srqn_op_en,
44264511 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
44274512 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
@@ -4468,15 +4553,11 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, · 40 unmodified lines
44684553 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
44694554 V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
44704555
4471− if (is_udp)
4472− roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4473− V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2);
4474− else
4475− roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4476− V2_QPC_BYTE_24_TC_S, grh->traffic_class);
4477−
4556+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4557+ V2_QPC_BYTE_24_TC_S, get_tclass(&attr->ah_attr.grh));
44784558 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
44794559 V2_QPC_BYTE_24_TC_S, 0);
4560+
44804561 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
44814562 V2_QPC_BYTE_28_FL_S, grh->flow_label);
44824563 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
@@ -4758,6 +4839,9 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, · 275 unmodified lines
47584839 unsigned long rq_flag = 0;
47594840 int ret;
47604841
4842+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
4843+ return -EOPNOTSUPP;
4844+
47614845 /*
47624846 * In v2 engine, software pass context and context mask to hardware
47634847 * when modifying qp. If software need modify some fields in context,
@@ -4818,7 +4902,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, · 54 unmodified lines
48184902 /* SW pass context to HW */
48194903 ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);
48204904 if (ret) {
4821− ibdev_err(ibdev, "failed to modify QP, ret = %d\n", ret);
4905+ ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret);
48224906 goto out;
48234907 }
48244908
@@ -4911,7 +4995,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, · 86 unmodified lines
49114995
49124996 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
49134997 if (ret) {
4914− ibdev_err(ibdev, "failed to query QPC, ret = %d\n", ret);
4998+ ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);
49154999 ret = -EINVAL;
49165000 goto out;
49175001 }
@@ -5026,13 +5110,15 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, · 108 unmodified lines
50265110 unsigned long flags;
50275111 int ret = 0;
50285112
5029− if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
5113+ if ((hr_qp->ibqp.qp_type == IB_QPT_RC ||
5114+ hr_qp->ibqp.qp_type == IB_QPT_UD) &&
5115+ hr_qp->state != IB_QPS_RESET) {
50305116 /* Modify qp to reset before destroying qp */
50315117 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
50325118 hr_qp->state, IB_QPS_RESET);
50335119 if (ret)
50345120 ibdev_err(ibdev,
5035− "failed to modify QP to RST, ret = %d\n",
5121+ "failed to modify QP to RST, ret = %d.\n",
50365122 ret);
50375123 }
50385124
@@ -5071,7 +5157,7 @@ static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) · 32 unmodified lines
50715157 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
50725158 if (ret)
50735159 ibdev_err(&hr_dev->ib_dev,
5074− "failed to destroy QP 0x%06lx, ret = %d\n",
5160+ "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",
50755161 hr_qp->qpn, ret);
50765162
50775163 hns_roce_qp_destroy(hr_dev, hr_qp, udata);
@@ -5094,7 +5180,7 @@ static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, · 16 unmodified lines
50945180 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
50955181 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
50965182 if (ret) {
5097− ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d\n", ret);
5183+ ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);
50985184 goto out;
50995185 }
51005186
@@ -5104,7 +5190,7 @@ static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, · 3 unmodified lines
51045190 clr->qpn = cpu_to_le32(hr_qp->qpn);
51055191 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
51065192 if (ret) {
5107− ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d\n", ret);
5193+ ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);
51085194 goto out;
51095195 }
51105196
@@ -5353,7 +5439,7 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) · 242 unmodified lines
53535439 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
53545440 if (ret)
53555441 ibdev_err(&hr_dev->ib_dev,
5356− "failed to process cmd when modifying CQ, ret = %d\n",
5442+ "failed to process cmd when modifying CQ, ret = %d.\n",
53575443 ret);
53585444
53595445 return ret;
@@ -5364,8 +5450,6 @@ static void hns_roce_irq_work_handle(struct work_struct *work) · 4 unmodified lines
53645450 struct hns_roce_work *irq_work =
53655451 container_of(work, struct hns_roce_work, work);
53665452 struct ib_device *ibdev = &irq_work->hr_dev->ib_dev;
5367− u32 qpn = irq_work->qpn;
5368− u32 cqn = irq_work->cqn;
53695453
53705454 switch (irq_work->event_type) {
53715455 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
@@ -5381,15 +5465,15 @@ static void hns_roce_irq_work_handle(struct work_struct *work) · 9 unmodified lines
53815465 break;
53825466 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
53835467 ibdev_err(ibdev, "Local work queue 0x%x catast error, sub_event type is: %d\n",
5384− qpn, irq_work->sub_type);
5468+ irq_work->queue_num, irq_work->sub_type);
53855469 break;
53865470 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
53875471 ibdev_err(ibdev, "Invalid request local work queue 0x%x error.\n",
5388− qpn);
5472+ irq_work->queue_num);
53895473 break;
53905474 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
53915475 ibdev_err(ibdev, "Local access violation work queue 0x%x error, sub_event type is: %d\n",
5392− qpn, irq_work->sub_type);
5476+ irq_work->queue_num, irq_work->sub_type);
53935477 break;
53945478 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
53955479 ibdev_warn(ibdev, "SRQ limit reach.\n");
@@ -5401,10 +5485,10 @@ static void hns_roce_irq_work_handle(struct work_struct *work) · 5 unmodified lines
54015485 ibdev_err(ibdev, "SRQ catas error.\n");
54025486 break;
54035487 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5404− ibdev_err(ibdev, "CQ 0x%x access err.\n", cqn);
5488+ ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work->queue_num);
54055489 break;
54065490 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5407− ibdev_warn(ibdev, "CQ 0x%x overflow\n", cqn);
5491+ ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work->queue_num);
54085492 break;
54095493 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
54105494 ibdev_warn(ibdev, "DB overflow.\n");
@@ -5420,8 +5504,7 @@ static void hns_roce_irq_work_handle(struct work_struct *work) · 9 unmodified lines
54205504 }
54215505
54225506 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
5423− struct hns_roce_eq *eq,
5424− u32 qpn, u32 cqn)
5507+ struct hns_roce_eq *eq, u32 queue_num)
54255508 {
54265509 struct hns_roce_work *irq_work;
54275510
@@ -5431,10 +5514,9 @@ static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, · 3 unmodified lines
54315514
54325515 INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
54335516 irq_work->hr_dev = hr_dev;
5434− irq_work->qpn = qpn;
5435− irq_work->cqn = cqn;
54365517 irq_work->event_type = eq->event_type;
54375518 irq_work->sub_type = eq->sub_type;
5519+ irq_work->queue_num = queue_num;
54385520 queue_work(hr_dev->irq_workq, &(irq_work->work));
54395521 }
54405522
@@ -5486,10 +5568,8 @@ static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, · 45 unmodified lines
54865568 struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
54875569 int aeqe_found = 0;
54885570 int event_type;
5571+ u32 queue_num;
54895572 int sub_type;
5490− u32 srqn;
5491− u32 qpn;
5492− u32 cqn;
54935573
54945574 while (aeqe) {
54955575 /* Make sure we read AEQ entry after we have checked the
@@ -5503,15 +5583,9 @@ static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, · 7 unmodified lines
55035583 sub_type = roce_get_field(aeqe->asyn,
55045584 HNS_ROCE_V2_AEQE_SUB_TYPE_M,
55055585 HNS_ROCE_V2_AEQE_SUB_TYPE_S);
5506− qpn = roce_get_field(aeqe->event.qp_event.qp,
5507− HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5508− HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5509− cqn = roce_get_field(aeqe->event.cq_event.cq,
5510− HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5511− HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5512− srqn = roce_get_field(aeqe->event.srq_event.srq,
5513− HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5514− HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5586+ queue_num = roce_get_field(aeqe->event.queue_event.num,
5587+ HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5588+ HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
55155589
55165590 switch (event_type) {
55175591 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
@@ -5522,17 +5596,15 @@ static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, · 4 unmodified lines
55225596 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
55235597 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
55245598 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
5525− hns_roce_qp_event(hr_dev, qpn, event_type);
5599+ hns_roce_qp_event(hr_dev, queue_num, event_type);
55265600 break;
55275601 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
55285602 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
5529− hns_roce_srq_event(hr_dev, srqn, event_type);
5603+ hns_roce_srq_event(hr_dev, queue_num, event_type);
55305604 break;
55315605 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
55325606 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5533− hns_roce_cq_event(hr_dev, cqn, event_type);
5534− break;
5535− case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
5607+ hns_roce_cq_event(hr_dev, queue_num, event_type);
55365608 break;
55375609 case HNS_ROCE_EVENT_TYPE_MB:
55385610 hns_roce_cmd_event(hr_dev,
@@ -5540,8 +5612,7 @@ static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, · 1 unmodified line
55405612 aeqe->event.cmd.status,
55415613 le64_to_cpu(aeqe->event.cmd.out_param));
55425614 break;
5543− case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
5544− break;
5615+ case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
55455616 case HNS_ROCE_EVENT_TYPE_FLR:
55465617 break;
55475618 default:
@@ -5558,7 +5629,7 @@ static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, · 10 unmodified lines
55585629 if (eq->cons_index > (2 * eq->entries - 1))
55595630 eq->cons_index = 0;
55605631
5561− hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
5632+ hns_roce_v2_init_irq_work(hr_dev, eq, queue_num);
55625633
55635634 aeqe = next_aeqe_sw_v2(eq);
55645635 }
@@ -6193,6 +6264,7 @@ static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { · 628 unmodified lines
61936264 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
61946265 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
61956266 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6267+ {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
61966268 /* required last entry */
61976269 {0, }
61986270 };
drivers/infiniband/hw/hns/hns_roce_hw_v2.h+154−111
@@ -44,6 +44,7 @@ · 43 unmodified lines
4444 #define HNS_ROCE_VF_SMAC_NUM 32
4545 #define HNS_ROCE_VF_SGID_NUM 32
4646 #define HNS_ROCE_VF_SL_NUM 8
47+#define HNS_ROCE_VF_GMV_BT_NUM 256
4748
4849 #define HNS_ROCE_V2_MAX_QP_NUM 0x100000
4950 #define HNS_ROCE_V2_MAX_QPC_TIMER_NUM 0x200
@@ -89,6 +90,7 @@ · 39 unmodified lines
8990
9091 #define HNS_ROCE_V2_SCCC_SZ 32
9192 #define HNS_ROCE_V3_SCCC_SZ 64
93+#define HNS_ROCE_V3_GMV_ENTRY_SZ 32
9294
9395 #define HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ PAGE_SIZE
9496 #define HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ PAGE_SIZE
@@ -241,6 +243,8 @@ enum hns_roce_opcode_type { · 146 unmodified lines
241243 HNS_ROCE_OPC_CLR_SCCC = 0x8509,
242244 HNS_ROCE_OPC_QUERY_SCCC = 0x850a,
243245 HNS_ROCE_OPC_RESET_SCCC = 0x850b,
246+ HNS_ROCE_OPC_CFG_GMV_TBL = 0x850f,
247+ HNS_ROCE_OPC_CFG_GMV_BT = 0x8510,
244248 HNS_SWITCH_PARAMETER_CFG = 0x1033,
245249 };
246250
@@ -263,23 +267,24 @@ enum hns_roce_sgid_type { · 16 unmodified lines
263267 };
264268
265269 struct hns_roce_v2_cq_context {
266− __le32 byte_4_pg_ceqn;
267− __le32 byte_8_cqn;
268− __le32 cqe_cur_blk_addr;
269− __le32 byte_16_hop_addr;
270− __le32 cqe_nxt_blk_addr;
271− __le32 byte_24_pgsz_addr;
272− __le32 byte_28_cq_pi;
273− __le32 byte_32_cq_ci;
274− __le32 cqe_ba;
275− __le32 byte_40_cqe_ba;
276− __le32 byte_44_db_record;
277− __le32 db_record_addr;
278− __le32 byte_52_cqe_cnt;
279− __le32 byte_56_cqe_period_maxcnt;
280− __le32 cqe_report_timer;
281− __le32 byte_64_se_cqe_idx;
270+ __le32 byte_4_pg_ceqn;
271+ __le32 byte_8_cqn;
272+ __le32 cqe_cur_blk_addr;
273+ __le32 byte_16_hop_addr;
274+ __le32 cqe_nxt_blk_addr;
275+ __le32 byte_24_pgsz_addr;
276+ __le32 byte_28_cq_pi;
277+ __le32 byte_32_cq_ci;
278+ __le32 cqe_ba;
279+ __le32 byte_40_cqe_ba;
280+ __le32 byte_44_db_record;
281+ __le32 db_record_addr;
282+ __le32 byte_52_cqe_cnt;
283+ __le32 byte_56_cqe_period_maxcnt;
284+ __le32 cqe_report_timer;
285+ __le32 byte_64_se_cqe_idx;
282286 };
287+
283288 #define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0
284289 #define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL 0x0
285290
@@ -356,6 +361,10 @@ struct hns_roce_v2_cq_context { · 70 unmodified lines
356361 #define V2_CQC_BYTE_64_SE_CQE_IDX_S 0
357362 #define V2_CQC_BYTE_64_SE_CQE_IDX_M GENMASK(23, 0)
358363
364+#define CQC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_cq_context, h, l)
365+
366+#define CQC_STASH CQC_FIELD_LOC(63, 63)
367+
359368 struct hns_roce_srq_context {
360369 __le32 byte_4_srqn_srqst;
361370 __le32 byte_8_limit_wl;
@@ -440,7 +449,7 @@ struct hns_roce_srq_context { · 78 unmodified lines
440449 #define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_S 1
441450 #define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_M GENMASK(31, 1)
442451
443−enum{
452+enum {
444453 V2_MPT_ST_VALID = 0x1,
445454 V2_MPT_ST_FREE = 0x2,
446455 };
@@ -457,68 +466,72 @@ enum hns_roce_v2_qp_state { · 10 unmodified lines
457466 HNS_ROCE_QP_NUM_ST
458467 };
459468
469+struct hns_roce_v2_qp_context_ex {
470+ __le32 data[64];
471+};
460472 struct hns_roce_v2_qp_context {
461− __le32 byte_4_sqpn_tst;
462− __le32 wqe_sge_ba;
463− __le32 byte_12_sq_hop;
464− __le32 byte_16_buf_ba_pg_sz;
465− __le32 byte_20_smac_sgid_idx;
466− __le32 byte_24_mtu_tc;
467− __le32 byte_28_at_fl;
468− u8 dgid[GID_LEN_V2];
469− __le32 dmac;
470− __le32 byte_52_udpspn_dmac;
471− __le32 byte_56_dqpn_err;
472− __le32 byte_60_qpst_tempid;
473− __le32 qkey_xrcd;
474− __le32 byte_68_rq_db;
475− __le32 rq_db_record_addr;
476− __le32 byte_76_srqn_op_en;
477− __le32 byte_80_rnr_rx_cqn;
478− __le32 byte_84_rq_ci_pi;
479− __le32 rq_cur_blk_addr;
480− __le32 byte_92_srq_info;
481− __le32 byte_96_rx_reqmsn;
482− __le32 rq_nxt_blk_addr;
483− __le32 byte_104_rq_sge;
484− __le32 byte_108_rx_reqepsn;
485− __le32 rq_rnr_timer;
486− __le32 rx_msg_len;
487− __le32 rx_rkey_pkt_info;
488− __le64 rx_va;
489− __le32 byte_132_trrl;
490− __le32 trrl_ba;
491− __le32 byte_140_raq;
492− __le32 byte_144_raq;
493− __le32 byte_148_raq;
494− __le32 byte_152_raq;
495− __le32 byte_156_raq;
496− __le32 byte_160_sq_ci_pi;
497− __le32 sq_cur_blk_addr;
498− __le32 byte_168_irrl_idx;
499− __le32 byte_172_sq_psn;
500− __le32 byte_176_msg_pktn;
501− __le32 sq_cur_sge_blk_addr;
502− __le32 byte_184_irrl_idx;
503− __le32 cur_sge_offset;
504− __le32 byte_192_ext_sge;
505− __le32 byte_196_sq_psn;
506− __le32 byte_200_sq_max;
507− __le32 irrl_ba;
508− __le32 byte_208_irrl;
509− __le32 byte_212_lsn;
510− __le32 sq_timer;
511− __le32 byte_220_retry_psn_msn;
512− __le32 byte_224_retry_msg;
513− __le32 rx_sq_cur_blk_addr;
514− __le32 byte_232_irrl_sge;
515− __le32 irrl_cur_sge_offset;
516− __le32 byte_240_irrl_tail;
517− __le32 byte_244_rnr_rxack;
518− __le32 byte_248_ack_psn;
519− __le32 byte_252_err_txcqn;
520− __le32 byte_256_sqflush_rqcqe;
521− __le32 ext[64];
473+ __le32 byte_4_sqpn_tst;
474+ __le32 wqe_sge_ba;
475+ __le32 byte_12_sq_hop;
476+ __le32 byte_16_buf_ba_pg_sz;
477+ __le32 byte_20_smac_sgid_idx;
478+ __le32 byte_24_mtu_tc;
479+ __le32 byte_28_at_fl;
480+ u8 dgid[GID_LEN_V2];
481+ __le32 dmac;
482+ __le32 byte_52_udpspn_dmac;
483+ __le32 byte_56_dqpn_err;
484+ __le32 byte_60_qpst_tempid;
485+ __le32 qkey_xrcd;
486+ __le32 byte_68_rq_db;
487+ __le32 rq_db_record_addr;
488+ __le32 byte_76_srqn_op_en;
489+ __le32 byte_80_rnr_rx_cqn;
490+ __le32 byte_84_rq_ci_pi;
491+ __le32 rq_cur_blk_addr;
492+ __le32 byte_92_srq_info;
493+ __le32 byte_96_rx_reqmsn;
494+ __le32 rq_nxt_blk_addr;
495+ __le32 byte_104_rq_sge;
496+ __le32 byte_108_rx_reqepsn;
497+ __le32 rq_rnr_timer;
498+ __le32 rx_msg_len;
499+ __le32 rx_rkey_pkt_info;
500+ __le64 rx_va;
501+ __le32 byte_132_trrl;
502+ __le32 trrl_ba;
503+ __le32 byte_140_raq;
504+ __le32 byte_144_raq;
505+ __le32 byte_148_raq;
506+ __le32 byte_152_raq;
507+ __le32 byte_156_raq;
508+ __le32 byte_160_sq_ci_pi;
509+ __le32 sq_cur_blk_addr;
510+ __le32 byte_168_irrl_idx;
511+ __le32 byte_172_sq_psn;
512+ __le32 byte_176_msg_pktn;
513+ __le32 sq_cur_sge_blk_addr;
514+ __le32 byte_184_irrl_idx;
515+ __le32 cur_sge_offset;
516+ __le32 byte_192_ext_sge;
517+ __le32 byte_196_sq_psn;
518+ __le32 byte_200_sq_max;
519+ __le32 irrl_ba;
520+ __le32 byte_208_irrl;
521+ __le32 byte_212_lsn;
522+ __le32 sq_timer;
523+ __le32 byte_220_retry_psn_msn;
524+ __le32 byte_224_retry_msg;
525+ __le32 rx_sq_cur_blk_addr;
526+ __le32 byte_232_irrl_sge;
527+ __le32 irrl_cur_sge_offset;
528+ __le32 byte_240_irrl_tail;
529+ __le32 byte_244_rnr_rxack;
530+ __le32 byte_248_ack_psn;
531+ __le32 byte_252_err_txcqn;
532+ __le32 byte_256_sqflush_rqcqe;
533+
534+ struct hns_roce_v2_qp_context_ex ext;
522535 };
523536
524537 #define V2_QPC_BYTE_4_TST_S 0
@@ -887,6 +900,10 @@ struct hns_roce_v2_qp_context { · 362 unmodified lines
887900 #define V2_QPC_BYTE_256_SQ_FLUSH_IDX_S 16
888901 #define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16)
889902
903+#define QPCEX_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context_ex, h, l)
904+
905+#define QPCEX_STASH QPCEX_FIELD_LOC(82, 82)
906+
890907 #define V2_QP_RWE_S 1 /* rdma write enable */
891908 #define V2_QP_RRE_S 2 /* rdma read enable */
892909 #define V2_QP_ATE_S 3 /* rdma atomic enable */
@@ -1073,12 +1090,13 @@ struct hns_roce_v2_ud_send_wqe { · 180 unmodified lines
10731090 __le32 byte_32;
10741091 __le32 byte_36;
10751092 __le32 byte_40;
1076− __le32 dmac;
1077− __le32 byte_48;
1093+ u8 dmac[ETH_ALEN];
1094+ u8 sgid_index;
1095+ u8 smac_index;
10781096 u8 dgid[GID_LEN_V2];
1079−
10801097 };
1081−#define V2_UD_SEND_WQE_BYTE_4_OPCODE_S 0
1098+
1099+#define V2_UD_SEND_WQE_BYTE_4_OPCODE_S 0
10821100 #define V2_UD_SEND_WQE_BYTE_4_OPCODE_M GENMASK(4, 0)
10831101
10841102 #define V2_UD_SEND_WQE_BYTE_4_OWNER_S 7
@@ -1117,37 +1135,10 @@ struct hns_roce_v2_ud_send_wqe { · 32 unmodified lines
11171135 #define V2_UD_SEND_WQE_BYTE_40_SL_S 20
11181136 #define V2_UD_SEND_WQE_BYTE_40_SL_M GENMASK(23, 20)
11191137
1120−#define V2_UD_SEND_WQE_BYTE_40_PORTN_S 24
1121−#define V2_UD_SEND_WQE_BYTE_40_PORTN_M GENMASK(26, 24)
1122−
11231138 #define V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S 30
11241139
11251140 #define V2_UD_SEND_WQE_BYTE_40_LBI_S 31
11261141
1127−#define V2_UD_SEND_WQE_DMAC_0_S 0
1128−#define V2_UD_SEND_WQE_DMAC_0_M GENMASK(7, 0)
1129−
1130−#define V2_UD_SEND_WQE_DMAC_1_S 8
1131−#define V2_UD_SEND_WQE_DMAC_1_M GENMASK(15, 8)
1132−
1133−#define V2_UD_SEND_WQE_DMAC_2_S 16
1134−#define V2_UD_SEND_WQE_DMAC_2_M GENMASK(23, 16)
1135−
1136−#define V2_UD_SEND_WQE_DMAC_3_S 24
1137−#define V2_UD_SEND_WQE_DMAC_3_M GENMASK(31, 24)
1138−
1139−#define V2_UD_SEND_WQE_BYTE_48_DMAC_4_S 0
1140−#define V2_UD_SEND_WQE_BYTE_48_DMAC_4_M GENMASK(7, 0)
1141−
1142−#define V2_UD_SEND_WQE_BYTE_48_DMAC_5_S 8
1143−#define V2_UD_SEND_WQE_BYTE_48_DMAC_5_M GENMASK(15, 8)
1144−
1145−#define V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S 16
1146−#define V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M GENMASK(23, 16)
1147−
1148−#define V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_S 24
1149−#define V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_M GENMASK(31, 24)
1150−
11511142 struct hns_roce_v2_rc_send_wqe {
11521143 __le32 byte_4;
11531144 __le32 msg_len;
@@ -1334,7 +1325,7 @@ struct hns_roce_pf_res_b { · 180 unmodified lines
13341325 __le32 sgid_idx_num;
13351326 __le32 qid_idx_sl_num;
13361327 __le32 sccc_bt_idx_num;
1337− __le32 rsv;
1328+ __le32 gmv_idx_num;
13381329 };
13391330
13401331 #define PF_RES_DATA_1_PF_SMAC_IDX_S 0
@@ -1361,6 +1352,12 @@ struct hns_roce_pf_res_b { · 20 unmodified lines
13611352 #define PF_RES_DATA_4_PF_SCCC_BT_NUM_S 9
13621353 #define PF_RES_DATA_4_PF_SCCC_BT_NUM_M GENMASK(17, 9)
13631354
1355+#define PF_RES_DATA_5_PF_GMV_BT_IDX_S 0
1356+#define PF_RES_DATA_5_PF_GMV_BT_IDX_M GENMASK(7, 0)
1357+
1358+#define PF_RES_DATA_5_PF_GMV_BT_NUM_S 8
1359+#define PF_RES_DATA_5_PF_GMV_BT_NUM_M GENMASK(16, 8)
1360+
13641361 struct hns_roce_pf_timer_res_a {
13651362 __le32 rsv0;
13661363 __le32 qpc_timer_bt_idx_num;
@@ -1425,7 +1422,7 @@ struct hns_roce_vf_res_b { · 58 unmodified lines
14251422 __le32 vf_sgid_idx_num;
14261423 __le32 vf_qid_idx_sl_num;
14271424 __le32 vf_sccc_idx_num;
1428− __le32 rsv1;
1425+ __le32 vf_gmv_idx_num;
14291426 };
14301427
14311428 #define VF_RES_B_DATA_0_VF_ID_S 0
@@ -1455,6 +1452,12 @@ struct hns_roce_vf_res_b { · 23 unmodified lines
14551452 #define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S 9
14561453 #define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M GENMASK(17, 9)
14571454
1455+#define VF_RES_B_DATA_5_VF_GMV_BT_IDX_S 0
1456+#define VF_RES_B_DATA_5_VF_GMV_BT_IDX_M GENMASK(7, 0)
1457+
1458+#define VF_RES_B_DATA_5_VF_GMV_BT_NUM_S 16
1459+#define VF_RES_B_DATA_5_VF_GMV_BT_NUM_M GENMASK(24, 16)
1460+
14581461 struct hns_roce_vf_switch {
14591462 __le32 rocee_sel;
14601463 __le32 fun_id;
@@ -1577,6 +1580,46 @@ struct hns_roce_cfg_smac_tb { · 116 unmodified lines
15771580 #define CFG_SMAC_TB_VF_SMAC_H_S 0
15781581 #define CFG_SMAC_TB_VF_SMAC_H_M GENMASK(15, 0)
15791582
1583+struct hns_roce_cfg_gmv_bt {
1584+ __le32 gmv_ba_l;
1585+ __le32 gmv_ba_h;
1586+ __le32 gmv_bt_idx;
1587+ __le32 rsv[3];
1588+};
1589+
1590+#define CFG_GMV_BA_H_S 0
1591+#define CFG_GMV_BA_H_M GENMASK(19, 0)
1592+
1593+struct hns_roce_cfg_gmv_tb_a {
1594+ __le32 vf_sgid_l;
1595+ __le32 vf_sgid_ml;
1596+ __le32 vf_sgid_mh;
1597+ __le32 vf_sgid_h;
1598+ __le32 vf_sgid_type_vlan;
1599+ __le32 resv;
1600+};
1601+
1602+#define CFG_GMV_TB_SGID_IDX_S 0
1603+#define CFG_GMV_TB_SGID_IDX_M GENMASK(7, 0)
1604+
1605+#define CFG_GMV_TB_VF_SGID_TYPE_S 0
1606+#define CFG_GMV_TB_VF_SGID_TYPE_M GENMASK(1, 0)
1607+
1608+#define CFG_GMV_TB_VF_VLAN_EN_S 2
1609+
1610+#define CFG_GMV_TB_VF_VLAN_ID_S 16
1611+#define CFG_GMV_TB_VF_VLAN_ID_M GENMASK(27, 16)
1612+
1613+struct hns_roce_cfg_gmv_tb_b {
1614+ __le32 vf_smac_l;
1615+ __le32 vf_smac_h;
1616+ __le32 table_idx_rsv;
1617+ __le32 resv[3];
1618+};
1619+
1620+#define CFG_GMV_TB_SMAC_H_S 0
1621+#define CFG_GMV_TB_SMAC_H_M GENMASK(15, 0)
1622+
15801623 #define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM 5
15811624 struct hns_roce_query_pf_caps_a {
15821625 u8 number_ports;
drivers/infiniband/hw/hns/hns_roce_main.c+37−45
@@ -33,13 +33,13 @@ · 32 unmodified lines
3333 #include <linux/acpi.h>
3434 #include <linux/of_platform.h>
3535 #include <linux/module.h>
36+#include <linux/pci.h>
3637 #include <rdma/ib_addr.h>
3738 #include <rdma/ib_smi.h>
3839 #include <rdma/ib_user_verbs.h>
3940 #include <rdma/ib_cache.h>
4041 #include "hns_roce_common.h"
4142 #include "hns_roce_device.h"
42−#include <rdma/hns-abi.h>
4343 #include "hns_roce_hem.h"
4444
4545 /**
@@ -53,7 +53,7 @@ · 7 unmodified lines
5353 * GID[0][0], GID[1][0],.....GID[N - 1][0],
5454 * And so on
5555 */
56−int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index)
56+u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index)
5757 {
5858 return gid_index * hr_dev->caps.num_ports + port;
5959 }
@@ -61,7 +61,10 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index) · 1 unmodified line
6161 static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr)
6262 {
6363 u8 phy_port;
64− u32 i = 0;
64+ u32 i;
65+
66+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
67+ return 0;
6568
6669 if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN))
6770 return 0;
@@ -90,14 +93,13 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) · 22 unmodified lines
9093 static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)
9194 {
9295 struct hns_roce_dev *hr_dev = to_hr_dev(attr->device);
93− struct ib_gid_attr zattr = {};
9496 u8 port = attr->port_num - 1;
9597 int ret;
9698
9799 if (port >= hr_dev->caps.num_ports)
98100 return -EINVAL;
99101
100− ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &zgid, &zattr);
102+ ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, NULL, NULL);
101103
102104 return ret;
103105 }
@@ -325,7 +327,8 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, · 221 unmodified lines
325327
326328 resp.cqe_size = hr_dev->caps.cqe_sz;
327329
328− ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
330+ ret = ib_copy_to_udata(udata, &resp,
331+ min(udata->outlen, sizeof(resp)));
329332 if (ret)
330333 goto error_fail_copy_to_udata;
331334
@@ -421,6 +424,7 @@ static const struct ib_device_ops hns_roce_dev_ops = { · 89 unmodified lines
421424 .alloc_pd = hns_roce_alloc_pd,
422425 .alloc_ucontext = hns_roce_alloc_ucontext,
423426 .create_ah = hns_roce_create_ah,
427+ .create_user_ah = hns_roce_create_ah,
424428 .create_cq = hns_roce_create_cq,
425429 .create_qp = hns_roce_create_qp,
426430 .dealloc_pd = hns_roce_dealloc_pd,
@@ -491,36 +495,13 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) · 64 unmodified lines
491495 ib_dev->phys_port_cnt = hr_dev->caps.num_ports;
492496 ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey;
493497 ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors;
494− ib_dev->uverbs_cmd_mask =
495− (1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) |
496− (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) |
497− (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) |
498− (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) |
499− (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) |
500− (1ULL << IB_USER_VERBS_CMD_REG_MR) |
501− (1ULL << IB_USER_VERBS_CMD_DEREG_MR) |
502− (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
503− (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) |
504− (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) |
505− (1ULL << IB_USER_VERBS_CMD_CREATE_QP) |
506− (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) |
507− (1ULL << IB_USER_VERBS_CMD_QUERY_QP) |
508− (1ULL << IB_USER_VERBS_CMD_DESTROY_QP);
509−
510− ib_dev->uverbs_ex_cmd_mask |= (1ULL << IB_USER_VERBS_EX_CMD_MODIFY_CQ);
511−
512− if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR) {
513− ib_dev->uverbs_cmd_mask |= (1ULL << IB_USER_VERBS_CMD_REREG_MR);
498+
499+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR)
514500 ib_set_device_ops(ib_dev, &hns_roce_dev_mr_ops);
515− }
516501
517502 /* MW */
518− if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_MW) {
519− ib_dev->uverbs_cmd_mask |=
520− (1ULL << IB_USER_VERBS_CMD_ALLOC_MW) |
521− (1ULL << IB_USER_VERBS_CMD_DEALLOC_MW);
503+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_MW)
522504 ib_set_device_ops(ib_dev, &hns_roce_dev_mw_ops);
523− }
524505
525506 /* FRMR */
526507 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR)
@@ -528,12 +509,6 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) · 1 unmodified line
528509
529510 /* SRQ */
530511 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
531− ib_dev->uverbs_cmd_mask |=
532− (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) |
533− (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) |
534− (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) |
535− (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) |
536− (1ULL << IB_USER_VERBS_CMD_POST_SRQ_RECV);
537512 ib_set_device_ops(ib_dev, &hns_roce_dev_srq_ops);
538513 ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops);
539514 }
@@ -580,8 +555,8 @@ error_failed_setup_mtu_mac: · 40 unmodified lines
580555
581556 static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
582557 {
583− int ret;
584558 struct device *dev = hr_dev->dev;
559+ int ret;
585560
586561 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table,
587562 HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz,
@@ -631,7 +606,7 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) · 43 unmodified lines
631606 goto err_unmap_trrl;
632607 }
633608
634− if (hr_dev->caps.srqc_entry_sz) {
609+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
635610 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->srq_table.table,
636611 HEM_TYPE_SRQC,
637612 hr_dev->caps.srqc_entry_sz,
@@ -643,7 +618,7 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) · 5 unmodified lines
643618 }
644619 }
645620
646− if (hr_dev->caps.sccc_sz) {
621+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
647622 ret = hns_roce_init_hem_table(hr_dev,
648623 &hr_dev->qp_table.sccc_table,
649624 HEM_TYPE_SCCC,
@@ -680,18 +655,35 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) · 30 unmodified lines
680655 }
681656 }
682657
658+ if (hr_dev->caps.gmv_entry_sz) {
659+ ret = hns_roce_init_hem_table(hr_dev, &hr_dev->gmv_table,
660+ HEM_TYPE_GMV,
661+ hr_dev->caps.gmv_entry_sz,
662+ hr_dev->caps.gmv_entry_num, 1);
663+ if (ret) {
664+ dev_err(dev,
665+ "failed to init gmv table memory, ret = %d\n",
666+ ret);
667+ goto err_unmap_cqc_timer;
668+ }
669+ }
670+
683671 return 0;
684672
673+err_unmap_cqc_timer:
674+ if (hr_dev->caps.cqc_timer_entry_sz)
675+ hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cqc_timer_table);
676+
685677 err_unmap_qpc_timer:
686678 if (hr_dev->caps.qpc_timer_entry_sz)
687679 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qpc_timer_table);
688680
689681 err_unmap_ctx:
690− if (hr_dev->caps.sccc_sz)
682+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
691683 hns_roce_cleanup_hem_table(hr_dev,
692684 &hr_dev->qp_table.sccc_table);
693685 err_unmap_srq:
694− if (hr_dev->caps.srqc_entry_sz)
686+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
695687 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->srq_table.table);
696688
697689 err_unmap_cq:
@@ -721,8 +713,8 @@ err_unmap_dmpt: · 23 unmodified lines
721713 */
722714 static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)
723715 {
724− int ret;
725716 struct device *dev = hr_dev->dev;
717+ int ret;
726718
727719 spin_lock_init(&hr_dev->sm_lock);
728720 spin_lock_init(&hr_dev->bt_cmd_lock);
@@ -846,8 +838,8 @@ void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev) · 117 unmodified lines
846838
847839 int hns_roce_init(struct hns_roce_dev *hr_dev)
848840 {
849− int ret;
850841 struct device *dev = hr_dev->dev;
842+ int ret;
851843
852844 if (hr_dev->hw->reset) {
853845 ret = hr_dev->hw->reset(hr_dev, true);
drivers/infiniband/hw/hns/hns_roce_mr.c+33−46
@@ -167,10 +167,10 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, · 166 unmodified lines
167167 static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
168168 struct hns_roce_mr *mr)
169169 {
170− int ret;
171170 unsigned long mtpt_idx = key_to_hw_index(mr->key);
172− struct device *dev = hr_dev->dev;
173171 struct hns_roce_cmd_mailbox *mailbox;
172+ struct device *dev = hr_dev->dev;
173+ int ret;
174174
175175 /* Allocate mailbox memory */
176176 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
@@ -185,14 +185,14 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, · 8 unmodified lines
185185 else
186186 ret = hr_dev->hw->frmr_write_mtpt(hr_dev, mailbox->buf, mr);
187187 if (ret) {
188− dev_err(dev, "Write mtpt fail!\n");
188+ dev_err(dev, "failed to write mtpt, ret = %d.\n", ret);
189189 goto err_page;
190190 }
191191
192192 ret = hns_roce_hw_create_mpt(hr_dev, mailbox,
193193 mtpt_idx & (hr_dev->caps.num_mtpts - 1));
194194 if (ret) {
195− dev_err(dev, "CREATE_MPT failed (%d)\n", ret);
195+ dev_err(dev, "failed to create mpt, ret = %d.\n", ret);
196196 goto err_page;
197197 }
198198
@@ -328,9 +328,10 @@ static int rereg_mr_trans(struct ib_mr *ibmr, int flags, · 129 unmodified lines
328328 return ret;
329329 }
330330
331−int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,
332− u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
333− struct ib_udata *udata)
331+struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start,
332+ u64 length, u64 virt_addr,
333+ int mr_access_flags, struct ib_pd *pd,
334+ struct ib_udata *udata)
334335 {
335336 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
336337 struct ib_device *ib_dev = &hr_dev->ib_dev;
@@ -341,11 +342,11 @@ int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length, · 4 unmodified lines
341342 int ret;
342343
343344 if (!mr->enabled)
344− return -EINVAL;
345+ return ERR_PTR(-EINVAL);
345346
346347 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
347348 if (IS_ERR(mailbox))
348− return PTR_ERR(mailbox);
349+ return ERR_CAST(mailbox);
349350
350351 mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);
351352 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0,
@@ -390,12 +391,12 @@ int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length, · 38 unmodified lines
390391
391392 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
392393
393− return 0;
394+ return NULL;
394395
395396 free_cmd_mbox:
396397 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
397398
398− return ret;
399+ return ERR_PTR(ret);
399400 }
400401
401402 int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
@@ -495,7 +496,7 @@ int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, · 93 unmodified lines
495496
496497 ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page);
497498 if (ret < 1) {
498− ibdev_err(ibdev, "failed to store sg pages %d %d, cnt = %d.\n",
499+ ibdev_err(ibdev, "failed to store sg pages %u %u, cnt = %d.\n",
499500 mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, ret);
500501 goto err_page_list;
501502 }
@@ -509,7 +510,7 @@ int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, · 7 unmodified lines
509510 ibdev_err(ibdev, "failed to map sg mtr, ret = %d.\n", ret);
510511 ret = 0;
511512 } else {
512− mr->pbl_mtr.hem_cfg.buf_pg_shift = ilog2(ibmr->page_size);
513+ mr->pbl_mtr.hem_cfg.buf_pg_shift = (u32)ilog2(ibmr->page_size);
513514 ret = mr->npages;
514515 }
515516
@@ -695,15 +696,6 @@ static inline size_t mtr_bufs_size(struct hns_roce_buf_attr *attr) · 179 unmodified lines
695696 return size;
696697 }
697698
698−static inline size_t mtr_kmem_direct_size(bool is_direct, size_t alloc_size,
699− unsigned int page_shift)
700−{
701− if (is_direct)
702− return ALIGN(alloc_size, 1 << page_shift);
703− else
704− return HNS_HW_DIRECT_PAGE_COUNT << page_shift;
705−}
706−
707699 /*
708700 * check the given pages in continuous address space
709701 * Returns 0 on success, or the error page num.
@@ -732,7 +724,6 @@ static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) · 22 unmodified lines
732724 /* release kernel buffers */
733725 if (mtr->kmem) {
734726 hns_roce_buf_free(hr_dev, mtr->kmem);
735− kfree(mtr->kmem);
736727 mtr->kmem = NULL;
737728 }
738729 }
@@ -744,13 +735,12 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 5 unmodified lines
744735 struct ib_device *ibdev = &hr_dev->ib_dev;
745736 unsigned int best_pg_shift;
746737 int all_pg_count = 0;
747− size_t direct_size;
748738 size_t total_size;
749739 int ret;
750740
751741 total_size = mtr_bufs_size(buf_attr);
752742 if (total_size < 1) {
753− ibdev_err(ibdev, "Failed to check mtr size\n");
743+ ibdev_err(ibdev, "failed to check mtr size\n.");
754744 return -EINVAL;
755745 }
756746
@@ -762,7 +752,7 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 5 unmodified lines
762752 mtr->umem = ib_umem_get(ibdev, user_addr, total_size,
763753 buf_attr->user_access);
764754 if (IS_ERR_OR_NULL(mtr->umem)) {
765− ibdev_err(ibdev, "Failed to get umem, ret %ld\n",
755+ ibdev_err(ibdev, "failed to get umem, ret = %ld.\n",
766756 PTR_ERR(mtr->umem));
767757 return -ENOMEM;
768758 }
@@ -780,19 +770,16 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 11 unmodified lines
780770 ret = 0;
781771 } else {
782772 mtr->umem = NULL;
783− mtr->kmem = kzalloc(sizeof(*mtr->kmem), GFP_KERNEL);
784− if (!mtr->kmem) {
785− ibdev_err(ibdev, "Failed to alloc kmem\n");
786− return -ENOMEM;
787− }
788− direct_size = mtr_kmem_direct_size(is_direct, total_size,
789− buf_attr->page_shift);
790− ret = hns_roce_buf_alloc(hr_dev, total_size, direct_size,
791− mtr->kmem, buf_attr->page_shift);
792− if (ret) {
793− ibdev_err(ibdev, "Failed to alloc kmem, ret %d\n", ret);
794− goto err_alloc_mem;
773+ mtr->kmem =
774+ hns_roce_buf_alloc(hr_dev, total_size,
775+ buf_attr->page_shift,
776+ is_direct ? HNS_ROCE_BUF_DIRECT : 0);
777+ if (IS_ERR(mtr->kmem)) {
778+ ibdev_err(ibdev, "failed to alloc kmem, ret = %ld.\n",
779+ PTR_ERR(mtr->kmem));
780+ return PTR_ERR(mtr->kmem);
795781 }
782+
796783 best_pg_shift = buf_attr->page_shift;
797784 all_pg_count = mtr->kmem->npages;
798785 }
@@ -800,7 +787,8 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 1 unmodified line
800787 /* must bigger than minimum hardware page shift */
801788 if (best_pg_shift < HNS_HW_PAGE_SHIFT || all_pg_count < 1) {
802789 ret = -EINVAL;
803− ibdev_err(ibdev, "Failed to check mtr page shift %d count %d\n",
790+ ibdev_err(ibdev,
791+ "failed to check mtr, page shift = %u count = %d.\n",
804792 best_pg_shift, all_pg_count);
805793 goto err_alloc_mem;
806794 }
@@ -841,12 +829,12 @@ static int mtr_get_pages(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 34 unmodified lines
841829 }
842830
843831 int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
844− dma_addr_t *pages, int page_cnt)
832+ dma_addr_t *pages, unsigned int page_cnt)
845833 {
846834 struct ib_device *ibdev = &hr_dev->ib_dev;
847835 struct hns_roce_buf_region *r;
836+ unsigned int i;
848837 int err;
849− int i;
850838
851839 /*
852840 * Only use the first page address as root ba when hopnum is 0, this
@@ -862,7 +850,7 @@ int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 9 unmodified lines
862850 if (r->offset + r->count > page_cnt) {
863851 err = -EINVAL;
864852 ibdev_err(ibdev,
865− "Failed to check mtr%d end %d + %d, max %d\n",
853+ "failed to check mtr%u end %u + %u, max %u.\n",
866854 i, r->offset, r->count, page_cnt);
867855 return err;
868856 }
@@ -870,7 +858,7 @@ int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 1 unmodified line
870858 err = mtr_map_region(hr_dev, mtr, &pages[r->offset], r);
871859 if (err) {
872860 ibdev_err(ibdev,
873− "Failed to map mtr%d offset %d, err %d\n",
861+ "failed to map mtr%u offset %u, ret = %d.\n",
874862 i, r->offset, err);
875863 return err;
876864 }
@@ -883,13 +871,12 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 6 unmodified lines
883871 int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr)
884872 {
885873 struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;
874+ int mtt_count, left;
886875 int start_index;
887− int mtt_count;
888876 int total = 0;
889877 __le64 *mtts;
890− int npage;
878+ u32 npage;
891879 u64 addr;
892− int left;
893880
894881 if (!mtt_buf || mtt_max < 1)
895882 goto done;
drivers/infiniband/hw/hns/hns_roce_pd.c+7−7
@@ -32,7 +32,6 @@ · 31 unmodified lines
3232
3333 #include <linux/platform_device.h>
3434 #include <linux/pci.h>
35−#include <uapi/rdma/hns-abi.h>
3635 #include "hns_roce_device.h"
3736
3837 static int hns_roce_pd_alloc(struct hns_roce_dev *hr_dev, unsigned long *pdn)
@@ -65,21 +64,22 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) · 26 unmodified lines
6564
6665 ret = hns_roce_pd_alloc(to_hr_dev(ib_dev), &pd->pdn);
6766 if (ret) {
68− ibdev_err(ib_dev, "failed to alloc pd, ret = %d\n", ret);
67+ ibdev_err(ib_dev, "failed to alloc pd, ret = %d.\n", ret);
6968 return ret;
7069 }
7170
7271 if (udata) {
73− struct hns_roce_ib_alloc_pd_resp uresp = {.pdn = pd->pdn};
72+ struct hns_roce_ib_alloc_pd_resp resp = {.pdn = pd->pdn};
7473
75− if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
74+ ret = ib_copy_to_udata(udata, &resp,
75+ min(udata->outlen, sizeof(resp)));
76+ if (ret) {
7677 hns_roce_pd_free(to_hr_dev(ib_dev), pd->pdn);
77− ibdev_err(ib_dev, "failed to copy to udata\n");
78− return -EFAULT;
78+ ibdev_err(ib_dev, "failed to copy to udata, ret = %d\n", ret);
7979 }
8080 }
8181
82− return 0;
82+ return ret;
8383 }
8484
8585 int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
drivers/infiniband/hw/hns/hns_roce_qp.c+196−104
@@ -39,7 +39,6 @@ · 38 unmodified lines
3939 #include "hns_roce_common.h"
4040 #include "hns_roce_device.h"
4141 #include "hns_roce_hem.h"
42−#include <rdma/hns-abi.h>
4342
4443 static void flush_work_handle(struct work_struct *work)
4544 {
@@ -114,8 +113,8 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) · 68 unmodified lines
114113 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
115114 enum hns_roce_event type)
116115 {
117− struct ib_event event;
118116 struct ib_qp *ibqp = &hr_qp->ibqp;
117+ struct ib_event event;
119118
120119 if (ibqp->event_handler) {
121120 event.device = ibqp->device;
@@ -154,9 +153,50 @@ static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, · 32 unmodified lines
154153 }
155154 }
156155
156+static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank)
157+{
158+ u32 least_load = bank[0].inuse;
159+ u8 bankid = 0;
160+ u32 bankcnt;
161+ u8 i;
162+
163+ for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) {
164+ bankcnt = bank[i].inuse;
165+ if (bankcnt < least_load) {
166+ least_load = bankcnt;
167+ bankid = i;
168+ }
169+ }
170+
171+ return bankid;
172+}
173+
174+static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid,
175+ unsigned long *qpn)
176+{
177+ int id;
178+
179+ id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL);
180+ if (id < 0) {
181+ id = ida_alloc_range(&bank->ida, bank->min, bank->max,
182+ GFP_KERNEL);
183+ if (id < 0)
184+ return id;
185+ }
186+
187+ /* the QPN should keep increasing until the max value is reached. */
188+ bank->next = (id + 1) > bank->max ? bank->min : id + 1;
189+
190+ /* the lower 3 bits is bankid */
191+ *qpn = (id << 3) | bankid;
192+
193+ return 0;
194+}
157195 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
158196 {
197+ struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
159198 unsigned long num = 0;
199+ u8 bankid;
160200 int ret;
161201
162202 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
@@ -169,13 +209,21 @@ static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) · 6 unmodified lines
169209
170210 hr_qp->doorbell_qpn = 1;
171211 } else {
172− ret = hns_roce_bitmap_alloc_range(&hr_dev->qp_table.bitmap,
173− 1, 1, &num);
212+ spin_lock(&qp_table->bank_lock);
213+ bankid = get_least_load_bankid_for_qp(qp_table->bank);
214+
215+ ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid,
216+ &num);
174217 if (ret) {
175− ibdev_err(&hr_dev->ib_dev, "Failed to alloc bitmap\n");
176− return -ENOMEM;
218+ ibdev_err(&hr_dev->ib_dev,
219+ "failed to alloc QPN, ret = %d\n", ret);
220+ spin_unlock(&qp_table->bank_lock);
221+ return ret;
177222 }
178223
224+ qp_table->bank[bankid].inuse++;
225+ spin_unlock(&qp_table->bank_lock);
226+
179227 hr_qp->doorbell_qpn = (u32)num;
180228 }
181229
@@ -286,7 +334,7 @@ static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) · 104 unmodified lines
286334 }
287335 }
288336
289− if (hr_dev->caps.sccc_sz) {
337+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
290338 /* Alloc memory for SCC CTX */
291339 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table,
292340 hr_qp->qpn);
@@ -340,9 +388,15 @@ static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) · 47 unmodified lines
340388 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
341389 }
342390
391+static inline u8 get_qp_bankid(unsigned long qpn)
392+{
393+ /* The lower 3 bits of QPN are used to hash to different banks */
394+ return (u8)(qpn & GENMASK(2, 0));
395+}
396+
343397 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
344398 {
345− struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
399+ u8 bankid;
346400
347401 if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
348402 return;
@@ -350,7 +404,13 @@ static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) · 1 unmodified line
350404 if (hr_qp->qpn < hr_dev->caps.reserved_qps)
351405 return;
352406
353− hns_roce_bitmap_free_range(&qp_table->bitmap, hr_qp->qpn, 1, BITMAP_RR);
407+ bankid = get_qp_bankid(hr_qp->qpn);
408+
409+ ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3);
410+
411+ spin_lock(&hr_dev->qp_table.bank_lock);
412+ hr_dev->qp_table.bank[bankid].inuse--;
413+ spin_unlock(&hr_dev->qp_table.bank_lock);
354414 }
355415
356416 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap,
@@ -404,37 +464,43 @@ static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, · 47 unmodified lines
404464 return 0;
405465 }
406466
407−static int set_extend_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
408− struct hns_roce_qp *hr_qp,
409− struct ib_qp_cap *cap)
467+static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp)
410468 {
411− u32 cnt;
469+ /* GSI/UD QP only has extended sge */
470+ if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD)
471+ return qp->sq.max_gs;
412472
413− cnt = max(1U, cap->max_send_sge);
414− if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
415− hr_qp->sq.max_gs = roundup_pow_of_two(cnt);
416− hr_qp->sge.sge_cnt = 0;
473+ if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE)
474+ return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE;
417475
418− return 0;
419− }
476+ return 0;
477+}
420478
421− hr_qp->sq.max_gs = cnt;
479+static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
480+ struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap)
481+{
482+ u32 total_sge_cnt;
483+ u32 wqe_sge_cnt;
422484
423− /* UD sqwqe's sge use extend sge */
424− if (hr_qp->ibqp.qp_type == IB_QPT_GSI ||
425− hr_qp->ibqp.qp_type == IB_QPT_UD) {
426− cnt = roundup_pow_of_two(sq_wqe_cnt * hr_qp->sq.max_gs);
427− } else if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) {
428− cnt = roundup_pow_of_two(sq_wqe_cnt *
429− (hr_qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE));
430− } else {
431− cnt = 0;
485+ hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
486+
487+ if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
488+ hr_qp->sq.max_gs = HNS_ROCE_SGE_IN_WQE;
489+ return;
432490 }
433491
434− hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
435− hr_qp->sge.sge_cnt = cnt;
492+ hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
436493
437− return 0;
494+ wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp);
495+
496+ /* If the number of extended sge is not zero, they MUST use the
497+ * space of HNS_HW_PAGE_SIZE at least.
498+ */
499+ if (wqe_sge_cnt) {
500+ total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt);
501+ hr_qp->sge.sge_cnt = max(total_sge_cnt,
502+ (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);
503+ }
438504 }
439505
440506 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
@@ -447,12 +513,12 @@ static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, · 6 unmodified lines
447513 /* Sanity check SQ size before proceeding */
448514 if (ucmd->log_sq_stride > max_sq_stride ||
449515 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
450− ibdev_err(&hr_dev->ib_dev, "Failed to check SQ stride size\n");
516+ ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n");
451517 return -EINVAL;
452518 }
453519
454520 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) {
455− ibdev_err(&hr_dev->ib_dev, "Failed to check SQ SGE size %d\n",
521+ ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n",
456522 cap->max_send_sge);
457523 return -EINVAL;
458524 }
@@ -479,9 +545,7 @@ static int set_user_sq_size(struct hns_roce_dev *hr_dev, · 20 unmodified lines
479545 return ret;
480546 }
481547
482− ret = set_extend_sge_param(hr_dev, cnt, hr_qp, cap);
483− if (ret)
484− return ret;
548+ set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
485549
486550 hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
487551 hr_qp->sq.wqe_cnt = cnt;
@@ -546,7 +610,6 @@ static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, · 58 unmodified lines
546610 {
547611 struct ib_device *ibdev = &hr_dev->ib_dev;
548612 u32 cnt;
549− int ret;
550613
551614 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes ||
552615 cap->max_send_sge > hr_dev->caps.max_sq_sg) {
@@ -558,7 +621,7 @@ static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, · 5 unmodified lines
558621
559622 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes));
560623 if (cnt > hr_dev->caps.max_wqes) {
561− ibdev_err(ibdev, "failed to check WQE num, WQE num = %d.\n",
624+ ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n",
562625 cnt);
563626 return -EINVAL;
564627 }
@@ -566,9 +629,7 @@ static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, · 1 unmodified line
566629 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
567630 hr_qp->sq.wqe_cnt = cnt;
568631
569− ret = set_extend_sge_param(hr_dev, cnt, hr_qp, cap);
570− if (ret)
571− return ret;
632+ set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
572633
573634 /* sync the parameters of kernel QP to user's configuration */
574635 cap->max_send_wr = cnt;
@@ -725,13 +786,17 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 150 unmodified lines
725786 struct ib_device *ibdev = &hr_dev->ib_dev;
726787 int ret;
727788
789+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE)
790+ hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;
791+
728792 if (udata) {
729793 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) {
730794 ret = hns_roce_db_map_user(uctx, udata, ucmd->sdb_addr,
731795 &hr_qp->sdb);
732796 if (ret) {
733797 ibdev_err(ibdev,
734− "Failed to map user SQ doorbell\n");
798+ "failed to map user SQ doorbell, ret = %d.\n",
799+ ret);
735800 goto err_out;
736801 }
737802 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
@@ -743,7 +808,8 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 5 unmodified lines
743808 &hr_qp->rdb);
744809 if (ret) {
745810 ibdev_err(ibdev,
746− "Failed to map user RQ doorbell\n");
811+ "failed to map user RQ doorbell, ret = %d.\n",
812+ ret);
747813 goto err_sdb;
748814 }
749815 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
@@ -760,7 +826,8 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 10 unmodified lines
760826 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0);
761827 if (ret) {
762828 ibdev_err(ibdev,
763− "Failed to alloc kernel RQ doorbell\n");
829+ "failed to alloc kernel RQ doorbell, ret = %d.\n",
830+ ret);
764831 goto err_out;
765832 }
766833 *hr_qp->rdb.db_record = 0;
@@ -803,14 +870,14 @@ static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev, · 36 unmodified lines
803870
804871 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL);
805872 if (ZERO_OR_NULL_PTR(sq_wrid)) {
806− ibdev_err(ibdev, "Failed to alloc SQ wrid\n");
873+ ibdev_err(ibdev, "failed to alloc SQ wrid.\n");
807874 return -ENOMEM;
808875 }
809876
810877 if (hr_qp->rq.wqe_cnt) {
811878 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL);
812879 if (ZERO_OR_NULL_PTR(rq_wrid)) {
813− ibdev_err(ibdev, "Failed to alloc RQ wrid\n");
880+ ibdev_err(ibdev, "failed to alloc RQ wrid.\n");
814881 ret = -ENOMEM;
815882 goto err_sq;
816883 }
@@ -860,29 +927,25 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 43 unmodified lines
860927 }
861928
862929 if (udata) {
863− if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd))) {
864− ibdev_err(ibdev, "Failed to copy QP ucmd\n");
865− return -EFAULT;
930+ ret = ib_copy_from_udata(ucmd, udata,
931+ min(udata->inlen, sizeof(*ucmd)));
932+ if (ret) {
933+ ibdev_err(ibdev,
934+ "failed to copy QP ucmd, ret = %d\n", ret);
935+ return ret;
866936 }
867937
868938 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);
869939 if (ret)
870− ibdev_err(ibdev, "Failed to set user SQ size\n");
940+ ibdev_err(ibdev,
941+ "failed to set user SQ size, ret = %d.\n",
942+ ret);
871943 } else {
872− if (init_attr->create_flags &
873− IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
874− ibdev_err(ibdev, "Failed to check multicast loopback\n");
875− return -EINVAL;
876− }
877−
878− if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) {
879− ibdev_err(ibdev, "Failed to check ipoib ud lso\n");
880− return -EINVAL;
881− }
882−
883944 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);
884945 if (ret)
885− ibdev_err(ibdev, "Failed to set kernel SQ size\n");
946+ ibdev_err(ibdev,
947+ "failed to set kernel SQ size, ret = %d.\n",
948+ ret);
886949 }
887950
888951 return ret;
@@ -906,47 +969,53 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, · 17 unmodified lines
906969 hr_qp->state = IB_QPS_RESET;
907970 hr_qp->flush_flag = 0;
908971
972+ if (init_attr->create_flags)
973+ return -EOPNOTSUPP;
974+
909975 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd);
910976 if (ret) {
911− ibdev_err(ibdev, "Failed to set QP param\n");
977+ ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret);
912978 return ret;
913979 }
914980
915981 if (!udata) {
916982 ret = alloc_kernel_wrid(hr_dev, hr_qp);
917983 if (ret) {
918− ibdev_err(ibdev, "Failed to alloc wrid\n");
984+ ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n",
985+ ret);
919986 return ret;
920987 }
921988 }
922989
923990 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp);
924991 if (ret) {
925− ibdev_err(ibdev, "Failed to alloc QP doorbell\n");
992+ ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n",
993+ ret);
926994 goto err_wrid;
927995 }
928996
929997 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);
930998 if (ret) {
931− ibdev_err(ibdev, "Failed to alloc QP buffer\n");
999+ ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret);
9321000 goto err_db;
9331001 }
9341002
9351003 ret = alloc_qpn(hr_dev, hr_qp);
9361004 if (ret) {
937− ibdev_err(ibdev, "Failed to alloc QPN\n");
1005+ ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret);
9381006 goto err_buf;
9391007 }
9401008
9411009 ret = alloc_qpc(hr_dev, hr_qp);
9421010 if (ret) {
943− ibdev_err(ibdev, "Failed to alloc QP context\n");
1011+ ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n",
1012+ ret);
9441013 goto err_qpn;
9451014 }
9461015
9471016 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr);
9481017 if (ret) {
949− ibdev_err(ibdev, "Failed to store QP\n");
1018+ ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret);
9501019 goto err_qpc;
9511020 }
9521021
@@ -1003,6 +1072,30 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 50 unmodified lines
10031072 kfree(hr_qp);
10041073 }
10051074
1075+static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type,
1076+ bool is_user)
1077+{
1078+ switch (type) {
1079+ case IB_QPT_UD:
1080+ if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 &&
1081+ is_user)
1082+ goto out;
1083+ fallthrough;
1084+ case IB_QPT_RC:
1085+ case IB_QPT_GSI:
1086+ break;
1087+ default:
1088+ goto out;
1089+ }
1090+
1091+ return 0;
1092+
1093+out:
1094+ ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type);
1095+
1096+ return -EOPNOTSUPP;
1097+}
1098+
10061099 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
10071100 struct ib_qp_init_attr *init_attr,
10081101 struct ib_udata *udata)
@@ -1012,15 +1105,9 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, · 3 unmodified lines
10121105 struct hns_roce_qp *hr_qp;
10131106 int ret;
10141107
1015− switch (init_attr->qp_type) {
1016− case IB_QPT_RC:
1017− case IB_QPT_GSI:
1018− break;
1019− default:
1020− ibdev_err(ibdev, "not support QP type %d\n",
1021− init_attr->qp_type);
1022− return ERR_PTR(-EOPNOTSUPP);
1023− }
1108+ ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata);
1109+ if (ret)
1110+ return ERR_PTR(ret);
10241111
10251112 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
10261113 if (!hr_qp)
@@ -1035,10 +1122,11 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, · 8 unmodified lines
10351122 if (ret) {
10361123 ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n",
10371124 init_attr->qp_type, ret);
1038− ibdev_err(ibdev, "Create GSI QP failed!\n");
1125+
10391126 kfree(hr_qp);
10401127 return ERR_PTR(ret);
10411128 }
1129+
10421130 return &hr_qp->ibqp;
10431131 }
10441132
@@ -1091,9 +1179,8 @@ static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 46 unmodified lines
10911179
10921180 if ((attr_mask & IB_QP_PORT) &&
10931181 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
1094− ibdev_err(&hr_dev->ib_dev,
1095− "attr port_num invalid.attr->port_num=%d\n",
1096− attr->port_num);
1182+ ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n",
1183+ attr->port_num);
10971184 return -EINVAL;
10981185 }
10991186
@@ -1101,8 +1188,8 @@ static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 1 unmodified line
11011188 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
11021189 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
11031190 ibdev_err(&hr_dev->ib_dev,
1104− "attr pkey_index invalid.attr->pkey_index=%d\n",
1105− attr->pkey_index);
1191+ "invalid attr, pkey_index = %u.\n",
1192+ attr->pkey_index);
11061193 return -EINVAL;
11071194 }
11081195 }
@@ -1110,16 +1197,16 @@ static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 1 unmodified line
11101197 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
11111198 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
11121199 ibdev_err(&hr_dev->ib_dev,
1113− "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n",
1114− attr->max_rd_atomic);
1200+ "invalid attr, max_rd_atomic = %u.\n",
1201+ attr->max_rd_atomic);
11151202 return -EINVAL;
11161203 }
11171204
11181205 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
11191206 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
11201207 ibdev_err(&hr_dev->ib_dev,
1121− "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n",
1122− attr->max_dest_rd_atomic);
1208+ "invalid attr, max_dest_rd_atomic = %u.\n",
1209+ attr->max_dest_rd_atomic);
11231210 return -EINVAL;
11241211 }
11251212
@@ -1244,22 +1331,22 @@ static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset) · 118 unmodified lines
12441331 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);
12451332 }
12461333
1247−void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, int n)
1334+void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
12481335 {
12491336 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
12501337 }
12511338
1252−void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, int n)
1339+void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
12531340 {
12541341 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
12551342 }
12561343
1257−void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, int n)
1344+void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n)
12581345 {
12591346 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift));
12601347 }
12611348
1262−bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
1349+bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,
12631350 struct ib_cq *ib_cq)
12641351 {
12651352 struct hns_roce_cq *hr_cq;
@@ -1280,22 +1367,24 @@ bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, · 14 unmodified lines
12801367 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
12811368 {
12821369 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
1283− int reserved_from_top = 0;
1284− int reserved_from_bot;
1285− int ret;
1370+ unsigned int reserved_from_bot;
1371+ unsigned int i;
12861372
12871373 mutex_init(&qp_table->scc_mutex);
12881374 xa_init(&hr_dev->qp_table_xa);
12891375
12901376 reserved_from_bot = hr_dev->caps.reserved_qps;
12911377
1292− ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps,
1293− hr_dev->caps.num_qps - 1, reserved_from_bot,
1294− reserved_from_top);
1295− if (ret) {
1296− dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n",
1297− ret);
1298− return ret;
1378+ for (i = 0; i < reserved_from_bot; i++) {
1379+ hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++;
1380+ hr_dev->qp_table.bank[get_qp_bankid(i)].min++;
1381+ }
1382+
1383+ for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) {
1384+ ida_init(&hr_dev->qp_table.bank[i].ida);
1385+ hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps /
1386+ HNS_ROCE_QP_BANK_NUM - 1;
1387+ hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min;
12991388 }
13001389
13011390 return 0;
@@ -1303,5 +1392,8 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) · 1 unmodified line
13031392
13041393 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
13051394 {
1306− hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap);
1395+ int i;
1396+
1397+ for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++)
1398+ ida_destroy(&hr_dev->qp_table.bank[i].ida);
13071399 }
drivers/infiniband/hw/hns/hns_roce_srq.c+31−22
@@ -4,7 +4,6 @@ · 3 unmodified lines
44 */
55
66 #include <rdma/ib_umem.h>
7−#include <rdma/hns-abi.h>
87 #include "hns_roce_device.h"
98 #include "hns_roce_cmd.h"
109 #include "hns_roce_hem.h"
@@ -93,7 +92,8 @@ static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, · 82 unmodified lines
9392 ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe,
9493 ARRAY_SIZE(mtts_wqe), &dma_handle_wqe);
9594 if (ret < 1) {
96− ibdev_err(ibdev, "Failed to find mtr for SRQ WQE\n");
95+ ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n",
96+ ret);
9797 return -ENOBUFS;
9898 }
9999
@@ -101,32 +101,34 @@ static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, · 1 unmodified line
101101 ret = hns_roce_mtr_find(hr_dev, &srq->idx_que.mtr, 0, mtts_idx,
102102 ARRAY_SIZE(mtts_idx), &dma_handle_idx);
103103 if (ret < 1) {
104− ibdev_err(ibdev, "Failed to find mtr for SRQ idx\n");
104+ ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n",
105+ ret);
105106 return -ENOBUFS;
106107 }
107108
108109 ret = hns_roce_bitmap_alloc(&srq_table->bitmap, &srq->srqn);
109110 if (ret) {
110− ibdev_err(ibdev, "Failed to alloc SRQ number, err %d\n", ret);
111+ ibdev_err(ibdev,
112+ "failed to alloc SRQ number, ret = %d.\n", ret);
111113 return -ENOMEM;
112114 }
113115
114116 ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);
115117 if (ret) {
116− ibdev_err(ibdev, "Failed to get SRQC table, err %d\n", ret);
118+ ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret);
117119 goto err_out;
118120 }
119121
120122 ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));
121123 if (ret) {
122− ibdev_err(ibdev, "Failed to store SRQC, err %d\n", ret);
124+ ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret);
123125 goto err_put;
124126 }
125127
126128 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
127129 if (IS_ERR_OR_NULL(mailbox)) {
128130 ret = -ENOMEM;
129− ibdev_err(ibdev, "Failed to alloc mailbox for SRQC\n");
131+ ibdev_err(ibdev, "failed to alloc mailbox for SRQC.\n");
130132 goto err_xa;
131133 }
132134
@@ -137,7 +139,7 @@ static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, · 4 unmodified lines
137139 ret = hns_roce_hw_create_srq(hr_dev, mailbox, srq->srqn);
138140 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
139141 if (ret) {
140− ibdev_err(ibdev, "Failed to config SRQC, err %d\n", ret);
142+ ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret);
141143 goto err_xa;
142144 }
143145
@@ -198,7 +200,8 @@ static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, · 54 unmodified lines
198200 hr_dev->caps.srqwqe_ba_pg_sz +
199201 HNS_HW_PAGE_SHIFT, udata, addr);
200202 if (err)
201− ibdev_err(ibdev, "Failed to alloc SRQ buf mtr, err %d\n", err);
203+ ibdev_err(ibdev,
204+ "failed to alloc SRQ buf mtr, ret = %d.\n", err);
202205
203206 return err;
204207 }
@@ -229,18 +232,18 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, · 24 unmodified lines
229232 hr_dev->caps.idx_ba_pg_sz + HNS_HW_PAGE_SHIFT,
230233 udata, addr);
231234 if (err) {
232− ibdev_err(ibdev, "Failed to alloc SRQ idx mtr, err %d\n", err);
235+ ibdev_err(ibdev,
236+ "failed to alloc SRQ idx mtr, ret = %d.\n", err);
233237 return err;
234238 }
235239
236240 if (!udata) {
237241 idx_que->bitmap = bitmap_zalloc(srq->wqe_cnt, GFP_KERNEL);
238242 if (!idx_que->bitmap) {
239− ibdev_err(ibdev, "Failed to alloc SRQ idx bitmap\n");
243+ ibdev_err(ibdev, "failed to alloc SRQ idx bitmap.\n");
240244 err = -ENOMEM;
241245 goto err_idx_mtr;
242246 }
243−
244247 }
245248
246249 return 0;
@@ -288,6 +291,10 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, · 41 unmodified lines
288291 int ret;
289292 u32 cqn;
290293
294+ if (init_attr->srq_type != IB_SRQT_BASIC &&
295+ init_attr->srq_type != IB_SRQT_XRC)
296+ return -EOPNOTSUPP;
297+
291298 /* Check the actual SRQ wqe and SRQ sge num */
292299 if (init_attr->attr.max_wr >= hr_dev->caps.max_srq_wrs ||
293300 init_attr->attr.max_sge > hr_dev->caps.max_srq_sges)
@@ -300,9 +307,10 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, · 6 unmodified lines
300307 srq->max_gs = init_attr->attr.max_sge;
301308
302309 if (udata) {
303− ret = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
310+ ret = ib_copy_from_udata(&ucmd, udata,
311+ min(udata->inlen, sizeof(ucmd)));
304312 if (ret) {
305− ibdev_err(ibdev, "Failed to copy SRQ udata, err %d\n",
313+ ibdev_err(ibdev, "failed to copy SRQ udata, ret = %d.\n",
306314 ret);
307315 return ret;
308316 }
@@ -310,20 +318,21 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, · 1 unmodified line
310318
311319 ret = alloc_srq_buf(hr_dev, srq, udata, ucmd.buf_addr);
312320 if (ret) {
313− ibdev_err(ibdev, "Failed to alloc SRQ buffer, err %d\n", ret);
321+ ibdev_err(ibdev,
322+ "failed to alloc SRQ buffer, ret = %d.\n", ret);
314323 return ret;
315324 }
316325
317326 ret = alloc_srq_idx(hr_dev, srq, udata, ucmd.que_addr);
318327 if (ret) {
319− ibdev_err(ibdev, "Failed to alloc SRQ idx, err %d\n", ret);
328+ ibdev_err(ibdev, "failed to alloc SRQ idx, ret = %d.\n", ret);
320329 goto err_buf_alloc;
321330 }
322331
323332 if (!udata) {
324333 ret = alloc_srq_wrid(hr_dev, srq);
325334 if (ret) {
326− ibdev_err(ibdev, "Failed to alloc SRQ wrid, err %d\n",
335+ ibdev_err(ibdev, "failed to alloc SRQ wrid, ret = %d.\n",
327336 ret);
328337 goto err_idx_alloc;
329338 }
@@ -335,7 +344,8 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, · 5 unmodified lines
335344
336345 ret = alloc_srqc(hr_dev, srq, to_hr_pd(ib_srq->pd)->pdn, cqn, 0, 0);
337346 if (ret) {
338− ibdev_err(ibdev, "Failed to alloc SRQ context, err %d\n", ret);
347+ ibdev_err(ibdev,
348+ "failed to alloc SRQ context, ret = %d.\n", ret);
339349 goto err_wrid_alloc;
340350 }
341351
@@ -343,11 +353,10 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, · 1 unmodified line
343353 resp.srqn = srq->srqn;
344354
345355 if (udata) {
346− if (ib_copy_to_udata(udata, &resp,
347− min(udata->outlen, sizeof(resp)))) {
348− ret = -EFAULT;
356+ ret = ib_copy_to_udata(udata, &resp,
357+ min(udata->outlen, sizeof(resp)));
358+ if (ret)
349359 goto err_srqc_alloc;
350− }
351360 }
352361
353362 return 0;
drivers/infiniband/hw/i40iw/i40iw.h+0−1
@@ -274,7 +274,6 @@ struct i40iw_device { · 273 unmodified lines
274274 u8 max_sge;
275275 u8 iw_status;
276276 u8 send_term_ok;
277− bool push_mode; /* Initialized from parameter passed to driver */
278277
279278 /* x710 specific */
280279 struct mutex pbl_mutex;
drivers/infiniband/hw/i40iw/i40iw_cm.c+3−3
@@ -2426,7 +2426,7 @@ static void i40iw_handle_rst_pkt(struct i40iw_cm_node *cm_node, · 2425 unmodified lines
24262426 }
24272427 break;
24282428 case I40IW_CM_STATE_MPAREQ_RCVD:
2429− atomic_add_return(1, &cm_node->passive_state);
2429+ atomic_inc(&cm_node->passive_state);
24302430 break;
24312431 case I40IW_CM_STATE_ESTABLISHED:
24322432 case I40IW_CM_STATE_SYN_RCVD:
@@ -3020,7 +3020,7 @@ static int i40iw_cm_reject(struct i40iw_cm_node *cm_node, const void *pdata, u8 · 587 unmodified lines
30203020 i40iw_cleanup_retrans_entry(cm_node);
30213021
30223022 if (!loopback) {
3023− passive_state = atomic_add_return(1, &cm_node->passive_state);
3023+ passive_state = atomic_inc_return(&cm_node->passive_state);
30243024 if (passive_state == I40IW_SEND_RESET_EVENT) {
30253025 cm_node->state = I40IW_CM_STATE_CLOSED;
30263026 i40iw_rem_ref_cm_node(cm_node);
@@ -3678,7 +3678,7 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) · 651 unmodified lines
36783678 return -EINVAL;
36793679 }
36803680
3681− passive_state = atomic_add_return(1, &cm_node->passive_state);
3681+ passive_state = atomic_inc_return(&cm_node->passive_state);
36823682 if (passive_state == I40IW_SEND_RESET_EVENT) {
36833683 i40iw_rem_ref_cm_node(cm_node);
36843684 return -ECONNRESET;
drivers/infiniband/hw/i40iw/i40iw_ctrl.c+11−61
@@ -819,46 +819,6 @@ static enum i40iw_status_code i40iw_sc_poll_for_cqp_op_done( · 818 unmodified lines
819819 return ret_code;
820820 }
821821
822−/**
823− * i40iw_sc_manage_push_page - Handle push page
824− * @cqp: struct for cqp hw
825− * @info: push page info
826− * @scratch: u64 saved to be used during cqp completion
827− * @post_sq: flag for cqp db to ring
828− */
829−static enum i40iw_status_code i40iw_sc_manage_push_page(
830− struct i40iw_sc_cqp *cqp,
831− struct i40iw_cqp_manage_push_page_info *info,
832− u64 scratch,
833− bool post_sq)
834−{
835− u64 *wqe;
836− u64 header;
837−
838− if (info->push_idx >= I40IW_MAX_PUSH_PAGE_COUNT)
839− return I40IW_ERR_INVALID_PUSH_PAGE_INDEX;
840−
841− wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
842− if (!wqe)
843− return I40IW_ERR_RING_FULL;
844−
845− set_64bit_val(wqe, 16, info->qs_handle);
846−
847− header = LS_64(info->push_idx, I40IW_CQPSQ_MPP_PPIDX) |
848− LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES, I40IW_CQPSQ_OPCODE) |
849− LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
850− LS_64(info->free_page, I40IW_CQPSQ_MPP_FREE_PAGE);
851−
852− i40iw_insert_wqe_hdr(wqe, header);
853−
854− i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",
855− wqe, I40IW_CQP_WQE_SIZE * 8);
856−
857− if (post_sq)
858− i40iw_sc_cqp_post_sq(cqp);
859− return 0;
860−}
861−
862822 /**
863823 * i40iw_sc_manage_hmc_pm_func_table - manage of function table
864824 * @cqp: struct for cqp hw
@@ -2859,9 +2819,7 @@ static enum i40iw_status_code i40iw_sc_qp_setctx( · 1994 unmodified lines
28592819 LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) |
28602820 LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) |
28612821 LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) |
2862− LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN) |
2863− LS_64(info->push_idx, I40IWQPC_PPIDX) |
2864− LS_64(info->push_mode_en, I40IWQPC_PMENA);
2822+ LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN);
28652823
28662824 set_64bit_val(qp_ctx, 8, qp->sq_pa);
28672825 set_64bit_val(qp_ctx, 16, qp->rq_pa);
@@ -4291,13 +4249,6 @@ static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev, · 1423 unmodified lines
42914249 pcmdinfo->in.u.add_arp_cache_entry.scratch,
42924250 pcmdinfo->post_sq);
42934251 break;
4294− case OP_MANAGE_PUSH_PAGE:
4295− status = i40iw_sc_manage_push_page(
4296− pcmdinfo->in.u.manage_push_page.cqp,
4297− &pcmdinfo->in.u.manage_push_page.info,
4298− pcmdinfo->in.u.manage_push_page.scratch,
4299− pcmdinfo->post_sq);
4300− break;
43014252 case OP_UPDATE_PE_SDS:
43024253 /* case I40IW_CQP_OP_UPDATE_PE_SDS */
43034254 status = i40iw_update_pe_sds(
@@ -5098,7 +5049,7 @@ void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi) · 794 unmodified lines
50985049 i40iw_hw_stats_stop_timer(vsi);
50995050 }
51005051
5101−static struct i40iw_cqp_ops iw_cqp_ops = {
5052+static const struct i40iw_cqp_ops iw_cqp_ops = {
51025053 .cqp_init = i40iw_sc_cqp_init,
51035054 .cqp_create = i40iw_sc_cqp_create,
51045055 .cqp_post_sq = i40iw_sc_cqp_post_sq,
@@ -5107,7 +5058,7 @@ static struct i40iw_cqp_ops iw_cqp_ops = { · 2 unmodified lines
51075058 .poll_for_cqp_op_done = i40iw_sc_poll_for_cqp_op_done
51085059 };
51095060
5110−static struct i40iw_ccq_ops iw_ccq_ops = {
5061+static const struct i40iw_ccq_ops iw_ccq_ops = {
51115062 .ccq_init = i40iw_sc_ccq_init,
51125063 .ccq_create = i40iw_sc_ccq_create,
51135064 .ccq_destroy = i40iw_sc_ccq_destroy,
@@ -5116,7 +5067,7 @@ static struct i40iw_ccq_ops iw_ccq_ops = { · 2 unmodified lines
51165067 .ccq_arm = i40iw_sc_ccq_arm
51175068 };
51185069
5119−static struct i40iw_ceq_ops iw_ceq_ops = {
5070+static const struct i40iw_ceq_ops iw_ceq_ops = {
51205071 .ceq_init = i40iw_sc_ceq_init,
51215072 .ceq_create = i40iw_sc_ceq_create,
51225073 .cceq_create_done = i40iw_sc_cceq_create_done,
@@ -5126,7 +5077,7 @@ static struct i40iw_ceq_ops iw_ceq_ops = { · 3 unmodified lines
51265077 .process_ceq = i40iw_sc_process_ceq
51275078 };
51285079
5129−static struct i40iw_aeq_ops iw_aeq_ops = {
5080+static const struct i40iw_aeq_ops iw_aeq_ops = {
51305081 .aeq_init = i40iw_sc_aeq_init,
51315082 .aeq_create = i40iw_sc_aeq_create,
51325083 .aeq_destroy = i40iw_sc_aeq_destroy,
@@ -5137,11 +5088,11 @@ static struct i40iw_aeq_ops iw_aeq_ops = { · 4 unmodified lines
51375088 };
51385089
51395090 /* iwarp pd ops */
5140−static struct i40iw_pd_ops iw_pd_ops = {
5091+static const struct i40iw_pd_ops iw_pd_ops = {
51415092 .pd_init = i40iw_sc_pd_init,
51425093 };
51435094
5144−static struct i40iw_priv_qp_ops iw_priv_qp_ops = {
5095+static const struct i40iw_priv_qp_ops iw_priv_qp_ops = {
51455096 .qp_init = i40iw_sc_qp_init,
51465097 .qp_create = i40iw_sc_qp_create,
51475098 .qp_modify = i40iw_sc_qp_modify,
@@ -5156,14 +5107,14 @@ static struct i40iw_priv_qp_ops iw_priv_qp_ops = { · 8 unmodified lines
51565107 .iw_mr_fast_register = i40iw_sc_mr_fast_register
51575108 };
51585109
5159−static struct i40iw_priv_cq_ops iw_priv_cq_ops = {
5110+static const struct i40iw_priv_cq_ops iw_priv_cq_ops = {
51605111 .cq_init = i40iw_sc_cq_init,
51615112 .cq_create = i40iw_sc_cq_create,
51625113 .cq_destroy = i40iw_sc_cq_destroy,
51635114 .cq_modify = i40iw_sc_cq_modify,
51645115 };
51655116
5166−static struct i40iw_mr_ops iw_mr_ops = {
5117+static const struct i40iw_mr_ops iw_mr_ops = {
51675118 .alloc_stag = i40iw_sc_alloc_stag,
51685119 .mr_reg_non_shared = i40iw_sc_mr_reg_non_shared,
51695120 .mr_reg_shared = i40iw_sc_mr_reg_shared,
@@ -5172,8 +5123,7 @@ static struct i40iw_mr_ops iw_mr_ops = { · 2 unmodified lines
51725123 .mw_alloc = i40iw_sc_mw_alloc
51735124 };
51745125
5175−static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {
5176− .manage_push_page = i40iw_sc_manage_push_page,
5126+static const struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {
51775127 .manage_hmc_pm_func_table = i40iw_sc_manage_hmc_pm_func_table,
51785128 .set_hmc_resource_profile = i40iw_sc_set_hmc_resource_profile,
51795129 .commit_fpm_values = i40iw_sc_commit_fpm_values,
@@ -5195,7 +5145,7 @@ static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = { · 15 unmodified lines
51955145 .update_resume_qp = i40iw_sc_resume_qp
51965146 };
51975147
5198−static struct i40iw_hmc_ops iw_hmc_ops = {
5148+static const struct i40iw_hmc_ops iw_hmc_ops = {
51995149 .init_iw_hmc = i40iw_sc_init_iw_hmc,
52005150 .parse_fpm_query_buf = i40iw_sc_parse_fpm_query_buf,
52015151 .configure_iw_fpm = i40iw_sc_configure_iw_fpm,
drivers/infiniband/hw/i40iw/i40iw_d.h+12−23
@@ -40,11 +40,6 @@ · 39 unmodified lines
4040 #define I40IW_DB_ADDR_OFFSET (4 * 1024 * 1024 - 64 * 1024)
4141 #define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)
4242
43−#define I40IW_PUSH_OFFSET (4 * 1024 * 1024)
44−#define I40IW_PF_FIRST_PUSH_PAGE_INDEX 16
45−#define I40IW_VF_PUSH_OFFSET ((8 + 64) * 1024)
46−#define I40IW_VF_FIRST_PUSH_PAGE_INDEX 2
47−
4843 #define I40IW_PE_DB_SIZE_4M 1
4944 #define I40IW_PE_DB_SIZE_8M 2
5045
@@ -402,7 +397,6 @@ · 351 unmodified lines
402397 #define I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE 0x0e
403398 #define I40IW_CQP_OP_MANAGE_ARP 0x0f
404399 #define I40IW_CQP_OP_MANAGE_VF_PBLE_BP 0x10
405−#define I40IW_CQP_OP_MANAGE_PUSH_PAGES 0x11
406400 #define I40IW_CQP_OP_QUERY_RDMA_FEATURES 0x12
407401 #define I40IW_CQP_OP_UPLOAD_CONTEXT 0x13
408402 #define I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY 0x14
@@ -843,7 +837,6 @@ · 434 unmodified lines
843837 #define I40IW_CQPSQ_MVPBP_PD_PLPBA_MASK \
844838 (0x1fffffffffffffffULL << I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT)
845839
846−/* Manage Push Page - MPP */
847840 #define I40IW_INVALID_PUSH_PAGE_INDEX 0xffff
848841
849842 #define I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT 0
@@ -1352,9 +1345,6 @@ · 502 unmodified lines
13521345 #define I40IWQPSQ_ADDFRAGCNT_SHIFT 38
13531346 #define I40IWQPSQ_ADDFRAGCNT_MASK (0x7ULL << I40IWQPSQ_ADDFRAGCNT_SHIFT)
13541347
1355−#define I40IWQPSQ_PUSHWQE_SHIFT 56
1356−#define I40IWQPSQ_PUSHWQE_MASK (1ULL << I40IWQPSQ_PUSHWQE_SHIFT)
1357−
13581348 #define I40IWQPSQ_STREAMMODE_SHIFT 58
13591349 #define I40IWQPSQ_STREAMMODE_MASK (1ULL << I40IWQPSQ_STREAMMODE_SHIFT)
13601350
@@ -1740,18 +1730,17 @@ enum i40iw_alignment { · 379 unmodified lines
17401730 #define OP_MW_ALLOC 20
17411731 #define OP_QP_FLUSH_WQES 21
17421732 #define OP_ADD_ARP_CACHE_ENTRY 22
1743−#define OP_MANAGE_PUSH_PAGE 23
1744−#define OP_UPDATE_PE_SDS 24
1745−#define OP_MANAGE_HMC_PM_FUNC_TABLE 25
1746−#define OP_SUSPEND 26
1747−#define OP_RESUME 27
1748−#define OP_MANAGE_VF_PBLE_BP 28
1749−#define OP_QUERY_FPM_VALUES 29
1750−#define OP_COMMIT_FPM_VALUES 30
1751−#define OP_REQUESTED_COMMANDS 31
1752−#define OP_COMPLETED_COMMANDS 32
1753−#define OP_GEN_AE 33
1754−#define OP_QUERY_RDMA_FEATURES 34
1755−#define OP_SIZE_CQP_STAT_ARRAY 35
1733+#define OP_UPDATE_PE_SDS 23
1734+#define OP_MANAGE_HMC_PM_FUNC_TABLE 24
1735+#define OP_SUSPEND 25
1736+#define OP_RESUME 26
1737+#define OP_MANAGE_VF_PBLE_BP 27
1738+#define OP_QUERY_FPM_VALUES 28
1739+#define OP_COMMIT_FPM_VALUES 29
1740+#define OP_REQUESTED_COMMANDS 30
1741+#define OP_COMPLETED_COMMANDS 31
1742+#define OP_GEN_AE 32
1743+#define OP_QUERY_RDMA_FEATURES 33
1744+#define OP_SIZE_CQP_STAT_ARRAY 34
17561745
17571746 #endif
drivers/infiniband/hw/i40iw/i40iw_status.h+0−1
@@ -61,7 +61,6 @@ enum i40iw_status_code { · 60 unmodified lines
6161 I40IW_ERR_QUEUE_EMPTY = -22,
6262 I40IW_ERR_INVALID_ALIGNMENT = -23,
6363 I40IW_ERR_FLUSHED_QUEUE = -24,
64− I40IW_ERR_INVALID_PUSH_PAGE_INDEX = -25,
6564 I40IW_ERR_INVALID_INLINE_DATA_SIZE = -26,
6665 I40IW_ERR_TIMEOUT = -27,
6766 I40IW_ERR_OPCODE_MISMATCH = -28,
drivers/infiniband/hw/i40iw/i40iw_type.h+10−28
@@ -387,7 +387,6 @@ struct i40iw_sc_qp { · 386 unmodified lines
387387 u8 *q2_buf;
388388 u64 qp_compl_ctx;
389389 u16 qs_handle;
390− u16 push_idx;
391390 u8 sq_tph_val;
392391 u8 rq_tph_val;
393392 u8 qp_state;
@@ -493,16 +492,16 @@ struct i40iw_sc_dev { · 99 unmodified lines
493492 struct i40iw_sc_aeq *aeq;
494493 struct i40iw_sc_ceq *ceq[I40IW_CEQ_MAX_COUNT];
495494 struct i40iw_sc_cq *ccq;
496− struct i40iw_cqp_ops *cqp_ops;
497− struct i40iw_ccq_ops *ccq_ops;
498− struct i40iw_ceq_ops *ceq_ops;
499− struct i40iw_aeq_ops *aeq_ops;
500− struct i40iw_pd_ops *iw_pd_ops;
501− struct i40iw_priv_qp_ops *iw_priv_qp_ops;
502− struct i40iw_priv_cq_ops *iw_priv_cq_ops;
503− struct i40iw_mr_ops *mr_ops;
504− struct i40iw_cqp_misc_ops *cqp_misc_ops;
505− struct i40iw_hmc_ops *hmc_ops;
495+ const struct i40iw_cqp_ops *cqp_ops;
496+ const struct i40iw_ccq_ops *ccq_ops;
497+ const struct i40iw_ceq_ops *ceq_ops;
498+ const struct i40iw_aeq_ops *aeq_ops;
499+ const struct i40iw_pd_ops *iw_pd_ops;
500+ const struct i40iw_priv_qp_ops *iw_priv_qp_ops;
501+ const struct i40iw_priv_cq_ops *iw_priv_cq_ops;
502+ const struct i40iw_mr_ops *mr_ops;
503+ const struct i40iw_cqp_misc_ops *cqp_misc_ops;
504+ const struct i40iw_hmc_ops *hmc_ops;
506505 struct i40iw_vchnl_if vchnl_if;
507506 const struct i40iw_vf_cqp_ops *iw_vf_cqp_ops;
508507
@@ -749,8 +748,6 @@ struct i40iw_qp_host_ctx_info { · 240 unmodified lines
749748 struct i40iwarp_offload_info *iwarp_info;
750749 u32 send_cq_num;
751750 u32 rcv_cq_num;
752− u16 push_idx;
753− bool push_mode_en;
754751 bool tcp_info_valid;
755752 bool iwarp_info_valid;
756753 bool err_rq_idx_valid;
@@ -937,12 +934,6 @@ struct i40iw_local_mac_ipaddr_entry_info { · 180 unmodified lines
937934 u8 entry_idx;
938935 };
939936
940−struct i40iw_cqp_manage_push_page_info {
941− u32 push_idx;
942− u16 qs_handle;
943− u8 free_page;
944−};
945−
946937 struct i40iw_qp_flush_info {
947938 u16 sq_minor_code;
948939 u16 sq_major_code;
@@ -1114,9 +1105,6 @@ struct i40iw_mr_ops { · 165 unmodified lines
11141105 };
11151106
11161107 struct i40iw_cqp_misc_ops {
1117− enum i40iw_status_code (*manage_push_page)(struct i40iw_sc_cqp *,
1118− struct i40iw_cqp_manage_push_page_info *,
1119− u64, bool);
11201108 enum i40iw_status_code (*manage_hmc_pm_func_table)(struct i40iw_sc_cqp *,
11211109 u64, u8, bool, bool);
11221110 enum i40iw_status_code (*set_hmc_resource_profile)(struct i40iw_sc_cqp *,
@@ -1253,12 +1241,6 @@ struct cqp_info { · 130 unmodified lines
12531241 u64 scratch;
12541242 } manage_vf_pble_bp;
12551243
1256− struct {
1257− struct i40iw_sc_cqp *cqp;
1258− struct i40iw_cqp_manage_push_page_info info;
1259− u64 scratch;
1260− } manage_push_page;
1261−
12621244 struct {
12631245 struct i40iw_sc_dev *dev;
12641246 struct i40iw_upload_context_info info;
drivers/infiniband/hw/i40iw/i40iw_uk.c+4−37
@@ -114,17 +114,6 @@ void i40iw_qp_post_wr(struct i40iw_qp_uk *qp) · 113 unmodified lines
114114 qp->initial_ring.head = qp->sq_ring.head;
115115 }
116116
117−/**
118− * i40iw_qp_ring_push_db - ring qp doorbell
119− * @qp: hw qp ptr
120− * @wqe_idx: wqe index
121− */
122−static void i40iw_qp_ring_push_db(struct i40iw_qp_uk *qp, u32 wqe_idx)
123−{
124− set_32bit_val(qp->push_db, 0, LS_32((wqe_idx >> 2), I40E_PFPE_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id);
125− qp->initial_ring.head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
126−}
127−
128117 /**
129118 * i40iw_qp_get_next_send_wqe - return next wqe ptr
130119 * @qp: hw qp ptr
@@ -426,7 +415,6 @@ static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp, · 295 unmodified lines
426415 u64 *wqe;
427416 u8 *dest, *src;
428417 struct i40iw_inline_rdma_write *op_info;
429− u64 *push;
430418 u64 header = 0;
431419 u32 wqe_idx;
432420 enum i40iw_status_code ret_code;
@@ -453,7 +441,6 @@ static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp, · 20 unmodified lines
453441 LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
454442 LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) |
455443 LS_64(1, I40IWQPSQ_INLINEDATAFLAG) |
456− LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) |
457444 LS_64(read_fence, I40IWQPSQ_READFENCE) |
458445 LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
459446 LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
@@ -475,14 +462,8 @@ static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp, · 15 unmodified lines
475462
476463 set_64bit_val(wqe, 24, header);
477464
478− if (qp->push_db) {
479− push = (u64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) * 0x20);
480− memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32);
481− i40iw_qp_ring_push_db(qp, wqe_idx);
482− } else {
483− if (post_sq)
484− i40iw_qp_post_wr(qp);
485− }
465+ if (post_sq)
466+ i40iw_qp_post_wr(qp);
486467
487468 return 0;
488469 }
@@ -507,7 +488,6 @@ static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp, · 18 unmodified lines
507488 enum i40iw_status_code ret_code;
508489 bool read_fence = false;
509490 u8 wqe_size;
510− u64 *push;
511491
512492 op_info = &info->op.inline_send;
513493 if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
@@ -526,7 +506,6 @@ static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp, · 12 unmodified lines
526506 LS_64(info->op_type, I40IWQPSQ_OPCODE) |
527507 LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) |
528508 LS_64(1, I40IWQPSQ_INLINEDATAFLAG) |
529− LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) |
530509 LS_64(read_fence, I40IWQPSQ_READFENCE) |
531510 LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
532511 LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
@@ -548,14 +527,8 @@ static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp, · 15 unmodified lines
548527
549528 set_64bit_val(wqe, 24, header);
550529
551− if (qp->push_db) {
552− push = (u64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) * 0x20);
553− memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32);
554− i40iw_qp_ring_push_db(qp, wqe_idx);
555− } else {
556− if (post_sq)
557− i40iw_qp_post_wr(qp);
558− }
530+ if (post_sq)
531+ i40iw_qp_post_wr(qp);
559532
560533 return 0;
561534 }
@@ -772,7 +745,6 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq, · 210 unmodified lines
772745
773746 q_type = (u8)RS_64(qword3, I40IW_CQ_SQ);
774747 info->error = (bool)RS_64(qword3, I40IW_CQ_ERROR);
775− info->push_dropped = (bool)RS_64(qword3, I40IWCQ_PSHDROP);
776748 if (info->error) {
777749 info->comp_status = I40IW_COMPL_STATUS_FLUSHED;
778750 info->major_err = (bool)RS_64(qword3, I40IW_CQ_MAJERR);
@@ -951,7 +923,6 @@ enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth) · 172 unmodified lines
951923
952924 static const struct i40iw_qp_uk_ops iw_qp_uk_ops = {
953925 .iw_qp_post_wr = i40iw_qp_post_wr,
954− .iw_qp_ring_push_db = i40iw_qp_ring_push_db,
955926 .iw_rdma_write = i40iw_rdma_write,
956927 .iw_rdma_read = i40iw_rdma_read,
957928 .iw_send = i40iw_send,
@@ -1009,11 +980,7 @@ enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp, · 51 unmodified lines
1009980
1010981 qp->wqe_alloc_reg = info->wqe_alloc_reg;
1011982 qp->qp_id = info->qp_id;
1012−
1013983 qp->sq_size = info->sq_size;
1014− qp->push_db = info->push_db;
1015− qp->push_wqe = info->push_wqe;
1016−
1017984 qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
1018985 sq_ring_size = qp->sq_size << sqshift;
1019986
drivers/infiniband/hw/i40iw/i40iw_user.h+0−8
@@ -64,13 +64,11 @@ enum i40iw_device_capabilities_const { · 63 unmodified lines
6464 I40IW_MAX_SGE_RD = 1,
6565 I40IW_MAX_OUTBOUND_MESSAGE_SIZE = 2147483647,
6666 I40IW_MAX_INBOUND_MESSAGE_SIZE = 2147483647,
67− I40IW_MAX_PUSH_PAGE_COUNT = 4096,
6867 I40IW_MAX_PE_ENABLED_VF_COUNT = 32,
6968 I40IW_MAX_VF_FPM_ID = 47,
7069 I40IW_MAX_VF_PER_PF = 127,
7170 I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496,
7271 I40IW_MAX_INLINE_DATA_SIZE = 48,
73− I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 48,
7472 I40IW_MAX_IRD_SIZE = 64,
7573 I40IW_MAX_ORD_SIZE = 127,
7674 I40IW_MAX_WQ_ENTRIES = 2048,
@@ -272,7 +270,6 @@ struct i40iw_cq_poll_info { · 195 unmodified lines
272270 u16 minor_err;
273271 u8 op_type;
274272 bool stag_invalid_set;
275− bool push_dropped;
276273 bool error;
277274 bool is_srq;
278275 bool solicited_event;
@@ -280,7 +277,6 @@ struct i40iw_cq_poll_info { · 1 unmodified line
280277
281278 struct i40iw_qp_uk_ops {
282279 void (*iw_qp_post_wr)(struct i40iw_qp_uk *);
283− void (*iw_qp_ring_push_db)(struct i40iw_qp_uk *, u32);
284280 enum i40iw_status_code (*iw_rdma_write)(struct i40iw_qp_uk *,
285281 struct i40iw_post_sq_info *, bool);
286282 enum i40iw_status_code (*iw_rdma_read)(struct i40iw_qp_uk *,
@@ -340,8 +336,6 @@ struct i40iw_qp_uk { · 53 unmodified lines
340336 struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
341337 u64 *rq_wrid_array;
342338 u64 *shadow_area;
343− u32 *push_db;
344− u64 *push_wqe;
345339 struct i40iw_ring sq_ring;
346340 struct i40iw_ring rq_ring;
347341 struct i40iw_ring initial_ring;
@@ -381,8 +375,6 @@ struct i40iw_qp_uk_init_info { · 33 unmodified lines
381375 u64 *shadow_area;
382376 struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
383377 u64 *rq_wrid_array;
384− u32 *push_db;
385− u64 *push_wqe;
386378 u32 qp_id;
387379 u32 sq_size;
388380 u32 rq_size;
drivers/infiniband/hw/i40iw/i40iw_verbs.c+11−110
@@ -179,78 +179,6 @@ static int i40iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) · 178 unmodified lines
179179 pgprot_noncached(vma->vm_page_prot), NULL);
180180 }
181181
182−/**
183− * i40iw_alloc_push_page - allocate a push page for qp
184− * @iwdev: iwarp device
185− * @qp: hardware control qp
186− */
187−static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)
188−{
189− struct i40iw_cqp_request *cqp_request;
190− struct cqp_commands_info *cqp_info;
191− enum i40iw_status_code status;
192−
193− if (qp->push_idx != I40IW_INVALID_PUSH_PAGE_INDEX)
194− return;
195−
196− cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
197− if (!cqp_request)
198− return;
199−
200− atomic_inc(&cqp_request->refcount);
201−
202− cqp_info = &cqp_request->info;
203− cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
204− cqp_info->post_sq = 1;
205−
206− cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
207− cqp_info->in.u.manage_push_page.info.free_page = 0;
208− cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
209− cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
210−
211− status = i40iw_handle_cqp_op(iwdev, cqp_request);
212− if (!status)
213− qp->push_idx = cqp_request->compl_info.op_ret_val;
214− else
215− i40iw_pr_err("CQP-OP Push page fail");
216− i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
217−}
218−
219−/**
220− * i40iw_dealloc_push_page - free a push page for qp
221− * @iwdev: iwarp device
222− * @qp: hardware control qp
223− */
224−static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)
225−{
226− struct i40iw_cqp_request *cqp_request;
227− struct cqp_commands_info *cqp_info;
228− enum i40iw_status_code status;
229−
230− if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX)
231− return;
232−
233− cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);
234− if (!cqp_request)
235− return;
236−
237− cqp_info = &cqp_request->info;
238− cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
239− cqp_info->post_sq = 1;
240−
241− cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
242− cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
243− cqp_info->in.u.manage_push_page.info.free_page = 1;
244− cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
245− cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
246−
247− status = i40iw_handle_cqp_op(iwdev, cqp_request);
248− if (!status)
249− qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
250− else
251− i40iw_pr_err("CQP-OP Push page fail");
252−}
253−
254182 /**
255183 * i40iw_alloc_pd - allocate protection domain
256184 * @pd: PD pointer
@@ -348,7 +276,6 @@ void i40iw_free_qp_resources(struct i40iw_qp *iwqp) · 91 unmodified lines
348276 u32 qp_num = iwqp->ibqp.qp_num;
349277
350278 i40iw_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
351− i40iw_dealloc_push_page(iwdev, &iwqp->sc_qp);
352279 if (qp_num)
353280 i40iw_free_resource(iwdev, iwdev->allocated_qps, qp_num);
354281 if (iwpbl->pbl_allocated)
@@ -533,7 +460,7 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, · 178 unmodified lines
533460 return ERR_PTR(-ENODEV);
534461
535462 if (init_attr->create_flags)
536− return ERR_PTR(-EINVAL);
463+ return ERR_PTR(-EOPNOTSUPP);
537464 if (init_attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE)
538465 init_attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
539466
@@ -561,8 +488,6 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, · 21 unmodified lines
561488
562489 qp = &iwqp->sc_qp;
563490 qp->back_qp = (void *)iwqp;
564− qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
565−
566491 iwqp->iwdev = iwdev;
567492 iwqp->ctx_info.iwarp_info = &iwqp->iwarp_info;
568493
@@ -606,8 +531,6 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, · 37 unmodified lines
606531 err_code = -EOPNOTSUPP;
607532 goto error;
608533 }
609− if (iwdev->push_mode)
610− i40iw_alloc_push_page(iwdev, qp);
611534 if (udata) {
612535 err_code = ib_copy_from_udata(&req, udata, sizeof(req));
613536 if (err_code) {
@@ -666,13 +589,6 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, · 52 unmodified lines
666589 ctx_info->iwarp_info_valid = true;
667590 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
668591 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
669− if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) {
670− ctx_info->push_mode_en = false;
671− } else {
672− ctx_info->push_mode_en = true;
673− ctx_info->push_idx = qp->push_idx;
674− }
675−
676592 ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
677593 (u64 *)iwqp->host_ctx.va,
678594 ctx_info);
@@ -712,7 +628,7 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, · 33 unmodified lines
712628 uresp.actual_sq_size = sq_size;
713629 uresp.actual_rq_size = rq_size;
714630 uresp.qp_id = qp_num;
715− uresp.push_idx = qp->push_idx;
631+ uresp.push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
716632 err_code = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
717633 if (err_code) {
718634 i40iw_pr_err("copy_to_udata failed\n");
@@ -832,6 +748,9 @@ int i40iw_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 113 unmodified lines
832748 u32 err;
833749 unsigned long flags;
834750
751+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
752+ return -EOPNOTSUPP;
753+
835754 memset(&info, 0, sizeof(info));
836755 ctx_info = &iwqp->ctx_info;
837756 iwarp_info = &iwqp->iwarp_info;
@@ -1081,6 +1000,9 @@ static int i40iw_create_cq(struct ib_cq *ibcq, · 243 unmodified lines
10811000 int err_code;
10821001 int entries = attr->cqe;
10831002
1003+ if (attr->flags)
1004+ return -EOPNOTSUPP;
1005+
10841006 if (iwdev->closing)
10851007 return -ENODEV;
10861008
@@ -2033,7 +1955,7 @@ static ssize_t hw_rev_show(struct device *dev, · 946 unmodified lines
20331955 rdma_device_to_drv_device(dev, struct i40iw_ib_device, ibdev);
20341956 u32 hw_rev = iwibdev->iwdev->sc_dev.hw_rev;
20351957
2036− return sprintf(buf, "%x\n", hw_rev);
1958+ return sysfs_emit(buf, "%x\n", hw_rev);
20371959 }
20381960 static DEVICE_ATTR_RO(hw_rev);
20391961
@@ -2043,7 +1965,7 @@ static DEVICE_ATTR_RO(hw_rev); · 3 unmodified lines
20431965 static ssize_t hca_type_show(struct device *dev,
20441966 struct device_attribute *attr, char *buf)
20451967 {
2046− return sprintf(buf, "I40IW\n");
1968+ return sysfs_emit(buf, "I40IW\n");
20471969 }
20481970 static DEVICE_ATTR_RO(hca_type);
20491971
@@ -2053,7 +1975,7 @@ static DEVICE_ATTR_RO(hca_type); · 3 unmodified lines
20531975 static ssize_t board_id_show(struct device *dev,
20541976 struct device_attribute *attr, char *buf)
20551977 {
2056− return sprintf(buf, "%.*s\n", 32, "I40IW Board ID");
1978+ return sysfs_emit(buf, "%.*s\n", 32, "I40IW Board ID");
20571979 }
20581980 static DEVICE_ATTR_RO(board_id);
20591981
@@ -2661,27 +2583,6 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev · 601 unmodified lines
26612583 iwibdev->ibdev.node_type = RDMA_NODE_RNIC;
26622584 ether_addr_copy((u8 *)&iwibdev->ibdev.node_guid, netdev->dev_addr);
26632585
2664− iwibdev->ibdev.uverbs_cmd_mask =
2665− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
2666− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
2667− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
2668− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
2669− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
2670− (1ull << IB_USER_VERBS_CMD_REG_MR) |
2671− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
2672− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
2673− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
2674− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
2675− (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
2676− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
2677− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
2678− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
2679− (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
2680− (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
2681− (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
2682− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
2683− (1ull << IB_USER_VERBS_CMD_POST_RECV) |
2684− (1ull << IB_USER_VERBS_CMD_POST_SEND);
26852586 iwibdev->ibdev.phys_port_cnt = 1;
26862587 iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count;
26872588 iwibdev->ibdev.dev.parent = &pcidev->dev;
drivers/infiniband/hw/mlx4/mad.c+1−0
@@ -1523,6 +1523,7 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc · 1522 unmodified lines
15231523 return;
15241524 } else
15251525 *slave_id = slave;
1526+ break;
15261527 default:
15271528 /* nothing */;
15281529 }
drivers/infiniband/hw/mlx4/main.c+7−57
@@ -2024,7 +2024,8 @@ static ssize_t hca_type_show(struct device *device, · 2023 unmodified lines
20242024 {
20252025 struct mlx4_ib_dev *dev =
20262026 rdma_device_to_drv_device(device, struct mlx4_ib_dev, ib_dev);
2027− return sprintf(buf, "MT%d\n", dev->dev->persist->pdev->device);
2027+
2028+ return sysfs_emit(buf, "MT%d\n", dev->dev->persist->pdev->device);
20282029 }
20292030 static DEVICE_ATTR_RO(hca_type);
20302031
@@ -2033,7 +2034,8 @@ static ssize_t hw_rev_show(struct device *device, · 2 unmodified lines
20332034 {
20342035 struct mlx4_ib_dev *dev =
20352036 rdma_device_to_drv_device(device, struct mlx4_ib_dev, ib_dev);
2036− return sprintf(buf, "%x\n", dev->dev->rev_id);
2037+
2038+ return sysfs_emit(buf, "%x\n", dev->dev->rev_id);
20372039 }
20382040 static DEVICE_ATTR_RO(hw_rev);
20392041
@@ -2043,8 +2045,7 @@ static ssize_t board_id_show(struct device *device, · 3 unmodified lines
20432045 struct mlx4_ib_dev *dev =
20442046 rdma_device_to_drv_device(device, struct mlx4_ib_dev, ib_dev);
20452047
2046− return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN,
2047− dev->dev->board_id);
2048+ return sysfs_emit(buf, "%.*s\n", MLX4_BOARD_ID_LEN, dev->dev->board_id);
20482049 }
20492050 static DEVICE_ATTR_RO(board_id);
20502051
@@ -2264,10 +2265,7 @@ static void mlx4_ib_update_qps(struct mlx4_ib_dev *ibdev, · 213 unmodified lines
22642265 u64 release_mac = MLX4_IB_INVALID_MAC;
22652266 struct mlx4_ib_qp *qp;
22662267
2267− read_lock(&dev_base_lock);
22682268 new_smac = mlx4_mac_to_u64(dev->dev_addr);
2269− read_unlock(&dev_base_lock);
2270−
22712269 atomic64_set(&ibdev->iboe.mac[port - 1], new_smac);
22722270
22732271 /* no need for update QP1 and mac registration in non-SRIOV */
@@ -2657,73 +2655,25 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) · 383 unmodified lines
26572655 ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors;
26582656 ibdev->ib_dev.dev.parent = &dev->persist->pdev->dev;
26592657
2660− ibdev->ib_dev.uverbs_cmd_mask =
2661− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
2662− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
2663− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
2664− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
2665− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
2666− (1ull << IB_USER_VERBS_CMD_REG_MR) |
2667− (1ull << IB_USER_VERBS_CMD_REREG_MR) |
2668− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
2669− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
2670− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
2671− (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
2672− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
2673− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
2674− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
2675− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
2676− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
2677− (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
2678− (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
2679− (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
2680− (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
2681− (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
2682− (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
2683− (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) |
2684− (1ull << IB_USER_VERBS_CMD_OPEN_QP);
2685−
26862658 ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_ops);
2687− ibdev->ib_dev.uverbs_ex_cmd_mask |=
2688− (1ull << IB_USER_VERBS_EX_CMD_MODIFY_CQ) |
2689− (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE) |
2690− (1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ) |
2691− (1ull << IB_USER_VERBS_EX_CMD_CREATE_QP);
26922659
26932660 if ((dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS) &&
26942661 ((mlx4_ib_port_link_layer(&ibdev->ib_dev, 1) ==
26952662 IB_LINK_LAYER_ETHERNET) ||
26962663 (mlx4_ib_port_link_layer(&ibdev->ib_dev, 2) ==
2697− IB_LINK_LAYER_ETHERNET))) {
2698− ibdev->ib_dev.uverbs_ex_cmd_mask |=
2699− (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
2700− (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
2701− (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
2702− (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
2703− (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
2664+ IB_LINK_LAYER_ETHERNET)))
27042665 ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_wq_ops);
2705− }
27062666
27072667 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
2708− dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
2709− ibdev->ib_dev.uverbs_cmd_mask |=
2710− (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
2711− (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
2668+ dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN)
27122669 ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_mw_ops);
2713− }
27142670
27152671 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) {
2716− ibdev->ib_dev.uverbs_cmd_mask |=
2717− (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
2718− (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
27192672 ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_xrc_ops);
27202673 }
27212674
27222675 if (check_flow_steering_support(dev)) {
27232676 ibdev->steering_support = MLX4_STEERING_MODE_DEVICE_MANAGED;
2724− ibdev->ib_dev.uverbs_ex_cmd_mask |=
2725− (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
2726− (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
27272677 ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_fs_ops);
27282678 }
27292679
drivers/infiniband/hw/mlx4/mcg.c+46−36
@@ -988,53 +988,63 @@ int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, · 987 unmodified lines
988988 }
989989
990990 static ssize_t sysfs_show_group(struct device *dev,
991− struct device_attribute *attr, char *buf)
991+ struct device_attribute *attr, char *buf)
992992 {
993993 struct mcast_group *group =
994994 container_of(attr, struct mcast_group, dentry);
995995 struct mcast_req *req = NULL;
996− char pending_str[40];
997996 char state_str[40];
998− ssize_t len = 0;
999− int f;
997+ char pending_str[40];
998+ int len;
999+ int i;
1000+ u32 hoplimit;
10001001
10011002 if (group->state == MCAST_IDLE)
1002− sprintf(state_str, "%s", get_state_string(group->state));
1003+ scnprintf(state_str, sizeof(state_str), "%s",
1004+ get_state_string(group->state));
10031005 else
1004− sprintf(state_str, "%s(TID=0x%llx)",
1005− get_state_string(group->state),
1006− be64_to_cpu(group->last_req_tid));
1006+ scnprintf(state_str, sizeof(state_str), "%s(TID=0x%llx)",
1007+ get_state_string(group->state),
1008+ be64_to_cpu(group->last_req_tid));
1009+
10071010 if (list_empty(&group->pending_list)) {
1008− sprintf(pending_str, "No");
1011+ scnprintf(pending_str, sizeof(pending_str), "No");
10091012 } else {
1010− req = list_first_entry(&group->pending_list, struct mcast_req, group_list);
1011− sprintf(pending_str, "Yes(TID=0x%llx)",
1012− be64_to_cpu(req->sa_mad.mad_hdr.tid));
1013+ req = list_first_entry(&group->pending_list, struct mcast_req,
1014+ group_list);
1015+ scnprintf(pending_str, sizeof(pending_str), "Yes(TID=0x%llx)",
1016+ be64_to_cpu(req->sa_mad.mad_hdr.tid));
10131017 }
1014− len += sprintf(buf + len, "%1d [%02d,%02d,%02d] %4d %4s %5s ",
1015− group->rec.scope_join_state & 0xf,
1016− group->members[2], group->members[1], group->members[0],
1017− atomic_read(&group->refcount),
1018− pending_str,
1019− state_str);
1020− for (f = 0; f < MAX_VFS; ++f)
1021− if (group->func[f].state == MCAST_MEMBER)
1022− len += sprintf(buf + len, "%d[%1x] ",
1023− f, group->func[f].join_state);
1024−
1025− len += sprintf(buf + len, "\t\t(%4hx %4x %2x %2x %2x %2x %2x "
1026− "%4x %4x %2x %2x)\n",
1027− be16_to_cpu(group->rec.pkey),
1028− be32_to_cpu(group->rec.qkey),
1029− (group->rec.mtusel_mtu & 0xc0) >> 6,
1030− group->rec.mtusel_mtu & 0x3f,
1031− group->rec.tclass,
1032− (group->rec.ratesel_rate & 0xc0) >> 6,
1033− group->rec.ratesel_rate & 0x3f,
1034− (be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0xf0000000) >> 28,
1035− (be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0x0fffff00) >> 8,
1036− be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0x000000ff,
1037− group->rec.proxy_join);
1018+
1019+ len = sysfs_emit(buf, "%1d [%02d,%02d,%02d] %4d %4s %5s ",
1020+ group->rec.scope_join_state & 0xf,
1021+ group->members[2],
1022+ group->members[1],
1023+ group->members[0],
1024+ atomic_read(&group->refcount),
1025+ pending_str,
1026+ state_str);
1027+
1028+ for (i = 0; i < MAX_VFS; i++) {
1029+ if (group->func[i].state == MCAST_MEMBER)
1030+ len += sysfs_emit_at(buf, len, "%d[%1x] ", i,
1031+ group->func[i].join_state);
1032+ }
1033+
1034+ hoplimit = be32_to_cpu(group->rec.sl_flowlabel_hoplimit);
1035+ len += sysfs_emit_at(buf, len,
1036+ "\t\t(%4hx %4x %2x %2x %2x %2x %2x %4x %4x %2x %2x)\n",
1037+ be16_to_cpu(group->rec.pkey),
1038+ be32_to_cpu(group->rec.qkey),
1039+ (group->rec.mtusel_mtu & 0xc0) >> 6,
1040+ (group->rec.mtusel_mtu & 0x3f),
1041+ group->rec.tclass,
1042+ (group->rec.ratesel_rate & 0xc0) >> 6,
1043+ (group->rec.ratesel_rate & 0x3f),
1044+ (hoplimit & 0xf0000000) >> 28,
1045+ (hoplimit & 0x0fffff00) >> 8,
1046+ (hoplimit & 0x000000ff),
1047+ group->rec.proxy_join);
10381048
10391049 return len;
10401050 }
drivers/infiniband/hw/mlx4/mlx4_ib.h+4−4
@@ -908,10 +908,10 @@ int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn); · 907 unmodified lines
908908 void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count);
909909 int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,
910910 int is_attach);
911−int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
912− u64 start, u64 length, u64 virt_addr,
913− int mr_access_flags, struct ib_pd *pd,
914− struct ib_udata *udata);
911+struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,
912+ u64 length, u64 virt_addr,
913+ int mr_access_flags, struct ib_pd *pd,
914+ struct ib_udata *udata);
915915 int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,
916916 const struct ib_gid_attr *attr);
917917
drivers/infiniband/hw/mlx4/mr.c+8−8
@@ -456,10 +456,10 @@ err_free: · 455 unmodified lines
456456 return ERR_PTR(err);
457457 }
458458
459−int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
460− u64 start, u64 length, u64 virt_addr,
461− int mr_access_flags, struct ib_pd *pd,
462− struct ib_udata *udata)
459+struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,
460+ u64 length, u64 virt_addr,
461+ int mr_access_flags, struct ib_pd *pd,
462+ struct ib_udata *udata)
463463 {
464464 struct mlx4_ib_dev *dev = to_mdev(mr->device);
465465 struct mlx4_ib_mr *mmr = to_mmr(mr);
@@ -472,9 +472,8 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, · 6 unmodified lines
472472 * race exists.
473473 */
474474 err = mlx4_mr_hw_get_mpt(dev->dev, &mmr->mmr, &pmpt_entry);
475−
476475 if (err)
477− return err;
476+ return ERR_PTR(err);
478477
479478 if (flags & IB_MR_REREG_PD) {
480479 err = mlx4_mr_hw_change_pd(dev->dev, *pmpt_entry,
@@ -542,8 +541,9 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, · 61 unmodified lines
542541
543542 release_mpt_entry:
544543 mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry);
545−
546− return err;
544+ if (err)
545+ return ERR_PTR(err);
546+ return NULL;
547547 }
548548
549549 static int
drivers/infiniband/hw/mlx4/qp.c+12−2
@@ -1493,7 +1493,7 @@ static int _mlx4_ib_create_qp(struct ib_pd *pd, struct mlx4_ib_qp *qp, · 1492 unmodified lines
14931493 MLX4_IB_SRIOV_SQP |
14941494 MLX4_IB_QP_NETIF |
14951495 MLX4_IB_QP_CREATE_ROCE_V2_GSI))
1496− return -EINVAL;
1496+ return -EOPNOTSUPP;
14971497
14981498 if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
14991499 if (init_attr->qp_type != IB_QPT_UD)
@@ -1561,6 +1561,11 @@ static int _mlx4_ib_create_qp(struct ib_pd *pd, struct mlx4_ib_qp *qp, · 61 unmodified lines
15611561 if (err)
15621562 return err;
15631563
1564+ if (init_attr->create_flags &
1565+ (MLX4_IB_SRIOV_SQP | MLX4_IB_SRIOV_TUNNEL_QP))
1566+ /* Internal QP created with ib_create_qp */
1567+ rdma_restrack_no_track(&qp->ibqp.res);
1568+
15641569 qp->port = init_attr->port_num;
15651570 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
15661571 init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI ? sqpn : 1;
@@ -2787,6 +2792,9 @@ int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 1220 unmodified lines
27872792 struct mlx4_ib_qp *mqp = to_mqp(ibqp);
27882793 int ret;
27892794
2795+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2796+ return -EOPNOTSUPP;
2797+
27902798 ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
27912799
27922800 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
@@ -4007,7 +4015,9 @@ int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr · 1214 unmodified lines
40074015 qp_attr->qp_access_flags =
40084016 to_ib_qp_access_flags(be32_to_cpu(context.params2));
40094017
4010− if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
4018+ if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC ||
4019+ qp->ibqp.qp_type == IB_QPT_XRC_INI ||
4020+ qp->ibqp.qp_type == IB_QPT_XRC_TGT) {
40114021 to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
40124022 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
40134023 qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
drivers/infiniband/hw/mlx4/srq.c+4−0
@@ -86,6 +86,10 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, · 85 unmodified lines
8686 int err;
8787 int i;
8888
89+ if (init_attr->srq_type != IB_SRQT_BASIC &&
90+ init_attr->srq_type != IB_SRQT_XRC)
91+ return -EOPNOTSUPP;
92+
8993 /* Sanity check SRQ size before proceeding */
9094 if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes ||
9195 init_attr->attr.max_sge > dev->dev->caps.max_srq_sge)
drivers/infiniband/hw/mlx4/sysfs.c+28−38
@@ -56,7 +56,7 @@ static ssize_t show_admin_alias_guid(struct device *dev, · 55 unmodified lines
5656 mlx4_ib_iov_dentry->entry_num,
5757 port->num);
5858
59− return sprintf(buf, "%llx\n", be64_to_cpu(sysadmin_ag_val));
59+ return sysfs_emit(buf, "%llx\n", be64_to_cpu(sysadmin_ag_val));
6060 }
6161
6262 /* store_admin_alias_guid stores the (new) administratively assigned value of that GUID.
@@ -117,22 +117,24 @@ static ssize_t show_port_gid(struct device *dev, · 54 unmodified lines
117117 struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx;
118118 struct mlx4_ib_dev *mdev = port->dev;
119119 union ib_gid gid;
120− ssize_t ret;
120+ int ret;
121+ __be16 *raw;
121122
122123 ret = __mlx4_ib_query_gid(&mdev->ib_dev, port->num,
123124 mlx4_ib_iov_dentry->entry_num, &gid, 1);
124125 if (ret)
125126 return ret;
126− ret = sprintf(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
127− be16_to_cpu(((__be16 *) gid.raw)[0]),
128− be16_to_cpu(((__be16 *) gid.raw)[1]),
129− be16_to_cpu(((__be16 *) gid.raw)[2]),
130− be16_to_cpu(((__be16 *) gid.raw)[3]),
131− be16_to_cpu(((__be16 *) gid.raw)[4]),
132− be16_to_cpu(((__be16 *) gid.raw)[5]),
133− be16_to_cpu(((__be16 *) gid.raw)[6]),
134− be16_to_cpu(((__be16 *) gid.raw)[7]));
135− return ret;
127+
128+ raw = (__be16 *)gid.raw;
129+ return sysfs_emit(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
130+ be16_to_cpu(raw[0]),
131+ be16_to_cpu(raw[1]),
132+ be16_to_cpu(raw[2]),
133+ be16_to_cpu(raw[3]),
134+ be16_to_cpu(raw[4]),
135+ be16_to_cpu(raw[5]),
136+ be16_to_cpu(raw[6]),
137+ be16_to_cpu(raw[7]));
136138 }
137139
138140 static ssize_t show_phys_port_pkey(struct device *dev,
@@ -151,7 +153,7 @@ static ssize_t show_phys_port_pkey(struct device *dev, · 12 unmodified lines
151153 if (ret)
152154 return ret;
153155
154− return sprintf(buf, "0x%04x\n", pkey);
156+ return sysfs_emit(buf, "0x%04x\n", pkey);
155157 }
156158
157159 #define DENTRY_REMOVE(_dentry) \
@@ -441,16 +443,12 @@ static ssize_t show_port_pkey(struct mlx4_port *p, struct port_attribute *attr, · 283 unmodified lines
441443 {
442444 struct port_table_attribute *tab_attr =
443445 container_of(attr, struct port_table_attribute, attr);
444− ssize_t ret = -ENODEV;
445−
446− if (p->dev->pkeys.virt2phys_pkey[p->slave][p->port_num - 1][tab_attr->index] >=
447− (p->dev->dev->caps.pkey_table_len[p->port_num]))
448− ret = sprintf(buf, "none\n");
449− else
450− ret = sprintf(buf, "%d\n",
451− p->dev->pkeys.virt2phys_pkey[p->slave]
452− [p->port_num - 1][tab_attr->index]);
453− return ret;
446+ struct pkey_mgt *m = &p->dev->pkeys;
447+ u8 key = m->virt2phys_pkey[p->slave][p->port_num - 1][tab_attr->index];
448+
449+ if (key >= p->dev->dev->caps.pkey_table_len[p->port_num])
450+ return sysfs_emit(buf, "none\n");
451+ return sysfs_emit(buf, "%d\n", key);
454452 }
455453
456454 static ssize_t store_port_pkey(struct mlx4_port *p, struct port_attribute *attr,
@@ -488,7 +486,7 @@ static ssize_t store_port_pkey(struct mlx4_port *p, struct port_attribute *attr, · 31 unmodified lines
488486 static ssize_t show_port_gid_idx(struct mlx4_port *p,
489487 struct port_attribute *attr, char *buf)
490488 {
491− return sprintf(buf, "%d\n", p->slave);
489+ return sysfs_emit(buf, "%d\n", p->slave);
492490 }
493491
494492 static struct attribute **
@@ -542,14 +540,10 @@ static ssize_t sysfs_show_smi_enabled(struct device *dev, · 47 unmodified lines
542540 {
543541 struct mlx4_port *p =
544542 container_of(attr, struct mlx4_port, smi_enabled);
545− ssize_t len = 0;
546543
547− if (mlx4_vf_smi_enabled(p->dev->dev, p->slave, p->port_num))
548− len = sprintf(buf, "%d\n", 1);
549− else
550− len = sprintf(buf, "%d\n", 0);
551−
552− return len;
544+ return sysfs_emit(buf, "%d\n",
545+ !!mlx4_vf_smi_enabled(p->dev->dev, p->slave,
546+ p->port_num));
553547 }
554548
555549 static ssize_t sysfs_show_enable_smi_admin(struct device *dev,
@@ -558,14 +552,10 @@ static ssize_t sysfs_show_enable_smi_admin(struct device *dev, · 2 unmodified lines
558552 {
559553 struct mlx4_port *p =
560554 container_of(attr, struct mlx4_port, enable_smi_admin);
561− ssize_t len = 0;
562−
563− if (mlx4_vf_get_enable_smi_admin(p->dev->dev, p->slave, p->port_num))
564− len = sprintf(buf, "%d\n", 1);
565− else
566− len = sprintf(buf, "%d\n", 0);
567555
568− return len;
556+ return sysfs_emit(buf, "%d\n",
557+ !!mlx4_vf_get_enable_smi_admin(p->dev->dev, p->slave,
558+ p->port_num));
569559 }
570560
571561 static ssize_t sysfs_store_enable_smi_admin(struct device *dev,
drivers/infiniband/hw/mlx5/cq.c+49−28
@@ -707,10 +707,10 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, · 706 unmodified lines
707707 int *cqe_size, int *index, int *inlen)
708708 {
709709 struct mlx5_ib_create_cq ucmd = {};
710+ unsigned long page_size;
711+ unsigned int page_offset_quantized;
710712 size_t ucmdlen;
711− int page_shift;
712713 __be64 *pas;
713− int npages;
714714 int ncont;
715715 void *cqc;
716716 int err;
@@ -742,14 +742,24 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, · 25 unmodified lines
742742 return err;
743743 }
744744
745+ page_size = mlx5_umem_find_best_cq_quantized_pgoff(
746+ cq->buf.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,
747+ page_offset, 64, &page_offset_quantized);
748+ if (!page_size) {
749+ err = -EINVAL;
750+ goto err_umem;
751+ }
752+
745753 err = mlx5_ib_db_map_user(context, udata, ucmd.db_addr, &cq->db);
746754 if (err)
747755 goto err_umem;
748756
749− mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, 0, &npages, &page_shift,
750− &ncont, NULL);
751− mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
752− ucmd.buf_addr, entries * ucmd.cqe_size, npages, page_shift, ncont);
757+ ncont = ib_umem_num_dma_blocks(cq->buf.umem, page_size);
758+ mlx5_ib_dbg(
759+ dev,
760+ "addr 0x%llx, size %u, npages %zu, page_size %lu, ncont %d\n",
761+ ucmd.buf_addr, entries * ucmd.cqe_size,
762+ ib_umem_num_pages(cq->buf.umem), page_size, ncont);
753763
754764 *inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
755765 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont;
@@ -760,11 +770,12 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, · 4 unmodified lines
760770 }
761771
762772 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);
763− mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, pas, 0);
773+ mlx5_ib_populate_pas(cq->buf.umem, page_size, pas, 0);
764774
765775 cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
766776 MLX5_SET(cqc, cqc, log_page_size,
767− page_shift - MLX5_ADAPTER_PAGE_SHIFT);
777+ order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
778+ MLX5_SET(cqc, cqc, page_offset, page_offset_quantized);
768779
769780 if (ucmd.flags & MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX) {
770781 *index = ucmd.uar_page_index;
@@ -1128,13 +1139,12 @@ int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) · 357 unmodified lines
11281139 }
11291140
11301141 static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1131− int entries, struct ib_udata *udata, int *npas,
1132− int *page_shift, int *cqe_size)
1142+ int entries, struct ib_udata *udata,
1143+ int *cqe_size)
11331144 {
11341145 struct mlx5_ib_resize_cq ucmd;
11351146 struct ib_umem *umem;
11361147 int err;
1137− int npages;
11381148
11391149 err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
11401150 if (err)
@@ -1155,9 +1165,6 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, · 14 unmodified lines
11551165 return err;
11561166 }
11571167
1158− mlx5_ib_cont_pages(umem, ucmd.buf_addr, 0, &npages, page_shift,
1159− npas, NULL);
1160−
11611168 cq->resize_umem = umem;
11621169 *cqe_size = ucmd.cqe_size;
11631170
@@ -1250,7 +1257,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) · 86 unmodified lines
12501257 int err;
12511258 int npas;
12521259 __be64 *pas;
1253− int page_shift;
1260+ unsigned int page_offset_quantized = 0;
1261+ unsigned int page_shift;
12541262 int inlen;
12551263 int cqe_size;
12561264 unsigned long flags;
@@ -1277,22 +1285,34 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) · 20 unmodified lines
12771285
12781286 mutex_lock(&cq->resize_mutex);
12791287 if (udata) {
1280− err = resize_user(dev, cq, entries, udata, &npas, &page_shift,
1281− &cqe_size);
1288+ unsigned long page_size;
1289+
1290+ err = resize_user(dev, cq, entries, udata, &cqe_size);
1291+ if (err)
1292+ goto ex;
1293+
1294+ page_size = mlx5_umem_find_best_cq_quantized_pgoff(
1295+ cq->resize_umem, cqc, log_page_size,
1296+ MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64,
1297+ &page_offset_quantized);
1298+ if (!page_size) {
1299+ err = -EINVAL;
1300+ goto ex_resize;
1301+ }
1302+ npas = ib_umem_num_dma_blocks(cq->resize_umem, page_size);
1303+ page_shift = order_base_2(page_size);
12821304 } else {
1305+ struct mlx5_frag_buf *frag_buf;
1306+
12831307 cqe_size = 64;
12841308 err = resize_kernel(dev, cq, entries, cqe_size);
1285− if (!err) {
1286− struct mlx5_frag_buf *frag_buf = &cq->resize_buf->frag_buf;
1287−
1288− npas = frag_buf->npages;
1289− page_shift = frag_buf->page_shift;
1290− }
1309+ if (err)
1310+ goto ex;
1311+ frag_buf = &cq->resize_buf->frag_buf;
1312+ npas = frag_buf->npages;
1313+ page_shift = frag_buf->page_shift;
12911314 }
12921315
1293− if (err)
1294− goto ex;
1295−
12961316 inlen = MLX5_ST_SZ_BYTES(modify_cq_in) +
12971317 MLX5_FLD_SZ_BYTES(modify_cq_in, pas[0]) * npas;
12981318
@@ -1304,8 +1324,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) · 5 unmodified lines
13041324
13051325 pas = (__be64 *)MLX5_ADDR_OF(modify_cq_in, in, pas);
13061326 if (udata)
1307− mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift,
1308− pas, 0);
1327+ mlx5_ib_populate_pas(cq->resize_umem, 1UL << page_shift, pas,
1328+ 0);
13091329 else
13101330 mlx5_fill_page_frag_array(&cq->resize_buf->frag_buf, pas);
13111331
@@ -1319,6 +1339,7 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) · 7 unmodified lines
13191339
13201340 MLX5_SET(cqc, cqc, log_page_size,
13211341 page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1342+ MLX5_SET(cqc, cqc, page_offset, page_offset_quantized);
13221343 MLX5_SET(cqc, cqc, cqe_sz,
13231344 cqe_sz_to_mlx_sz(cqe_size,
13241345 cq->private_flags &
drivers/infiniband/hw/mlx5/devx.c+43−47
@@ -93,9 +93,6 @@ struct devx_async_event_file { · 92 unmodified lines
9393 struct devx_umem {
9494 struct mlx5_core_dev *mdev;
9595 struct ib_umem *umem;
96− u32 page_offset;
97− int page_shift;
98− int ncont;
9996 u32 dinlen;
10097 u32 dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
10198 };
@@ -1311,7 +1308,7 @@ static int devx_obj_cleanup(struct ib_uobject *uobject, · 1209 unmodified lines
13111308 else
13121309 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
13131310 obj->dinlen, out, sizeof(out));
1314− if (ib_is_destroy_retryable(ret, why, uobject))
1311+ if (ret)
13151312 return ret;
13161313
13171314 devx_event_table = &dev->devx_event_table;
@@ -2057,9 +2054,7 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, · 739 unmodified lines
20572054 u64 addr;
20582055 size_t size;
20592056 u32 access;
2060− int npages;
20612057 int err;
2062− u32 page_mask;
20632058
20642059 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
20652060 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
@@ -2073,57 +2068,62 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, · 7 unmodified lines
20732068 if (err)
20742069 return err;
20752070
2076− err = ib_check_mr_access(access);
2071+ err = ib_check_mr_access(&dev->ib_dev, access);
20772072 if (err)
20782073 return err;
20792074
20802075 obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access);
20812076 if (IS_ERR(obj->umem))
20822077 return PTR_ERR(obj->umem);
2083−
2084− mlx5_ib_cont_pages(obj->umem, obj->umem->address,
2085− MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
2086− &obj->page_shift, &obj->ncont, NULL);
2087−
2088− if (!npages) {
2089− ib_umem_release(obj->umem);
2090− return -EINVAL;
2091− }
2092−
2093− page_mask = (1 << obj->page_shift) - 1;
2094− obj->page_offset = obj->umem->address & page_mask;
2095−
20962078 return 0;
20972079 }
20982080
2099−static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
2081+static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev,
2082+ struct uverbs_attr_bundle *attrs,
21002083 struct devx_umem *obj,
21012084 struct devx_umem_reg_cmd *cmd)
21022085 {
2086+ unsigned int page_size;
2087+ __be64 *mtt;
2088+ void *umem;
2089+
2090+ /*
2091+ * We don't know what the user intends to use this umem for, but the HW
2092+ * restrictions must be met. MR, doorbell records, QP, WQ and CQ all
2093+ * have different requirements. Since we have no idea how to sort this
2094+ * out, only support PAGE_SIZE with the expectation that userspace will
2095+ * provide the necessary alignments inside the known PAGE_SIZE and that
2096+ * FW will check everything.
2097+ */
2098+ page_size = ib_umem_find_best_pgoff(
2099+ obj->umem, PAGE_SIZE,
2100+ __mlx5_page_offset_to_bitmask(__mlx5_bit_sz(umem, page_offset),
2101+ 0));
2102+ if (!page_size)
2103+ return -EINVAL;
2104+
21032105 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2104− (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
2106+ (MLX5_ST_SZ_BYTES(mtt) *
2107+ ib_umem_num_dma_blocks(obj->umem, page_size));
21052108 cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2106− return PTR_ERR_OR_ZERO(cmd->in);
2107−}
2108−
2109−static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
2110− struct devx_umem *obj,
2111− struct devx_umem_reg_cmd *cmd)
2112−{
2113− void *umem;
2114− __be64 *mtt;
2109+ if (IS_ERR(cmd->in))
2110+ return PTR_ERR(cmd->in);
21152111
21162112 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
21172113 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
21182114
21192115 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2120− MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
2121− MLX5_SET(umem, umem, log_page_size, obj->page_shift -
2122− MLX5_ADAPTER_PAGE_SHIFT);
2123− MLX5_SET(umem, umem, page_offset, obj->page_offset);
2124− mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
2116+ MLX5_SET64(umem, umem, num_of_mtt,
2117+ ib_umem_num_dma_blocks(obj->umem, page_size));
2118+ MLX5_SET(umem, umem, log_page_size,
2119+ order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
2120+ MLX5_SET(umem, umem, page_offset,
2121+ ib_umem_dma_offset(obj->umem, page_size));
2122+
2123+ mlx5_ib_populate_pas(obj->umem, page_size, mtt,
21252124 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2126− MLX5_IB_MTT_READ);
2125+ MLX5_IB_MTT_READ);
2126+ return 0;
21272127 }
21282128
21292129 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
@@ -2150,12 +2150,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)( · 20 unmodified lines
21502150 if (err)
21512151 goto err_obj_free;
21522152
2153− err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
2153+ err = devx_umem_reg_cmd_alloc(dev, attrs, obj, &cmd);
21542154 if (err)
21552155 goto err_umem_release;
21562156
2157− devx_umem_reg_cmd_build(dev, obj, &cmd);
2158−
21592157 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
21602158 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
21612159 sizeof(cmd.out));
@@ -2187,7 +2185,7 @@ static int devx_umem_cleanup(struct ib_uobject *uobject, · 25 unmodified lines
21872185 int err;
21882186
21892187 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2190− if (ib_is_destroy_retryable(err, why, uobject))
2188+ if (err)
21912189 return err;
21922190
21932191 ib_umem_release(obj->umem);
@@ -2600,8 +2598,8 @@ static const struct file_operations devx_async_event_fops = { · 406 unmodified lines
26002598 .llseek = no_llseek,
26012599 };
26022600
2603−static int devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2604− enum rdma_remove_reason why)
2601+static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2602+ enum rdma_remove_reason why)
26052603 {
26062604 struct devx_async_cmd_event_file *comp_ev_file =
26072605 container_of(uobj, struct devx_async_cmd_event_file,
@@ -2623,11 +2621,10 @@ static int devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj, · 15 unmodified lines
26232621 kvfree(entry);
26242622 }
26252623 spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2626− return 0;
26272624 };
26282625
2629−static int devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2630− enum rdma_remove_reason why)
2626+static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2627+ enum rdma_remove_reason why)
26312628 {
26322629 struct devx_async_event_file *ev_file =
26332630 container_of(uobj, struct devx_async_event_file,
@@ -2671,7 +2668,6 @@ static int devx_async_event_destroy_uobj(struct ib_uobject *uobj, · 37 unmodified lines
26712668 mutex_unlock(&dev->devx_event_table.event_xa_lock);
26722669
26732670 put_device(&dev->ib_dev.dev);
2674− return 0;
26752671 };
26762672
26772673 DECLARE_UVERBS_NAMED_METHOD(
drivers/infiniband/hw/mlx5/fs.c+2−4
@@ -2035,11 +2035,9 @@ static int flow_matcher_cleanup(struct ib_uobject *uobject, · 2034 unmodified lines
20352035 struct uverbs_attr_bundle *attrs)
20362036 {
20372037 struct mlx5_ib_flow_matcher *obj = uobject->object;
2038− int ret;
20392038
2040− ret = ib_destroy_usecnt(&obj->usecnt, why, uobject);
2041− if (ret)
2042− return ret;
2039+ if (atomic_read(&obj->usecnt))
2040+ return -EBUSY;
20432041
20442042 kfree(obj);
20452043 return 0;
drivers/infiniband/hw/mlx5/main.c+24−81
@@ -75,12 +75,6 @@ static LIST_HEAD(mlx5_ib_dev_list); · 74 unmodified lines
7575 */
7676 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
7777
78−/* We can't use an array for xlt_emergency_page because dma_map_single
79− * doesn't work on kernel modules memory
80− */
81−static unsigned long xlt_emergency_page;
82−static struct mutex xlt_emergency_page_mutex;
83−
8478 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
8579 {
8680 struct mlx5_ib_dev *dev;
@@ -425,10 +419,22 @@ static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed, · 338 unmodified lines
425419 *active_width = IB_WIDTH_2X;
426420 *active_speed = IB_SPEED_HDR;
427421 break;
422+ case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):
423+ *active_width = IB_WIDTH_1X;
424+ *active_speed = IB_SPEED_NDR;
425+ break;
428426 case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
429427 *active_width = IB_WIDTH_4X;
430428 *active_speed = IB_SPEED_HDR;
431429 break;
430+ case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):
431+ *active_width = IB_WIDTH_2X;
432+ *active_speed = IB_SPEED_NDR;
433+ break;
434+ case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):
435+ *active_width = IB_WIDTH_4X;
436+ *active_speed = IB_SPEED_NDR;
437+ break;
432438 default:
433439 return -EINVAL;
434440 }
@@ -2628,7 +2634,7 @@ static ssize_t fw_pages_show(struct device *device, · 2193 unmodified lines
26282634 struct mlx5_ib_dev *dev =
26292635 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
26302636
2631− return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
2637+ return sysfs_emit(buf, "%d\n", dev->mdev->priv.fw_pages);
26322638 }
26332639 static DEVICE_ATTR_RO(fw_pages);
26342640
@@ -2638,7 +2644,7 @@ static ssize_t reg_pages_show(struct device *device, · 3 unmodified lines
26382644 struct mlx5_ib_dev *dev =
26392645 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
26402646
2641− return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
2647+ return sysfs_emit(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
26422648 }
26432649 static DEVICE_ATTR_RO(reg_pages);
26442650
@@ -2648,7 +2654,7 @@ static ssize_t hca_type_show(struct device *device, · 3 unmodified lines
26482654 struct mlx5_ib_dev *dev =
26492655 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
26502656
2651− return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
2657+ return sysfs_emit(buf, "MT%d\n", dev->mdev->pdev->device);
26522658 }
26532659 static DEVICE_ATTR_RO(hca_type);
26542660
@@ -2658,7 +2664,7 @@ static ssize_t hw_rev_show(struct device *device, · 3 unmodified lines
26582664 struct mlx5_ib_dev *dev =
26592665 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
26602666
2661− return sprintf(buf, "%x\n", dev->mdev->rev_id);
2667+ return sysfs_emit(buf, "%x\n", dev->mdev->rev_id);
26622668 }
26632669 static DEVICE_ATTR_RO(hw_rev);
26642670
@@ -2668,8 +2674,8 @@ static ssize_t board_id_show(struct device *device, · 3 unmodified lines
26682674 struct mlx5_ib_dev *dev =
26692675 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
26702676
2671− return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2672− dev->mdev->board_id);
2677+ return sysfs_emit(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2678+ dev->mdev->board_id);
26732679 }
26742680 static DEVICE_ATTR_RO(board_id);
26752681
@@ -4024,6 +4030,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 1348 unmodified lines
40244030 .create_cq = mlx5_ib_create_cq,
40254031 .create_qp = mlx5_ib_create_qp,
40264032 .create_srq = mlx5_ib_create_srq,
4033+ .create_user_ah = mlx5_ib_create_ah,
40274034 .dealloc_pd = mlx5_ib_dealloc_pd,
40284035 .dealloc_ucontext = mlx5_ib_dealloc_ucontext,
40294036 .del_gid = mlx5_ib_del_gid,
@@ -4141,42 +4148,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) · 111 unmodified lines
41414148 struct mlx5_core_dev *mdev = dev->mdev;
41424149 int err;
41434150
4144− dev->ib_dev.uverbs_cmd_mask =
4145− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
4146− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
4147− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
4148− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
4149− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
4150− (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
4151− (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
4152− (1ull << IB_USER_VERBS_CMD_REG_MR) |
4153− (1ull << IB_USER_VERBS_CMD_REREG_MR) |
4154− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
4155− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
4156− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
4157− (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
4158− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
4159− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
4160− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
4161− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
4162− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
4163− (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
4164− (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
4165− (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
4166− (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
4167− (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
4168− (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
4169− (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) |
4170− (1ull << IB_USER_VERBS_CMD_OPEN_QP);
4171− dev->ib_dev.uverbs_ex_cmd_mask =
4172− (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE) |
4173− (1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ) |
4174− (1ull << IB_USER_VERBS_EX_CMD_CREATE_QP) |
4175− (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP) |
4176− (1ull << IB_USER_VERBS_EX_CMD_MODIFY_CQ) |
4177− (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
4178− (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
4179−
41804151 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
41814152 IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))
41824153 ib_set_device_ops(&dev->ib_dev,
@@ -4187,19 +4158,11 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) · 4 unmodified lines
41874158
41884159 dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
41894160
4190− if (MLX5_CAP_GEN(mdev, imaicl)) {
4191− dev->ib_dev.uverbs_cmd_mask |=
4192− (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
4193− (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
4161+ if (MLX5_CAP_GEN(mdev, imaicl))
41944162 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_mw_ops);
4195− }
41964163
4197− if (MLX5_CAP_GEN(mdev, xrc)) {
4198− dev->ib_dev.uverbs_cmd_mask |=
4199− (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
4200− (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
4164+ if (MLX5_CAP_GEN(mdev, xrc))
42014165 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
4202− }
42034166
42044167 if (MLX5_CAP_DEV_MEM(mdev, memic) ||
42054168 MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
@@ -4278,12 +4241,6 @@ static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev) · 72 unmodified lines
42784241 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
42794242
42804243 if (ll == IB_LINK_LAYER_ETHERNET) {
4281− dev->ib_dev.uverbs_ex_cmd_mask |=
4282− (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
4283− (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
4284− (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
4285− (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
4286− (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
42874244 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
42884245
42894246 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
@@ -4878,30 +4835,17 @@ static struct auxiliary_driver mlx5r_driver = { · 588 unmodified lines
48784835 .id_table = mlx5r_id_table,
48794836 };
48804837
4881−unsigned long mlx5_ib_get_xlt_emergency_page(void)
4882−{
4883− mutex_lock(&xlt_emergency_page_mutex);
4884− return xlt_emergency_page;
4885−}
4886−
4887−void mlx5_ib_put_xlt_emergency_page(void)
4888−{
4889− mutex_unlock(&xlt_emergency_page_mutex);
4890−}
4891−
48924838 static int __init mlx5_ib_init(void)
48934839 {
48944840 int ret;
48954841
4896− xlt_emergency_page = __get_free_page(GFP_KERNEL);
4842+ xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL);
48974843 if (!xlt_emergency_page)
48984844 return -ENOMEM;
48994845
4900− mutex_init(&xlt_emergency_page_mutex);
4901−
49024846 mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
49034847 if (!mlx5_ib_event_wq) {
4904− free_page(xlt_emergency_page);
4848+ free_page((unsigned long)xlt_emergency_page);
49054849 return -ENOMEM;
49064850 }
49074851
@@ -4934,8 +4878,7 @@ static void __exit mlx5_ib_cleanup(void) · 26 unmodified lines
49344878 mlx5r_rep_cleanup();
49354879
49364880 destroy_workqueue(mlx5_ib_event_wq);
4937− mutex_destroy(&xlt_emergency_page_mutex);
4938− free_page(xlt_emergency_page);
4881+ free_page((unsigned long)xlt_emergency_page);
49394882 }
49404883
49414884 module_init(mlx5_ib_init);
drivers/infiniband/hw/mlx5/mem.c+48−144
@@ -36,161 +36,65 @@ · 35 unmodified lines
3636 #include "mlx5_ib.h"
3737 #include <linux/jiffies.h>
3838
39−/* @umem: umem object to scan
40− * @addr: ib virtual address requested by the user
41− * @max_page_shift: high limit for page_shift - 0 means no limit
42− * @count: number of PAGE_SIZE pages covered by umem
43− * @shift: page shift for the compound pages found in the region
44− * @ncont: number of compund pages
45− * @order: log2 of the number of compound pages
39+/*
40+ * Fill in a physical address list. ib_umem_num_dma_blocks() entries will be
41+ * filled in the pas array.
4642 */
47−void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
48− unsigned long max_page_shift,
49− int *count, int *shift,
50− int *ncont, int *order)
43+void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas,
44+ u64 access_flags)
5145 {
52− unsigned long tmp;
53− unsigned long m;
54− u64 base = ~0, p = 0;
55− u64 len, pfn;
56− int i = 0;
57− struct scatterlist *sg;
58− int entry;
59−
60− addr = addr >> PAGE_SHIFT;
61− tmp = (unsigned long)addr;
62− m = find_first_bit(&tmp, BITS_PER_LONG);
63− if (max_page_shift)
64− m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m);
65−
66− for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
67− len = sg_dma_len(sg) >> PAGE_SHIFT;
68− pfn = sg_dma_address(sg) >> PAGE_SHIFT;
69− if (base + p != pfn) {
70− /* If either the offset or the new
71− * base are unaligned update m
72− */
73− tmp = (unsigned long)(pfn | p);
74− if (!IS_ALIGNED(tmp, 1 << m))
75− m = find_first_bit(&tmp, BITS_PER_LONG);
76−
77− base = pfn;
78− p = 0;
79− }
80−
81− p += len;
82− i += len;
83− }
84−
85− if (i) {
86− m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m);
87−
88− if (order)
89− *order = ilog2(roundup_pow_of_two(i) >> m);
90−
91− *ncont = DIV_ROUND_UP(i, (1 << m));
92− } else {
93− m = 0;
46+ struct ib_block_iter biter;
9447
95− if (order)
96− *order = 0;
97−
98− *ncont = 0;
48+ rdma_umem_for_each_dma_block (umem, &biter, page_size) {
49+ *pas = cpu_to_be64(rdma_block_iter_dma_address(&biter) |
50+ access_flags);
51+ pas++;
9952 }
100− *shift = PAGE_SHIFT + m;
101− *count = i;
10253 }
10354
10455 /*
105− * Populate the given array with bus addresses from the umem.
106− *
107− * dev - mlx5_ib device
108− * umem - umem to use to fill the pages
109− * page_shift - determines the page size used in the resulting array
110− * offset - offset into the umem to start from,
111− * only implemented for ODP umems
112− * num_pages - total number of pages to fill
113− * pas - bus addresses array to fill
114− * access_flags - access flags to set on all present pages.
115− use enum mlx5_ib_mtt_access_flags for this.
56+ * Compute the page shift and page_offset for mailboxes that use a quantized
57+ * page_offset. The granulatity of the page offset scales according to page
58+ * size.
11659 */
117−void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
118− int page_shift, size_t offset, size_t num_pages,
119− __be64 *pas, int access_flags)
60+unsigned long __mlx5_umem_find_best_quantized_pgoff(
61+ struct ib_umem *umem, unsigned long pgsz_bitmap,
62+ unsigned int page_offset_bits, u64 pgoff_bitmask, unsigned int scale,
63+ unsigned int *page_offset_quantized)
12064 {
121− int shift = page_shift - PAGE_SHIFT;
122− int mask = (1 << shift) - 1;
123− int i, k, idx;
124− u64 cur = 0;
125− u64 base;
126− int len;
127− struct scatterlist *sg;
128− int entry;
129−
130− i = 0;
131− for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
132− len = sg_dma_len(sg) >> PAGE_SHIFT;
133− base = sg_dma_address(sg);
134−
135− /* Skip elements below offset */
136− if (i + len < offset << shift) {
137− i += len;
138− continue;
139− }
140−
141− /* Skip pages below offset */
142− if (i < offset << shift) {
143− k = (offset << shift) - i;
144− i = offset << shift;
145− } else {
146− k = 0;
147− }
148−
149− for (; k < len; k++) {
150− if (!(i & mask)) {
151− cur = base + (k << PAGE_SHIFT);
152− cur |= access_flags;
153− idx = (i >> shift) - offset;
154−
155− pas[idx] = cpu_to_be64(cur);
156− mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n",
157− i >> shift, be64_to_cpu(pas[idx]));
158− }
159− i++;
160−
161− /* Stop after num_pages reached */
162− if (i >> shift >= offset + num_pages)
163− return;
164− }
65+ const u64 page_offset_mask = (1UL << page_offset_bits) - 1;
66+ unsigned long page_size;
67+ u64 page_offset;
68+
69+ page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, pgoff_bitmask);
70+ if (!page_size)
71+ return 0;
72+
73+ /*
74+ * page size is the largest possible page size.
75+ *
76+ * Reduce the page_size, and thus the page_offset and quanta, until the
77+ * page_offset fits into the mailbox field. Once page_size < scale this
78+ * loop is guaranteed to terminate.
79+ */
80+ page_offset = ib_umem_dma_offset(umem, page_size);
81+ while (page_offset & ~(u64)(page_offset_mask * (page_size / scale))) {
82+ page_size /= 2;
83+ page_offset = ib_umem_dma_offset(umem, page_size);
16584 }
166−}
16785
168−void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
169− int page_shift, __be64 *pas, int access_flags)
170−{
171− return __mlx5_ib_populate_pas(dev, umem, page_shift, 0,
172− ib_umem_num_dma_blocks(umem, PAGE_SIZE),
173− pas, access_flags);
174−}
175−int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset)
176−{
177− u64 page_size;
178− u64 page_mask;
179− u64 off_size;
180− u64 off_mask;
181− u64 buf_off;
182−
183− page_size = (u64)1 << page_shift;
184− page_mask = page_size - 1;
185− buf_off = addr & page_mask;
186− off_size = page_size >> 6;
187− off_mask = off_size - 1;
188−
189− if (buf_off & off_mask)
190− return -EINVAL;
191−
192− *offset = buf_off >> ilog2(off_size);
193− return 0;
86+ /*
87+ * The address is not aligned, or otherwise cannot be represented by the
88+ * page_offset.
89+ */
90+ if (!(pgsz_bitmap & page_size))
91+ return 0;
92+
93+ *page_offset_quantized =
94+ (unsigned long)page_offset / (page_size / scale);
95+ if (WARN_ON(*page_offset_quantized > page_offset_mask))
96+ return 0;
97+ return page_size;
19498 }
19599
196100 #define WR_ID_BF 0xBF
drivers/infiniband/hw/mlx5/mlx5_ib.h+81−21
@@ -40,7 +40,73 @@ · 39 unmodified lines
4040 #define MLX5_IB_DEFAULT_UIDX 0xffffff
4141 #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
4242
43−#define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
43+static __always_inline unsigned long
44+__mlx5_log_page_size_to_bitmap(unsigned int log_pgsz_bits,
45+ unsigned int pgsz_shift)
46+{
47+ unsigned int largest_pg_shift =
48+ min_t(unsigned long, (1ULL << log_pgsz_bits) - 1 + pgsz_shift,
49+ BITS_PER_LONG - 1);
50+
51+ /*
52+ * Despite a command allowing it, the device does not support lower than
53+ * 4k page size.
54+ */
55+ pgsz_shift = max_t(unsigned int, MLX5_ADAPTER_PAGE_SHIFT, pgsz_shift);
56+ return GENMASK(largest_pg_shift, pgsz_shift);
57+}
58+
59+/*
60+ * For mkc users, instead of a page_offset the command has a start_iova which
61+ * specifies both the page_offset and the on-the-wire IOVA
62+ */
63+#define mlx5_umem_find_best_pgsz(umem, typ, log_pgsz_fld, pgsz_shift, iova) \
64+ ib_umem_find_best_pgsz(umem, \
65+ __mlx5_log_page_size_to_bitmap( \
66+ __mlx5_bit_sz(typ, log_pgsz_fld), \
67+ pgsz_shift), \
68+ iova)
69+
70+static __always_inline unsigned long
71+__mlx5_page_offset_to_bitmask(unsigned int page_offset_bits,
72+ unsigned int offset_shift)
73+{
74+ unsigned int largest_offset_shift =
75+ min_t(unsigned long, page_offset_bits - 1 + offset_shift,
76+ BITS_PER_LONG - 1);
77+
78+ return GENMASK(largest_offset_shift, offset_shift);
79+}
80+
81+/*
82+ * QP/CQ/WQ/etc type commands take a page offset that satisifies:
83+ * page_offset_quantized * (page_size/scale) = page_offset
84+ * Which restricts allowed page sizes to ones that satisify the above.
85+ */
86+unsigned long __mlx5_umem_find_best_quantized_pgoff(
87+ struct ib_umem *umem, unsigned long pgsz_bitmap,
88+ unsigned int page_offset_bits, u64 pgoff_bitmask, unsigned int scale,
89+ unsigned int *page_offset_quantized);
90+#define mlx5_umem_find_best_quantized_pgoff(umem, typ, log_pgsz_fld, \
91+ pgsz_shift, page_offset_fld, \
92+ scale, page_offset_quantized) \
93+ __mlx5_umem_find_best_quantized_pgoff( \
94+ umem, \
95+ __mlx5_log_page_size_to_bitmap( \
96+ __mlx5_bit_sz(typ, log_pgsz_fld), pgsz_shift), \
97+ __mlx5_bit_sz(typ, page_offset_fld), \
98+ GENMASK(31, order_base_2(scale)), scale, \
99+ page_offset_quantized)
100+
101+#define mlx5_umem_find_best_cq_quantized_pgoff(umem, typ, log_pgsz_fld, \
102+ pgsz_shift, page_offset_fld, \
103+ scale, page_offset_quantized) \
104+ __mlx5_umem_find_best_quantized_pgoff( \
105+ umem, \
106+ __mlx5_log_page_size_to_bitmap( \
107+ __mlx5_bit_sz(typ, log_pgsz_fld), pgsz_shift), \
108+ __mlx5_bit_sz(typ, page_offset_fld), 0, scale, \
109+ page_offset_quantized)
44110
45111 enum {
46112 MLX5_IB_MMAP_OFFSET_START = 9,
@@ -597,14 +663,12 @@ struct mlx5_ib_mr { · 550 unmodified lines
597663 int max_descs;
598664 int desc_size;
599665 int access_mode;
666+ unsigned int page_shift;
600667 struct mlx5_core_mkey mmkey;
601668 struct ib_umem *umem;
602669 struct mlx5_shared_mr_info *smr_info;
603670 struct list_head list;
604− unsigned int order;
605671 struct mlx5_cache_ent *cache_ent;
606− int npages;
607− struct mlx5_ib_dev *dev;
608672 u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
609673 struct mlx5_core_sig_ctx *sig;
610674 void *descs_alloc;
@@ -1042,6 +1106,11 @@ static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev) · 431 unmodified lines
10421106 return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
10431107 }
10441108
1109+static inline struct mlx5_ib_dev *mr_to_mdev(struct mlx5_ib_mr *mr)
1110+{
1111+ return to_mdev(mr->ibmr.device);
1112+}
1113+
10451114 static inline struct mlx5_ib_dev *mlx5_udata_to_mdev(struct ib_udata *udata)
10461115 {
10471116 struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
@@ -1189,9 +1258,9 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, · 141 unmodified lines
11891258 int access_flags);
11901259 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr);
11911260 void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr);
1192−int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1193− u64 length, u64 virt_addr, int access_flags,
1194− struct ib_pd *pd, struct ib_udata *udata);
1261+struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1262+ u64 length, u64 virt_addr, int access_flags,
1263+ struct ib_pd *pd, struct ib_udata *udata);
11951264 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
11961265 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
11971266 u32 max_num_sg);
@@ -1210,7 +1279,6 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, · 12 unmodified lines
12101279 size_t *out_mad_size, u16 *out_mad_pkey_index);
12111280 int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
12121281 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
1213−int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
12141282 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
12151283 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
12161284 struct ib_smp *out_mad);
@@ -1230,15 +1298,8 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port, · 13 unmodified lines
12301298 struct ib_port_attr *props);
12311299 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
12321300 struct ib_port_attr *props);
1233−void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
1234− unsigned long max_page_shift,
1235− int *count, int *shift,
1236− int *ncont, int *order);
1237−void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1238− int page_shift, size_t offset, size_t num_pages,
1239− __be64 *pas, int access_flags);
1240−void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1241− int page_shift, __be64 *pas, int access_flags);
1301+void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas,
1302+ u64 access_flags);
12421303 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
12431304 int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);
12441305 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
@@ -1283,7 +1344,7 @@ void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, · 38 unmodified lines
12831344 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
12841345 enum ib_uverbs_advise_mr_advice advice,
12851346 u32 flags, struct ib_sge *sg_list, u32 num_sge);
1286−int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, bool enable);
1347+int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr);
12871348 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
12881349 static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
12891350 {
@@ -1305,7 +1366,7 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, · 15 unmodified lines
13051366 {
13061367 return -EOPNOTSUPP;
13071368 }
1308−static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, bool enable)
1369+static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr)
13091370 {
13101371 return -EOPNOTSUPP;
13111372 }
@@ -1456,8 +1517,7 @@ static inline int get_num_static_uars(struct mlx5_ib_dev *dev, · 144 unmodified lines
14561517 return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
14571518 }
14581519
1459−unsigned long mlx5_ib_get_xlt_emergency_page(void);
1460−void mlx5_ib_put_xlt_emergency_page(void);
1520+extern void *xlt_emergency_page;
14611521
14621522 int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
14631523 struct mlx5_bfreg_info *bfregi, u32 bfregn,
drivers/infiniband/hw/mlx5/mr.c+571−389
@@ -41,6 +41,13 @@ · 40 unmodified lines
4141 #include <rdma/ib_verbs.h>
4242 #include "mlx5_ib.h"
4343
44+/*
45+ * We can't use an array for xlt_emergency_page because dma_map_single doesn't
46+ * work on kernel modules memory
47+ */
48+void *xlt_emergency_page;
49+static DEFINE_MUTEX(xlt_emergency_page_mutex);
50+
4451 enum {
4552 MAX_PENDING_REG_MR = 8,
4653 };
@@ -49,6 +56,9 @@ enum { · 2 unmodified lines
4956
5057 static void
5158 create_mkey_callback(int status, struct mlx5_async_work *context);
59+static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
60+ u64 iova, int access_flags,
61+ unsigned int page_size, bool populate);
5262
5363 static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
5464 struct ib_pd *pd)
@@ -123,19 +133,12 @@ static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) · 68 unmodified lines
123133 return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
124134 }
125135
126−static inline bool mlx5_ib_pas_fits_in_mr(struct mlx5_ib_mr *mr, u64 start,
127− u64 length)
128−{
129− return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
130− length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1));
131−}
132−
133136 static void create_mkey_callback(int status, struct mlx5_async_work *context)
134137 {
135138 struct mlx5_ib_mr *mr =
136139 container_of(context, struct mlx5_ib_mr, cb_work);
137− struct mlx5_ib_dev *dev = mr->dev;
138140 struct mlx5_cache_ent *ent = mr->cache_ent;
141+ struct mlx5_ib_dev *dev = ent->dev;
139142 unsigned long flags;
140143
141144 if (status) {
@@ -172,9 +175,7 @@ static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc) · 30 unmodified lines
172175 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
173176 if (!mr)
174177 return NULL;
175− mr->order = ent->order;
176178 mr->cache_ent = ent;
177− mr->dev = ent->dev;
178179
179180 set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd);
180181 MLX5_SET(mkc, mkc, free, 1);
@@ -642,6 +643,7 @@ void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) · 461 unmodified lines
642643 if (mlx5_mr_cache_invalidate(mr)) {
643644 detach_mr_from_cache(mr);
644645 destroy_mkey(dev, mr);
646+ kfree(mr);
645647 return;
646648 }
647649
@@ -867,57 +869,6 @@ static int mr_cache_max_order(struct mlx5_ib_dev *dev) · 219 unmodified lines
867869 return MLX5_MAX_UMR_SHIFT;
868870 }
869871
870−static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
871− int access_flags, struct ib_umem **umem, int *npages,
872− int *page_shift, int *ncont, int *order)
873−{
874− struct ib_umem *u;
875−
876− *umem = NULL;
877−
878− if (access_flags & IB_ACCESS_ON_DEMAND) {
879− struct ib_umem_odp *odp;
880−
881− odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags,
882− &mlx5_mn_ops);
883− if (IS_ERR(odp)) {
884− mlx5_ib_dbg(dev, "umem get failed (%ld)\n",
885− PTR_ERR(odp));
886− return PTR_ERR(odp);
887− }
888−
889− u = &odp->umem;
890−
891− *page_shift = odp->page_shift;
892− *ncont = ib_umem_odp_num_pages(odp);
893− *npages = *ncont << (*page_shift - PAGE_SHIFT);
894− if (order)
895− *order = ilog2(roundup_pow_of_two(*ncont));
896− } else {
897− u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
898− if (IS_ERR(u)) {
899− mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
900− return PTR_ERR(u);
901− }
902−
903− mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
904− page_shift, ncont, order);
905− }
906−
907− if (!*npages) {
908− mlx5_ib_warn(dev, "avoid zero region\n");
909− ib_umem_release(u);
910− return -EINVAL;
911− }
912−
913− *umem = u;
914−
915− mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
916− *npages, *ncont, *order, *page_shift);
917−
918− return 0;
919−}
920−
921872 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
922873 {
923874 struct mlx5_ib_umr_context *context =
@@ -974,25 +925,49 @@ static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev, · 50 unmodified lines
974925 return &cache->ent[order];
975926 }
976927
977−static struct mlx5_ib_mr *
978−alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
979− u64 len, int npages, int page_shift, unsigned int order,
980− int access_flags)
928+static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
929+ u64 length, int access_flags)
930+{
931+ mr->ibmr.lkey = mr->mmkey.key;
932+ mr->ibmr.rkey = mr->mmkey.key;
933+ mr->ibmr.length = length;
934+ mr->ibmr.device = &dev->ib_dev;
935+ mr->access_flags = access_flags;
936+}
937+
938+static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
939+ struct ib_umem *umem, u64 iova,
940+ int access_flags)
981941 {
982942 struct mlx5_ib_dev *dev = to_mdev(pd->device);
983− struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);
943+ struct mlx5_cache_ent *ent;
984944 struct mlx5_ib_mr *mr;
945+ unsigned int page_size;
985946
986− if (!ent)
987− return ERR_PTR(-E2BIG);
988−
989− /* Matches access in alloc_cache_mr() */
990− if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags))
991− return ERR_PTR(-EOPNOTSUPP);
947+ page_size = mlx5_umem_find_best_pgsz(umem, mkc, log_page_size, 0, iova);
948+ if (WARN_ON(!page_size))
949+ return ERR_PTR(-EINVAL);
950+ ent = mr_cache_ent_from_order(
951+ dev, order_base_2(ib_umem_num_dma_blocks(umem, page_size)));
952+ /*
953+ * Matches access in alloc_cache_mr(). If the MR can't come from the
954+ * cache then synchronously create an uncached one.
955+ */
956+ if (!ent || ent->limit == 0 ||
957+ !mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags)) {
958+ mutex_lock(&dev->slow_path_mutex);
959+ mr = reg_create(pd, umem, iova, access_flags, page_size, false);
960+ mutex_unlock(&dev->slow_path_mutex);
961+ return mr;
962+ }
992963
993964 mr = get_cache_mr(ent);
994965 if (!mr) {
995966 mr = create_cache_mr(ent);
967+ /*
968+ * The above already tried to do the same stuff as reg_create(),
969+ * no reason to try it again.
970+ */
996971 if (IS_ERR(mr))
997972 return mr;
998973 }
@@ -1001,9 +976,12 @@ alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr, · 2 unmodified lines
1001976 mr->umem = umem;
1002977 mr->access_flags = access_flags;
1003978 mr->desc_size = sizeof(struct mlx5_mtt);
1004− mr->mmkey.iova = virt_addr;
1005− mr->mmkey.size = len;
979+ mr->mmkey.iova = iova;
980+ mr->mmkey.size = umem->length;
1006981 mr->mmkey.pd = to_mpd(pd)->pdn;
982+ mr->page_shift = order_base_2(page_size);
983+ mr->umem = umem;
984+ set_mr_fields(dev, mr, umem->length, access_flags);
1007985
1008986 return mr;
1009987 }
@@ -1012,14 +990,144 @@ alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr, · 2 unmodified lines
1012990 MLX5_UMR_MTT_ALIGNMENT)
1013991 #define MLX5_SPARE_UMR_CHUNK 0x10000
1014992
993+/*
994+ * Allocate a temporary buffer to hold the per-page information to transfer to
995+ * HW. For efficiency this should be as large as it can be, but buffer
996+ * allocation failure is not allowed, so try smaller sizes.
997+ */
998+static void *mlx5_ib_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask)
999+{
1000+ const size_t xlt_chunk_align =
1001+ MLX5_UMR_MTT_ALIGNMENT / sizeof(ent_size);
1002+ size_t size;
1003+ void *res = NULL;
1004+
1005+ static_assert(PAGE_SIZE % MLX5_UMR_MTT_ALIGNMENT == 0);
1006+
1007+ /*
1008+ * MLX5_IB_UPD_XLT_ATOMIC doesn't signal an atomic context just that the
1009+ * allocation can't trigger any kind of reclaim.
1010+ */
1011+ might_sleep();
1012+
1013+ gfp_mask |= __GFP_ZERO;
1014+
1015+ /*
1016+ * If the system already has a suitable high order page then just use
1017+ * that, but don't try hard to create one. This max is about 1M, so a
1018+ * free x86 huge page will satisfy it.
1019+ */
1020+ size = min_t(size_t, ent_size * ALIGN(*nents, xlt_chunk_align),
1021+ MLX5_MAX_UMR_CHUNK);
1022+ *nents = size / ent_size;
1023+ res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
1024+ get_order(size));
1025+ if (res)
1026+ return res;
1027+
1028+ if (size > MLX5_SPARE_UMR_CHUNK) {
1029+ size = MLX5_SPARE_UMR_CHUNK;
1030+ *nents = get_order(size) / ent_size;
1031+ res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
1032+ get_order(size));
1033+ if (res)
1034+ return res;
1035+ }
1036+
1037+ *nents = PAGE_SIZE / ent_size;
1038+ res = (void *)__get_free_page(gfp_mask);
1039+ if (res)
1040+ return res;
1041+
1042+ mutex_lock(&xlt_emergency_page_mutex);
1043+ memset(xlt_emergency_page, 0, PAGE_SIZE);
1044+ return xlt_emergency_page;
1045+}
1046+
1047+static void mlx5_ib_free_xlt(void *xlt, size_t length)
1048+{
1049+ if (xlt == xlt_emergency_page) {
1050+ mutex_unlock(&xlt_emergency_page_mutex);
1051+ return;
1052+ }
1053+
1054+ free_pages((unsigned long)xlt, get_order(length));
1055+}
1056+
1057+/*
1058+ * Create a MLX5_IB_SEND_UMR_UPDATE_XLT work request and XLT buffer ready for
1059+ * submission.
1060+ */
1061+static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr,
1062+ struct mlx5_umr_wr *wr, struct ib_sge *sg,
1063+ size_t nents, size_t ent_size,
1064+ unsigned int flags)
1065+{
1066+ struct mlx5_ib_dev *dev = mr_to_mdev(mr);
1067+ struct device *ddev = &dev->mdev->pdev->dev;
1068+ dma_addr_t dma;
1069+ void *xlt;
1070+
1071+ xlt = mlx5_ib_alloc_xlt(&nents, ent_size,
1072+ flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC :
1073+ GFP_KERNEL);
1074+ sg->length = nents * ent_size;
1075+ dma = dma_map_single(ddev, xlt, sg->length, DMA_TO_DEVICE);
1076+ if (dma_mapping_error(ddev, dma)) {
1077+ mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
1078+ mlx5_ib_free_xlt(xlt, sg->length);
1079+ return NULL;
1080+ }
1081+ sg->addr = dma;
1082+ sg->lkey = dev->umrc.pd->local_dma_lkey;
1083+
1084+ memset(wr, 0, sizeof(*wr));
1085+ wr->wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;
1086+ if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
1087+ wr->wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1088+ wr->wr.sg_list = sg;
1089+ wr->wr.num_sge = 1;
1090+ wr->wr.opcode = MLX5_IB_WR_UMR;
1091+ wr->pd = mr->ibmr.pd;
1092+ wr->mkey = mr->mmkey.key;
1093+ wr->length = mr->mmkey.size;
1094+ wr->virt_addr = mr->mmkey.iova;
1095+ wr->access_flags = mr->access_flags;
1096+ wr->page_shift = mr->page_shift;
1097+ wr->xlt_size = sg->length;
1098+ return xlt;
1099+}
1100+
1101+static void mlx5_ib_unmap_free_xlt(struct mlx5_ib_dev *dev, void *xlt,
1102+ struct ib_sge *sg)
1103+{
1104+ struct device *ddev = &dev->mdev->pdev->dev;
1105+
1106+ dma_unmap_single(ddev, sg->addr, sg->length, DMA_TO_DEVICE);
1107+ mlx5_ib_free_xlt(xlt, sg->length);
1108+}
1109+
1110+static unsigned int xlt_wr_final_send_flags(unsigned int flags)
1111+{
1112+ unsigned int res = 0;
1113+
1114+ if (flags & MLX5_IB_UPD_XLT_ENABLE)
1115+ res |= MLX5_IB_SEND_UMR_ENABLE_MR |
1116+ MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |
1117+ MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1118+ if (flags & MLX5_IB_UPD_XLT_PD || flags & MLX5_IB_UPD_XLT_ACCESS)
1119+ res |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1120+ if (flags & MLX5_IB_UPD_XLT_ADDR)
1121+ res |= MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1122+ return res;
1123+}
1124+
10151125 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
10161126 int page_shift, int flags)
10171127 {
1018− struct mlx5_ib_dev *dev = mr->dev;
1019− struct device *ddev = dev->ib_dev.dev.parent;
1020− int size;
1128+ struct mlx5_ib_dev *dev = mr_to_mdev(mr);
1129+ struct device *ddev = &dev->mdev->pdev->dev;
10211130 void *xlt;
1022− dma_addr_t dma;
10231131 struct mlx5_umr_wr wr;
10241132 struct ib_sge sg;
10251133 int err = 0;
@@ -1030,15 +1138,17 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, · 4 unmodified lines
10301138 const int page_mask = page_align - 1;
10311139 size_t pages_mapped = 0;
10321140 size_t pages_to_map = 0;
1033− size_t pages_iter = 0;
1141+ size_t pages_iter;
10341142 size_t size_to_map = 0;
1035− gfp_t gfp;
1036− bool use_emergency_page = false;
1143+ size_t orig_sg_length;
10371144
10381145 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
10391146 !umr_can_use_indirect_mkey(dev))
10401147 return -EPERM;
10411148
1149+ if (WARN_ON(!mr->umem->is_odp))
1150+ return -EINVAL;
1151+
10421152 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
10431153 * so we need to align the offset and length accordingly
10441154 */
@@ -1046,63 +1156,21 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, · 1 unmodified line
10461156 npages += idx & page_mask;
10471157 idx &= ~page_mask;
10481158 }
1049−
1050− gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL;
1051− gfp |= __GFP_ZERO | __GFP_NOWARN;
1052−
10531159 pages_to_map = ALIGN(npages, page_align);
1054− size = desc_size * pages_to_map;
1055− size = min_t(int, size, MLX5_MAX_UMR_CHUNK);
1056−
1057− xlt = (void *)__get_free_pages(gfp, get_order(size));
1058− if (!xlt && size > MLX5_SPARE_UMR_CHUNK) {
1059− mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n",
1060− size, get_order(size), MLX5_SPARE_UMR_CHUNK);
1061−
1062− size = MLX5_SPARE_UMR_CHUNK;
1063− xlt = (void *)__get_free_pages(gfp, get_order(size));
1064− }
10651160
1066− if (!xlt) {
1067− mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
1068− xlt = (void *)mlx5_ib_get_xlt_emergency_page();
1069− size = PAGE_SIZE;
1070− memset(xlt, 0, size);
1071− use_emergency_page = true;
1072− }
1073− pages_iter = size / desc_size;
1074− dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
1075− if (dma_mapping_error(ddev, dma)) {
1076− mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
1077− err = -ENOMEM;
1078− goto free_xlt;
1079− }
1161+ xlt = mlx5_ib_create_xlt_wr(mr, &wr, &sg, npages, desc_size, flags);
1162+ if (!xlt)
1163+ return -ENOMEM;
1164+ pages_iter = sg.length / desc_size;
1165+ orig_sg_length = sg.length;
10801166
1081− if (mr->umem->is_odp) {
1082− if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) {
1083− struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
1084− size_t max_pages = ib_umem_odp_num_pages(odp) - idx;
1167+ if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) {
1168+ struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
1169+ size_t max_pages = ib_umem_odp_num_pages(odp) - idx;
10851170
1086− pages_to_map = min_t(size_t, pages_to_map, max_pages);
1087− }
1171+ pages_to_map = min_t(size_t, pages_to_map, max_pages);
10881172 }
10891173
1090− sg.addr = dma;
1091− sg.lkey = dev->umrc.pd->local_dma_lkey;
1092−
1093− memset(&wr, 0, sizeof(wr));
1094− wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;
1095− if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
1096− wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1097− wr.wr.sg_list = &sg;
1098− wr.wr.num_sge = 1;
1099− wr.wr.opcode = MLX5_IB_WR_UMR;
1100−
1101− wr.pd = mr->ibmr.pd;
1102− wr.mkey = mr->mmkey.key;
1103− wr.length = mr->mmkey.size;
1104− wr.virt_addr = mr->mmkey.iova;
1105− wr.access_flags = mr->access_flags;
11061174 wr.page_shift = page_shift;
11071175
11081176 for (pages_mapped = 0;
@@ -1110,50 +1178,87 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, · 1 unmodified line
11101178 pages_mapped += pages_iter, idx += pages_iter) {
11111179 npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
11121180 size_to_map = npages * desc_size;
1113− dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE);
1114− if (mr->umem->is_odp) {
1115− mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
1116− } else {
1117− __mlx5_ib_populate_pas(dev, mr->umem, page_shift, idx,
1118− npages, xlt,
1119− MLX5_IB_MTT_PRESENT);
1120− /* Clear padding after the pages
1121− * brought from the umem.
1122− */
1123− memset(xlt + size_to_map, 0, size - size_to_map);
1124− }
1125− dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
1181+ dma_sync_single_for_cpu(ddev, sg.addr, sg.length,
1182+ DMA_TO_DEVICE);
1183+ mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
1184+ dma_sync_single_for_device(ddev, sg.addr, sg.length,
1185+ DMA_TO_DEVICE);
11261186
11271187 sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT);
11281188
1129− if (pages_mapped + pages_iter >= pages_to_map) {
1130− if (flags & MLX5_IB_UPD_XLT_ENABLE)
1131− wr.wr.send_flags |=
1132− MLX5_IB_SEND_UMR_ENABLE_MR |
1133− MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |
1134− MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1135− if (flags & MLX5_IB_UPD_XLT_PD ||
1136− flags & MLX5_IB_UPD_XLT_ACCESS)
1137− wr.wr.send_flags |=
1138− MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1139− if (flags & MLX5_IB_UPD_XLT_ADDR)
1140− wr.wr.send_flags |=
1141− MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1142− }
1189+ if (pages_mapped + pages_iter >= pages_to_map)
1190+ wr.wr.send_flags |= xlt_wr_final_send_flags(flags);
11431191
11441192 wr.offset = idx * desc_size;
11451193 wr.xlt_size = sg.length;
11461194
11471195 err = mlx5_ib_post_send_wait(dev, &wr);
11481196 }
1149− dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1197+ sg.length = orig_sg_length;
1198+ mlx5_ib_unmap_free_xlt(dev, xlt, &sg);
1199+ return err;
1200+}
1201+
1202+/*
1203+ * Send the DMA list to the HW for a normal MR using UMR.
1204+ */
1205+static int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags)
1206+{
1207+ struct mlx5_ib_dev *dev = mr_to_mdev(mr);
1208+ struct device *ddev = &dev->mdev->pdev->dev;
1209+ struct ib_block_iter biter;
1210+ struct mlx5_mtt *cur_mtt;
1211+ struct mlx5_umr_wr wr;
1212+ size_t orig_sg_length;
1213+ struct mlx5_mtt *mtt;
1214+ size_t final_size;
1215+ struct ib_sge sg;
1216+ int err = 0;
1217+
1218+ if (WARN_ON(mr->umem->is_odp))
1219+ return -EINVAL;
1220+
1221+ mtt = mlx5_ib_create_xlt_wr(mr, &wr, &sg,
1222+ ib_umem_num_dma_blocks(mr->umem,
1223+ 1 << mr->page_shift),
1224+ sizeof(*mtt), flags);
1225+ if (!mtt)
1226+ return -ENOMEM;
1227+ orig_sg_length = sg.length;
1228+
1229+ cur_mtt = mtt;
1230+ rdma_for_each_block (mr->umem->sg_head.sgl, &biter, mr->umem->nmap,
1231+ BIT(mr->page_shift)) {
1232+ if (cur_mtt == (void *)mtt + sg.length) {
1233+ dma_sync_single_for_device(ddev, sg.addr, sg.length,
1234+ DMA_TO_DEVICE);
1235+ err = mlx5_ib_post_send_wait(dev, &wr);
1236+ if (err)
1237+ goto err;
1238+ dma_sync_single_for_cpu(ddev, sg.addr, sg.length,
1239+ DMA_TO_DEVICE);
1240+ wr.offset += sg.length;
1241+ cur_mtt = mtt;
1242+ }
1243+
1244+ cur_mtt->ptag =
1245+ cpu_to_be64(rdma_block_iter_dma_address(&biter) |
1246+ MLX5_IB_MTT_PRESENT);
1247+ cur_mtt++;
1248+ }
11501249
1151−free_xlt:
1152− if (use_emergency_page)
1153− mlx5_ib_put_xlt_emergency_page();
1154− else
1155− free_pages((unsigned long)xlt, get_order(size));
1250+ final_size = (void *)cur_mtt - (void *)mtt;
1251+ sg.length = ALIGN(final_size, MLX5_UMR_MTT_ALIGNMENT);
1252+ memset(cur_mtt, 0, sg.length - final_size);
1253+ wr.wr.send_flags |= xlt_wr_final_send_flags(flags);
1254+ wr.xlt_size = sg.length;
11561255
1256+ dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE);
1257+ err = mlx5_ib_post_send_wait(dev, &wr);
1258+
1259+err:
1260+ sg.length = orig_sg_length;
1261+ mlx5_ib_unmap_free_xlt(dev, mtt, &sg);
11571262 return err;
11581263 }
11591264
@@ -1161,11 +1266,9 @@ free_xlt: · 1 unmodified line
11611266 * If ibmr is NULL it will be allocated by reg_create.
11621267 * Else, the given ibmr will be used.
11631268 */
1164−static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
1165− u64 virt_addr, u64 length,
1166− struct ib_umem *umem, int npages,
1167− int page_shift, int access_flags,
1168− bool populate)
1269+static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
1270+ u64 iova, int access_flags,
1271+ unsigned int page_size, bool populate)
11691272 {
11701273 struct mlx5_ib_dev *dev = to_mdev(pd->device);
11711274 struct mlx5_ib_mr *mr;
@@ -1176,16 +1279,20 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, · 4 unmodified lines
11761279 int err;
11771280 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
11781281
1179− mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);
1282+ if (!page_size)
1283+ return ERR_PTR(-EINVAL);
1284+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
11801285 if (!mr)
11811286 return ERR_PTR(-ENOMEM);
11821287
11831288 mr->ibmr.pd = pd;
11841289 mr->access_flags = access_flags;
1290+ mr->page_shift = order_base_2(page_size);
11851291
11861292 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
11871293 if (populate)
1188− inlen += sizeof(*pas) * roundup(npages, 2);
1294+ inlen += sizeof(*pas) *
1295+ roundup(ib_umem_num_dma_blocks(umem, page_size), 2);
11891296 in = kvzalloc(inlen, GFP_KERNEL);
11901297 if (!in) {
11911298 err = -ENOMEM;
@@ -1197,7 +1304,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, · 5 unmodified lines
11971304 err = -EINVAL;
11981305 goto err_2;
11991306 }
1200− mlx5_ib_populate_pas(dev, umem, page_shift, pas,
1307+ mlx5_ib_populate_pas(umem, 1UL << mr->page_shift, pas,
12011308 pg_cap ? MLX5_IB_MTT_PRESENT : 0);
12021309 }
12031310
@@ -1206,20 +1313,20 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, · 2 unmodified lines
12061313 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
12071314
12081315 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1209− set_mkc_access_pd_addr_fields(mkc, access_flags, virt_addr,
1316+ set_mkc_access_pd_addr_fields(mkc, access_flags, iova,
12101317 populate ? pd : dev->umrc.pd);
12111318 MLX5_SET(mkc, mkc, free, !populate);
12121319 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
12131320 MLX5_SET(mkc, mkc, umr_en, 1);
12141321
1215− MLX5_SET64(mkc, mkc, len, length);
1322+ MLX5_SET64(mkc, mkc, len, umem->length);
12161323 MLX5_SET(mkc, mkc, bsf_octword_size, 0);
12171324 MLX5_SET(mkc, mkc, translations_octword_size,
1218− get_octo_len(virt_addr, length, page_shift));
1219− MLX5_SET(mkc, mkc, log_page_size, page_shift);
1325+ get_octo_len(iova, umem->length, mr->page_shift));
1326+ MLX5_SET(mkc, mkc, log_page_size, mr->page_shift);
12201327 if (populate) {
12211328 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
1222− get_octo_len(virt_addr, length, page_shift));
1329+ get_octo_len(iova, umem->length, mr->page_shift));
12231330 }
12241331
12251332 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
@@ -1229,7 +1336,8 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, · 3 unmodified lines
12291336 }
12301337 mr->mmkey.type = MLX5_MKEY_MR;
12311338 mr->desc_size = sizeof(struct mlx5_mtt);
1232− mr->dev = dev;
1339+ mr->umem = umem;
1340+ set_mr_fields(dev, mr, umem->length, access_flags);
12331341 kvfree(in);
12341342
12351343 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
@@ -1238,25 +1346,11 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, · 2 unmodified lines
12381346
12391347 err_2:
12401348 kvfree(in);
1241−
12421349 err_1:
1243− if (!ibmr)
1244− kfree(mr);
1245−
1350+ kfree(mr);
12461351 return ERR_PTR(err);
12471352 }
12481353
1249−static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
1250− int npages, u64 length, int access_flags)
1251−{
1252− mr->npages = npages;
1253− atomic_add(npages, &dev->mdev->priv.reg_pages);
1254− mr->ibmr.lkey = mr->mmkey.key;
1255− mr->ibmr.rkey = mr->mmkey.key;
1256− mr->ibmr.length = length;
1257− mr->access_flags = access_flags;
1258−}
1259−
12601354 static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,
12611355 u64 length, int acc, int mode)
12621356 {
@@ -1290,8 +1384,7 @@ static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr, · 27 unmodified lines
12901384
12911385 kfree(in);
12921386
1293− mr->umem = NULL;
1294− set_mr_fields(dev, mr, 0, length, acc);
1387+ set_mr_fields(dev, mr, length, acc);
12951388
12961389 return &mr->ibmr;
12971390
@@ -1352,116 +1445,128 @@ struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, · 54 unmodified lines
13521445 attr->access_flags, mode);
13531446 }
13541447
1355−struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1356− u64 virt_addr, int access_flags,
1357− struct ib_udata *udata)
1448+static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem,
1449+ u64 iova, int access_flags)
13581450 {
13591451 struct mlx5_ib_dev *dev = to_mdev(pd->device);
13601452 struct mlx5_ib_mr *mr = NULL;
13611453 bool xlt_with_umr;
1362− struct ib_umem *umem;
1363− int page_shift;
1364− int npages;
1365− int ncont;
1366− int order;
13671454 int err;
13681455
1369− if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
1370− return ERR_PTR(-EOPNOTSUPP);
1371−
1372− mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1373− start, virt_addr, length, access_flags);
1374−
1375− xlt_with_umr = mlx5_ib_can_load_pas_with_umr(dev, length);
1376− /* ODP requires xlt update via umr to work. */
1377− if (!xlt_with_umr && (access_flags & IB_ACCESS_ON_DEMAND))
1378− return ERR_PTR(-EINVAL);
1379−
1380− if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && !start &&
1381− length == U64_MAX) {
1382− if (virt_addr != start)
1383− return ERR_PTR(-EINVAL);
1384− if (!(access_flags & IB_ACCESS_ON_DEMAND) ||
1385− !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1386− return ERR_PTR(-EINVAL);
1387−
1388− mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags);
1389− if (IS_ERR(mr))
1390− return ERR_CAST(mr);
1391− return &mr->ibmr;
1392− }
1393−
1394− err = mr_umem_get(dev, start, length, access_flags, &umem,
1395− &npages, &page_shift, &ncont, &order);
1396−
1397− if (err < 0)
1398− return ERR_PTR(err);
1399−
1456+ xlt_with_umr = mlx5_ib_can_load_pas_with_umr(dev, umem->length);
14001457 if (xlt_with_umr) {
1401− mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
1402− page_shift, order, access_flags);
1403− if (IS_ERR(mr))
1404− mr = NULL;
1405− }
1458+ mr = alloc_cacheable_mr(pd, umem, iova, access_flags);
1459+ } else {
1460+ unsigned int page_size = mlx5_umem_find_best_pgsz(
1461+ umem, mkc, log_page_size, 0, iova);
14061462
1407− if (!mr) {
14081463 mutex_lock(&dev->slow_path_mutex);
1409− mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
1410− page_shift, access_flags, !xlt_with_umr);
1464+ mr = reg_create(pd, umem, iova, access_flags, page_size, true);
14111465 mutex_unlock(&dev->slow_path_mutex);
14121466 }
1413−
14141467 if (IS_ERR(mr)) {
1415− err = PTR_ERR(mr);
1416− goto error;
1468+ ib_umem_release(umem);
1469+ return ERR_CAST(mr);
14171470 }
14181471
14191472 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);
14201473
1421− mr->umem = umem;
1422− set_mr_fields(dev, mr, npages, length, access_flags);
1474+ atomic_add(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages);
14231475
1424− if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) {
1476+ if (xlt_with_umr) {
14251477 /*
14261478 * If the MR was created with reg_create then it will be
14271479 * configured properly but left disabled. It is safe to go ahead
14281480 * and configure it again via UMR while enabling it.
14291481 */
1430− int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
1431−
1432− err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
1433− update_xlt_flags);
1482+ err = mlx5_ib_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE);
14341483 if (err) {
14351484 dereg_mr(dev, mr);
14361485 return ERR_PTR(err);
14371486 }
14381487 }
1488+ return &mr->ibmr;
1489+}
14391490
1440− if (is_odp_mr(mr)) {
1441− to_ib_umem_odp(mr->umem)->private = mr;
1442− init_waitqueue_head(&mr->q_deferred_work);
1443− atomic_set(&mr->num_deferred_work, 0);
1444− err = xa_err(xa_store(&dev->odp_mkeys,
1445− mlx5_base_mkey(mr->mmkey.key), &mr->mmkey,
1446− GFP_KERNEL));
1447− if (err) {
1448− dereg_mr(dev, mr);
1449− return ERR_PTR(err);
1450− }
1491+static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length,
1492+ u64 iova, int access_flags,
1493+ struct ib_udata *udata)
1494+{
1495+ struct mlx5_ib_dev *dev = to_mdev(pd->device);
1496+ struct ib_umem_odp *odp;
1497+ struct mlx5_ib_mr *mr;
1498+ int err;
14511499
1452− err = mlx5_ib_init_odp_mr(mr, xlt_with_umr);
1453− if (err) {
1454− dereg_mr(dev, mr);
1455− return ERR_PTR(err);
1456− }
1500+ if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1501+ return ERR_PTR(-EOPNOTSUPP);
1502+
1503+ if (!start && length == U64_MAX) {
1504+ if (iova != 0)
1505+ return ERR_PTR(-EINVAL);
1506+ if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1507+ return ERR_PTR(-EINVAL);
1508+
1509+ mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags);
1510+ if (IS_ERR(mr))
1511+ return ERR_CAST(mr);
1512+ return &mr->ibmr;
1513+ }
1514+
1515+ /* ODP requires xlt update via umr to work. */
1516+ if (!mlx5_ib_can_load_pas_with_umr(dev, length))
1517+ return ERR_PTR(-EINVAL);
1518+
1519+ odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags,
1520+ &mlx5_mn_ops);
1521+ if (IS_ERR(odp))
1522+ return ERR_CAST(odp);
1523+
1524+ mr = alloc_cacheable_mr(pd, &odp->umem, iova, access_flags);
1525+ if (IS_ERR(mr)) {
1526+ ib_umem_release(&odp->umem);
1527+ return ERR_CAST(mr);
14571528 }
14581529
1530+ odp->private = mr;
1531+ init_waitqueue_head(&mr->q_deferred_work);
1532+ atomic_set(&mr->num_deferred_work, 0);
1533+ err = xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key),
1534+ &mr->mmkey, GFP_KERNEL));
1535+ if (err)
1536+ goto err_dereg_mr;
1537+
1538+ err = mlx5_ib_init_odp_mr(mr);
1539+ if (err)
1540+ goto err_dereg_mr;
14591541 return &mr->ibmr;
1460−error:
1461− ib_umem_release(umem);
1542+
1543+err_dereg_mr:
1544+ dereg_mr(dev, mr);
14621545 return ERR_PTR(err);
14631546 }
14641547
1548+struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1549+ u64 iova, int access_flags,
1550+ struct ib_udata *udata)
1551+{
1552+ struct mlx5_ib_dev *dev = to_mdev(pd->device);
1553+ struct ib_umem *umem;
1554+
1555+ if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
1556+ return ERR_PTR(-EOPNOTSUPP);
1557+
1558+ mlx5_ib_dbg(dev, "start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n",
1559+ start, iova, length, access_flags);
1560+
1561+ if (access_flags & IB_ACCESS_ON_DEMAND)
1562+ return create_user_odp_mr(pd, start, length, iova, access_flags,
1563+ udata);
1564+ umem = ib_umem_get(&dev->ib_dev, start, length, access_flags);
1565+ if (IS_ERR(umem))
1566+ return ERR_CAST(umem);
1567+ return create_real_mr(pd, umem, iova, access_flags);
1568+}
1569+
14651570 /**
14661571 * mlx5_mr_cache_invalidate - Fence all DMA on the MR
14671572 * @mr: The MR to fence
@@ -1474,151 +1579,224 @@ int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr) · 6 unmodified lines
14741579 {
14751580 struct mlx5_umr_wr umrwr = {};
14761581
1477− if (mr->dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1582+ if (mr_to_mdev(mr)->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
14781583 return 0;
14791584
14801585 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
14811586 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
14821587 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1483− umrwr.pd = mr->dev->umrc.pd;
1588+ umrwr.pd = mr_to_mdev(mr)->umrc.pd;
14841589 umrwr.mkey = mr->mmkey.key;
14851590 umrwr.ignore_free_state = 1;
14861591
1487− return mlx5_ib_post_send_wait(mr->dev, &umrwr);
1592+ return mlx5_ib_post_send_wait(mr_to_mdev(mr), &umrwr);
14881593 }
14891594
1490−static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1491− int access_flags, int flags)
1492−{
1493− struct mlx5_ib_dev *dev = to_mdev(pd->device);
1494− struct mlx5_umr_wr umrwr = {};
1595+/*
1596+ * True if the change in access flags can be done via UMR, only some access
1597+ * flags can be updated.
1598+ */
1599+static bool can_use_umr_rereg_access(struct mlx5_ib_dev *dev,
1600+ unsigned int current_access_flags,
1601+ unsigned int target_access_flags)
1602+{
1603+ unsigned int diffs = current_access_flags ^ target_access_flags;
1604+
1605+ if (diffs & ~(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE |
1606+ IB_ACCESS_REMOTE_READ | IB_ACCESS_RELAXED_ORDERING))
1607+ return false;
1608+ return mlx5_ib_can_reconfig_with_umr(dev, current_access_flags,
1609+ target_access_flags);
1610+}
1611+
1612+static int umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd,
1613+ int access_flags)
1614+{
1615+ struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);
1616+ struct mlx5_umr_wr umrwr = {
1617+ .wr = {
1618+ .send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE |
1619+ MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS,
1620+ .opcode = MLX5_IB_WR_UMR,
1621+ },
1622+ .mkey = mr->mmkey.key,
1623+ .pd = pd,
1624+ .access_flags = access_flags,
1625+ };
14951626 int err;
14961627
1497− umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1498−
1499− umrwr.wr.opcode = MLX5_IB_WR_UMR;
1500− umrwr.mkey = mr->mmkey.key;
1501−
1502− if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
1503− umrwr.pd = pd;
1504− umrwr.access_flags = access_flags;
1505− umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1506− }
1507−
15081628 err = mlx5_ib_post_send_wait(dev, &umrwr);
1629+ if (err)
1630+ return err;
15091631
1510− return err;
1632+ mr->access_flags = access_flags;
1633+ mr->mmkey.pd = to_mpd(pd)->pdn;
1634+ return 0;
15111635 }
15121636
1513−int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1514− u64 length, u64 virt_addr, int new_access_flags,
1515− struct ib_pd *new_pd, struct ib_udata *udata)
1637+static bool can_use_umr_rereg_pas(struct mlx5_ib_mr *mr,
1638+ struct ib_umem *new_umem,
1639+ int new_access_flags, u64 iova,
1640+ unsigned long *page_size)
15161641 {
1517− struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
1518− struct mlx5_ib_mr *mr = to_mmr(ib_mr);
1519− struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd;
1520− int access_flags = flags & IB_MR_REREG_ACCESS ?
1521− new_access_flags :
1522− mr->access_flags;
1523− int page_shift = 0;
1524− int upd_flags = 0;
1525− int npages = 0;
1526− int ncont = 0;
1527− int order = 0;
1528− u64 addr, len;
1529− int err;
1642+ struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);
15301643
1531− mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1532− start, virt_addr, length, access_flags);
1644+ /* We only track the allocated sizes of MRs from the cache */
1645+ if (!mr->cache_ent)
1646+ return false;
1647+ if (!mlx5_ib_can_load_pas_with_umr(dev, new_umem->length))
1648+ return false;
15331649
1534− atomic_sub(mr->npages, &dev->mdev->priv.reg_pages);
1650+ *page_size =
1651+ mlx5_umem_find_best_pgsz(new_umem, mkc, log_page_size, 0, iova);
1652+ if (WARN_ON(!*page_size))
1653+ return false;
1654+ return (1ULL << mr->cache_ent->order) >=
1655+ ib_umem_num_dma_blocks(new_umem, *page_size);
1656+}
15351657
1536− if (!mr->umem)
1537− return -EINVAL;
1658+static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd,
1659+ int access_flags, int flags, struct ib_umem *new_umem,
1660+ u64 iova, unsigned long page_size)
1661+{
1662+ struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);
1663+ int upd_flags = MLX5_IB_UPD_XLT_ADDR | MLX5_IB_UPD_XLT_ENABLE;
1664+ struct ib_umem *old_umem = mr->umem;
1665+ int err;
15381666
1539− if (is_odp_mr(mr))
1540− return -EOPNOTSUPP;
1667+ /*
1668+ * To keep everything simple the MR is revoked before we start to mess
1669+ * with it. This ensure the change is atomic relative to any use of the
1670+ * MR.
1671+ */
1672+ err = mlx5_mr_cache_invalidate(mr);
1673+ if (err)
1674+ return err;
15411675
1542− if (flags & IB_MR_REREG_TRANS) {
1543− addr = virt_addr;
1544− len = length;
1545− } else {
1546− addr = mr->umem->address;
1547− len = mr->umem->length;
1676+ if (flags & IB_MR_REREG_PD) {
1677+ mr->ibmr.pd = pd;
1678+ mr->mmkey.pd = to_mpd(pd)->pdn;
1679+ upd_flags |= MLX5_IB_UPD_XLT_PD;
1680+ }
1681+ if (flags & IB_MR_REREG_ACCESS) {
1682+ mr->access_flags = access_flags;
1683+ upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
15481684 }
15491685
1550− if (flags != IB_MR_REREG_PD) {
1686+ mr->ibmr.length = new_umem->length;
1687+ mr->mmkey.iova = iova;
1688+ mr->mmkey.size = new_umem->length;
1689+ mr->page_shift = order_base_2(page_size);
1690+ mr->umem = new_umem;
1691+ err = mlx5_ib_update_mr_pas(mr, upd_flags);
1692+ if (err) {
15511693 /*
1552− * Replace umem. This needs to be done whether or not UMR is
1553− * used.
1694+ * The MR is revoked at this point so there is no issue to free
1695+ * new_umem.
15541696 */
1555− flags |= IB_MR_REREG_TRANS;
1556− ib_umem_release(mr->umem);
1557− mr->umem = NULL;
1558− err = mr_umem_get(dev, addr, len, access_flags, &mr->umem,
1559− &npages, &page_shift, &ncont, &order);
1560− if (err)
1561− goto err;
1697+ mr->umem = old_umem;
1698+ return err;
15621699 }
15631700
1564− if (!mlx5_ib_can_reconfig_with_umr(dev, mr->access_flags,
1565− access_flags) ||
1566− !mlx5_ib_can_load_pas_with_umr(dev, len) ||
1567− (flags & IB_MR_REREG_TRANS &&
1568− !mlx5_ib_pas_fits_in_mr(mr, addr, len))) {
1569− /*
1570− * UMR can't be used - MKey needs to be replaced.
1571− */
1572− if (mr->cache_ent)
1573− detach_mr_from_cache(mr);
1574− err = destroy_mkey(dev, mr);
1575− if (err)
1576− goto err;
1701+ atomic_sub(ib_umem_num_pages(old_umem), &dev->mdev->priv.reg_pages);
1702+ ib_umem_release(old_umem);
1703+ atomic_add(ib_umem_num_pages(new_umem), &dev->mdev->priv.reg_pages);
1704+ return 0;
1705+}
15771706
1578− mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
1579− page_shift, access_flags, true);
1707+struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1708+ u64 length, u64 iova, int new_access_flags,
1709+ struct ib_pd *new_pd,
1710+ struct ib_udata *udata)
1711+{
1712+ struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
1713+ struct mlx5_ib_mr *mr = to_mmr(ib_mr);
1714+ int err;
15801715
1581− if (IS_ERR(mr)) {
1582− err = PTR_ERR(mr);
1583− mr = to_mmr(ib_mr);
1584− goto err;
1585− }
1586− } else {
1587− /*
1588− * Send a UMR WQE
1589− */
1590− mr->ibmr.pd = pd;
1591− mr->access_flags = access_flags;
1592− mr->mmkey.iova = addr;
1593− mr->mmkey.size = len;
1594− mr->mmkey.pd = to_mpd(pd)->pdn;
1716+ if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
1717+ return ERR_PTR(-EOPNOTSUPP);
15951718
1596− if (flags & IB_MR_REREG_TRANS) {
1597− upd_flags = MLX5_IB_UPD_XLT_ADDR;
1598− if (flags & IB_MR_REREG_PD)
1599− upd_flags |= MLX5_IB_UPD_XLT_PD;
1600− if (flags & IB_MR_REREG_ACCESS)
1601− upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
1602− err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
1603− upd_flags);
1604− } else {
1605− err = rereg_umr(pd, mr, access_flags, flags);
1719+ mlx5_ib_dbg(
1720+ dev,
1721+ "start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n",
1722+ start, iova, length, new_access_flags);
1723+
1724+ if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
1725+ return ERR_PTR(-EOPNOTSUPP);
1726+
1727+ if (!(flags & IB_MR_REREG_ACCESS))
1728+ new_access_flags = mr->access_flags;
1729+ if (!(flags & IB_MR_REREG_PD))
1730+ new_pd = ib_mr->pd;
1731+
1732+ if (!(flags & IB_MR_REREG_TRANS)) {
1733+ struct ib_umem *umem;
1734+
1735+ /* Fast path for PD/access change */
1736+ if (can_use_umr_rereg_access(dev, mr->access_flags,
1737+ new_access_flags)) {
1738+ err = umr_rereg_pd_access(mr, new_pd, new_access_flags);
1739+ if (err)
1740+ return ERR_PTR(err);
1741+ return NULL;
16061742 }
1743+ /* DM or ODP MR's don't have a umem so we can't re-use it */
1744+ if (!mr->umem || is_odp_mr(mr))
1745+ goto recreate;
16071746
1747+ /*
1748+ * Only one active MR can refer to a umem at one time, revoke
1749+ * the old MR before assigning the umem to the new one.
1750+ */
1751+ err = mlx5_mr_cache_invalidate(mr);
16081752 if (err)
1609− goto err;
1610− }
1611−
1612− set_mr_fields(dev, mr, npages, len, access_flags);
1753+ return ERR_PTR(err);
1754+ umem = mr->umem;
1755+ mr->umem = NULL;
1756+ atomic_sub(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages);
16131757
1614− return 0;
1758+ return create_real_mr(new_pd, umem, mr->mmkey.iova,
1759+ new_access_flags);
1760+ }
16151761
1616−err:
1617− ib_umem_release(mr->umem);
1618− mr->umem = NULL;
1762+ /*
1763+ * DM doesn't have a PAS list so we can't re-use it, odp does but the
1764+ * logic around releasing the umem is different
1765+ */
1766+ if (!mr->umem || is_odp_mr(mr))
1767+ goto recreate;
1768+
1769+ if (!(new_access_flags & IB_ACCESS_ON_DEMAND) &&
1770+ can_use_umr_rereg_access(dev, mr->access_flags, new_access_flags)) {
1771+ struct ib_umem *new_umem;
1772+ unsigned long page_size;
1773+
1774+ new_umem = ib_umem_get(&dev->ib_dev, start, length,
1775+ new_access_flags);
1776+ if (IS_ERR(new_umem))
1777+ return ERR_CAST(new_umem);
1778+
1779+ /* Fast path for PAS change */
1780+ if (can_use_umr_rereg_pas(mr, new_umem, new_access_flags, iova,
1781+ &page_size)) {
1782+ err = umr_rereg_pas(mr, new_pd, new_access_flags, flags,
1783+ new_umem, iova, page_size);
1784+ if (err) {
1785+ ib_umem_release(new_umem);
1786+ return ERR_PTR(err);
1787+ }
1788+ return NULL;
1789+ }
1790+ return create_real_mr(new_pd, new_umem, iova, new_access_flags);
1791+ }
16191792
1620− clean_mr(dev, mr);
1621− return err;
1793+ /*
1794+ * Everything else has no state we can preserve, just create a new MR
1795+ * from scratch
1796+ */
1797+recreate:
1798+ return mlx5_ib_reg_user_mr(new_pd, start, length, iova,
1799+ new_access_flags, udata);
16221800 }
16231801
16241802 static int
@@ -1627,6 +1805,8 @@ mlx5_alloc_priv_descs(struct ib_device *device, · 2 unmodified lines
16271805 int ndescs,
16281806 int desc_size)
16291807 {
1808+ struct mlx5_ib_dev *dev = to_mdev(device);
1809+ struct device *ddev = &dev->mdev->pdev->dev;
16301810 int size = ndescs * desc_size;
16311811 int add_size;
16321812 int ret;
@@ -1639,9 +1819,8 @@ mlx5_alloc_priv_descs(struct ib_device *device, · 6 unmodified lines
16391819
16401820 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN);
16411821
1642− mr->desc_map = dma_map_single(device->dev.parent, mr->descs,
1643− size, DMA_TO_DEVICE);
1644− if (dma_mapping_error(device->dev.parent, mr->desc_map)) {
1822+ mr->desc_map = dma_map_single(ddev, mr->descs, size, DMA_TO_DEVICE);
1823+ if (dma_mapping_error(ddev, mr->desc_map)) {
16451824 ret = -ENOMEM;
16461825 goto err;
16471826 }
@@ -1659,9 +1838,10 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr) · 11 unmodified lines
16591838 if (mr->descs) {
16601839 struct ib_device *device = mr->ibmr.device;
16611840 int size = mr->max_descs * mr->desc_size;
1841+ struct mlx5_ib_dev *dev = to_mdev(device);
16621842
1663− dma_unmap_single(device->dev.parent, mr->desc_map,
1664− size, DMA_TO_DEVICE);
1843+ dma_unmap_single(&dev->mdev->pdev->dev, mr->desc_map, size,
1844+ DMA_TO_DEVICE);
16651845 kfree(mr->descs_alloc);
16661846 mr->descs = NULL;
16671847 }
@@ -1691,7 +1871,6 @@ static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) · 23 unmodified lines
16911871
16921872 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
16931873 {
1694− int npages = mr->npages;
16951874 struct ib_umem *umem = mr->umem;
16961875
16971876 /* Stop all DMA */
@@ -1700,14 +1879,17 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) · 2 unmodified lines
17001879 else
17011880 clean_mr(dev, mr);
17021881
1882+ if (umem) {
1883+ if (!is_odp_mr(mr))
1884+ atomic_sub(ib_umem_num_pages(umem),
1885+ &dev->mdev->priv.reg_pages);
1886+ ib_umem_release(umem);
1887+ }
1888+
17031889 if (mr->cache_ent)
17041890 mlx5_mr_cache_free(dev, mr);
17051891 else
17061892 kfree(mr);
1707−
1708− ib_umem_release(umem);
1709− atomic_sub(npages, &dev->mdev->priv.reg_pages);
1710−
17111893 }
17121894
17131895 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
drivers/infiniband/hw/mlx5/odp.c+29−27
@@ -102,7 +102,7 @@ static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, · 101 unmodified lines
102102 if (flags & MLX5_IB_UPD_XLT_ZAP) {
103103 for (; pklm != end; pklm++, idx++) {
104104 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
105− pklm->key = cpu_to_be32(imr->dev->null_mkey);
105+ pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey);
106106 pklm->va = 0;
107107 }
108108 return;
@@ -129,7 +129,7 @@ static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, · 20 unmodified lines
129129 * locking around the xarray.
130130 */
131131 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex);
132− lockdep_assert_held(&imr->dev->odp_srcu);
132+ lockdep_assert_held(&mr_to_mdev(imr)->odp_srcu);
133133
134134 for (; pklm != end; pklm++, idx++) {
135135 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx);
@@ -139,7 +139,7 @@ static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, · 3 unmodified lines
139139 pklm->key = cpu_to_be32(mtt->ibmr.lkey);
140140 pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE);
141141 } else {
142− pklm->key = cpu_to_be32(imr->dev->null_mkey);
142+ pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey);
143143 pklm->va = 0;
144144 }
145145 }
@@ -199,7 +199,7 @@ static void dma_fence_odp_mr(struct mlx5_ib_mr *mr) · 53 unmodified lines
199199 mutex_unlock(&odp->umem_mutex);
200200
201201 if (!mr->cache_ent) {
202− mlx5_core_destroy_mkey(mr->dev->mdev, &mr->mmkey);
202+ mlx5_core_destroy_mkey(mr_to_mdev(mr)->mdev, &mr->mmkey);
203203 WARN_ON(mr->descs);
204204 }
205205 }
@@ -222,19 +222,19 @@ static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt) · 16 unmodified lines
222222 WARN_ON(atomic_read(&mr->num_deferred_work));
223223
224224 if (need_imr_xlt) {
225− srcu_key = srcu_read_lock(&mr->dev->odp_srcu);
225+ srcu_key = srcu_read_lock(&mr_to_mdev(mr)->odp_srcu);
226226 mutex_lock(&odp_imr->umem_mutex);
227227 mlx5_ib_update_xlt(mr->parent, idx, 1, 0,
228228 MLX5_IB_UPD_XLT_INDIRECT |
229229 MLX5_IB_UPD_XLT_ATOMIC);
230230 mutex_unlock(&odp_imr->umem_mutex);
231− srcu_read_unlock(&mr->dev->odp_srcu, srcu_key);
231+ srcu_read_unlock(&mr_to_mdev(mr)->odp_srcu, srcu_key);
232232 }
233233
234234 dma_fence_odp_mr(mr);
235235
236236 mr->parent = NULL;
237− mlx5_mr_cache_free(mr->dev, mr);
237+ mlx5_mr_cache_free(mr_to_mdev(mr), mr);
238238 ib_umem_odp_release(odp);
239239 if (atomic_dec_and_test(&imr->num_deferred_work))
240240 wake_up(&imr->q_deferred_work);
@@ -274,7 +274,7 @@ static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr) · 33 unmodified lines
274274 goto out_unlock;
275275
276276 atomic_inc(&imr->num_deferred_work);
277− call_srcu(&mr->dev->odp_srcu, &mr->odp_destroy.rcu,
277+ call_srcu(&mr_to_mdev(mr)->odp_srcu, &mr->odp_destroy.rcu,
278278 free_implicit_child_mr_rcu);
279279
280280 out_unlock:
@@ -476,12 +476,13 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, · 195 unmodified lines
476476 if (IS_ERR(odp))
477477 return ERR_CAST(odp);
478478
479− ret = mr = mlx5_mr_cache_alloc(imr->dev, MLX5_IMR_MTT_CACHE_ENTRY,
480− imr->access_flags);
479+ ret = mr = mlx5_mr_cache_alloc(
480+ mr_to_mdev(imr), MLX5_IMR_MTT_CACHE_ENTRY, imr->access_flags);
481481 if (IS_ERR(mr))
482482 goto out_umem;
483483
484484 mr->ibmr.pd = imr->ibmr.pd;
485+ mr->ibmr.device = &mr_to_mdev(imr)->ib_dev;
485486 mr->umem = &odp->umem;
486487 mr->ibmr.lkey = mr->mmkey.key;
487488 mr->ibmr.rkey = mr->mmkey.key;
@@ -517,11 +518,11 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, · 29 unmodified lines
517518 goto out_mr;
518519 }
519520
520− mlx5_ib_dbg(imr->dev, "key %x mr %p\n", mr->mmkey.key, mr);
521+ mlx5_ib_dbg(mr_to_mdev(imr), "key %x mr %p\n", mr->mmkey.key, mr);
521522 return mr;
522523
523524 out_mr:
524− mlx5_mr_cache_free(imr->dev, mr);
525+ mlx5_mr_cache_free(mr_to_mdev(imr), mr);
525526 out_umem:
526527 ib_umem_odp_release(odp);
527528 return ret;
@@ -536,6 +537,10 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, · 8 unmodified lines
536537 struct mlx5_ib_mr *imr;
537538 int err;
538539
540+ if (!mlx5_ib_can_load_pas_with_umr(dev,
541+ MLX5_IMR_MTT_ENTRIES * PAGE_SIZE))
542+ return ERR_PTR(-EOPNOTSUPP);
543+
539544 umem_odp = ib_umem_odp_alloc_implicit(&dev->ib_dev, access_flags);
540545 if (IS_ERR(umem_odp))
541546 return ERR_CAST(umem_odp);
@@ -551,6 +556,7 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, · 9 unmodified lines
551556 imr->umem = &umem_odp->umem;
552557 imr->ibmr.lkey = imr->mmkey.key;
553558 imr->ibmr.rkey = imr->mmkey.key;
559+ imr->ibmr.device = &dev->ib_dev;
554560 imr->umem = &umem_odp->umem;
555561 imr->is_odp_implicit = true;
556562 atomic_set(&imr->num_deferred_work, 0);
@@ -584,7 +590,7 @@ out_umem: · 27 unmodified lines
584590 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)
585591 {
586592 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem);
587− struct mlx5_ib_dev *dev = imr->dev;
593+ struct mlx5_ib_dev *dev = mr_to_mdev(imr);
588594 struct list_head destroy_list;
589595 struct mlx5_ib_mr *mtt;
590596 struct mlx5_ib_mr *tmp;
@@ -654,10 +660,10 @@ void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) · 63 unmodified lines
654660 void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr)
655661 {
656662 /* Prevent new page faults and prefetch requests from succeeding */
657− xa_erase(&mr->dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key));
663+ xa_erase(&mr_to_mdev(mr)->odp_mkeys, mlx5_base_mkey(mr->mmkey.key));
658664
659665 /* Wait for all running page-fault handlers to finish. */
660− synchronize_srcu(&mr->dev->odp_srcu);
666+ synchronize_srcu(&mr_to_mdev(mr)->odp_srcu);
661667
662668 wait_event(mr->q_deferred_work, !atomic_read(&mr->num_deferred_work));
663669
@@ -701,7 +707,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, · 37 unmodified lines
701707
702708 if (ret < 0) {
703709 if (ret != -EAGAIN)
704− mlx5_ib_err(mr->dev,
710+ mlx5_ib_err(mr_to_mdev(mr),
705711 "Failed to update mkey page tables\n");
706712 goto out;
707713 }
@@ -791,7 +797,7 @@ out: · 83 unmodified lines
791797 MLX5_IB_UPD_XLT_ATOMIC);
792798 mutex_unlock(&odp_imr->umem_mutex);
793799 if (err) {
794− mlx5_ib_err(imr->dev, "Failed to update PAS\n");
800+ mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n");
795801 return err;
796802 }
797803 return ret;
@@ -811,7 +817,7 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, · 13 unmodified lines
811817 {
812818 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
813819
814− lockdep_assert_held(&mr->dev->odp_srcu);
820+ lockdep_assert_held(&mr_to_mdev(mr)->odp_srcu);
815821 if (unlikely(io_virt < mr->mmkey.iova))
816822 return -EFAULT;
817823
@@ -831,17 +837,13 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, · 13 unmodified lines
831837 flags);
832838 }
833839
834−int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, bool enable)
840+int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr)
835841 {
836− u32 flags = MLX5_PF_FLAGS_SNAPSHOT;
837842 int ret;
838843
839− if (enable)
840− flags |= MLX5_PF_FLAGS_ENABLE;
841−
842− ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem),
843− mr->umem->address, mr->umem->length, NULL,
844− flags);
844+ ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem), mr->umem->address,
845+ mr->umem->length, NULL,
846+ MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE);
845847 return ret >= 0 ? 0 : ret;
846848 }
847849
@@ -1783,7 +1785,7 @@ static void mlx5_ib_prefetch_mr_work(struct work_struct *w) · 935 unmodified lines
17831785
17841786 /* We rely on IB/core that work is executed if we have num_sge != 0 only. */
17851787 WARN_ON(!work->num_sge);
1786− dev = work->frags[0].mr->dev;
1788+ dev = mr_to_mdev(work->frags[0].mr);
17871789 /* SRCU should be held when calling to mlx5_odp_populate_xlt() */
17881790 srcu_key = srcu_read_lock(&dev->odp_srcu);
17891791 for (i = 0; i < work->num_sge; ++i) {
drivers/infiniband/hw/mlx5/qp.c+89−108
@@ -778,39 +778,6 @@ int bfregn_to_uar_index(struct mlx5_ib_dev *dev, · 777 unmodified lines
778778 return bfregi->sys_pages[index_of_sys_page] + offset;
779779 }
780780
781−static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev, struct ib_udata *udata,
782− unsigned long addr, size_t size,
783− struct ib_umem **umem, int *npages, int *page_shift,
784− int *ncont, u32 *offset)
785−{
786− int err;
787−
788− *umem = ib_umem_get(&dev->ib_dev, addr, size, 0);
789− if (IS_ERR(*umem)) {
790− mlx5_ib_dbg(dev, "umem_get failed\n");
791− return PTR_ERR(*umem);
792− }
793−
794− mlx5_ib_cont_pages(*umem, addr, 0, npages, page_shift, ncont, NULL);
795−
796− err = mlx5_ib_get_buf_offset(addr, *page_shift, offset);
797− if (err) {
798− mlx5_ib_warn(dev, "bad offset\n");
799− goto err_umem;
800− }
801−
802− mlx5_ib_dbg(dev, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n",
803− addr, size, *npages, *page_shift, *ncont, *offset);
804−
805− return 0;
806−
807−err_umem:
808− ib_umem_release(*umem);
809− *umem = NULL;
810−
811− return err;
812−}
813−
814781 static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
815782 struct mlx5_ib_rwq *rwq, struct ib_udata *udata)
816783 {
@@ -833,10 +800,8 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, · 16 unmodified lines
833800 {
834801 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
835802 udata, struct mlx5_ib_ucontext, ibucontext);
836− int page_shift = 0;
837− int npages;
803+ unsigned long page_size = 0;
838804 u32 offset = 0;
839− int ncont = 0;
840805 int err;
841806
842807 if (!ucmd->buf_addr)
@@ -849,23 +814,26 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, · 6 unmodified lines
849814 return err;
850815 }
851816
852− mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, 0, &npages, &page_shift,
853− &ncont, NULL);
854− err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift,
855− &rwq->rq_page_offset);
856− if (err) {
817+ page_size = mlx5_umem_find_best_quantized_pgoff(
818+ rwq->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
819+ page_offset, 64, &rwq->rq_page_offset);
820+ if (!page_size) {
857821 mlx5_ib_warn(dev, "bad offset\n");
822+ err = -EINVAL;
858823 goto err_umem;
859824 }
860825
861− rwq->rq_num_pas = ncont;
862− rwq->page_shift = page_shift;
863− rwq->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
826+ rwq->rq_num_pas = ib_umem_num_dma_blocks(rwq->umem, page_size);
827+ rwq->page_shift = order_base_2(page_size);
828+ rwq->log_page_size = rwq->page_shift - MLX5_ADAPTER_PAGE_SHIFT;
864829 rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
865830
866− mlx5_ib_dbg(dev, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n",
867− (unsigned long long)ucmd->buf_addr, rwq->buf_size,
868− npages, page_shift, ncont, offset);
831+ mlx5_ib_dbg(
832+ dev,
833+ "addr 0x%llx, size %zd, npages %zu, page_size %ld, ncont %d, offset %d\n",
834+ (unsigned long long)ucmd->buf_addr, rwq->buf_size,
835+ ib_umem_num_pages(rwq->umem), page_size, rwq->rq_num_pas,
836+ offset);
869837
870838 err = mlx5_ib_db_map_user(ucontext, udata, ucmd->db_addr, &rwq->db);
871839 if (err) {
@@ -896,10 +864,9 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, · 24 unmodified lines
896864 {
897865 struct mlx5_ib_ucontext *context;
898866 struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
899− int page_shift = 0;
867+ unsigned int page_offset_quantized = 0;
868+ unsigned long page_size = 0;
900869 int uar_index = 0;
901− int npages;
902− u32 offset = 0;
903870 int bfregn;
904871 int ncont = 0;
905872 __be64 *pas;
@@ -950,11 +917,21 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, · 44 unmodified lines
950917
951918 if (ucmd->buf_addr && ubuffer->buf_size) {
952919 ubuffer->buf_addr = ucmd->buf_addr;
953− err = mlx5_ib_umem_get(dev, udata, ubuffer->buf_addr,
954− ubuffer->buf_size, &ubuffer->umem,
955− &npages, &page_shift, &ncont, &offset);
956− if (err)
920+ ubuffer->umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr,
921+ ubuffer->buf_size, 0);
922+ if (IS_ERR(ubuffer->umem)) {
923+ err = PTR_ERR(ubuffer->umem);
957924 goto err_bfreg;
925+ }
926+ page_size = mlx5_umem_find_best_quantized_pgoff(
927+ ubuffer->umem, qpc, log_page_size,
928+ MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64,
929+ &page_offset_quantized);
930+ if (!page_size) {
931+ err = -EINVAL;
932+ goto err_umem;
933+ }
934+ ncont = ib_umem_num_dma_blocks(ubuffer->umem, page_size);
958935 } else {
959936 ubuffer->umem = NULL;
960937 }
@@ -969,15 +946,14 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, · 8 unmodified lines
969946
970947 uid = (attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
971948 MLX5_SET(create_qp_in, *in, uid, uid);
972− pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
973− if (ubuffer->umem)
974− mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
975−
976949 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
977−
978− MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
979− MLX5_SET(qpc, qpc, page_offset, offset);
980−
950+ pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
951+ if (ubuffer->umem) {
952+ mlx5_ib_populate_pas(ubuffer->umem, page_size, pas, 0);
953+ MLX5_SET(qpc, qpc, log_page_size,
954+ order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
955+ MLX5_SET(qpc, qpc, page_offset, page_offset_quantized);
956+ }
981957 MLX5_SET(qpc, qpc, uar_page, uar_index);
982958 if (bfregn != MLX5_IB_INVALID_BFREG)
983959 resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
@@ -1209,18 +1185,24 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, · 225 unmodified lines
12091185 void *wq;
12101186 int inlen;
12111187 int err;
1212− int page_shift = 0;
1213− int npages;
1214− int ncont = 0;
1215− u32 offset = 0;
1216−
1217− err = mlx5_ib_umem_get(dev, udata, ubuffer->buf_addr, ubuffer->buf_size,
1218− &sq->ubuffer.umem, &npages, &page_shift, &ncont,
1219− &offset);
1220− if (err)
1221− return err;
1188+ unsigned int page_offset_quantized;
1189+ unsigned long page_size;
1190+
1191+ sq->ubuffer.umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr,
1192+ ubuffer->buf_size, 0);
1193+ if (IS_ERR(sq->ubuffer.umem))
1194+ return PTR_ERR(sq->ubuffer.umem);
1195+ page_size = mlx5_umem_find_best_quantized_pgoff(
1196+ ubuffer->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
1197+ page_offset, 64, &page_offset_quantized);
1198+ if (!page_size) {
1199+ err = -EINVAL;
1200+ goto err_umem;
1201+ }
12221202
1223− inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * ncont;
1203+ inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1204+ sizeof(u64) *
1205+ ib_umem_num_dma_blocks(sq->ubuffer.umem, page_size);
12241206 in = kvzalloc(inlen, GFP_KERNEL);
12251207 if (!in) {
12261208 err = -ENOMEM;
@@ -1248,11 +1230,12 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, · 21 unmodified lines
12481230 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
12491231 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
12501232 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
1251− MLX5_SET(wq, wq, log_wq_pg_sz, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1252− MLX5_SET(wq, wq, page_offset, offset);
1233+ MLX5_SET(wq, wq, log_wq_pg_sz,
1234+ order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1235+ MLX5_SET(wq, wq, page_offset, page_offset_quantized);
12531236
12541237 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1255− mlx5_ib_populate_pas(dev, sq->ubuffer.umem, page_shift, pas, 0);
1238+ mlx5_ib_populate_pas(sq->ubuffer.umem, page_size, pas, 0);
12561239
12571240 err = mlx5_core_create_sq_tracked(dev, in, inlen, &sq->base.mqp);
12581241
@@ -1278,40 +1261,31 @@ static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev, · 19 unmodified lines
12781261 ib_umem_release(sq->ubuffer.umem);
12791262 }
12801263
1281−static size_t get_rq_pas_size(void *qpc)
1282−{
1283− u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12;
1284− u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride);
1285− u32 log_rq_size = MLX5_GET(qpc, qpc, log_rq_size);
1286− u32 page_offset = MLX5_GET(qpc, qpc, page_offset);
1287− u32 po_quanta = 1 << (log_page_size - 6);
1288− u32 rq_sz = 1 << (log_rq_size + 4 + log_rq_stride);
1289− u32 page_size = 1 << log_page_size;
1290− u32 rq_sz_po = rq_sz + (page_offset * po_quanta);
1291− u32 rq_num_pas = (rq_sz_po + page_size - 1) / page_size;
1292−
1293− return rq_num_pas * sizeof(u64);
1294−}
1295−
12961264 static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
12971265 struct mlx5_ib_rq *rq, void *qpin,
1298− size_t qpinlen, struct ib_pd *pd)
1266+ struct ib_pd *pd)
12991267 {
13001268 struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
13011269 __be64 *pas;
1302− __be64 *qp_pas;
13031270 void *in;
13041271 void *rqc;
13051272 void *wq;
13061273 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1307− size_t rq_pas_size = get_rq_pas_size(qpc);
1274+ struct ib_umem *umem = rq->base.ubuffer.umem;
1275+ unsigned int page_offset_quantized;
1276+ unsigned long page_size = 0;
13081277 size_t inlen;
13091278 int err;
13101279
1311− if (qpinlen < rq_pas_size + MLX5_BYTE_OFF(create_qp_in, pas))
1280+ page_size = mlx5_umem_find_best_quantized_pgoff(umem, wq, log_wq_pg_sz,
1281+ MLX5_ADAPTER_PAGE_SHIFT,
1282+ page_offset, 64,
1283+ &page_offset_quantized);
1284+ if (!page_size)
13121285 return -EINVAL;
13131286
1314− inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size;
1287+ inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1288+ sizeof(u64) * ib_umem_num_dma_blocks(umem, page_size);
13151289 in = kvzalloc(inlen, GFP_KERNEL);
13161290 if (!in)
13171291 return -ENOMEM;
@@ -1333,16 +1307,16 @@ static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev, · 15 unmodified lines
13331307 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
13341308 if (rq->flags & MLX5_IB_RQ_PCI_WRITE_END_PADDING)
13351309 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1336− MLX5_SET(wq, wq, page_offset, MLX5_GET(qpc, qpc, page_offset));
1310+ MLX5_SET(wq, wq, page_offset, page_offset_quantized);
13371311 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
13381312 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
13391313 MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
1340− MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(qpc, qpc, log_page_size));
1314+ MLX5_SET(wq, wq, log_wq_pg_sz,
1315+ order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
13411316 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
13421317
13431318 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1344− qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas);
1345− memcpy(pas, qp_pas, rq_pas_size);
1319+ mlx5_ib_populate_pas(umem, page_size, pas, 0);
13461320
13471321 err = mlx5_core_create_rq_tracked(dev, in, inlen, &rq->base.mqp);
13481322
@@ -1463,7 +1437,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, · 114 unmodified lines
14631437 rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
14641438 if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING)
14651439 rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
1466− err = create_raw_packet_qp_rq(dev, rq, in, inlen, pd);
1440+ err = create_raw_packet_qp_rq(dev, rq, in, pd);
14671441 if (err)
14681442 goto err_destroy_sq;
14691443
@@ -2436,7 +2410,7 @@ static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd, · 966 unmodified lines
24362410 }
24372411
24382412 qp->state = IB_QPS_RESET;
2439−
2413+ rdma_restrack_no_track(&qp->ibqp.res);
24402414 return 0;
24412415 }
24422416
@@ -2460,6 +2434,7 @@ static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, · 17 unmodified lines
24602434 case IB_QPT_GSI:
24612435 if (dev->profile == &raw_eth_profile)
24622436 goto out;
2437+ fallthrough;
24632438 case IB_QPT_RAW_PACKET:
24642439 case IB_QPT_UD:
24652440 case MLX5_IB_QPT_REG_UMR:
@@ -2712,11 +2687,12 @@ static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, · 246 unmodified lines
27122687 process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,
27132688 true, qp);
27142689
2715− if (create_flags)
2690+ if (create_flags) {
27162691 mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",
27172692 create_flags);
2718−
2719− return (create_flags) ? -EINVAL : 0;
2693+ return -EOPNOTSUPP;
2694+ }
2695+ return 0;
27202696 }
27212697
27222698 static int process_udata_size(struct mlx5_ib_dev *dev,
@@ -3102,7 +3078,7 @@ static int ib_to_mlx5_rate_map(u8 rate) · 379 unmodified lines
31023078 return 5;
31033079 default:
31043080 return rate + MLX5_STAT_RATE_OFFSET;
3105− };
3081+ }
31063082
31073083 return 0;
31083084 }
@@ -4247,6 +4223,9 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 1138 unmodified lines
42474223 int err = -EINVAL;
42484224 int port;
42494225
4226+ if (attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
4227+ return -EOPNOTSUPP;
4228+
42504229 if (ibqp->rwq_ind_tbl)
42514230 return -ENOSYS;
42524231
@@ -4576,7 +4555,9 @@ static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, · 323 unmodified lines
45764555 pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
45774556 alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
45784557
4579− if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
4558+ if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC ||
4559+ qp->ibqp.qp_type == IB_QPT_XRC_INI ||
4560+ qp->ibqp.qp_type == IB_QPT_XRC_TGT) {
45804561 to_rdma_ah_attr(dev, &qp_attr->ah_attr, pri_path);
45814562 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, alt_path);
45824563 qp_attr->alt_pkey_index = MLX5_GET(ads, alt_path, pkey_index);
@@ -4882,7 +4863,7 @@ static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd, · 299 unmodified lines
48824863 MLX5_SET(rqc, rqc, delay_drop_en, 1);
48834864 }
48844865 rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
4885− mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);
4866+ mlx5_ib_populate_pas(rwq->umem, 1UL << rwq->page_shift, rq_pas0, 0);
48864867 err = mlx5_core_create_rq_tracked(dev, in, inlen, &rwq->core_qp);
48874868 if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
48884869 err = set_delay_drop(dev);
drivers/infiniband/hw/mlx5/restrack.c+1−1
@@ -116,7 +116,7 @@ static int fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ibmr) · 115 unmodified lines
116116 {
117117 struct mlx5_ib_mr *mr = to_mmr(ibmr);
118118
119− return fill_res_raw(msg, mr->dev, MLX5_SGMT_TYPE_PRM_QUERY_MKEY,
119+ return fill_res_raw(msg, mr_to_mdev(mr), MLX5_SGMT_TYPE_PRM_QUERY_MKEY,
120120 mlx5_mkey_to_idx(mr->mmkey.key));
121121 }
122122
drivers/infiniband/hw/mlx5/srq.c+7−27
@@ -51,10 +51,6 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, · 50 unmodified lines
5151 udata, struct mlx5_ib_ucontext, ibucontext);
5252 size_t ucmdlen;
5353 int err;
54− int npages;
55− int page_shift;
56− int ncont;
57− u32 offset;
5854 u32 uidx = MLX5_IB_DEFAULT_UIDX;
5955
6056 ucmdlen = min(udata->inlen, sizeof(ucmd));
@@ -86,32 +82,14 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, · 25 unmodified lines
8682 err = PTR_ERR(srq->umem);
8783 return err;
8884 }
89−
90− mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages,
91− &page_shift, &ncont, NULL);
92− err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift,
93− &offset);
94− if (err) {
95− mlx5_ib_warn(dev, "bad offset\n");
96− goto err_umem;
97− }
98−
99− in->pas = kvcalloc(ncont, sizeof(*in->pas), GFP_KERNEL);
100− if (!in->pas) {
101− err = -ENOMEM;
102− goto err_umem;
103− }
104−
105− mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0);
85+ in->umem = srq->umem;
10686
10787 err = mlx5_ib_db_map_user(ucontext, udata, ucmd.db_addr, &srq->db);
10888 if (err) {
10989 mlx5_ib_dbg(dev, "map doorbell failed\n");
110− goto err_in;
90+ goto err_umem;
11191 }
11292
113− in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
114− in->page_offset = offset;
11593 in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0;
11694 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
11795 in->type != IB_SRQT_BASIC)
@@ -119,9 +97,6 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, · 1 unmodified line
11997
12098 return 0;
12199
122−err_in:
123− kvfree(in->pas);
124−
125100 err_umem:
126101 ib_umem_release(srq->umem);
127102
@@ -226,6 +201,11 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, · 98 unmodified lines
226201 struct mlx5_srq_attr in = {};
227202 __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
228203
204+ if (init_attr->srq_type != IB_SRQT_BASIC &&
205+ init_attr->srq_type != IB_SRQT_XRC &&
206+ init_attr->srq_type != IB_SRQT_TM)
207+ return -EOPNOTSUPP;
208+
229209 /* Sanity check SRQ size before proceeding */
230210 if (init_attr->attr.max_wr >= max_srq_wqes) {
231211 mlx5_ib_dbg(dev, "max_wr %d, cap %d\n",
drivers/infiniband/hw/mlx5/srq.h+1−0
@@ -28,6 +28,7 @@ struct mlx5_srq_attr { · 27 unmodified lines
2828 u32 user_index;
2929 u64 db_record;
3030 __be64 *pas;
31+ struct ib_umem *umem;
3132 u32 tm_log_list_size;
3233 u32 tm_next_tag;
3334 u32 tm_hw_phase_cnt;
drivers/infiniband/hw/mlx5/srq_cmd.c+76−4
@@ -92,6 +92,25 @@ struct mlx5_core_srq *mlx5_cmd_get_srq(struct mlx5_ib_dev *dev, u32 srqn) · 91 unmodified lines
9292 return srq;
9393 }
9494
95+static int __set_srq_page_size(struct mlx5_srq_attr *in,
96+ unsigned long page_size)
97+{
98+ if (!page_size)
99+ return -EINVAL;
100+ in->log_page_size = order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT;
101+
102+ if (WARN_ON(get_pas_size(in) !=
103+ ib_umem_num_dma_blocks(in->umem, page_size) * sizeof(u64)))
104+ return -EINVAL;
105+ return 0;
106+}
107+
108+#define set_srq_page_size(in, typ, log_pgsz_fld) \
109+ __set_srq_page_size(in, mlx5_umem_find_best_quantized_pgoff( \
110+ (in)->umem, typ, log_pgsz_fld, \
111+ MLX5_ADAPTER_PAGE_SHIFT, page_offset, \
112+ 64, &(in)->page_offset))
113+
95114 static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
96115 struct mlx5_srq_attr *in)
97116 {
@@ -103,6 +122,12 @@ static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 5 unmodified lines
103122 int inlen;
104123 int err;
105124
125+ if (in->umem) {
126+ err = set_srq_page_size(in, srqc, log_page_size);
127+ if (err)
128+ return err;
129+ }
130+
106131 pas_size = get_pas_size(in);
107132 inlen = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;
108133 create_in = kvzalloc(inlen, GFP_KERNEL);
@@ -114,7 +139,13 @@ static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 5 unmodified lines
114139 pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);
115140
116141 set_srqc(srqc, in);
117− memcpy(pas, in->pas, pas_size);
142+ if (in->umem)
143+ mlx5_ib_populate_pas(
144+ in->umem,
145+ 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
146+ pas, 0);
147+ else
148+ memcpy(pas, in->pas, pas_size);
118149
119150 MLX5_SET(create_srq_in, create_in, opcode,
120151 MLX5_CMD_OP_CREATE_SRQ);
@@ -194,6 +225,12 @@ static int create_xrc_srq_cmd(struct mlx5_ib_dev *dev, · 73 unmodified lines
194225 int inlen;
195226 int err;
196227
228+ if (in->umem) {
229+ err = set_srq_page_size(in, xrc_srqc, log_page_size);
230+ if (err)
231+ return err;
232+ }
233+
197234 pas_size = get_pas_size(in);
198235 inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
199236 create_in = kvzalloc(inlen, GFP_KERNEL);
@@ -207,7 +244,13 @@ static int create_xrc_srq_cmd(struct mlx5_ib_dev *dev, · 7 unmodified lines
207244
208245 set_srqc(xrc_srqc, in);
209246 MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);
210− memcpy(pas, in->pas, pas_size);
247+ if (in->umem)
248+ mlx5_ib_populate_pas(
249+ in->umem,
250+ 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
251+ pas, 0);
252+ else
253+ memcpy(pas, in->pas, pas_size);
211254 MLX5_SET(create_xrc_srq_in, create_in, opcode,
212255 MLX5_CMD_OP_CREATE_XRC_SRQ);
213256
@@ -289,11 +332,18 @@ static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 75 unmodified lines
289332 void *create_in = NULL;
290333 void *rmpc;
291334 void *wq;
335+ void *pas;
292336 int pas_size;
293337 int outlen;
294338 int inlen;
295339 int err;
296340
341+ if (in->umem) {
342+ err = set_srq_page_size(in, wq, log_wq_pg_sz);
343+ if (err)
344+ return err;
345+ }
346+
297347 pas_size = get_pas_size(in);
298348 inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
299349 outlen = MLX5_ST_SZ_BYTES(create_rmp_out);
@@ -309,8 +359,16 @@ static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 9 unmodified lines
309359
310360 MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
311361 MLX5_SET(create_rmp_in, create_in, uid, in->uid);
362+ pas = MLX5_ADDR_OF(rmpc, rmpc, wq.pas);
363+
312364 set_wq(wq, in);
313− memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);
365+ if (in->umem)
366+ mlx5_ib_populate_pas(
367+ in->umem,
368+ 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
369+ pas, 0);
370+ else
371+ memcpy(pas, in->pas, pas_size);
314372
315373 MLX5_SET(create_rmp_in, create_in, opcode, MLX5_CMD_OP_CREATE_RMP);
316374 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen);
@@ -421,10 +479,17 @@ static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 104 unmodified lines
421479 void *create_in;
422480 void *xrqc;
423481 void *wq;
482+ void *pas;
424483 int pas_size;
425484 int inlen;
426485 int err;
427486
487+ if (in->umem) {
488+ err = set_srq_page_size(in, wq, log_wq_pg_sz);
489+ if (err)
490+ return err;
491+ }
492+
428493 pas_size = get_pas_size(in);
429494 inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;
430495 create_in = kvzalloc(inlen, GFP_KERNEL);
@@ -433,9 +498,16 @@ static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 2 unmodified lines
433498
434499 xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);
435500 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
501+ pas = MLX5_ADDR_OF(xrqc, xrqc, wq.pas);
436502
437503 set_wq(wq, in);
438− memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size);
504+ if (in->umem)
505+ mlx5_ib_populate_pas(
506+ in->umem,
507+ 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
508+ pas, 0);
509+ else
510+ memcpy(pas, in->pas, pas_size);
439511
440512 if (in->type == IB_SRQT_TM) {
441513 MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);
drivers/infiniband/hw/mthca/mthca_cq.c+1−1
@@ -604,7 +604,7 @@ static inline int mthca_poll_one(struct mthca_dev *dev, · 603 unmodified lines
604604 entry->byte_len = MTHCA_ATOMIC_BYTE_LEN;
605605 break;
606606 default:
607− entry->opcode = MTHCA_OPCODE_INVALID;
607+ entry->opcode = 0xFF;
608608 break;
609609 }
610610 } else {
drivers/infiniband/hw/mthca/mthca_dev.h+0−1
@@ -105,7 +105,6 @@ enum { · 104 unmodified lines
105105 MTHCA_OPCODE_ATOMIC_CS = 0x11,
106106 MTHCA_OPCODE_ATOMIC_FA = 0x12,
107107 MTHCA_OPCODE_BIND_MW = 0x18,
108− MTHCA_OPCODE_INVALID = 0xff
109108 };
110109
111110 enum {
drivers/infiniband/hw/mthca/mthca_provider.c+21−40
@@ -470,7 +470,7 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd, · 469 unmodified lines
470470 int err;
471471
472472 if (init_attr->create_flags)
473− return ERR_PTR(-EINVAL);
473+ return ERR_PTR(-EOPNOTSUPP);
474474
475475 switch (init_attr->qp_type) {
476476 case IB_QPT_RC:
@@ -612,7 +612,7 @@ static int mthca_create_cq(struct ib_cq *ibcq, · 135 unmodified lines
612612 udata, struct mthca_ucontext, ibucontext);
613613
614614 if (attr->flags)
615− return -EINVAL;
615+ return -EOPNOTSUPP;
616616
617617 if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
618618 return -EINVAL;
@@ -961,29 +961,34 @@ static ssize_t hw_rev_show(struct device *device, · 342 unmodified lines
961961 struct mthca_dev *dev =
962962 rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
963963
964− return sprintf(buf, "%x\n", dev->rev_id);
964+ return sysfs_emit(buf, "%x\n", dev->rev_id);
965965 }
966966 static DEVICE_ATTR_RO(hw_rev);
967967
968−static ssize_t hca_type_show(struct device *device,
969− struct device_attribute *attr, char *buf)
968+static const char *hca_type_string(int hca_type)
970969 {
971− struct mthca_dev *dev =
972− rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
973−
974− switch (dev->pdev->device) {
970+ switch (hca_type) {
975971 case PCI_DEVICE_ID_MELLANOX_TAVOR:
976− return sprintf(buf, "MT23108\n");
972+ return "MT23108";
977973 case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT:
978− return sprintf(buf, "MT25208 (MT23108 compat mode)\n");
974+ return "MT25208 (MT23108 compat mode)";
979975 case PCI_DEVICE_ID_MELLANOX_ARBEL:
980− return sprintf(buf, "MT25208\n");
976+ return "MT25208";
981977 case PCI_DEVICE_ID_MELLANOX_SINAI:
982978 case PCI_DEVICE_ID_MELLANOX_SINAI_OLD:
983− return sprintf(buf, "MT25204\n");
984− default:
985− return sprintf(buf, "unknown\n");
979+ return "MT25204";
986980 }
981+
982+ return "unknown";
983+}
984+
985+static ssize_t hca_type_show(struct device *device,
986+ struct device_attribute *attr, char *buf)
987+{
988+ struct mthca_dev *dev =
989+ rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
990+
991+ return sysfs_emit(buf, "%s\n", hca_type_string(dev->pdev->device));
987992 }
988993 static DEVICE_ATTR_RO(hca_type);
989994
@@ -993,7 +998,7 @@ static ssize_t board_id_show(struct device *device, · 3 unmodified lines
993998 struct mthca_dev *dev =
994999 rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
9951000
996− return sprintf(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id);
1001+ return sysfs_emit(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id);
9971002 }
9981003 static DEVICE_ATTR_RO(board_id);
9991004
@@ -1158,36 +1163,12 @@ int mthca_register_device(struct mthca_dev *dev) · 158 unmodified lines
11581163 if (ret)
11591164 return ret;
11601165
1161− dev->ib_dev.uverbs_cmd_mask =
1162− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1163− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1164− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1165− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1166− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1167− (1ull << IB_USER_VERBS_CMD_REG_MR) |
1168− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1169− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1170− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1171− (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
1172− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1173− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1174− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
1175− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1176− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1177− (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
1178− (1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
11791166 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
11801167 dev->ib_dev.phys_port_cnt = dev->limits.num_ports;
11811168 dev->ib_dev.num_comp_vectors = 1;
11821169 dev->ib_dev.dev.parent = &dev->pdev->dev;
11831170
11841171 if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
1185− dev->ib_dev.uverbs_cmd_mask |=
1186− (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
1187− (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
1188− (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
1189− (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
1190−
11911172 if (mthca_is_memfree(dev))
11921173 ib_set_device_ops(&dev->ib_dev,
11931174 &mthca_dev_arbel_srq_ops);
drivers/infiniband/hw/mthca/mthca_qp.c+3−0
@@ -863,6 +863,9 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, · 862 unmodified lines
863863 enum ib_qp_state cur_state, new_state;
864864 int err = -EINVAL;
865865
866+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
867+ return -EOPNOTSUPP;
868+
866869 mutex_lock(&qp->mutex);
867870 if (attr_mask & IB_QP_CUR_STATE) {
868871 cur_state = attr->cur_qp_state;
drivers/infiniband/hw/ocrdma/ocrdma_main.c+5−37
@@ -119,7 +119,7 @@ static ssize_t hw_rev_show(struct device *device, · 118 unmodified lines
119119 struct ocrdma_dev *dev =
120120 rdma_device_to_drv_device(device, struct ocrdma_dev, ibdev);
121121
122− return scnprintf(buf, PAGE_SIZE, "0x%x\n", dev->nic_info.pdev->vendor);
122+ return sysfs_emit(buf, "0x%x\n", dev->nic_info.pdev->vendor);
123123 }
124124 static DEVICE_ATTR_RO(hw_rev);
125125
@@ -129,7 +129,7 @@ static ssize_t hca_type_show(struct device *device, · 3 unmodified lines
129129 struct ocrdma_dev *dev =
130130 rdma_device_to_drv_device(device, struct ocrdma_dev, ibdev);
131131
132− return scnprintf(buf, PAGE_SIZE, "%s\n", &dev->model_number[0]);
132+ return sysfs_emit(buf, "%s\n", &dev->model_number[0]);
133133 }
134134 static DEVICE_ATTR_RO(hca_type);
135135
@@ -154,6 +154,7 @@ static const struct ib_device_ops ocrdma_dev_ops = { · 18 unmodified lines
154154 .create_ah = ocrdma_create_ah,
155155 .create_cq = ocrdma_create_cq,
156156 .create_qp = ocrdma_create_qp,
157+ .create_user_ah = ocrdma_create_ah,
157158 .dealloc_pd = ocrdma_dealloc_pd,
158159 .dealloc_ucontext = ocrdma_dealloc_ucontext,
159160 .dereg_mr = ocrdma_dereg_mr,
@@ -204,32 +205,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) · 44 unmodified lines
204205 BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
205206 memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC,
206207 sizeof(OCRDMA_NODE_DESC));
207− dev->ibdev.uverbs_cmd_mask =
208− OCRDMA_UVERBS(GET_CONTEXT) |
209− OCRDMA_UVERBS(QUERY_DEVICE) |
210− OCRDMA_UVERBS(QUERY_PORT) |
211− OCRDMA_UVERBS(ALLOC_PD) |
212− OCRDMA_UVERBS(DEALLOC_PD) |
213− OCRDMA_UVERBS(REG_MR) |
214− OCRDMA_UVERBS(DEREG_MR) |
215− OCRDMA_UVERBS(CREATE_COMP_CHANNEL) |
216− OCRDMA_UVERBS(CREATE_CQ) |
217− OCRDMA_UVERBS(RESIZE_CQ) |
218− OCRDMA_UVERBS(DESTROY_CQ) |
219− OCRDMA_UVERBS(REQ_NOTIFY_CQ) |
220− OCRDMA_UVERBS(CREATE_QP) |
221− OCRDMA_UVERBS(MODIFY_QP) |
222− OCRDMA_UVERBS(QUERY_QP) |
223− OCRDMA_UVERBS(DESTROY_QP) |
224− OCRDMA_UVERBS(POLL_CQ) |
225− OCRDMA_UVERBS(POST_SEND) |
226− OCRDMA_UVERBS(POST_RECV);
227−
228− dev->ibdev.uverbs_cmd_mask |=
229− OCRDMA_UVERBS(CREATE_AH) |
230− OCRDMA_UVERBS(MODIFY_AH) |
231− OCRDMA_UVERBS(QUERY_AH) |
232− OCRDMA_UVERBS(DESTROY_AH);
233208
234209 dev->ibdev.node_type = RDMA_NODE_IB_CA;
235210 dev->ibdev.phys_port_cnt = 1;
@@ -240,16 +215,9 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) · 4 unmodified lines
240215
241216 ib_set_device_ops(&dev->ibdev, &ocrdma_dev_ops);
242217
243− if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
244− dev->ibdev.uverbs_cmd_mask |=
245− OCRDMA_UVERBS(CREATE_SRQ) |
246− OCRDMA_UVERBS(MODIFY_SRQ) |
247− OCRDMA_UVERBS(QUERY_SRQ) |
248− OCRDMA_UVERBS(DESTROY_SRQ) |
249− OCRDMA_UVERBS(POST_SRQ_RECV);
250−
218+ if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R)
251219 ib_set_device_ops(&dev->ibdev, &ocrdma_dev_srq_ops);
252− }
220+
253221 rdma_set_device_sysfs_group(&dev->ibdev, &ocrdma_attr_group);
254222 ret = ib_device_set_netdev(&dev->ibdev, dev->nic_info.netdev, 1);
255223 if (ret)
drivers/infiniband/hw/ocrdma/ocrdma_verbs.c+10−1
@@ -974,7 +974,7 @@ int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 973 unmodified lines
974974 struct ocrdma_create_cq_ureq ureq;
975975
976976 if (attr->flags)
977− return -EINVAL;
977+ return -EOPNOTSUPP;
978978
979979 if (udata) {
980980 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
@@ -1299,6 +1299,9 @@ struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd, · 318 unmodified lines
12991299 struct ocrdma_create_qp_ureq ureq;
13001300 u16 dpp_credit_lmt, dpp_offset;
13011301
1302+ if (attrs->create_flags)
1303+ return ERR_PTR(-EOPNOTSUPP);
1304+
13021305 status = ocrdma_check_qp_params(ibpd, dev, attrs, udata);
13031306 if (status)
13041307 goto gen_err;
@@ -1391,6 +1394,9 @@ int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 86 unmodified lines
13911394 struct ocrdma_dev *dev;
13921395 enum ib_qp_state old_qps, new_qps;
13931396
1397+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1398+ return -EOPNOTSUPP;
1399+
13941400 qp = get_ocrdma_qp(ibqp);
13951401 dev = get_ocrdma_dev(ibqp->device);
13961402
@@ -1770,6 +1776,9 @@ int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, · 373 unmodified lines
17701776 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
17711777 struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq);
17721778
1779+ if (init_attr->srq_type != IB_SRQT_BASIC)
1780+ return -EOPNOTSUPP;
1781+
17731782 if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
17741783 return -EINVAL;
17751784 if (init_attr->attr.max_wr > dev->attr.max_rqe)
drivers/infiniband/hw/qedr/main.c+5−34
@@ -124,7 +124,7 @@ static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr, · 123 unmodified lines
124124 struct qedr_dev *dev =
125125 rdma_device_to_drv_device(device, struct qedr_dev, ibdev);
126126
127− return scnprintf(buf, PAGE_SIZE, "0x%x\n", dev->attr.hw_ver);
127+ return sysfs_emit(buf, "0x%x\n", dev->attr.hw_ver);
128128 }
129129 static DEVICE_ATTR_RO(hw_rev);
130130
@@ -134,10 +134,9 @@ static ssize_t hca_type_show(struct device *device, · 3 unmodified lines
134134 struct qedr_dev *dev =
135135 rdma_device_to_drv_device(device, struct qedr_dev, ibdev);
136136
137− return scnprintf(buf, PAGE_SIZE, "FastLinQ QL%x %s\n",
138− dev->pdev->device,
139− rdma_protocol_iwarp(&dev->ibdev, 1) ?
140− "iWARP" : "RoCE");
137+ return sysfs_emit(buf, "FastLinQ QL%x %s\n", dev->pdev->device,
138+ rdma_protocol_iwarp(&dev->ibdev, 1) ? "iWARP" :
139+ "RoCE");
141140 }
142141 static DEVICE_ATTR_RO(hca_type);
143142
@@ -188,10 +187,6 @@ static void qedr_roce_register_device(struct qedr_dev *dev) · 44 unmodified lines
188187 dev->ibdev.node_type = RDMA_NODE_IB_CA;
189188
190189 ib_set_device_ops(&dev->ibdev, &qedr_roce_dev_ops);
191−
192− dev->ibdev.uverbs_cmd_mask |= QEDR_UVERBS(OPEN_XRCD) |
193− QEDR_UVERBS(CLOSE_XRCD) |
194− QEDR_UVERBS(CREATE_XSRQ);
195190 }
196191
197192 static const struct ib_device_ops qedr_dev_ops = {
@@ -249,31 +244,6 @@ static int qedr_register_device(struct qedr_dev *dev) · 51 unmodified lines
249244 dev->ibdev.node_guid = dev->attr.node_guid;
250245 memcpy(dev->ibdev.node_desc, QEDR_NODE_DESC, sizeof(QEDR_NODE_DESC));
251246
252− dev->ibdev.uverbs_cmd_mask = QEDR_UVERBS(GET_CONTEXT) |
253− QEDR_UVERBS(QUERY_DEVICE) |
254− QEDR_UVERBS(QUERY_PORT) |
255− QEDR_UVERBS(ALLOC_PD) |
256− QEDR_UVERBS(DEALLOC_PD) |
257− QEDR_UVERBS(CREATE_COMP_CHANNEL) |
258− QEDR_UVERBS(CREATE_CQ) |
259− QEDR_UVERBS(RESIZE_CQ) |
260− QEDR_UVERBS(DESTROY_CQ) |
261− QEDR_UVERBS(REQ_NOTIFY_CQ) |
262− QEDR_UVERBS(CREATE_QP) |
263− QEDR_UVERBS(MODIFY_QP) |
264− QEDR_UVERBS(QUERY_QP) |
265− QEDR_UVERBS(DESTROY_QP) |
266− QEDR_UVERBS(CREATE_SRQ) |
267− QEDR_UVERBS(DESTROY_SRQ) |
268− QEDR_UVERBS(QUERY_SRQ) |
269− QEDR_UVERBS(MODIFY_SRQ) |
270− QEDR_UVERBS(POST_SRQ_RECV) |
271− QEDR_UVERBS(REG_MR) |
272− QEDR_UVERBS(DEREG_MR) |
273− QEDR_UVERBS(POLL_CQ) |
274− QEDR_UVERBS(POST_SEND) |
275− QEDR_UVERBS(POST_RECV);
276−
277247 if (IS_IWARP(dev)) {
278248 rc = qedr_iw_register_device(dev);
279249 if (rc)
@@ -796,6 +766,7 @@ static void qedr_affiliated_event(void *context, u8 e_code, void *fw_handle) · 516 unmodified lines
796766 }
797767 xa_unlock_irqrestore(&dev->srqs, flags);
798768 DP_NOTICE(dev, "SRQ event %d on handle %p\n", e_code, srq);
769+ break;
799770 default:
800771 break;
801772 }
drivers/infiniband/hw/qedr/verbs.c+13−0
@@ -928,6 +928,9 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 927 unmodified lines
928928 "create_cq: called from %s. entries=%d, vector=%d\n",
929929 udata ? "User Lib" : "Kernel", entries, vector);
930930
931+ if (attr->flags)
932+ return -EOPNOTSUPP;
933+
931934 if (entries > QEDR_MAX_CQES) {
932935 DP_ERR(dev,
933936 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
@@ -1546,6 +1549,10 @@ int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, · 612 unmodified lines
15461549 "create SRQ called from %s (pd %p)\n",
15471550 (udata) ? "User lib" : "kernel", pd);
15481551
1552+ if (init_attr->srq_type != IB_SRQT_BASIC &&
1553+ init_attr->srq_type != IB_SRQT_XRC)
1554+ return -EOPNOTSUPP;
1555+
15491556 rc = qedr_check_srq_params(dev, init_attr, udata);
15501557 if (rc)
15511558 return -EINVAL;
@@ -2241,6 +2248,9 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, · 689 unmodified lines
22412248 struct ib_qp *ibqp;
22422249 int rc = 0;
22432250
2251+ if (attrs->create_flags)
2252+ return ERR_PTR(-EOPNOTSUPP);
2253+
22442254 if (attrs->qp_type == IB_QPT_XRC_TGT) {
22452255 xrcd = get_qedr_xrcd(attrs->xrcd);
22462256 dev = get_qedr_dev(xrcd->ibxrcd.device);
@@ -2477,6 +2487,9 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 230 unmodified lines
24772487 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
24782488 attr->qp_state);
24792489
2490+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2491+ return -EOPNOTSUPP;
2492+
24802493 old_qp_state = qedr_get_ibqp_state(qp->state);
24812494 if (attr_mask & IB_QP_STATE)
24822495 new_qp_state = attr->qp_state;
drivers/infiniband/hw/qib/qib_pcie.c+2−9
@@ -90,25 +90,18 @@ int qib_pcie_init(struct pci_dev *pdev, const struct pci_device_id *ent) · 89 unmodified lines
9090 goto bail;
9191 }
9292
93− ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
93+ ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
9494 if (ret) {
9595 /*
9696 * If the 64 bit setup fails, try 32 bit. Some systems
9797 * do not setup 64 bit maps on systems with 2GB or less
9898 * memory installed.
9999 */
100− ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
100+ ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
101101 if (ret) {
102102 qib_devinfo(pdev, "Unable to set DMA mask: %d\n", ret);
103103 goto bail;
104104 }
105− ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
106− } else
107− ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
108− if (ret) {
109− qib_early_err(&pdev->dev,
110− "Unable to set DMA consistent mask: %d\n", ret);
111− goto bail;
112105 }
113106
114107 pci_set_master(pdev);
drivers/infiniband/hw/qib/qib_sysfs.c+43−53
@@ -43,11 +43,8 @@ · 42 unmodified lines
4343 static ssize_t show_hrtbt_enb(struct qib_pportdata *ppd, char *buf)
4444 {
4545 struct qib_devdata *dd = ppd->dd;
46− int ret;
4746
48− ret = dd->f_get_ib_cfg(ppd, QIB_IB_CFG_HRTBT);
49− ret = scnprintf(buf, PAGE_SIZE, "%d\n", ret);
50− return ret;
47+ return sysfs_emit(buf, "%d\n", dd->f_get_ib_cfg(ppd, QIB_IB_CFG_HRTBT));
5148 }
5249
5350 static ssize_t store_hrtbt_enb(struct qib_pportdata *ppd, const char *buf,
@@ -106,14 +103,10 @@ static ssize_t store_led_override(struct qib_pportdata *ppd, const char *buf, · 52 unmodified lines
106103
107104 static ssize_t show_status(struct qib_pportdata *ppd, char *buf)
108105 {
109− ssize_t ret;
110−
111106 if (!ppd->statusp)
112− ret = -EINVAL;
113− else
114− ret = scnprintf(buf, PAGE_SIZE, "0x%llx\n",
115− (unsigned long long) *(ppd->statusp));
116− return ret;
107+ return -EINVAL;
108+
109+ return sysfs_emit(buf, "0x%llx\n", (unsigned long long)*(ppd->statusp));
117110 }
118111
119112 /*
@@ -392,7 +385,7 @@ static ssize_t sl2vl_attr_show(struct kobject *kobj, struct attribute *attr, · 272 unmodified lines
392385 container_of(kobj, struct qib_pportdata, sl2vl_kobj);
393386 struct qib_ibport *qibp = &ppd->ibport_data;
394387
395− return sprintf(buf, "%u\n", qibp->sl_to_vl[sattr->sl]);
388+ return sysfs_emit(buf, "%u\n", qibp->sl_to_vl[sattr->sl]);
396389 }
397390
398391 static const struct sysfs_ops qib_sl2vl_ops = {
@@ -501,17 +494,18 @@ static ssize_t diagc_attr_show(struct kobject *kobj, struct attribute *attr, · 102 unmodified lines
501494 struct qib_pportdata *ppd =
502495 container_of(kobj, struct qib_pportdata, diagc_kobj);
503496 struct qib_ibport *qibp = &ppd->ibport_data;
497+ u64 val;
504498
505499 if (!strncmp(dattr->attr.name, "rc_acks", 7))
506− return sprintf(buf, "%llu\n", READ_PER_CPU_CNTR(rc_acks));
500+ val = READ_PER_CPU_CNTR(rc_acks);
507501 else if (!strncmp(dattr->attr.name, "rc_qacks", 8))
508− return sprintf(buf, "%llu\n", READ_PER_CPU_CNTR(rc_qacks));
502+ val = READ_PER_CPU_CNTR(rc_qacks);
509503 else if (!strncmp(dattr->attr.name, "rc_delayed_comp", 15))
510− return sprintf(buf, "%llu\n",
511− READ_PER_CPU_CNTR(rc_delayed_comp));
504+ val = READ_PER_CPU_CNTR(rc_delayed_comp);
512505 else
513− return sprintf(buf, "%u\n",
514− *(u32 *)((char *)qibp + dattr->counter));
506+ val = *(u32 *)((char *)qibp + dattr->counter);
507+
508+ return sysfs_emit(buf, "%llu\n", val);
515509 }
516510
517511 static ssize_t diagc_attr_store(struct kobject *kobj, struct attribute *attr,
@@ -565,7 +559,7 @@ static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr, · 47 unmodified lines
565559 struct qib_ibdev *dev =
566560 rdma_device_to_drv_device(device, struct qib_ibdev, rdi.ibdev);
567561
568− return sprintf(buf, "%x\n", dd_from_dev(dev)->minrev);
562+ return sysfs_emit(buf, "%x\n", dd_from_dev(dev)->minrev);
569563 }
570564 static DEVICE_ATTR_RO(hw_rev);
571565
@@ -575,13 +569,10 @@ static ssize_t hca_type_show(struct device *device, · 3 unmodified lines
575569 struct qib_ibdev *dev =
576570 rdma_device_to_drv_device(device, struct qib_ibdev, rdi.ibdev);
577571 struct qib_devdata *dd = dd_from_dev(dev);
578− int ret;
579572
580573 if (!dd->boardname)
581− ret = -EINVAL;
582− else
583− ret = scnprintf(buf, PAGE_SIZE, "%s\n", dd->boardname);
584− return ret;
574+ return -EINVAL;
575+ return sysfs_emit(buf, "%s\n", dd->boardname);
585576 }
586577 static DEVICE_ATTR_RO(hca_type);
587578 static DEVICE_ATTR(board_id, 0444, hca_type_show, NULL);
@@ -590,7 +581,7 @@ static ssize_t version_show(struct device *device, · 2 unmodified lines
590581 struct device_attribute *attr, char *buf)
591582 {
592583 /* The string printed here is already newline-terminated. */
593− return scnprintf(buf, PAGE_SIZE, "%s", (char *)ib_qib_version);
584+ return sysfs_emit(buf, "%s", (char *)ib_qib_version);
594585 }
595586 static DEVICE_ATTR_RO(version);
596587
@@ -602,7 +593,7 @@ static ssize_t boardversion_show(struct device *device, · 5 unmodified lines
602593 struct qib_devdata *dd = dd_from_dev(dev);
603594
604595 /* The string printed here is already newline-terminated. */
605− return scnprintf(buf, PAGE_SIZE, "%s", dd->boardversion);
596+ return sysfs_emit(buf, "%s", dd->boardversion);
606597 }
607598 static DEVICE_ATTR_RO(boardversion);
608599
@@ -614,7 +605,7 @@ static ssize_t localbus_info_show(struct device *device, · 5 unmodified lines
614605 struct qib_devdata *dd = dd_from_dev(dev);
615606
616607 /* The string printed here is already newline-terminated. */
617− return scnprintf(buf, PAGE_SIZE, "%s", dd->lbus_info);
608+ return sysfs_emit(buf, "%s", dd->lbus_info);
618609 }
619610 static DEVICE_ATTR_RO(localbus_info);
620611
@@ -628,9 +619,10 @@ static ssize_t nctxts_show(struct device *device, · 7 unmodified lines
628619 /* Return the number of user ports (contexts) available. */
629620 /* The calculation below deals with a special case where
630621 * cfgctxts is set to 1 on a single-port board. */
631− return scnprintf(buf, PAGE_SIZE, "%u\n",
632− (dd->first_user_ctxt > dd->cfgctxts) ? 0 :
633− (dd->cfgctxts - dd->first_user_ctxt));
622+ return sysfs_emit(buf, "%u\n",
623+ (dd->first_user_ctxt > dd->cfgctxts) ?
624+ 0 :
625+ (dd->cfgctxts - dd->first_user_ctxt));
634626 }
635627 static DEVICE_ATTR_RO(nctxts);
636628
@@ -642,21 +634,20 @@ static ssize_t nfreectxts_show(struct device *device, · 5 unmodified lines
642634 struct qib_devdata *dd = dd_from_dev(dev);
643635
644636 /* Return the number of free user ports (contexts) available. */
645− return scnprintf(buf, PAGE_SIZE, "%u\n", dd->freectxts);
637+ return sysfs_emit(buf, "%u\n", dd->freectxts);
646638 }
647639 static DEVICE_ATTR_RO(nfreectxts);
648640
649−static ssize_t serial_show(struct device *device,
650− struct device_attribute *attr, char *buf)
641+static ssize_t serial_show(struct device *device, struct device_attribute *attr,
642+ char *buf)
651643 {
652644 struct qib_ibdev *dev =
653645 rdma_device_to_drv_device(device, struct qib_ibdev, rdi.ibdev);
654646 struct qib_devdata *dd = dd_from_dev(dev);
647+ const u8 *end = memchr(dd->serial, 0, ARRAY_SIZE(dd->serial));
648+ int size = end ? end - dd->serial : ARRAY_SIZE(dd->serial);
655649
656− buf[sizeof(dd->serial)] = '\0';
657− memcpy(buf, dd->serial, sizeof(dd->serial));
658− strcat(buf, "\n");
659− return strlen(buf);
650+ return sysfs_emit(buf, ".%*s\n", size, dd->serial);
660651 }
661652 static DEVICE_ATTR_RO(serial);
662653
@@ -689,27 +680,26 @@ static ssize_t tempsense_show(struct device *device, · 26 unmodified lines
689680 struct qib_ibdev *dev =
690681 rdma_device_to_drv_device(device, struct qib_ibdev, rdi.ibdev);
691682 struct qib_devdata *dd = dd_from_dev(dev);
692− int ret;
693− int idx;
683+ int i;
694684 u8 regvals[8];
695685
696− ret = -ENXIO;
697− for (idx = 0; idx < 8; ++idx) {
698− if (idx == 6)
686+ for (i = 0; i < 8; i++) {
687+ int ret;
688+
689+ if (i == 6)
699690 continue;
700− ret = dd->f_tempsense_rd(dd, idx);
691+ ret = dd->f_tempsense_rd(dd, i);
701692 if (ret < 0)
702− break;
703− regvals[idx] = ret;
693+ return ret; /* return error on bad read */
694+ regvals[i] = ret;
704695 }
705− if (idx == 8)
706− ret = scnprintf(buf, PAGE_SIZE, "%d %d %02X %02X %d %d\n",
707− *(signed char *)(regvals),
708− *(signed char *)(regvals + 1),
709− regvals[2], regvals[3],
710− *(signed char *)(regvals + 5),
711− *(signed char *)(regvals + 7));
712− return ret;
696+ return sysfs_emit(buf, "%d %d %02X %02X %d %d\n",
697+ (signed char)regvals[0],
698+ (signed char)regvals[1],
699+ regvals[2],
700+ regvals[3],
701+ (signed char)regvals[5],
702+ (signed char)regvals[7]);
713703 }
714704 static DEVICE_ATTR_RO(tempsense);
715705
drivers/infiniband/hw/usnic/usnic_ib_main.c+0−19
@@ -398,25 +398,6 @@ static void *usnic_ib_device_add(struct pci_dev *dev) · 397 unmodified lines
398398 us_ibdev->ib_dev.num_comp_vectors = USNIC_IB_NUM_COMP_VECTORS;
399399 us_ibdev->ib_dev.dev.parent = &dev->dev;
400400
401− us_ibdev->ib_dev.uverbs_cmd_mask =
402− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
403− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
404− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
405− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
406− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
407− (1ull << IB_USER_VERBS_CMD_REG_MR) |
408− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
409− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
410− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
411− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
412− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
413− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
414− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
415− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
416− (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
417− (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
418− (1ull << IB_USER_VERBS_CMD_OPEN_QP);
419−
420401 ib_set_device_ops(&us_ibdev->ib_dev, &usnic_dev_ops);
421402
422403 rdma_set_device_sysfs_group(&us_ibdev->ib_dev, &usnic_attr_group);
drivers/infiniband/hw/usnic/usnic_ib_sysfs.c+40−60
@@ -57,7 +57,7 @@ static ssize_t board_id_show(struct device *device, · 56 unmodified lines
5757 subsystem_device_id = us_ibdev->pdev->subsystem_device;
5858 mutex_unlock(&us_ibdev->usdev_lock);
5959
60− return scnprintf(buf, PAGE_SIZE, "%hu\n", subsystem_device_id);
60+ return sysfs_emit(buf, "%u\n", subsystem_device_id);
6161 }
6262 static DEVICE_ATTR_RO(board_id);
6363
@@ -69,19 +69,13 @@ config_show(struct device *device, struct device_attribute *attr, char *buf) · 5 unmodified lines
6969 {
7070 struct usnic_ib_dev *us_ibdev =
7171 rdma_device_to_drv_device(device, struct usnic_ib_dev, ib_dev);
72− char *ptr;
73− unsigned left;
74− unsigned n;
7572 enum usnic_vnic_res_type res_type;
76−
77− /* Buffer space limit is 1 page */
78− ptr = buf;
79− left = PAGE_SIZE;
73+ int len;
8074
8175 mutex_lock(&us_ibdev->usdev_lock);
8276 if (kref_read(&us_ibdev->vf_cnt) > 0) {
8377 char *busname;
84−
78+ char *sep = "";
8579 /*
8680 * bus name seems to come with annoying prefix.
8781 * Remove it if it is predictable
@@ -90,39 +84,35 @@ config_show(struct device *device, struct device_attribute *attr, char *buf) · 2 unmodified lines
9084 if (strncmp(busname, "PCI Bus ", 8) == 0)
9185 busname += 8;
9286
93− n = scnprintf(ptr, left,
94− "%s: %s:%d.%d, %s, %pM, %u VFs\n Per VF:",
95− dev_name(&us_ibdev->ib_dev.dev),
96− busname,
97− PCI_SLOT(us_ibdev->pdev->devfn),
98− PCI_FUNC(us_ibdev->pdev->devfn),
99− netdev_name(us_ibdev->netdev),
100− us_ibdev->ufdev->mac,
101− kref_read(&us_ibdev->vf_cnt));
102− UPDATE_PTR_LEFT(n, ptr, left);
87+ len = sysfs_emit(buf, "%s: %s:%d.%d, %s, %pM, %u VFs\n",
88+ dev_name(&us_ibdev->ib_dev.dev),
89+ busname,
90+ PCI_SLOT(us_ibdev->pdev->devfn),
91+ PCI_FUNC(us_ibdev->pdev->devfn),
92+ netdev_name(us_ibdev->netdev),
93+ us_ibdev->ufdev->mac,
94+ kref_read(&us_ibdev->vf_cnt));
10395
96+ len += sysfs_emit_at(buf, len, " Per VF:");
10497 for (res_type = USNIC_VNIC_RES_TYPE_EOL;
105− res_type < USNIC_VNIC_RES_TYPE_MAX;
106− res_type++) {
98+ res_type < USNIC_VNIC_RES_TYPE_MAX; res_type++) {
10799 if (us_ibdev->vf_res_cnt[res_type] == 0)
108100 continue;
109− n = scnprintf(ptr, left, " %d %s%s",
110− us_ibdev->vf_res_cnt[res_type],
111− usnic_vnic_res_type_to_str(res_type),
112− (res_type < (USNIC_VNIC_RES_TYPE_MAX - 1)) ?
113− "," : "");
114− UPDATE_PTR_LEFT(n, ptr, left);
101+ len += sysfs_emit_at(buf, len, "%s %d %s",
102+ sep,
103+ us_ibdev->vf_res_cnt[res_type],
104+ usnic_vnic_res_type_to_str(res_type));
105+ sep = ",";
115106 }
116− n = scnprintf(ptr, left, "\n");
117− UPDATE_PTR_LEFT(n, ptr, left);
107+ len += sysfs_emit_at(buf, len, "\n");
118108 } else {
119− n = scnprintf(ptr, left, "%s: no VFs\n",
120− dev_name(&us_ibdev->ib_dev.dev));
121− UPDATE_PTR_LEFT(n, ptr, left);
109+ len = sysfs_emit(buf, "%s: no VFs\n",
110+ dev_name(&us_ibdev->ib_dev.dev));
122111 }
112+
123113 mutex_unlock(&us_ibdev->usdev_lock);
124114
125− return ptr - buf;
115+ return len;
126116 }
127117 static DEVICE_ATTR_RO(config);
128118
@@ -132,8 +122,7 @@ iface_show(struct device *device, struct device_attribute *attr, char *buf) · 3 unmodified lines
132122 struct usnic_ib_dev *us_ibdev =
133123 rdma_device_to_drv_device(device, struct usnic_ib_dev, ib_dev);
134124
135− return scnprintf(buf, PAGE_SIZE, "%s\n",
136− netdev_name(us_ibdev->netdev));
125+ return sysfs_emit(buf, "%s\n", netdev_name(us_ibdev->netdev));
137126 }
138127 static DEVICE_ATTR_RO(iface);
139128
@@ -143,8 +132,7 @@ max_vf_show(struct device *device, struct device_attribute *attr, char *buf) · 3 unmodified lines
143132 struct usnic_ib_dev *us_ibdev =
144133 rdma_device_to_drv_device(device, struct usnic_ib_dev, ib_dev);
145134
146− return scnprintf(buf, PAGE_SIZE, "%u\n",
147− kref_read(&us_ibdev->vf_cnt));
135+ return sysfs_emit(buf, "%u\n", kref_read(&us_ibdev->vf_cnt));
148136 }
149137 static DEVICE_ATTR_RO(max_vf);
150138
@@ -158,8 +146,7 @@ qp_per_vf_show(struct device *device, struct device_attribute *attr, char *buf) · 7 unmodified lines
158146 qp_per_vf = max(us_ibdev->vf_res_cnt[USNIC_VNIC_RES_TYPE_WQ],
159147 us_ibdev->vf_res_cnt[USNIC_VNIC_RES_TYPE_RQ]);
160148
161− return scnprintf(buf, PAGE_SIZE,
162− "%d\n", qp_per_vf);
149+ return sysfs_emit(buf, "%d\n", qp_per_vf);
163150 }
164151 static DEVICE_ATTR_RO(qp_per_vf);
165152
@@ -169,8 +156,8 @@ cq_per_vf_show(struct device *device, struct device_attribute *attr, char *buf) · 3 unmodified lines
169156 struct usnic_ib_dev *us_ibdev =
170157 rdma_device_to_drv_device(device, struct usnic_ib_dev, ib_dev);
171158
172− return scnprintf(buf, PAGE_SIZE, "%d\n",
173− us_ibdev->vf_res_cnt[USNIC_VNIC_RES_TYPE_CQ]);
159+ return sysfs_emit(buf, "%d\n",
160+ us_ibdev->vf_res_cnt[USNIC_VNIC_RES_TYPE_CQ]);
174161 }
175162 static DEVICE_ATTR_RO(cq_per_vf);
176163
@@ -217,43 +204,36 @@ struct qpn_attribute qpn_attr_##NAME = __ATTR_RO(NAME) · 40 unmodified lines
217204
218205 static ssize_t context_show(struct usnic_ib_qp_grp *qp_grp, char *buf)
219206 {
220− return scnprintf(buf, PAGE_SIZE, "0x%p\n", qp_grp->ctx);
207+ return sysfs_emit(buf, "0x%p\n", qp_grp->ctx);
221208 }
222209
223210 static ssize_t summary_show(struct usnic_ib_qp_grp *qp_grp, char *buf)
224211 {
225− int i, j, n;
226− int left;
227− char *ptr;
212+ int i, j;
228213 struct usnic_vnic_res_chunk *res_chunk;
229214 struct usnic_vnic_res *vnic_res;
215+ int len;
230216
231− left = PAGE_SIZE;
232− ptr = buf;
233−
234− n = scnprintf(ptr, left,
235− "QPN: %d State: (%s) PID: %u VF Idx: %hu ",
236− qp_grp->ibqp.qp_num,
237− usnic_ib_qp_grp_state_to_string(qp_grp->state),
238− qp_grp->owner_pid,
239− usnic_vnic_get_index(qp_grp->vf->vnic));
240− UPDATE_PTR_LEFT(n, ptr, left);
217+ len = sysfs_emit(buf, "QPN: %d State: (%s) PID: %u VF Idx: %hu ",
218+ qp_grp->ibqp.qp_num,
219+ usnic_ib_qp_grp_state_to_string(qp_grp->state),
220+ qp_grp->owner_pid,
221+ usnic_vnic_get_index(qp_grp->vf->vnic));
241222
242223 for (i = 0; qp_grp->res_chunk_list[i]; i++) {
243224 res_chunk = qp_grp->res_chunk_list[i];
244225 for (j = 0; j < res_chunk->cnt; j++) {
245226 vnic_res = res_chunk->res[j];
246− n = scnprintf(ptr, left, "%s[%d] ",
227+ len += sysfs_emit_at(
228+ buf, len, "%s[%d] ",
247229 usnic_vnic_res_type_to_str(vnic_res->type),
248230 vnic_res->vnic_idx);
249− UPDATE_PTR_LEFT(n, ptr, left);
250231 }
251232 }
252233
253− n = scnprintf(ptr, left, "\n");
254− UPDATE_PTR_LEFT(n, ptr, left);
234+ len = sysfs_emit_at(buf, len, "\n");
255235
256− return ptr - buf;
236+ return len;
257237 }
258238
259239 static QPN_ATTR_RO(context);
drivers/infiniband/hw/usnic/usnic_ib_verbs.c+5−2
@@ -474,7 +474,7 @@ struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, · 473 unmodified lines
474474 us_ibdev = to_usdev(pd->device);
475475
476476 if (init_attr->create_flags)
477− return ERR_PTR(-EINVAL);
477+ return ERR_PTR(-EOPNOTSUPP);
478478
479479 err = ib_copy_from_udata(&cmd, udata, sizeof(cmd));
480480 if (err) {
@@ -557,6 +557,9 @@ int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 76 unmodified lines
557557 int status;
558558 usnic_dbg("\n");
559559
560+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
561+ return -EOPNOTSUPP;
562+
560563 qp_grp = to_uqp_grp(ibqp);
561564
562565 mutex_lock(&qp_grp->vf->pf->usdev_lock);
@@ -581,7 +584,7 @@ int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 18 unmodified lines
581584 struct ib_udata *udata)
582585 {
583586 if (attr->flags)
584− return -EINVAL;
587+ return -EOPNOTSUPP;
585588
586589 return 0;
587590 }
drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c+3−0
@@ -119,6 +119,9 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 118 unmodified lines
119119
120120 BUILD_BUG_ON(sizeof(struct pvrdma_cqe) != 64);
121121
122+ if (attr->flags)
123+ return -EOPNOTSUPP;
124+
122125 entries = roundup_pow_of_two(entries);
123126 if (entries < 1 || entries > dev->dsr->caps.max_cqe)
124127 return -EINVAL;
drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c+3−31
@@ -68,21 +68,21 @@ static int pvrdma_del_gid(const struct ib_gid_attr *attr, void **context); · 67 unmodified lines
6868 static ssize_t hca_type_show(struct device *device,
6969 struct device_attribute *attr, char *buf)
7070 {
71− return sprintf(buf, "VMW_PVRDMA-%s\n", DRV_VERSION);
71+ return sysfs_emit(buf, "VMW_PVRDMA-%s\n", DRV_VERSION);
7272 }
7373 static DEVICE_ATTR_RO(hca_type);
7474
7575 static ssize_t hw_rev_show(struct device *device,
7676 struct device_attribute *attr, char *buf)
7777 {
78− return sprintf(buf, "%d\n", PVRDMA_REV_ID);
78+ return sysfs_emit(buf, "%d\n", PVRDMA_REV_ID);
7979 }
8080 static DEVICE_ATTR_RO(hw_rev);
8181
8282 static ssize_t board_id_show(struct device *device,
8383 struct device_attribute *attr, char *buf)
8484 {
85− return sprintf(buf, "%d\n", PVRDMA_BOARD_ID);
85+ return sysfs_emit(buf, "%d\n", PVRDMA_BOARD_ID);
8686 }
8787 static DEVICE_ATTR_RO(board_id);
8888
@@ -205,27 +205,6 @@ static int pvrdma_register_device(struct pvrdma_dev *dev) · 116 unmodified lines
205205 dev->flags = 0;
206206 dev->ib_dev.num_comp_vectors = 1;
207207 dev->ib_dev.dev.parent = &dev->pdev->dev;
208− dev->ib_dev.uverbs_cmd_mask =
209− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
210− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
211− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
212− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
213− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
214− (1ull << IB_USER_VERBS_CMD_REG_MR) |
215− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
216− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
217− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
218− (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
219− (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
220− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
221− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
222− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
223− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
224− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
225− (1ull << IB_USER_VERBS_CMD_POST_SEND) |
226− (1ull << IB_USER_VERBS_CMD_POST_RECV) |
227− (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
228− (1ull << IB_USER_VERBS_CMD_DESTROY_AH);
229208
230209 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
231210 dev->ib_dev.phys_port_cnt = dev->dsr->caps.phys_port_cnt;
@@ -249,13 +228,6 @@ static int pvrdma_register_device(struct pvrdma_dev *dev) · 17 unmodified lines
249228
250229 /* Check if SRQ is supported by backend */
251230 if (dev->dsr->caps.max_srq) {
252− dev->ib_dev.uverbs_cmd_mask |=
253− (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
254− (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
255− (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
256− (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
257− (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
258−
259231 ib_set_device_ops(&dev->ib_dev, &pvrdma_dev_srq_ops);
260232
261233 dev->srq_tbl = kcalloc(dev->dsr->caps.max_srq,
drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c+4−1
@@ -209,7 +209,7 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, · 208 unmodified lines
209209 dev_warn(&dev->pdev->dev,
210210 "invalid create queuepair flags %#x\n",
211211 init_attr->create_flags);
212− return ERR_PTR(-EINVAL);
212+ return ERR_PTR(-EOPNOTSUPP);
213213 }
214214
215215 if (init_attr->qp_type != IB_QPT_RC &&
@@ -544,6 +544,9 @@ int pvrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 328 unmodified lines
544544 enum ib_qp_state cur_state, next_state;
545545 int ret;
546546
547+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
548+ return -EOPNOTSUPP;
549+
547550 /* Sanity checking. Should need lock here */
548551 mutex_lock(&qp->mutex);
549552 cur_state = (attr_mask & IB_QP_CUR_STATE) ? attr->cur_qp_state :
drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c+1−1
@@ -121,7 +121,7 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, · 120 unmodified lines
121121 dev_warn(&dev->pdev->dev,
122122 "shared receive queue type %d not supported\n",
123123 init_attr->srq_type);
124− return -EINVAL;
124+ return -EOPNOTSUPP;
125125 }
126126
127127 if (init_attr->attr.max_wr > dev->dsr->caps.max_srq_wr ||
drivers/infiniband/sw/rdmavt/Kconfig+0−1
@@ -4,6 +4,5 @@ config INFINIBAND_RDMAVT · 3 unmodified lines
44 depends on INFINIBAND_VIRT_DMA
55 depends on X86_64
66 depends on PCI
7− select DMA_VIRT_OPS
87 help
98 This is a common software verbs provider for RDMA networks.
drivers/infiniband/sw/rdmavt/ah.c+1−2
@@ -126,10 +126,9 @@ int rvt_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, · 125 unmodified lines
126126 }
127127
128128 /**
129− * rvt_destory_ah - Destory an address handle
129+ * rvt_destroy_ah - Destroy an address handle
130130 * @ibah: address handle
131131 * @destroy_flags: destroy address handle flags (see enum rdma_destroy_ah_flags)
132− *
133132 * Return: 0 on success
134133 */
135134 int rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags)
drivers/infiniband/sw/rdmavt/cq.c+1−1
@@ -211,7 +211,7 @@ int rvt_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 210 unmodified lines
211211 int err;
212212
213213 if (attr->flags)
214− return -EINVAL;
214+ return -EOPNOTSUPP;
215215
216216 if (entries < 1 || entries > rdi->dparms.props.max_cqe)
217217 return -EINVAL;
drivers/infiniband/sw/rdmavt/mcast.c+6−6
@@ -54,7 +54,7 @@ · 53 unmodified lines
5454 #include "mcast.h"
5555
5656 /**
57− * rvt_driver_mcast - init resources for multicast
57+ * rvt_driver_mcast_init - init resources for multicast
5858 * @rdi: rvt dev struct
5959 *
6060 * This is per device that registers with rdmavt
@@ -69,7 +69,7 @@ void rvt_driver_mcast_init(struct rvt_dev_info *rdi) · 8 unmodified lines
6969 }
7070
7171 /**
72− * mcast_qp_alloc - alloc a struct to link a QP to mcast GID struct
72+ * rvt_mcast_qp_alloc - alloc a struct to link a QP to mcast GID struct
7373 * @qp: the QP to link
7474 */
7575 static struct rvt_mcast_qp *rvt_mcast_qp_alloc(struct rvt_qp *qp)
@@ -98,7 +98,7 @@ static void rvt_mcast_qp_free(struct rvt_mcast_qp *mqp) · 22 unmodified lines
9898 }
9999
100100 /**
101− * mcast_alloc - allocate the multicast GID structure
101+ * rvt_mcast_alloc - allocate the multicast GID structure
102102 * @mgid: the multicast GID
103103 * @lid: the muilticast LID (host order)
104104 *
@@ -181,7 +181,7 @@ struct rvt_mcast *rvt_mcast_find(struct rvt_ibport *ibp, union ib_gid *mgid, · 76 unmodified lines
181181 EXPORT_SYMBOL(rvt_mcast_find);
182182
183183 /**
184− * mcast_add - insert mcast GID into table and attach QP struct
184+ * rvt_mcast_add - insert mcast GID into table and attach QP struct
185185 * @mcast: the mcast GID table
186186 * @mqp: the QP to attach
187187 *
@@ -426,8 +426,8 @@ int rvt_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) · 238 unmodified lines
426426 }
427427
428428 /**
429− *rvt_mast_tree_empty - determine if any qps are attached to any mcast group
430− *@rdi: rvt dev struct
429+ * rvt_mcast_tree_empty - determine if any qps are attached to any mcast group
430+ * @rdi: rvt dev struct
431431 *
432432 * Return: in use count
433433 */
drivers/infiniband/sw/rdmavt/mr.c+2−4
@@ -324,8 +324,6 @@ static void __rvt_free_mr(struct rvt_mr *mr) · 323 unmodified lines
324324 * @acc: access flags
325325 *
326326 * Return: the memory region on success, otherwise returns an errno.
327− * Note that all DMA addresses should be created via the functions in
328− * struct dma_virt_ops.
329327 */
330328 struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc)
331329 {
@@ -766,7 +764,7 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd, · 434 unmodified lines
766764
767765 /*
768766 * We use LKEY == zero for kernel virtual addresses
769− * (see rvt_get_dma_mr() and dma_virt_ops).
767+ * (see rvt_get_dma_mr()).
770768 */
771769 if (sge->lkey == 0) {
772770 struct rvt_dev_info *dev = ib_to_rvt(pd->ibpd.device);
@@ -877,7 +875,7 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge, · 104 unmodified lines
877875
878876 /*
879877 * We use RKEY == zero for kernel virtual addresses
880− * (see rvt_get_dma_mr() and dma_virt_ops).
878+ * (see rvt_get_dma_mr()).
881879 */
882880 rcu_read_lock();
883881 if (rkey == 0) {
drivers/infiniband/sw/rdmavt/qp.c+11−7
@@ -1083,10 +1083,11 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, · 1082 unmodified lines
10831083 if (!rdi)
10841084 return ERR_PTR(-EINVAL);
10851085
1086+ if (init_attr->create_flags & ~IB_QP_CREATE_NETDEV_USE)
1087+ return ERR_PTR(-EOPNOTSUPP);
1088+
10861089 if (init_attr->cap.max_send_sge > rdi->dparms.props.max_send_sge ||
1087− init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr ||
1088− (init_attr->create_flags &&
1089− init_attr->create_flags != IB_QP_CREATE_NETDEV_USE))
1090+ init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr)
10901091 return ERR_PTR(-EINVAL);
10911092
10921093 /* Check receive queue parameters if no SRQ is specified. */
@@ -1469,6 +1470,9 @@ int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 376 unmodified lines
14691470 int pmtu = 0; /* for gcc warning only */
14701471 int opa_ah;
14711472
1473+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1474+ return -EOPNOTSUPP;
1475+
14721476 spin_lock_irq(&qp->r_lock);
14731477 spin_lock(&qp->s_hlock);
14741478 spin_lock(&qp->s_lock);
@@ -1823,7 +1827,7 @@ int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 348 unmodified lines
18231827 }
18241828
18251829 /**
1826− * rvt_post_receive - post a receive on a QP
1830+ * rvt_post_recv - post a receive on a QP
18271831 * @ibqp: the QP to post the receive on
18281832 * @wr: the WR to post
18291833 * @bad_wr: the first bad WR is put here
@@ -2245,7 +2249,7 @@ bail: · 415 unmodified lines
22452249 }
22462250
22472251 /**
2248− * rvt_post_srq_receive - post a receive on a shared receive queue
2252+ * rvt_post_srq_recv - post a receive on a shared receive queue
22492253 * @ibsrq: the SRQ to post the receive on
22502254 * @wr: the list of work requests to post
22512255 * @bad_wr: A pointer to the first WR to cause a problem is put here
@@ -2497,7 +2501,7 @@ bail: · 245 unmodified lines
24972501 EXPORT_SYMBOL(rvt_get_rwqe);
24982502
24992503 /**
2500− * qp_comm_est - handle trap with QP established
2504+ * rvt_comm_est - handle trap with QP established
25012505 * @qp: the QP
25022506 */
25032507 void rvt_comm_est(struct rvt_qp *qp)
@@ -2943,7 +2947,7 @@ static enum ib_wc_status loopback_qp_drop(struct rvt_ibport *rvp, · 439 unmodified lines
29432947 }
29442948
29452949 /**
2946− * ruc_loopback - handle UC and RC loopback requests
2950+ * rvt_ruc_loopback - handle UC and RC loopback requests
29472951 * @sqp: the sending QP
29482952 *
29492953 * This is called from rvt_do_send() to forward a WQE addressed to the same HFI
drivers/infiniband/sw/rdmavt/vt.c+2−34
@@ -384,6 +384,7 @@ static const struct ib_device_ops rvt_dev_ops = { · 383 unmodified lines
384384 .create_cq = rvt_create_cq,
385385 .create_qp = rvt_create_qp,
386386 .create_srq = rvt_create_srq,
387+ .create_user_ah = rvt_create_ah,
387388 .dealloc_pd = rvt_dealloc_pd,
388389 .dealloc_ucontext = rvt_dealloc_ucontext,
389390 .dereg_mr = rvt_dereg_mr,
@@ -524,7 +525,6 @@ static noinline int check_support(struct rvt_dev_info *rdi, int verb) · 134 unmodified lines
524525 int rvt_register_device(struct rvt_dev_info *rdi)
525526 {
526527 int ret = 0, i;
527− u64 dma_mask;
528528
529529 if (!rdi)
530530 return -EINVAL;
@@ -579,13 +579,6 @@ int rvt_register_device(struct rvt_dev_info *rdi) · 48 unmodified lines
579579 /* Completion queues */
580580 spin_lock_init(&rdi->n_cqs_lock);
581581
582− /* DMA Operations */
583− rdi->ibdev.dev.dma_parms = rdi->ibdev.dev.parent->dma_parms;
584− dma_mask = IS_ENABLED(CONFIG_64BIT) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32);
585− ret = dma_coerce_mask_and_coherent(&rdi->ibdev.dev, dma_mask);
586− if (ret)
587− goto bail_wss;
588−
589582 /* Protection Domain */
590583 spin_lock_init(&rdi->n_pds_lock);
591584 rdi->n_pds_allocated = 0;
@@ -596,36 +589,11 @@ int rvt_register_device(struct rvt_dev_info *rdi) · 4 unmodified lines
596589 * exactly which functions rdmavt supports, nor do they know the ABI
597590 * version, so we do all of this sort of stuff here.
598591 */
599− rdi->ibdev.uverbs_cmd_mask =
600− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
601− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
602− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
603− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
604− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
605− (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
606− (1ull << IB_USER_VERBS_CMD_MODIFY_AH) |
607− (1ull << IB_USER_VERBS_CMD_QUERY_AH) |
608− (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
609− (1ull << IB_USER_VERBS_CMD_REG_MR) |
610− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
611− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
612− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
613− (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
614− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
592+ rdi->ibdev.uverbs_cmd_mask |=
615593 (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
616594 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
617− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
618− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
619− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
620− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
621595 (1ull << IB_USER_VERBS_CMD_POST_SEND) |
622596 (1ull << IB_USER_VERBS_CMD_POST_RECV) |
623− (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
624− (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
625− (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
626− (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
627− (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
628− (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
629597 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
630598 rdi->ibdev.node_type = RDMA_NODE_IB_CA;
631599 if (!rdi->ibdev.num_comp_vectors)
drivers/infiniband/sw/rxe/Kconfig+0−1
@@ -5,7 +5,6 @@ config RDMA_RXE · 4 unmodified lines
55 depends on INFINIBAND_VIRT_DMA
66 select NET_UDP_TUNNEL
77 select CRYPTO_CRC32
8− select DMA_VIRT_OPS
98 help
109 This driver implements the InfiniBand RDMA transport over
1110 the Linux network stack. It enables a system with a
drivers/infiniband/sw/rxe/rxe_cq.c+0−5
@@ -123,11 +123,6 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited) · 122 unmodified lines
123123
124124 memcpy(producer_addr(cq->queue), cqe, sizeof(*cqe));
125125
126− /* make sure all changes to the CQ are written before we update the
127− * producer pointer
128− */
129− smp_wmb();
130−
131126 advance_producer(cq->queue);
132127 spin_unlock_irqrestore(&cq->cq_lock, flags);
133128
drivers/infiniband/sw/rxe/rxe_mr.c+0−1
@@ -31,7 +31,6 @@ int mem_check_range(struct rxe_mem *mem, u64 iova, size_t length) · 30 unmodified lines
3131 return 0;
3232
3333 case RXE_MEM_TYPE_MR:
34− case RXE_MEM_TYPE_FMR:
3534 if (iova < mem->iova ||
3635 length > mem->length ||
3736 iova > mem->iova + mem->length - length)
drivers/infiniband/sw/rxe/rxe_net.c+0−18
@@ -8,7 +8,6 @@ · 7 unmodified lines
88 #include <linux/if_arp.h>
99 #include <linux/netdevice.h>
1010 #include <linux/if.h>
11−#include <linux/if_vlan.h>
1211 #include <net/udp_tunnel.h>
1312 #include <net/sch_generic.h>
1413 #include <linux/netfilter.h>
@@ -20,18 +19,6 @@ · 5 unmodified lines
2019
2120 static struct rxe_recv_sockets recv_sockets;
2221
23−struct device *rxe_dma_device(struct rxe_dev *rxe)
24−{
25− struct net_device *ndev;
26−
27− ndev = rxe->ndev;
28−
29− if (is_vlan_dev(ndev))
30− ndev = vlan_dev_real_dev(ndev);
31−
32− return ndev->dev.parent;
33−}
34−
3522 int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)
3623 {
3724 int err;
@@ -166,14 +153,9 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) · 128 unmodified lines
166153 {
167154 struct udphdr *udph;
168155 struct net_device *ndev = skb->dev;
169− struct net_device *rdev = ndev;
170156 struct rxe_dev *rxe = rxe_get_dev_from_net(ndev);
171157 struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
172158
173− if (!rxe && is_vlan_dev(rdev)) {
174− rdev = vlan_dev_real_dev(ndev);
175− rxe = rxe_get_dev_from_net(rdev);
176− }
177159 if (!rxe)
178160 goto drop;
179161
drivers/infiniband/sw/rxe/rxe_queue.h+60−34
@@ -7,9 +7,11 @@ · 6 unmodified lines
77 #ifndef RXE_QUEUE_H
88 #define RXE_QUEUE_H
99
10+/* for definition of shared struct rxe_queue_buf */
11+#include <uapi/rdma/rdma_user_rxe.h>
12+
1013 /* implements a simple circular buffer that can optionally be
1114 * shared between user space and the kernel and can be resized
12−
1315 * the requested element size is rounded up to a power of 2
1416 * and the number of elements in the buffer is also rounded
1517 * up to a power of 2. Since the queue is empty when the
@@ -17,28 +19,6 @@ · 1 unmodified line
1719 * of the queue is one less than the number of element slots
1820 */
1921
20−/* this data structure is shared between user space and kernel
21− * space for those cases where the queue is shared. It contains
22− * the producer and consumer indices. Is also contains a copy
23− * of the queue size parameters for user space to use but the
24− * kernel must use the parameters in the rxe_queue struct
25− * this MUST MATCH the corresponding librxe struct
26− * for performance reasons arrange to have producer and consumer
27− * pointers in separate cache lines
28− * the kernel should always mask the indices to avoid accessing
29− * memory outside of the data area
30− */
31−struct rxe_queue_buf {
32− __u32 log2_elem_size;
33− __u32 index_mask;
34− __u32 pad_1[30];
35− __u32 producer_index;
36− __u32 pad_2[31];
37− __u32 consumer_index;
38− __u32 pad_3[31];
39− __u8 data[];
40−};
41−
4222 struct rxe_queue {
4323 struct rxe_dev *rxe;
4424 struct rxe_queue_buf *buf;
@@ -46,7 +26,7 @@ struct rxe_queue { · 1 unmodified line
4626 size_t buf_size;
4727 size_t elem_size;
4828 unsigned int log2_elem_size;
49− unsigned int index_mask;
29+ u32 index_mask;
5030 };
5131
5232 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
@@ -76,26 +56,56 @@ static inline int next_index(struct rxe_queue *q, int index) · 23 unmodified lines
7656
7757 static inline int queue_empty(struct rxe_queue *q)
7858 {
79− return ((q->buf->producer_index - q->buf->consumer_index)
80− & q->index_mask) == 0;
59+ u32 prod;
60+ u32 cons;
61+
62+ /* make sure all changes to queue complete before
63+ * testing queue empty
64+ */
65+ prod = smp_load_acquire(&q->buf->producer_index);
66+ /* same */
67+ cons = smp_load_acquire(&q->buf->consumer_index);
68+
69+ return ((prod - cons) & q->index_mask) == 0;
8170 }
8271
8372 static inline int queue_full(struct rxe_queue *q)
8473 {
85− return ((q->buf->producer_index + 1 - q->buf->consumer_index)
86− & q->index_mask) == 0;
74+ u32 prod;
75+ u32 cons;
76+
77+ /* make sure all changes to queue complete before
78+ * testing queue full
79+ */
80+ prod = smp_load_acquire(&q->buf->producer_index);
81+ /* same */
82+ cons = smp_load_acquire(&q->buf->consumer_index);
83+
84+ return ((prod + 1 - cons) & q->index_mask) == 0;
8785 }
8886
8987 static inline void advance_producer(struct rxe_queue *q)
9088 {
91− q->buf->producer_index = (q->buf->producer_index + 1)
92− & q->index_mask;
89+ u32 prod;
90+
91+ prod = (q->buf->producer_index + 1) & q->index_mask;
92+
93+ /* make sure all changes to queue complete before
94+ * changing producer index
95+ */
96+ smp_store_release(&q->buf->producer_index, prod);
9397 }
9498
9599 static inline void advance_consumer(struct rxe_queue *q)
96100 {
97− q->buf->consumer_index = (q->buf->consumer_index + 1)
98− & q->index_mask;
101+ u32 cons;
102+
103+ cons = (q->buf->consumer_index + 1) & q->index_mask;
104+
105+ /* make sure all changes to queue complete before
106+ * changing consumer index
107+ */
108+ smp_store_release(&q->buf->consumer_index, cons);
99109 }
100110
101111 static inline void *producer_addr(struct rxe_queue *q)
@@ -112,12 +122,28 @@ static inline void *consumer_addr(struct rxe_queue *q) · 10 unmodified lines
112122
113123 static inline unsigned int producer_index(struct rxe_queue *q)
114124 {
115− return q->buf->producer_index;
125+ u32 index;
126+
127+ /* make sure all changes to queue
128+ * complete before getting producer index
129+ */
130+ index = smp_load_acquire(&q->buf->producer_index);
131+ index &= q->index_mask;
132+
133+ return index;
116134 }
117135
118136 static inline unsigned int consumer_index(struct rxe_queue *q)
119137 {
120− return q->buf->consumer_index;
138+ u32 index;
139+
140+ /* make sure all changes to queue
141+ * complete before getting consumer index
142+ */
143+ index = smp_load_acquire(&q->buf->consumer_index);
144+ index &= q->index_mask;
145+
146+ return index;
121147 }
122148
123149 static inline void *addr_from_index(struct rxe_queue *q, unsigned int index)
drivers/infiniband/sw/rxe/rxe_req.c+2−1
@@ -634,7 +634,8 @@ next_wqe: · 633 unmodified lines
634634 }
635635
636636 if (unlikely(qp_type(qp) == IB_QPT_RC &&
637− qp->req.psn > (qp->comp.psn + RXE_MAX_UNACKED_PSNS))) {
637+ psn_compare(qp->req.psn, (qp->comp.psn +
638+ RXE_MAX_UNACKED_PSNS)) > 0)) {
638639 qp->req.wait_psn = 1;
639640 goto exit;
640641 }
drivers/infiniband/sw/rxe/rxe_resp.c+0−5
@@ -872,11 +872,6 @@ static enum resp_states do_complete(struct rxe_qp *qp, · 871 unmodified lines
872872 else
873873 wc->network_hdr_type = RDMA_NETWORK_IPV6;
874874
875− if (is_vlan_dev(skb->dev)) {
876− wc->wc_flags |= IB_WC_WITH_VLAN;
877− wc->vlan_id = vlan_dev_vlan_id(skb->dev);
878− }
879−
880875 if (pkt->mask & RXE_IMMDT_MASK) {
881876 wc->wc_flags |= IB_WC_WITH_IMM;
882877 wc->ex.imm_data = immdt_imm(pkt);
drivers/infiniband/sw/rxe/rxe_verbs.c+14−53
@@ -244,11 +244,6 @@ static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr) · 243 unmodified lines
244244 recv_wqe->dma.cur_sge = 0;
245245 recv_wqe->dma.sge_offset = 0;
246246
247− /* make sure all changes to the work queue are written before we
248− * update the producer pointer
249− */
250− smp_wmb();
251−
252247 advance_producer(rq->queue);
253248 return 0;
254249
@@ -265,6 +260,9 @@ static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init, · 10 unmodified lines
265260 struct rxe_srq *srq = to_rsrq(ibsrq);
266261 struct rxe_create_srq_resp __user *uresp = NULL;
267262
263+ if (init->srq_type != IB_SRQT_BASIC)
264+ return -EOPNOTSUPP;
265+
268266 if (udata) {
269267 if (udata->outlen < sizeof(*uresp))
270268 return -EINVAL;
@@ -392,6 +390,9 @@ static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd, · 121 unmodified lines
392390 uresp = udata->outbuf;
393391 }
394392
393+ if (init->create_flags)
394+ return ERR_PTR(-EOPNOTSUPP);
395+
395396 err = rxe_qp_chk_init(rxe, init);
396397 if (err)
397398 goto err1;
@@ -433,6 +434,9 @@ static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 35 unmodified lines
433434 struct rxe_dev *rxe = to_rdev(ibqp->device);
434435 struct rxe_qp *qp = to_rqp(ibqp);
435436
437+ if (mask & ~IB_QP_ATTR_STANDARD_BITS)
438+ return -EOPNOTSUPP;
439+
436440 err = rxe_qp_chk_attr(rxe, qp, attr, mask);
437441 if (err)
438442 goto err1;
@@ -624,12 +628,6 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr, · 185 unmodified lines
624628 if (unlikely(err))
625629 goto err1;
626630
627− /*
628− * make sure all changes to the work queue are
629− * written before we update the producer pointer
630− */
631− smp_wmb();
632−
633631 advance_producer(sq->queue);
634632 spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
635633
@@ -765,7 +763,7 @@ static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, · 129 unmodified lines
765763 }
766764
767765 if (attr->flags)
768− return -EINVAL;
766+ return -EOPNOTSUPP;
769767
770768 err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector);
771769 if (err)
@@ -1033,7 +1031,7 @@ static ssize_t parent_show(struct device *device, · 261 unmodified lines
10331031 struct rxe_dev *rxe =
10341032 rdma_device_to_drv_device(device, struct rxe_dev, ib_dev);
10351033
1036− return scnprintf(buf, PAGE_SIZE, "%s\n", rxe_parent_name(rxe, 1));
1034+ return sysfs_emit(buf, "%s\n", rxe_parent_name(rxe, 1));
10371035 }
10381036
10391037 static DEVICE_ATTR_RO(parent);
@@ -1070,6 +1068,7 @@ static const struct ib_device_ops rxe_dev_ops = { · 30 unmodified lines
10701068 .create_cq = rxe_create_cq,
10711069 .create_qp = rxe_create_qp,
10721070 .create_srq = rxe_create_srq,
1071+ .create_user_ah = rxe_create_ah,
10731072 .dealloc_driver = rxe_dealloc,
10741073 .dealloc_pd = rxe_dealloc_pd,
10751074 .dealloc_ucontext = rxe_dealloc_ucontext,
@@ -1118,56 +1117,18 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name) · 42 unmodified lines
11181117 int err;
11191118 struct ib_device *dev = &rxe->ib_dev;
11201119 struct crypto_shash *tfm;
1121− u64 dma_mask;
11221120
11231121 strlcpy(dev->node_desc, "rxe", sizeof(dev->node_desc));
11241122
11251123 dev->node_type = RDMA_NODE_IB_CA;
11261124 dev->phys_port_cnt = 1;
11271125 dev->num_comp_vectors = num_possible_cpus();
1128− dev->dev.parent = rxe_dma_device(rxe);
11291126 dev->local_dma_lkey = 0;
11301127 addrconf_addr_eui48((unsigned char *)&dev->node_guid,
11311128 rxe->ndev->dev_addr);
1132− dev->dev.dma_parms = &rxe->dma_parms;
1133− dma_set_max_seg_size(&dev->dev, UINT_MAX);
1134− dma_mask = IS_ENABLED(CONFIG_64BIT) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32);
1135− err = dma_coerce_mask_and_coherent(&dev->dev, dma_mask);
1136− if (err)
1137− return err;
11381129
1139− dev->uverbs_cmd_mask = BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT)
1140− | BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)
1141− | BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE)
1142− | BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT)
1143− | BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD)
1144− | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD)
1145− | BIT_ULL(IB_USER_VERBS_CMD_CREATE_SRQ)
1146− | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_SRQ)
1147− | BIT_ULL(IB_USER_VERBS_CMD_QUERY_SRQ)
1148− | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_SRQ)
1149− | BIT_ULL(IB_USER_VERBS_CMD_POST_SRQ_RECV)
1150− | BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP)
1151− | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP)
1152− | BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP)
1153− | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP)
1154− | BIT_ULL(IB_USER_VERBS_CMD_POST_SEND)
1155− | BIT_ULL(IB_USER_VERBS_CMD_POST_RECV)
1156− | BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ)
1157− | BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ)
1158− | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ)
1159− | BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ)
1160− | BIT_ULL(IB_USER_VERBS_CMD_PEEK_CQ)
1161− | BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ)
1162− | BIT_ULL(IB_USER_VERBS_CMD_REG_MR)
1163− | BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR)
1164− | BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH)
1165− | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_AH)
1166− | BIT_ULL(IB_USER_VERBS_CMD_QUERY_AH)
1167− | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH)
1168− | BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST)
1169− | BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST)
1170− ;
1130+ dev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POST_SEND) |
1131+ BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ);
11711132
11721133 ib_set_device_ops(dev, &rxe_dev_ops);
11731134 err = ib_device_set_netdev(&rxe->ib_dev, rxe->ndev, 1);
drivers/infiniband/sw/rxe/rxe_verbs.h+0−2
@@ -273,7 +273,6 @@ enum rxe_mem_type { · 272 unmodified lines
273273 RXE_MEM_TYPE_NONE,
274274 RXE_MEM_TYPE_DMA,
275275 RXE_MEM_TYPE_MR,
276− RXE_MEM_TYPE_FMR,
277276 RXE_MEM_TYPE_MW,
278277 };
279278
@@ -352,7 +351,6 @@ struct rxe_port { · 72 unmodified lines
352351 struct rxe_dev {
353352 struct ib_device ib_dev;
354353 struct ib_device_attr attr;
355− struct device_dma_parameters dma_parms;
356354 int max_ucontext;
357355 int max_inline_data;
358356 struct mutex usdev_lock;
drivers/infiniband/sw/siw/Kconfig+0−1
@@ -2,7 +2,6 @@ config RDMA_SIW · 1 unmodified line
22 tristate "Software RDMA over TCP/IP (iWARP) driver"
33 depends on INET && INFINIBAND && LIBCRC32C
44 depends on INFINIBAND_VIRT_DMA
5− select DMA_VIRT_OPS
65 help
76 This driver implements the iWARP RDMA transport over
87 the Linux TCP/IP network stack. It enables a system with a
drivers/infiniband/sw/siw/siw.h+0−1
@@ -69,7 +69,6 @@ struct siw_pd { · 68 unmodified lines
6969
7070 struct siw_device {
7171 struct ib_device base_dev;
72− struct device_dma_parameters dma_parms;
7372 struct net_device *netdev;
7473 struct siw_dev_cap attrs;
7574
drivers/infiniband/sw/siw/siw_cm.c+1−1
@@ -1047,7 +1047,7 @@ static void siw_cm_work_handler(struct work_struct *w) · 1046 unmodified lines
10471047 cep->state);
10481048 }
10491049 }
1050− if (rv && rv != EAGAIN)
1050+ if (rv && rv != -EAGAIN)
10511051 release_cep = 1;
10521052 break;
10531053
drivers/infiniband/sw/siw/siw_main.c+3−49
@@ -305,25 +305,8 @@ static struct siw_device *siw_device_create(struct net_device *netdev) · 304 unmodified lines
305305 {
306306 struct siw_device *sdev = NULL;
307307 struct ib_device *base_dev;
308− struct device *parent = netdev->dev.parent;
309− u64 dma_mask;
310308 int rv;
311309
312− if (!parent) {
313− /*
314− * The loopback device has no parent device,
315− * so it appears as a top-level device. To support
316− * loopback device connectivity, take this device
317− * as the parent device. Skip all other devices
318− * w/o parent device.
319− */
320− if (netdev->type != ARPHRD_LOOPBACK) {
321− pr_warn("siw: device %s error: no parent device\n",
322− netdev->name);
323− return NULL;
324− }
325− parent = &netdev->dev;
326− }
327310 sdev = ib_alloc_device(siw_device, base_dev);
328311 if (!sdev)
329312 return NULL;
@@ -347,30 +330,8 @@ static struct siw_device *siw_device_create(struct net_device *netdev) · 17 unmodified lines
347330 addrconf_addr_eui48((unsigned char *)&base_dev->node_guid,
348331 addr);
349332 }
350− base_dev->uverbs_cmd_mask =
351− (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
352− (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
353− (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
354− (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
355− (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
356− (1ull << IB_USER_VERBS_CMD_REG_MR) |
357− (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
358− (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
359− (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
360− (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
361− (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
362− (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
363− (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
364− (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
365− (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
366− (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
367− (1ull << IB_USER_VERBS_CMD_POST_SEND) |
368− (1ull << IB_USER_VERBS_CMD_POST_RECV) |
369− (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
370− (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV) |
371− (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
372− (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
373− (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
333+
334+ base_dev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POST_SEND);
374335
375336 base_dev->node_type = RDMA_NODE_RNIC;
376337 memcpy(base_dev->node_desc, SIW_NODE_DESC_COMMON,
@@ -382,13 +343,6 @@ static struct siw_device *siw_device_create(struct net_device *netdev) · 5 unmodified lines
382343 * per physical port.
383344 */
384345 base_dev->phys_port_cnt = 1;
385− base_dev->dev.parent = parent;
386− base_dev->dev.dma_parms = &sdev->dma_parms;
387− dma_set_max_seg_size(&base_dev->dev, UINT_MAX);
388− dma_mask = IS_ENABLED(CONFIG_64BIT) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32);
389− if (dma_coerce_mask_and_coherent(&base_dev->dev, dma_mask))
390− goto error;
391−
392346 base_dev->num_comp_vectors = num_possible_cpus();
393347
394348 xa_init_flags(&sdev->qp_xa, XA_FLAGS_ALLOC1);
@@ -430,7 +384,7 @@ static struct siw_device *siw_device_create(struct net_device *netdev) · 35 unmodified lines
430384 atomic_set(&sdev->num_mr, 0);
431385 atomic_set(&sdev->num_pd, 0);
432386
433− sdev->numa_node = dev_to_node(parent);
387+ sdev->numa_node = dev_to_node(&netdev->dev);
434388 spin_lock_init(&sdev->lock);
435389
436390 return sdev;
drivers/infiniband/sw/siw/siw_verbs.c+12−0
@@ -307,6 +307,9 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, · 306 unmodified lines
307307
308308 siw_dbg(base_dev, "create new QP\n");
309309
310+ if (attrs->create_flags)
311+ return ERR_PTR(-EOPNOTSUPP);
312+
310313 if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
311314 siw_dbg(base_dev, "too many QP's\n");
312315 rv = -ENOMEM;
@@ -544,6 +547,9 @@ int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr, · 231 unmodified lines
544547 if (!attr_mask)
545548 return 0;
546549
550+ if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
551+ return -EOPNOTSUPP;
552+
547553 memset(&new_attrs, 0, sizeof(new_attrs));
548554
549555 if (attr_mask & IB_QP_ACCESS_FLAGS) {
@@ -1094,6 +1100,9 @@ int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr, · 544 unmodified lines
10941100 struct siw_cq *cq = to_siw_cq(base_cq);
10951101 int rv, size = attr->cqe;
10961102
1103+ if (attr->flags)
1104+ return -EOPNOTSUPP;
1105+
10971106 if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
10981107 siw_dbg(base_cq->device, "too many CQ's\n");
10991108 rv = -ENOMEM;
@@ -1555,6 +1564,9 @@ int siw_create_srq(struct ib_srq *base_srq, · 455 unmodified lines
15551564 base_ucontext);
15561565 int rv;
15571566
1567+ if (init_attrs->srq_type != IB_SRQT_BASIC)
1568+ return -EOPNOTSUPP;
1569+
15581570 if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
15591571 siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
15601572 rv = -ENOMEM;
drivers/infiniband/ulp/ipoib/ipoib_cm.c+2−2
@@ -1514,9 +1514,9 @@ static ssize_t show_mode(struct device *d, struct device_attribute *attr, · 1513 unmodified lines
15141514 struct ipoib_dev_priv *priv = ipoib_priv(dev);
15151515
15161516 if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags))
1517− return sprintf(buf, "connected\n");
1517+ return sysfs_emit(buf, "connected\n");
15181518 else
1519− return sprintf(buf, "datagram\n");
1519+ return sysfs_emit(buf, "datagram\n");
15201520 }
15211521
15221522 static ssize_t set_mode(struct device *d, struct device_attribute *attr,
drivers/infiniband/ulp/ipoib/ipoib_ethtool.c+4−0
@@ -166,6 +166,10 @@ static inline int ib_speed_enum_to_int(int speed) · 165 unmodified lines
166166 return SPEED_14000;
167167 case IB_SPEED_EDR:
168168 return SPEED_25000;
169+ case IB_SPEED_HDR:
170+ return SPEED_50000;
171+ case IB_SPEED_NDR:
172+ return SPEED_100000;
169173 }
170174
171175 return SPEED_UNKNOWN;
drivers/infiniband/ulp/ipoib/ipoib_main.c+4−3
@@ -2266,7 +2266,7 @@ static ssize_t show_pkey(struct device *dev, · 2265 unmodified lines
22662266 struct net_device *ndev = to_net_dev(dev);
22672267 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
22682268
2269− return sprintf(buf, "0x%04x\n", priv->pkey);
2269+ return sysfs_emit(buf, "0x%04x\n", priv->pkey);
22702270 }
22712271 static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
22722272
@@ -2276,7 +2276,8 @@ static ssize_t show_umcast(struct device *dev, · 3 unmodified lines
22762276 struct net_device *ndev = to_net_dev(dev);
22772277 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
22782278
2279− return sprintf(buf, "%d\n", test_bit(IPOIB_FLAG_UMCAST, &priv->flags));
2279+ return sysfs_emit(buf, "%d\n",
2280+ test_bit(IPOIB_FLAG_UMCAST, &priv->flags));
22802281 }
22812282
22822283 void ipoib_set_umcast(struct net_device *ndev, int umcast_val)
@@ -2446,7 +2447,7 @@ static ssize_t dev_id_show(struct device *dev, · 163 unmodified lines
24462447 "\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",
24472448 current->comm);
24482449
2449− return sprintf(buf, "%#x\n", ndev->dev_id);
2450+ return sysfs_emit(buf, "%#x\n", ndev->dev_id);
24502451 }
24512452 static DEVICE_ATTR_RO(dev_id);
24522453
drivers/infiniband/ulp/ipoib/ipoib_verbs.c+2−2
@@ -158,6 +158,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca) · 157 unmodified lines
158158
159159 int ret, size, req_vec;
160160 int i;
161+ static atomic_t counter;
161162
162163 size = ipoib_recvq_size + 1;
163164 ret = ipoib_cm_dev_init(dev);
@@ -171,8 +172,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca) · 7 unmodified lines
171172 if (ret != -EOPNOTSUPP)
172173 return ret;
173174
174− req_vec = (priv->port - 1) * 2;
175−
175+ req_vec = atomic_inc_return(&counter) * 2;
176176 cq_attr.cqe = size;
177177 cq_attr.comp_vector = req_vec % priv->ca->num_comp_vectors;
178178 priv->recv_cq = ib_create_cq(priv->ca, ipoib_ib_rx_completion, NULL,
drivers/infiniband/ulp/ipoib/ipoib_vlan.c+1−1
@@ -46,7 +46,7 @@ static ssize_t show_parent(struct device *d, struct device_attribute *attr, · 45 unmodified lines
4646 struct net_device *dev = to_net_dev(d);
4747 struct ipoib_dev_priv *priv = ipoib_priv(dev);
4848
49− return sprintf(buf, "%s\n", priv->parent->name);
49+ return sysfs_emit(buf, "%s\n", priv->parent->name);
5050 }
5151 static DEVICE_ATTR(parent, S_IRUGO, show_parent, NULL);
5252
drivers/infiniband/ulp/iser/iscsi_iser.c+6−18
@@ -187,23 +187,14 @@ iser_initialize_task_headers(struct iscsi_task *task, · 186 unmodified lines
187187 struct iser_device *device = iser_conn->ib_conn.device;
188188 struct iscsi_iser_task *iser_task = task->dd_data;
189189 u64 dma_addr;
190− const bool mgmt_task = !task->sc && !in_interrupt();
191− int ret = 0;
192190
193− if (unlikely(mgmt_task))
194− mutex_lock(&iser_conn->state_mutex);
195−
196− if (unlikely(iser_conn->state != ISER_CONN_UP)) {
197− ret = -ENODEV;
198− goto out;
199− }
191+ if (unlikely(iser_conn->state != ISER_CONN_UP))
192+ return -ENODEV;
200193
201194 dma_addr = ib_dma_map_single(device->ib_device, (void *)tx_desc,
202195 ISER_HEADERS_LEN, DMA_TO_DEVICE);
203− if (ib_dma_mapping_error(device->ib_device, dma_addr)) {
204− ret = -ENOMEM;
205− goto out;
206− }
196+ if (ib_dma_mapping_error(device->ib_device, dma_addr))
197+ return -ENOMEM;
207198
208199 tx_desc->inv_wr.next = NULL;
209200 tx_desc->reg_wr.wr.next = NULL;
@@ -214,11 +205,8 @@ iser_initialize_task_headers(struct iscsi_task *task, · 4 unmodified lines
214205 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
215206
216207 iser_task->iser_conn = iser_conn;
217−out:
218− if (unlikely(mgmt_task))
219− mutex_unlock(&iser_conn->state_mutex);
220208
221− return ret;
209+ return 0;
222210 }
223211
224212 /**
@@ -739,7 +727,7 @@ iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn, · 514 unmodified lines
739727 }
740728
741729 /**
742− * iscsi_iser_set_param() - set class connection parameter
730+ * iscsi_iser_conn_get_stats() - get iscsi connection statistics
743731 * @cls_conn: iscsi class connection
744732 * @stats: iscsi stats to output
745733 *
drivers/infiniband/ulp/isert/ib_isert.c+27−2
@@ -28,6 +28,18 @@ static int isert_debug_level; · 27 unmodified lines
2828 module_param_named(debug_level, isert_debug_level, int, 0644);
2929 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");
3030
31+static int isert_sg_tablesize_set(const char *val,
32+ const struct kernel_param *kp);
33+static const struct kernel_param_ops sg_tablesize_ops = {
34+ .set = isert_sg_tablesize_set,
35+ .get = param_get_int,
36+};
37+
38+static int isert_sg_tablesize = ISCSI_ISER_DEF_SG_TABLESIZE;
39+module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644);
40+MODULE_PARM_DESC(sg_tablesize,
41+ "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 256, max: 4096)");
42+
3143 static DEFINE_MUTEX(device_list_mutex);
3244 static LIST_HEAD(device_list);
3345 static struct workqueue_struct *isert_comp_wq;
@@ -47,6 +59,19 @@ static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); · 13 unmodified lines
4759 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc);
4860 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc);
4961
62+static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp)
63+{
64+ int n = 0, ret;
65+
66+ ret = kstrtoint(val, 10, &n);
67+ if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE ||
68+ n > ISCSI_ISER_MAX_SG_TABLESIZE)
69+ return -EINVAL;
70+
71+ return param_set_int(val, kp);
72+}
73+
74+
5075 static inline bool
5176 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
5277 {
@@ -101,7 +126,7 @@ isert_create_qp(struct isert_conn *isert_conn, · 48 unmodified lines
101126 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
102127 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
103128 factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num,
104− ISCSI_ISER_MAX_SG_TABLESIZE);
129+ isert_sg_tablesize);
105130 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor;
106131 attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
107132 attr.cap.max_recv_sge = 1;
@@ -1076,7 +1101,7 @@ isert_handle_scsi_cmd(struct isert_conn *isert_conn, · 968 unmodified lines
10761101 sequence_cmd:
10771102 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
10781103
1079− if (!rc && dump_payload == false && unsol_data)
1104+ if (!rc && !dump_payload && unsol_data)
10801105 iscsit_set_unsolicited_dataout(cmd);
10811106 else if (dump_payload && imm_data)
10821107 target_put_sess_cmd(&cmd->se_cmd);
drivers/infiniband/ulp/isert/ib_isert.h+6−0
@@ -65,6 +65,12 @@ · 64 unmodified lines
6565 */
6666 #define ISER_RX_SIZE (ISCSI_DEF_MAX_RECV_SEG_LEN + 1024)
6767
68+/* Default I/O size is 1MB */
69+#define ISCSI_ISER_DEF_SG_TABLESIZE 256
70+
71+/* Minimum I/O size is 512KB */
72+#define ISCSI_ISER_MIN_SG_TABLESIZE 128
73+
6874 /* Maximum support is 16MB I/O size */
6975 #define ISCSI_ISER_MAX_SG_TABLESIZE 4096
7076
drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h+1−1
@@ -437,7 +437,7 @@ struct opa_veswport_trap { · 436 unmodified lines
437437 } __packed;
438438
439439 /**
440− * struct opa_vnic_iface_macs_entry - single entry in the mac list
440+ * struct opa_vnic_iface_mac_entry - single entry in the mac list
441441 * @mac_addr: MAC address
442442 */
443443 struct opa_vnic_iface_mac_entry {
drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c+1−1
@@ -74,7 +74,7 @@ void opa_vnic_vema_report_event(struct opa_vnic_adapter *adapter, u8 event) · 73 unmodified lines
7474 }
7575
7676 /**
77− * opa_vnic_get_error_counters - get summary counters
77+ * opa_vnic_get_summary_counters - get summary counters
7878 * @adapter: vnic port adapter
7979 * @cntrs: pointer to destination summary counters structure
8080 *
drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c+31−31
@@ -52,7 +52,8 @@ static ssize_t max_reconnect_attempts_show(struct device *dev, · 51 unmodified lines
5252 {
5353 struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);
5454
55− return sprintf(page, "%d\n", rtrs_clt_get_max_reconnect_attempts(clt));
55+ return sysfs_emit(page, "%d\n",
56+ rtrs_clt_get_max_reconnect_attempts(clt));
5657 }
5758
5859 static ssize_t max_reconnect_attempts_store(struct device *dev,
@@ -95,11 +96,13 @@ static ssize_t mpath_policy_show(struct device *dev, · 36 unmodified lines
9596
9697 switch (clt->mp_policy) {
9798 case MP_POLICY_RR:
98− return sprintf(page, "round-robin (RR: %d)\n", clt->mp_policy);
99+ return sysfs_emit(page, "round-robin (RR: %d)\n",
100+ clt->mp_policy);
99101 case MP_POLICY_MIN_INFLIGHT:
100− return sprintf(page, "min-inflight (MI: %d)\n", clt->mp_policy);
102+ return sysfs_emit(page, "min-inflight (MI: %d)\n",
103+ clt->mp_policy);
101104 default:
102− return sprintf(page, "Unknown (%d)\n", clt->mp_policy);
105+ return sysfs_emit(page, "Unknown (%d)\n", clt->mp_policy);
103106 }
104107 }
105108
@@ -138,9 +141,10 @@ static DEVICE_ATTR_RW(mpath_policy); · 32 unmodified lines
138141 static ssize_t add_path_show(struct device *dev,
139142 struct device_attribute *attr, char *page)
140143 {
141− return scnprintf(page, PAGE_SIZE,
142− "Usage: echo [<source addr>@]<destination addr> > %s\n\n*addr ::= [ ip:<ipv4|ipv6> | gid:<gid> ]\n",
143− attr->attr.name);
144+ return sysfs_emit(
145+ page,
146+ "Usage: echo [<source addr>@]<destination addr> > %s\n\n*addr ::= [ ip:<ipv4|ipv6> | gid:<gid> ]\n",
147+ attr->attr.name);
144148 }
145149
146150 static ssize_t add_path_store(struct device *dev,
@@ -184,20 +188,18 @@ static ssize_t rtrs_clt_state_show(struct kobject *kobj, · 37 unmodified lines
184188
185189 sess = container_of(kobj, struct rtrs_clt_sess, kobj);
186190 if (sess->state == RTRS_CLT_CONNECTED)
187− return sprintf(page, "connected\n");
191+ return sysfs_emit(page, "connected\n");
188192
189− return sprintf(page, "disconnected\n");
193+ return sysfs_emit(page, "disconnected\n");
190194 }
191195
192196 static struct kobj_attribute rtrs_clt_state_attr =
193197 __ATTR(state, 0444, rtrs_clt_state_show, NULL);
194198
195199 static ssize_t rtrs_clt_reconnect_show(struct kobject *kobj,
196− struct kobj_attribute *attr,
197− char *page)
200+ struct kobj_attribute *attr, char *buf)
198201 {
199− return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
200− attr->attr.name);
202+ return sysfs_emit(buf, "Usage: echo 1 > %s\n", attr->attr.name);
201203 }
202204
203205 static ssize_t rtrs_clt_reconnect_store(struct kobject *kobj,
@@ -225,11 +227,9 @@ static struct kobj_attribute rtrs_clt_reconnect_attr = · 21 unmodified lines
225227 rtrs_clt_reconnect_store);
226228
227229 static ssize_t rtrs_clt_disconnect_show(struct kobject *kobj,
228− struct kobj_attribute *attr,
229− char *page)
230+ struct kobj_attribute *attr, char *buf)
230231 {
231− return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
232− attr->attr.name);
232+ return sysfs_emit(buf, "Usage: echo 1 > %s\n", attr->attr.name);
233233 }
234234
235235 static ssize_t rtrs_clt_disconnect_store(struct kobject *kobj,
@@ -257,11 +257,9 @@ static struct kobj_attribute rtrs_clt_disconnect_attr = · 21 unmodified lines
257257 rtrs_clt_disconnect_store);
258258
259259 static ssize_t rtrs_clt_remove_path_show(struct kobject *kobj,
260− struct kobj_attribute *attr,
261− char *page)
260+ struct kobj_attribute *attr, char *buf)
262261 {
263− return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
264− attr->attr.name);
262+ return sysfs_emit(buf, "Usage: echo 1 > %s\n", attr->attr.name);
265263 }
266264
267265 static ssize_t rtrs_clt_remove_path_store(struct kobject *kobj,
@@ -324,7 +322,7 @@ static ssize_t rtrs_clt_hca_port_show(struct kobject *kobj, · 56 unmodified lines
324322
325323 sess = container_of(kobj, typeof(*sess), kobj);
326324
327− return scnprintf(page, PAGE_SIZE, "%u\n", sess->hca_port);
325+ return sysfs_emit(page, "%u\n", sess->hca_port);
328326 }
329327
330328 static struct kobj_attribute rtrs_clt_hca_port_attr =
@@ -338,7 +336,7 @@ static ssize_t rtrs_clt_hca_name_show(struct kobject *kobj, · 7 unmodified lines
338336
339337 sess = container_of(kobj, struct rtrs_clt_sess, kobj);
340338
341− return scnprintf(page, PAGE_SIZE, "%s\n", sess->hca_name);
339+ return sysfs_emit(page, "%s\n", sess->hca_name);
342340 }
343341
344342 static struct kobj_attribute rtrs_clt_hca_name_attr =
@@ -349,12 +347,13 @@ static ssize_t rtrs_clt_src_addr_show(struct kobject *kobj, · 4 unmodified lines
349347 char *page)
350348 {
351349 struct rtrs_clt_sess *sess;
352− int cnt;
350+ int len;
353351
354352 sess = container_of(kobj, struct rtrs_clt_sess, kobj);
355− cnt = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
356− page, PAGE_SIZE);
357− return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
353+ len = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr, page,
354+ PAGE_SIZE);
355+ len += sysfs_emit_at(page, len, "\n");
356+ return len;
358357 }
359358
360359 static struct kobj_attribute rtrs_clt_src_addr_attr =
@@ -365,12 +364,13 @@ static ssize_t rtrs_clt_dst_addr_show(struct kobject *kobj, · 4 unmodified lines
365364 char *page)
366365 {
367366 struct rtrs_clt_sess *sess;
368− int cnt;
367+ int len;
369368
370369 sess = container_of(kobj, struct rtrs_clt_sess, kobj);
371− cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
372− page, PAGE_SIZE);
373− return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
370+ len = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr, page,
371+ PAGE_SIZE);
372+ len += sysfs_emit_at(page, len, "\n");
373+ return len;
374374 }
375375
376376 static struct kobj_attribute rtrs_clt_dst_addr_attr =
drivers/infiniband/ulp/rtrs/rtrs-clt.c+36−38
@@ -1236,8 +1236,7 @@ static void free_sess_reqs(struct rtrs_clt_sess *sess) · 1235 unmodified lines
12361236 if (req->mr)
12371237 ib_dereg_mr(req->mr);
12381238 kfree(req->sge);
1239− rtrs_iu_free(req->iu, DMA_TO_DEVICE,
1240− sess->s.dev->ib_dev, 1);
1239+ rtrs_iu_free(req->iu, sess->s.dev->ib_dev, 1);
12411240 }
12421241 kfree(sess->reqs);
12431242 sess->reqs = NULL;
@@ -1499,6 +1498,7 @@ static int create_con(struct rtrs_clt_sess *sess, unsigned int cid) · 255 unmodified lines
14991498 con->c.cid = cid;
15001499 con->c.sess = &sess->s;
15011500 atomic_set(&con->io_cnt, 0);
1501+ mutex_init(&con->con_mutex);
15021502
15031503 sess->s.con[cid] = &con->c;
15041504
@@ -1510,6 +1510,7 @@ static void destroy_con(struct rtrs_clt_con *con) · 5 unmodified lines
15101510 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
15111511
15121512 sess->s.con[con->c.cid] = NULL;
1513+ mutex_destroy(&con->con_mutex);
15131514 kfree(con);
15141515 }
15151516
@@ -1520,15 +1521,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) · 4 unmodified lines
15201521 int err, cq_vector;
15211522 struct rtrs_msg_rkey_rsp *rsp;
15221523
1523− /*
1524− * This function can fail, but still destroy_con_cq_qp() should
1525− * be called, this is because create_con_cq_qp() is called on cm
1526− * event path, thus caller/waiter never knows: have we failed before
1527− * create_con_cq_qp() or after. To solve this dilemma without
1528− * creating any additional flags just allow destroy_con_cq_qp() be
1529− * called many times.
1530− */
1531−
1524+ lockdep_assert_held(&con->con_mutex);
15321525 if (con->c.cid == 0) {
15331526 /*
15341527 * One completion for each receive and two for each send
@@ -1602,11 +1595,10 @@ static void destroy_con_cq_qp(struct rtrs_clt_con *con) · 67 unmodified lines
16021595 * Be careful here: destroy_con_cq_qp() can be called even
16031596 * create_con_cq_qp() failed, see comments there.
16041597 */
1605−
1598+ lockdep_assert_held(&con->con_mutex);
16061599 rtrs_cq_qp_destroy(&con->c);
16071600 if (con->rsp_ius) {
1608− rtrs_iu_free(con->rsp_ius, DMA_FROM_DEVICE,
1609− sess->s.dev->ib_dev, con->queue_size);
1601+ rtrs_iu_free(con->rsp_ius, sess->s.dev->ib_dev, con->queue_size);
16101602 con->rsp_ius = NULL;
16111603 con->queue_size = 0;
16121604 }
@@ -1634,16 +1626,16 @@ static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con) · 21 unmodified lines
16341626 struct rtrs_sess *s = con->c.sess;
16351627 int err;
16361628
1629+ mutex_lock(&con->con_mutex);
16371630 err = create_con_cq_qp(con);
1631+ mutex_unlock(&con->con_mutex);
16381632 if (err) {
16391633 rtrs_err(s, "create_con_cq_qp(), err: %d\n", err);
16401634 return err;
16411635 }
16421636 err = rdma_resolve_route(con->c.cm_id, RTRS_CONNECT_TIMEOUT_MS);
1643− if (err) {
1637+ if (err)
16441638 rtrs_err(s, "Resolving route failed, err: %d\n", err);
1645− destroy_con_cq_qp(con);
1646− }
16471639
16481640 return err;
16491641 }
@@ -1837,8 +1829,8 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, · 187 unmodified lines
18371829 cm_err = rtrs_rdma_route_resolved(con);
18381830 break;
18391831 case RDMA_CM_EVENT_ESTABLISHED:
1840− con->cm_err = rtrs_rdma_conn_established(con, ev);
1841− if (likely(!con->cm_err)) {
1832+ cm_err = rtrs_rdma_conn_established(con, ev);
1833+ if (likely(!cm_err)) {
18421834 /*
18431835 * Report success and wake up. Here we abuse state_wq,
18441836 * i.e. wake up without state change, but we set cm_err.
@@ -1851,20 +1843,22 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, · 6 unmodified lines
18511843 case RDMA_CM_EVENT_REJECTED:
18521844 cm_err = rtrs_rdma_conn_rejected(con, ev);
18531845 break;
1846+ case RDMA_CM_EVENT_DISCONNECTED:
1847+ /* No message for disconnecting */
1848+ cm_err = -ECONNRESET;
1849+ break;
18541850 case RDMA_CM_EVENT_CONNECT_ERROR:
18551851 case RDMA_CM_EVENT_UNREACHABLE:
1852+ case RDMA_CM_EVENT_ADDR_CHANGE:
1853+ case RDMA_CM_EVENT_TIMEWAIT_EXIT:
18561854 rtrs_wrn(s, "CM error event %d\n", ev->event);
18571855 cm_err = -ECONNRESET;
18581856 break;
18591857 case RDMA_CM_EVENT_ADDR_ERROR:
18601858 case RDMA_CM_EVENT_ROUTE_ERROR:
1859+ rtrs_wrn(s, "CM error event %d\n", ev->event);
18611860 cm_err = -EHOSTUNREACH;
18621861 break;
1863− case RDMA_CM_EVENT_DISCONNECTED:
1864− case RDMA_CM_EVENT_ADDR_CHANGE:
1865− case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1866− cm_err = -ECONNRESET;
1867− break;
18681862 case RDMA_CM_EVENT_DEVICE_REMOVAL:
18691863 /*
18701864 * Device removal is a special case. Queue close and return 0.
@@ -1949,8 +1943,9 @@ static int create_cm(struct rtrs_clt_con *con) · 78 unmodified lines
19491943
19501944 errr:
19511945 stop_cm(con);
1952− /* Is safe to call destroy if cq_qp is not inited */
1946+ mutex_lock(&con->con_mutex);
19531947 destroy_con_cq_qp(con);
1948+ mutex_unlock(&con->con_mutex);
19541949 destroy_cm:
19551950 destroy_cm(con);
19561951
@@ -2057,7 +2052,9 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) · 100 unmodified lines
20572052 if (!sess->s.con[cid])
20582053 break;
20592054 con = to_clt_con(sess->s.con[cid]);
2055+ mutex_lock(&con->con_mutex);
20602056 destroy_con_cq_qp(con);
2057+ mutex_unlock(&con->con_mutex);
20612058 destroy_cm(con);
20622059 destroy_con(con);
20632060 }
@@ -2164,8 +2161,7 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) · 100 unmodified lines
21642161 mutex_unlock(&clt->paths_mutex);
21652162 }
21662163
2167−static void rtrs_clt_add_path_to_arr(struct rtrs_clt_sess *sess,
2168− struct rtrs_addr *addr)
2164+static void rtrs_clt_add_path_to_arr(struct rtrs_clt_sess *sess)
21692165 {
21702166 struct rtrs_clt *clt = sess->clt;
21712167
@@ -2224,7 +2220,10 @@ destroy: · 52 unmodified lines
22242220 struct rtrs_clt_con *con = to_clt_con(sess->s.con[cid]);
22252221
22262222 stop_cm(con);
2223+
2224+ mutex_lock(&con->con_mutex);
22272225 destroy_con_cq_qp(con);
2226+ mutex_unlock(&con->con_mutex);
22282227 destroy_cm(con);
22292228 destroy_con(con);
22302229 }
@@ -2245,7 +2244,7 @@ static void rtrs_clt_info_req_done(struct ib_cq *cq, struct ib_wc *wc) · 14 unmodified lines
22452244 struct rtrs_iu *iu;
22462245
22472246 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
2248− rtrs_iu_free(iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1);
2247+ rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);
22492248
22502249 if (unlikely(wc->status != IB_WC_SUCCESS)) {
22512250 rtrs_err(sess->clt, "Sess info request send failed: %s\n",
@@ -2264,8 +2263,12 @@ static int process_info_rsp(struct rtrs_clt_sess *sess, · 12 unmodified lines
22642263 int i, sgi;
22652264
22662265 sg_cnt = le16_to_cpu(msg->sg_cnt);
2267− if (unlikely(!sg_cnt))
2266+ if (unlikely(!sg_cnt || (sess->queue_depth % sg_cnt))) {
2267+ rtrs_err(sess->clt, "Incorrect sg_cnt %d, is not multiple\n",
2268+ sg_cnt);
22682269 return -EINVAL;
2270+ }
2271+
22692272 /*
22702273 * Check if IB immediate data size is enough to hold the mem_id and
22712274 * the offset inside the memory chunk.
@@ -2278,11 +2281,6 @@ static int process_info_rsp(struct rtrs_clt_sess *sess, · 6 unmodified lines
22782281 MAX_IMM_PAYL_BITS, sg_cnt, sess->chunk_size);
22792282 return -EINVAL;
22802283 }
2281− if (unlikely(!sg_cnt || (sess->queue_depth % sg_cnt))) {
2282− rtrs_err(sess->clt, "Incorrect sg_cnt %d, is not multiple\n",
2283− sg_cnt);
2284− return -EINVAL;
2285− }
22862284 total_len = 0;
22872285 for (sgi = 0, i = 0; sgi < sg_cnt && i < sess->queue_depth; sgi++) {
22882286 const struct rtrs_sg_desc *desc = &msg->desc[sgi];
@@ -2374,7 +2372,7 @@ static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) · 85 unmodified lines
23742372
23752373 out:
23762374 rtrs_clt_update_wc_stats(con);
2377− rtrs_iu_free(iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1);
2375+ rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);
23782376 rtrs_clt_change_state(sess, state);
23792377 }
23802378
@@ -2436,9 +2434,9 @@ static int rtrs_send_sess_info(struct rtrs_clt_sess *sess) · 55 unmodified lines
24362434
24372435 out:
24382436 if (tx_iu)
2439− rtrs_iu_free(tx_iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1);
2437+ rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1);
24402438 if (rx_iu)
2441− rtrs_iu_free(rx_iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1);
2439+ rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1);
24422440 if (unlikely(err))
24432441 /* If we've never taken async path because of malloc problems */
24442442 rtrs_clt_change_state(sess, RTRS_CLT_CONNECTING_ERR);
@@ -2938,7 +2936,7 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt, · 493 unmodified lines
29382936 * IO will never grab it. Also it is very important to add
29392937 * path before init, since init fires LINK_CONNECTED event.
29402938 */
2941− rtrs_clt_add_path_to_arr(sess, addr);
2939+ rtrs_clt_add_path_to_arr(sess);
29422940
29432941 err = init_sess(sess);
29442942 if (err)
drivers/infiniband/ulp/rtrs/rtrs-clt.h+1−0
@@ -72,6 +72,7 @@ struct rtrs_clt_con { · 71 unmodified lines
7272 struct rtrs_iu *rsp_ius;
7373 u32 queue_size;
7474 unsigned int cpu;
75+ struct mutex con_mutex;
7576 atomic_t io_cnt;
7677 int cm_err;
7778 };
drivers/infiniband/ulp/rtrs/rtrs-pri.h+1−2
@@ -287,8 +287,7 @@ struct rtrs_msg_rdma_hdr { · 286 unmodified lines
287287 struct rtrs_iu *rtrs_iu_alloc(u32 queue_size, size_t size, gfp_t t,
288288 struct ib_device *dev, enum dma_data_direction,
289289 void (*done)(struct ib_cq *cq, struct ib_wc *wc));
290−void rtrs_iu_free(struct rtrs_iu *iu, enum dma_data_direction dir,
291− struct ib_device *dev, u32 queue_size);
290+void rtrs_iu_free(struct rtrs_iu *iu, struct ib_device *dev, u32 queue_size);
292291 int rtrs_iu_post_recv(struct rtrs_con *con, struct rtrs_iu *iu);
293292 int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size,
294293 struct ib_send_wr *head);
drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c+9−12
@@ -27,11 +27,9 @@ static struct kobj_type ktype = { · 26 unmodified lines
2727 };
2828
2929 static ssize_t rtrs_srv_disconnect_show(struct kobject *kobj,
30− struct kobj_attribute *attr,
31− char *page)
30+ struct kobj_attribute *attr, char *buf)
3231 {
33− return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
34− attr->attr.name);
32+ return sysfs_emit(buf, "Usage: echo 1 > %s\n", attr->attr.name);
3533 }
3634
3735 static ssize_t rtrs_srv_disconnect_store(struct kobject *kobj,
@@ -72,8 +70,7 @@ static ssize_t rtrs_srv_hca_port_show(struct kobject *kobj, · 34 unmodified lines
7270 sess = container_of(kobj, typeof(*sess), kobj);
7371 usr_con = sess->s.con[0];
7472
75− return scnprintf(page, PAGE_SIZE, "%u\n",
76− usr_con->cm_id->port_num);
73+ return sysfs_emit(page, "%u\n", usr_con->cm_id->port_num);
7774 }
7875
7976 static struct kobj_attribute rtrs_srv_hca_port_attr =
@@ -87,8 +84,7 @@ static ssize_t rtrs_srv_hca_name_show(struct kobject *kobj, · 7 unmodified lines
8784
8885 sess = container_of(kobj, struct rtrs_srv_sess, kobj);
8986
90− return scnprintf(page, PAGE_SIZE, "%s\n",
91− sess->s.dev->ib_dev->name);
87+ return sysfs_emit(page, "%s\n", sess->s.dev->ib_dev->name);
9288 }
9389
9490 static struct kobj_attribute rtrs_srv_hca_name_attr =
@@ -115,12 +111,13 @@ static ssize_t rtrs_srv_dst_addr_show(struct kobject *kobj, · 20 unmodified lines
115111 char *page)
116112 {
117113 struct rtrs_srv_sess *sess;
118− int cnt;
114+ int len;
119115
120116 sess = container_of(kobj, struct rtrs_srv_sess, kobj);
121− cnt = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
122− page, PAGE_SIZE);
123− return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
117+ len = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr, page,
118+ PAGE_SIZE);
119+ len += sysfs_emit_at(page, len, "\n");
120+ return len;
124121 }
125122
126123 static struct kobj_attribute rtrs_srv_dst_addr_attr =
drivers/infiniband/ulp/rtrs/rtrs-srv.c+46−73
@@ -113,28 +113,18 @@ static bool __rtrs_srv_change_state(struct rtrs_srv_sess *sess, · 112 unmodified lines
113113 return changed;
114114 }
115115
116−static bool rtrs_srv_change_state_get_old(struct rtrs_srv_sess *sess,
117− enum rtrs_srv_state new_state,
118− enum rtrs_srv_state *old_state)
116+static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess,
117+ enum rtrs_srv_state new_state)
119118 {
120119 bool changed;
121120
122121 spin_lock_irq(&sess->state_lock);
123− *old_state = sess->state;
124122 changed = __rtrs_srv_change_state(sess, new_state);
125123 spin_unlock_irq(&sess->state_lock);
126124
127125 return changed;
128126 }
129127
130−static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess,
131− enum rtrs_srv_state new_state)
132−{
133− enum rtrs_srv_state old_state;
134−
135− return rtrs_srv_change_state_get_old(sess, new_state, &old_state);
136−}
137−
138128 static void free_id(struct rtrs_srv_op *id)
139129 {
140130 if (!id)
@@ -471,10 +461,7 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, · 330 unmodified lines
471461
472462 void close_sess(struct rtrs_srv_sess *sess)
473463 {
474− enum rtrs_srv_state old_state;
475−
476− if (rtrs_srv_change_state_get_old(sess, RTRS_SRV_CLOSING,
477− &old_state))
464+ if (rtrs_srv_change_state(sess, RTRS_SRV_CLOSING))
478465 queue_work(rtrs_wq, &sess->close_work);
479466 WARN_ON(sess->state != RTRS_SRV_CLOSING);
480467 }
@@ -577,8 +564,7 @@ static void unmap_cont_bufs(struct rtrs_srv_sess *sess) · 96 unmodified lines
577564 struct rtrs_srv_mr *srv_mr;
578565
579566 srv_mr = &sess->mrs[i];
580− rtrs_iu_free(srv_mr->iu, DMA_TO_DEVICE,
581− sess->s.dev->ib_dev, 1);
567+ rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1);
582568 ib_dereg_mr(srv_mr->mr);
583569 ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl,
584570 srv_mr->sgt.nents, DMA_BIDIRECTIONAL);
@@ -682,8 +668,7 @@ err: · 97 unmodified lines
682668 sgt = &srv_mr->sgt;
683669 mr = srv_mr->mr;
684670 free_iu:
685− rtrs_iu_free(srv_mr->iu, DMA_TO_DEVICE,
686− sess->s.dev->ib_dev, 1);
671+ rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1);
687672 dereg_mr:
688673 ib_dereg_mr(mr);
689674 unmap_sg:
@@ -735,7 +720,7 @@ static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) · 45 unmodified lines
735720 struct rtrs_iu *iu;
736721
737722 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
738− rtrs_iu_free(iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1);
723+ rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);
739724
740725 if (unlikely(wc->status != IB_WC_SUCCESS)) {
741726 rtrs_err(s, "Sess info response send failed: %s\n",
@@ -861,7 +846,7 @@ static int process_info_req(struct rtrs_srv_con *con, · 119 unmodified lines
861846 if (unlikely(err)) {
862847 rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err);
863848 iu_free:
864− rtrs_iu_free(tx_iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1);
849+ rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1);
865850 }
866851 rwr_free:
867852 kfree(rwr);
@@ -906,7 +891,7 @@ static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) · 38 unmodified lines
906891 goto close;
907892
908893 out:
909− rtrs_iu_free(iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1);
894+ rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);
910895 return;
911896 close:
912897 close_sess(sess);
@@ -929,7 +914,7 @@ static int post_recv_info_req(struct rtrs_srv_con *con) · 16 unmodified lines
929914 err = rtrs_iu_post_recv(&con->c, rx_iu);
930915 if (unlikely(err)) {
931916 rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err);
932− rtrs_iu_free(rx_iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1);
917+ rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1);
933918 return err;
934919 }
935920
@@ -1328,17 +1313,42 @@ static void rtrs_srv_dev_release(struct device *dev) · 392 unmodified lines
13281313 kfree(srv);
13291314 }
13301315
1331−static struct rtrs_srv *__alloc_srv(struct rtrs_srv_ctx *ctx,
1332− const uuid_t *paths_uuid)
1316+static void free_srv(struct rtrs_srv *srv)
1317+{
1318+ int i;
1319+
1320+ WARN_ON(refcount_read(&srv->refcount));
1321+ for (i = 0; i < srv->queue_depth; i++)
1322+ mempool_free(srv->chunks[i], chunk_pool);
1323+ kfree(srv->chunks);
1324+ mutex_destroy(&srv->paths_mutex);
1325+ mutex_destroy(&srv->paths_ev_mutex);
1326+ /* last put to release the srv structure */
1327+ put_device(&srv->dev);
1328+}
1329+
1330+static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx,
1331+ const uuid_t *paths_uuid)
13331332 {
13341333 struct rtrs_srv *srv;
13351334 int i;
13361335
1336+ mutex_lock(&ctx->srv_mutex);
1337+ list_for_each_entry(srv, &ctx->srv_list, ctx_list) {
1338+ if (uuid_equal(&srv->paths_uuid, paths_uuid) &&
1339+ refcount_inc_not_zero(&srv->refcount)) {
1340+ mutex_unlock(&ctx->srv_mutex);
1341+ return srv;
1342+ }
1343+ }
1344+
1345+ /* need to allocate a new srv */
13371346 srv = kzalloc(sizeof(*srv), GFP_KERNEL);
1338− if (!srv)
1347+ if (!srv) {
1348+ mutex_unlock(&ctx->srv_mutex);
13391349 return NULL;
1350+ }
13401351
1341− refcount_set(&srv->refcount, 1);
13421352 INIT_LIST_HEAD(&srv->paths_list);
13431353 mutex_init(&srv->paths_mutex);
13441354 mutex_init(&srv->paths_ev_mutex);
@@ -1347,6 +1357,8 @@ static struct rtrs_srv *__alloc_srv(struct rtrs_srv_ctx *ctx, · 2 unmodified lines
13471357 srv->ctx = ctx;
13481358 device_initialize(&srv->dev);
13491359 srv->dev.release = rtrs_srv_dev_release;
1360+ list_add(&srv->ctx_list, &ctx->srv_list);
1361+ mutex_unlock(&ctx->srv_mutex);
13501362
13511363 srv->chunks = kcalloc(srv->queue_depth, sizeof(*srv->chunks),
13521364 GFP_KERNEL);
@@ -1358,7 +1370,7 @@ static struct rtrs_srv *__alloc_srv(struct rtrs_srv_ctx *ctx, · 5 unmodified lines
13581370 if (!srv->chunks[i])
13591371 goto err_free_chunks;
13601372 }
1361− list_add(&srv->ctx_list, &ctx->srv_list);
1373+ refcount_set(&srv->refcount, 1);
13621374
13631375 return srv;
13641376
@@ -1369,52 +1381,9 @@ err_free_chunks: · 4 unmodified lines
13691381
13701382 err_free_srv:
13711383 kfree(srv);
1372−
1373− return NULL;
1374−}
1375−
1376−static void free_srv(struct rtrs_srv *srv)
1377−{
1378− int i;
1379−
1380− WARN_ON(refcount_read(&srv->refcount));
1381− for (i = 0; i < srv->queue_depth; i++)
1382− mempool_free(srv->chunks[i], chunk_pool);
1383− kfree(srv->chunks);
1384− mutex_destroy(&srv->paths_mutex);
1385− mutex_destroy(&srv->paths_ev_mutex);
1386− /* last put to release the srv structure */
1387− put_device(&srv->dev);
1388−}
1389−
1390−static inline struct rtrs_srv *__find_srv_and_get(struct rtrs_srv_ctx *ctx,
1391− const uuid_t *paths_uuid)
1392−{
1393− struct rtrs_srv *srv;
1394−
1395− list_for_each_entry(srv, &ctx->srv_list, ctx_list) {
1396− if (uuid_equal(&srv->paths_uuid, paths_uuid) &&
1397− refcount_inc_not_zero(&srv->refcount))
1398− return srv;
1399− }
1400−
14011384 return NULL;
14021385 }
14031386
1404−static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx,
1405− const uuid_t *paths_uuid)
1406−{
1407− struct rtrs_srv *srv;
1408−
1409− mutex_lock(&ctx->srv_mutex);
1410− srv = __find_srv_and_get(ctx, paths_uuid);
1411− if (!srv)
1412− srv = __alloc_srv(ctx, paths_uuid);
1413− mutex_unlock(&ctx->srv_mutex);
1414−
1415− return srv;
1416−}
1417−
14181387 static void put_srv(struct rtrs_srv *srv)
14191388 {
14201389 if (refcount_dec_and_test(&srv->refcount)) {
@@ -1813,7 +1782,11 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, · 392 unmodified lines
18131782 }
18141783 recon_cnt = le16_to_cpu(msg->recon_cnt);
18151784 srv = get_or_create_srv(ctx, &msg->paths_uuid);
1816− if (!srv) {
1785+ /*
1786+ * "refcount == 0" happens if a previous thread calls get_or_create_srv
1787+ * allocate srv, but chunks of srv are not allocated yet.
1788+ */
1789+ if (!srv || refcount_read(&srv->refcount) == 0) {
18171790 err = -ENOMEM;
18181791 goto reject_w_err;
18191792 }
drivers/infiniband/ulp/rtrs/rtrs-srv.h+1−1
@@ -62,7 +62,7 @@ struct rtrs_srv_op { · 61 unmodified lines
6262
6363 /*
6464 * server side memory region context, when always_invalidate=Y, we need
65− * queue_depth of memory regrion to invalidate each memory region.
65+ * queue_depth of memory region to invalidate each memory region.
6666 */
6767 struct rtrs_srv_mr {
6868 struct ib_mr *mr;
drivers/infiniband/ulp/rtrs/rtrs.c+23−38
@@ -31,6 +31,7 @@ struct rtrs_iu *rtrs_iu_alloc(u32 queue_size, size_t size, gfp_t gfp_mask, · 30 unmodified lines
3131 return NULL;
3232 for (i = 0; i < queue_size; i++) {
3333 iu = &ius[i];
34+ iu->direction = dir;
3435 iu->buf = kzalloc(size, gfp_mask);
3536 if (!iu->buf)
3637 goto err;
@@ -41,17 +42,15 @@ struct rtrs_iu *rtrs_iu_alloc(u32 queue_size, size_t size, gfp_t gfp_mask, · 4 unmodified lines
4142
4243 iu->cqe.done = done;
4344 iu->size = size;
44− iu->direction = dir;
4545 }
4646 return ius;
4747 err:
48− rtrs_iu_free(ius, dir, dma_dev, i);
48+ rtrs_iu_free(ius, dma_dev, i);
4949 return NULL;
5050 }
5151 EXPORT_SYMBOL_GPL(rtrs_iu_alloc);
5252
53−void rtrs_iu_free(struct rtrs_iu *ius, enum dma_data_direction dir,
54− struct ib_device *ibdev, u32 queue_size)
53+void rtrs_iu_free(struct rtrs_iu *ius, struct ib_device *ibdev, u32 queue_size)
5554 {
5655 struct rtrs_iu *iu;
5756 int i;
@@ -61,7 +60,7 @@ void rtrs_iu_free(struct rtrs_iu *ius, enum dma_data_direction dir, · 3 unmodified lines
6160
6261 for (i = 0; i < queue_size; i++) {
6362 iu = &ius[i];
64− ib_dma_unmap_single(ibdev, iu->dma_addr, iu->size, dir);
63+ ib_dma_unmap_single(ibdev, iu->dma_addr, iu->size, iu->direction);
6564 kfree(iu->buf);
6665 }
6766 kfree(ius);
@@ -105,6 +104,22 @@ int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe) · 37 unmodified lines
105104 }
106105 EXPORT_SYMBOL_GPL(rtrs_post_recv_empty);
107106
107+static int rtrs_post_send(struct ib_qp *qp, struct ib_send_wr *head,
108+ struct ib_send_wr *wr)
109+{
110+ if (head) {
111+ struct ib_send_wr *tail = head;
112+
113+ while (tail->next)
114+ tail = tail->next;
115+ tail->next = wr;
116+ } else {
117+ head = wr;
118+ }
119+
120+ return ib_post_send(qp, head, NULL);
121+}
122+
108123 int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size,
109124 struct ib_send_wr *head)
110125 {
@@ -127,17 +142,7 @@ int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size, · 16 unmodified lines
127142 .send_flags = IB_SEND_SIGNALED,
128143 };
129144
130− if (head) {
131− struct ib_send_wr *tail = head;
132−
133− while (tail->next)
134− tail = tail->next;
135− tail->next = &wr;
136− } else {
137− head = &wr;
138− }
139−
140− return ib_post_send(con->qp, head, NULL);
145+ return rtrs_post_send(con->qp, head, &wr);
141146 }
142147 EXPORT_SYMBOL_GPL(rtrs_iu_post_send);
143148
@@ -169,17 +174,7 @@ int rtrs_iu_post_rdma_write_imm(struct rtrs_con *con, struct rtrs_iu *iu, · 25 unmodified lines
169174 if (WARN_ON(sge[i].length == 0))
170175 return -EINVAL;
171176
172− if (head) {
173− struct ib_send_wr *tail = head;
174−
175− while (tail->next)
176− tail = tail->next;
177− tail->next = &wr.wr;
178− } else {
179− head = &wr.wr;
180− }
181−
182− return ib_post_send(con->qp, head, NULL);
177+ return rtrs_post_send(con->qp, head, &wr.wr);
183178 }
184179 EXPORT_SYMBOL_GPL(rtrs_iu_post_rdma_write_imm);
185180
@@ -196,17 +191,7 @@ int rtrs_post_rdma_write_imm_empty(struct rtrs_con *con, struct ib_cqe *cqe, · 10 unmodified lines
196191 .ex.imm_data = cpu_to_be32(imm_data),
197192 };
198193
199− if (head) {
200− struct ib_send_wr *tail = head;
201−
202− while (tail->next)
203− tail = tail->next;
204− tail->next = &wr;
205− } else {
206− head = &wr;
207− }
208−
209− return ib_post_send(con->qp, head, NULL);
194+ return rtrs_post_send(con->qp, head, &wr);
210195 }
211196 EXPORT_SYMBOL_GPL(rtrs_post_rdma_write_imm_empty);
212197
drivers/infiniband/ulp/srp/ib_srp.c+26−22
@@ -169,9 +169,9 @@ static int srp_tmo_get(char *buffer, const struct kernel_param *kp) · 168 unmodified lines
169169 int tmo = *(int *)kp->arg;
170170
171171 if (tmo >= 0)
172− return sprintf(buffer, "%d\n", tmo);
172+ return sysfs_emit(buffer, "%d\n", tmo);
173173 else
174− return sprintf(buffer, "off\n");
174+ return sysfs_emit(buffer, "off\n");
175175 }
176176
177177 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
@@ -2896,7 +2896,7 @@ static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr, · 2718 unmodified lines
28962896 {
28972897 struct srp_target_port *target = host_to_target(class_to_shost(dev));
28982898
2899− return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2899+ return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
29002900 }
29012901
29022902 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
@@ -2904,7 +2904,7 @@ static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr, · 1 unmodified line
29042904 {
29052905 struct srp_target_port *target = host_to_target(class_to_shost(dev));
29062906
2907− return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2907+ return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
29082908 }
29092909
29102910 static ssize_t show_service_id(struct device *dev,
@@ -2914,8 +2914,8 @@ static ssize_t show_service_id(struct device *dev, · 3 unmodified lines
29142914
29152915 if (target->using_rdma_cm)
29162916 return -ENOENT;
2917− return sprintf(buf, "0x%016llx\n",
2918− be64_to_cpu(target->ib_cm.service_id));
2917+ return sysfs_emit(buf, "0x%016llx\n",
2918+ be64_to_cpu(target->ib_cm.service_id));
29192919 }
29202920
29212921 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
@@ -2925,7 +2925,8 @@ static ssize_t show_pkey(struct device *dev, struct device_attribute *attr, · 3 unmodified lines
29252925
29262926 if (target->using_rdma_cm)
29272927 return -ENOENT;
2928− return sprintf(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey));
2928+
2929+ return sysfs_emit(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey));
29292930 }
29302931
29312932 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
@@ -2933,7 +2934,7 @@ static ssize_t show_sgid(struct device *dev, struct device_attribute *attr, · 1 unmodified line
29332934 {
29342935 struct srp_target_port *target = host_to_target(class_to_shost(dev));
29352936
2936− return sprintf(buf, "%pI6\n", target->sgid.raw);
2937+ return sysfs_emit(buf, "%pI6\n", target->sgid.raw);
29372938 }
29382939
29392940 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
@@ -2944,7 +2945,8 @@ static ssize_t show_dgid(struct device *dev, struct device_attribute *attr, · 4 unmodified lines
29442945
29452946 if (target->using_rdma_cm)
29462947 return -ENOENT;
2947− return sprintf(buf, "%pI6\n", ch->ib_cm.path.dgid.raw);
2948+
2949+ return sysfs_emit(buf, "%pI6\n", ch->ib_cm.path.dgid.raw);
29482950 }
29492951
29502952 static ssize_t show_orig_dgid(struct device *dev,
@@ -2954,7 +2956,8 @@ static ssize_t show_orig_dgid(struct device *dev, · 3 unmodified lines
29542956
29552957 if (target->using_rdma_cm)
29562958 return -ENOENT;
2957− return sprintf(buf, "%pI6\n", target->ib_cm.orig_dgid.raw);
2959+
2960+ return sysfs_emit(buf, "%pI6\n", target->ib_cm.orig_dgid.raw);
29582961 }
29592962
29602963 static ssize_t show_req_lim(struct device *dev,
@@ -2968,7 +2971,8 @@ static ssize_t show_req_lim(struct device *dev, · 7 unmodified lines
29682971 ch = &target->ch[i];
29692972 req_lim = min(req_lim, ch->req_lim);
29702973 }
2971− return sprintf(buf, "%d\n", req_lim);
2974+
2975+ return sysfs_emit(buf, "%d\n", req_lim);
29722976 }
29732977
29742978 static ssize_t show_zero_req_lim(struct device *dev,
@@ -2976,7 +2980,7 @@ static ssize_t show_zero_req_lim(struct device *dev, · 1 unmodified line
29762980 {
29772981 struct srp_target_port *target = host_to_target(class_to_shost(dev));
29782982
2979− return sprintf(buf, "%d\n", target->zero_req_lim);
2983+ return sysfs_emit(buf, "%d\n", target->zero_req_lim);
29802984 }
29812985
29822986 static ssize_t show_local_ib_port(struct device *dev,
@@ -2984,7 +2988,7 @@ static ssize_t show_local_ib_port(struct device *dev, · 1 unmodified line
29842988 {
29852989 struct srp_target_port *target = host_to_target(class_to_shost(dev));
29862990
2987− return sprintf(buf, "%d\n", target->srp_host->port);
2991+ return sysfs_emit(buf, "%d\n", target->srp_host->port);
29882992 }
29892993
29902994 static ssize_t show_local_ib_device(struct device *dev,
@@ -2992,8 +2996,8 @@ static ssize_t show_local_ib_device(struct device *dev, · 1 unmodified line
29922996 {
29932997 struct srp_target_port *target = host_to_target(class_to_shost(dev));
29942998
2995− return sprintf(buf, "%s\n",
2996− dev_name(&target->srp_host->srp_dev->dev->dev));
2999+ return sysfs_emit(buf, "%s\n",
3000+ dev_name(&target->srp_host->srp_dev->dev->dev));
29973001 }
29983002
29993003 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
@@ -3001,7 +3005,7 @@ static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr, · 1 unmodified line
30013005 {
30023006 struct srp_target_port *target = host_to_target(class_to_shost(dev));
30033007
3004− return sprintf(buf, "%d\n", target->ch_count);
3008+ return sysfs_emit(buf, "%d\n", target->ch_count);
30053009 }
30063010
30073011 static ssize_t show_comp_vector(struct device *dev,
@@ -3009,7 +3013,7 @@ static ssize_t show_comp_vector(struct device *dev, · 1 unmodified line
30093013 {
30103014 struct srp_target_port *target = host_to_target(class_to_shost(dev));
30113015
3012− return sprintf(buf, "%d\n", target->comp_vector);
3016+ return sysfs_emit(buf, "%d\n", target->comp_vector);
30133017 }
30143018
30153019 static ssize_t show_tl_retry_count(struct device *dev,
@@ -3017,7 +3021,7 @@ static ssize_t show_tl_retry_count(struct device *dev, · 1 unmodified line
30173021 {
30183022 struct srp_target_port *target = host_to_target(class_to_shost(dev));
30193023
3020− return sprintf(buf, "%d\n", target->tl_retry_count);
3024+ return sysfs_emit(buf, "%d\n", target->tl_retry_count);
30213025 }
30223026
30233027 static ssize_t show_cmd_sg_entries(struct device *dev,
@@ -3025,7 +3029,7 @@ static ssize_t show_cmd_sg_entries(struct device *dev, · 1 unmodified line
30253029 {
30263030 struct srp_target_port *target = host_to_target(class_to_shost(dev));
30273031
3028− return sprintf(buf, "%u\n", target->cmd_sg_cnt);
3032+ return sysfs_emit(buf, "%u\n", target->cmd_sg_cnt);
30293033 }
30303034
30313035 static ssize_t show_allow_ext_sg(struct device *dev,
@@ -3033,7 +3037,7 @@ static ssize_t show_allow_ext_sg(struct device *dev, · 1 unmodified line
30333037 {
30343038 struct srp_target_port *target = host_to_target(class_to_shost(dev));
30353039
3036− return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
3040+ return sysfs_emit(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
30373041 }
30383042
30393043 static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
@@ -3893,7 +3897,7 @@ static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr, · 853 unmodified lines
38933897 {
38943898 struct srp_host *host = container_of(dev, struct srp_host, dev);
38953899
3896− return sprintf(buf, "%s\n", dev_name(&host->srp_dev->dev->dev));
3900+ return sysfs_emit(buf, "%s\n", dev_name(&host->srp_dev->dev->dev));
38973901 }
38983902
38993903 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
@@ -3903,7 +3907,7 @@ static ssize_t show_port(struct device *dev, struct device_attribute *attr, · 3 unmodified lines
39033907 {
39043908 struct srp_host *host = container_of(dev, struct srp_host, dev);
39053909
3906− return sprintf(buf, "%d\n", host->port);
3910+ return sysfs_emit(buf, "%d\n", host->port);
39073911 }
39083912
39093913 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
drivers/infiniband/ulp/srpt/ib_srpt.c+7−7
@@ -3448,7 +3448,7 @@ static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item, · 3447 unmodified lines
34483448 struct se_portal_group *se_tpg = attrib_to_tpg(item);
34493449 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
34503450
3451− return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
3451+ return sysfs_emit(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
34523452 }
34533453
34543454 static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item,
@@ -3485,7 +3485,7 @@ static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item, · 30 unmodified lines
34853485 struct se_portal_group *se_tpg = attrib_to_tpg(item);
34863486 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
34873487
3488− return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
3488+ return sysfs_emit(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
34893489 }
34903490
34913491 static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item,
@@ -3522,7 +3522,7 @@ static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item, · 30 unmodified lines
35223522 struct se_portal_group *se_tpg = attrib_to_tpg(item);
35233523 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
35243524
3525− return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size);
3525+ return sysfs_emit(page, "%u\n", sport->port_attrib.srp_sq_size);
35263526 }
35273527
35283528 static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item,
@@ -3559,7 +3559,7 @@ static ssize_t srpt_tpg_attrib_use_srq_show(struct config_item *item, · 30 unmodified lines
35593559 struct se_portal_group *se_tpg = attrib_to_tpg(item);
35603560 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
35613561
3562− return sprintf(page, "%d\n", sport->port_attrib.use_srq);
3562+ return sysfs_emit(page, "%d\n", sport->port_attrib.use_srq);
35633563 }
35643564
35653565 static ssize_t srpt_tpg_attrib_use_srq_store(struct config_item *item,
@@ -3649,7 +3649,7 @@ out: · 83 unmodified lines
36493649
36503650 static ssize_t srpt_rdma_cm_port_show(struct config_item *item, char *page)
36513651 {
3652− return sprintf(page, "%d\n", rdma_cm_port);
3652+ return sysfs_emit(page, "%d\n", rdma_cm_port);
36533653 }
36543654
36553655 static ssize_t srpt_rdma_cm_port_store(struct config_item *item,
@@ -3705,7 +3705,7 @@ static ssize_t srpt_tpg_enable_show(struct config_item *item, char *page) · 49 unmodified lines
37053705 struct se_portal_group *se_tpg = to_tpg(item);
37063706 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
37073707
3708− return snprintf(page, PAGE_SIZE, "%d\n", sport->enabled);
3708+ return sysfs_emit(page, "%d\n", sport->enabled);
37093709 }
37103710
37113711 static ssize_t srpt_tpg_enable_store(struct config_item *item,
@@ -3812,7 +3812,7 @@ static void srpt_drop_tport(struct se_wwn *wwn) · 100 unmodified lines
38123812
38133813 static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf)
38143814 {
3815− return scnprintf(buf, PAGE_SIZE, "\n");
3815+ return sysfs_emit(buf, "\n");
38163816 }
38173817
38183818 CONFIGFS_ATTR_RO(srpt_wwn_, version);
drivers/infiniband/ulp/srpt/ib_srpt.h+1−1
@@ -347,7 +347,7 @@ struct srpt_nexus { · 346 unmodified lines
347347 };
348348
349349 /**
350− * struct srpt_port_attib - attributes for SRPT port
350+ * struct srpt_port_attrib - attributes for SRPT port
351351 * @srp_max_rdma_size: Maximum size of SRP RDMA transfers for new connections.
352352 * @srp_max_rsp_size: Maximum size of SRP response messages in bytes.
353353 * @srp_sq_size: Shared receive queue (SRQ) size.
drivers/nvme/host/rdma.c+1−1
@@ -853,7 +853,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, · 852 unmodified lines
853853 return error;
854854
855855 ctrl->device = ctrl->queues[0].device;
856− ctrl->ctrl.numa_node = dev_to_node(ctrl->device->dev->dma_device);
856+ ctrl->ctrl.numa_node = ibdev_to_node(ctrl->device->dev);
857857
858858 /* T10-PI support */
859859 if (ctrl->device->dev->attrs.device_cap_flags &
drivers/nvme/target/rdma.c+2−1
@@ -414,7 +414,8 @@ static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev, · 413 unmodified lines
414414 if (ib_dma_mapping_error(ndev->device, r->send_sge.addr))
415415 goto out_free_rsp;
416416
417− r->req.p2p_client = &ndev->device->dev;
417+ if (!ib_uses_virt_dma(ndev->device))
418+ r->req.p2p_client = &ndev->device->dev;
418419 r->send_sge.length = sizeof(*r->req.cqe);
419420 r->send_sge.lkey = ndev->pd->local_dma_lkey;
420421
drivers/pci/p2pdma.c+1−24
@@ -556,15 +556,6 @@ int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, · 555 unmodified lines
556556 return -1;
557557
558558 for (i = 0; i < num_clients; i++) {
559−#ifdef CONFIG_DMA_VIRT_OPS
560− if (clients[i]->dma_ops == &dma_virt_ops) {
561− if (verbose)
562− dev_warn(clients[i],
563− "cannot be used for peer-to-peer DMA because the driver makes use of dma_virt_ops\n");
564− return -1;
565− }
566−#endif
567−
568559 pci_client = find_parent_pci_dev(clients[i]);
569560 if (!pci_client) {
570561 if (verbose)
@@ -834,24 +825,10 @@ static int __pci_p2pdma_map_sg(struct pci_p2pdma_pagemap *p2p_pgmap, · 263 unmodified lines
834825 struct device *dev, struct scatterlist *sg, int nents)
835826 {
836827 struct scatterlist *s;
837− phys_addr_t paddr;
838828 int i;
839829
840− /*
841− * p2pdma mappings are not compatible with devices that use
842− * dma_virt_ops. If the upper layers do the right thing
843− * this should never happen because it will be prevented
844− * by the check in pci_p2pdma_distance_many()
845− */
846−#ifdef CONFIG_DMA_VIRT_OPS
847− if (WARN_ON_ONCE(dev->dma_ops == &dma_virt_ops))
848− return 0;
849−#endif
850−
851830 for_each_sg(sg, s, nents, i) {
852− paddr = sg_phys(s);
853−
854− s->dma_address = paddr - p2p_pgmap->bus_offset;
831+ s->dma_address = sg_phys(s) - p2p_pgmap->bus_offset;
855832 sg_dma_len(s) = s->length;
856833 }
857834
include/linux/dma-mapping.h+0−2
@@ -565,6 +565,4 @@ static inline int dma_mmap_wc(struct device *dev, · 564 unmodified lines
565565 int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start,
566566 dma_addr_t dma_start, u64 size);
567567
568−extern const struct dma_map_ops dma_virt_ops;
569−
570568 #endif /* _LINUX_DMA_MAPPING_H */
include/rdma/ib_umem.h+42−0
@@ -34,6 +34,13 @@ static inline int ib_umem_offset(struct ib_umem *umem) · 33 unmodified lines
3434 return umem->address & ~PAGE_MASK;
3535 }
3636
37+static inline unsigned long ib_umem_dma_offset(struct ib_umem *umem,
38+ unsigned long pgsz)
39+{
40+ return (sg_dma_address(umem->sg_head.sgl) + ib_umem_offset(umem)) &
41+ (pgsz - 1);
42+}
43+
3744 static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem,
3845 unsigned long pgsz)
3946 {
@@ -79,6 +86,35 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, · 39 unmodified lines
7986 unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
8087 unsigned long pgsz_bitmap,
8188 unsigned long virt);
89+/**
90+ * ib_umem_find_best_pgoff - Find best HW page size
91+ *
92+ * @umem: umem struct
93+ * @pgsz_bitmap bitmap of HW supported page sizes
94+ * @pgoff_bitmask: Mask of bits that can be represented with an offset
95+ *
96+ * This is very similar to ib_umem_find_best_pgsz() except instead of accepting
97+ * an IOVA it accepts a bitmask specifying what address bits can be represented
98+ * with a page offset.
99+ *
100+ * For instance if the HW has multiple page sizes, requires 64 byte alignemnt,
101+ * and can support aligned offsets up to 4032 then pgoff_bitmask would be
102+ * "111111000000".
103+ *
104+ * If the pgoff_bitmask requires either alignment in the low bit or an
105+ * unavailable page size for the high bits, this function returns 0.
106+ */
107+static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem,
108+ unsigned long pgsz_bitmap,
109+ u64 pgoff_bitmask)
110+{
111+ struct scatterlist *sg = umem->sg_head.sgl;
112+ dma_addr_t dma_addr;
113+
114+ dma_addr = sg_dma_address(sg) + (umem->address & ~PAGE_MASK);
115+ return ib_umem_find_best_pgsz(umem, pgsz_bitmap,
116+ dma_addr & pgoff_bitmask);
117+}
82118
83119 #else /* CONFIG_INFINIBAND_USER_MEM */
84120
@@ -101,6 +137,12 @@ static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, · 16 unmodified lines
101137 {
102138 return 0;
103139 }
140+static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem,
141+ unsigned long pgsz_bitmap,
142+ u64 pgoff_bitmask)
143+{
144+ return 0;
145+}
104146
105147 #endif /* CONFIG_INFINIBAND_USER_MEM */
106148
include/rdma/ib_verbs.h+97−101
@@ -1235,6 +1235,8 @@ enum ib_qp_attr_mask { · 1234 unmodified lines
12351235 IB_QP_RESERVED3 = (1<<23),
12361236 IB_QP_RESERVED4 = (1<<24),
12371237 IB_QP_RATE_LIMIT = (1<<25),
1238+
1239+ IB_QP_ATTR_STANDARD_BITS = GENMASK(20, 0),
12381240 };
12391241
12401242 enum ib_qp_state {
@@ -1470,6 +1472,8 @@ enum rdma_remove_reason { · 229 unmodified lines
14701472 RDMA_REMOVE_DRIVER_REMOVE,
14711473 /* uobj is being cleaned-up before being committed */
14721474 RDMA_REMOVE_ABORT,
1475+ /* The driver failed to destroy the uobject and is being disconnected */
1476+ RDMA_REMOVE_DRIVER_FAILURE,
14731477 };
14741478
14751479 struct ib_rdmacg_object {
@@ -1482,8 +1486,6 @@ struct ib_ucontext { · 6 unmodified lines
14821486 struct ib_device *device;
14831487 struct ib_uverbs_file *ufile;
14841488
1485− bool cleanup_retryable;
1486−
14871489 struct ib_rdmacg_object cg_obj;
14881490 /*
14891491 * Implementation details of the RDMA core, don't use in drivers:
@@ -2402,6 +2404,8 @@ struct ib_device_ops { · 912 unmodified lines
24022404 int (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
24032405 int (*create_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
24042406 struct ib_udata *udata);
2407+ int (*create_user_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
2408+ struct ib_udata *udata);
24052409 int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
24062410 int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
24072411 int (*destroy_ah)(struct ib_ah *ah, u32 flags);
@@ -2430,9 +2434,10 @@ struct ib_device_ops { · 22 unmodified lines
24302434 struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
24312435 u64 virt_addr, int mr_access_flags,
24322436 struct ib_udata *udata);
2433− int (*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start, u64 length,
2434− u64 virt_addr, int mr_access_flags,
2435− struct ib_pd *pd, struct ib_udata *udata);
2437+ struct ib_mr *(*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start,
2438+ u64 length, u64 virt_addr,
2439+ int mr_access_flags, struct ib_pd *pd,
2440+ struct ib_udata *udata);
24362441 int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
24372442 struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
24382443 u32 max_num_sg);
@@ -2666,7 +2671,6 @@ struct ib_device { · 227 unmodified lines
26662671 const struct attribute_group *groups[3];
26672672
26682673 u64 uverbs_cmd_mask;
2669− u64 uverbs_ex_cmd_mask;
26702674
26712675 char node_desc[IB_DEVICE_NODE_DESC_MAX];
26722676 __be64 node_guid;
@@ -2901,46 +2905,6 @@ static inline bool ib_is_udata_cleared(struct ib_udata *udata, · 228 unmodified lines
29012905 return ib_is_buffer_cleared(udata->inbuf + offset, len);
29022906 }
29032907
2904−/**
2905− * ib_is_destroy_retryable - Check whether the uobject destruction
2906− * is retryable.
2907− * @ret: The initial destruction return code
2908− * @why: remove reason
2909− * @uobj: The uobject that is destroyed
2910− *
2911− * This function is a helper function that IB layer and low-level drivers
2912− * can use to consider whether the destruction of the given uobject is
2913− * retry-able.
2914− * It checks the original return code, if it wasn't success the destruction
2915− * is retryable according to the ucontext state (i.e. cleanup_retryable) and
2916− * the remove reason. (i.e. why).
2917− * Must be called with the object locked for destroy.
2918− */
2919−static inline bool ib_is_destroy_retryable(int ret, enum rdma_remove_reason why,
2920− struct ib_uobject *uobj)
2921−{
2922− return ret && (why == RDMA_REMOVE_DESTROY ||
2923− uobj->context->cleanup_retryable);
2924−}
2925−
2926−/**
2927− * ib_destroy_usecnt - Called during destruction to check the usecnt
2928− * @usecnt: The usecnt atomic
2929− * @why: remove reason
2930− * @uobj: The uobject that is destroyed
2931− *
2932− * Non-zero usecnts will block destruction unless destruction was triggered by
2933− * a ucontext cleanup.
2934− */
2935−static inline int ib_destroy_usecnt(atomic_t *usecnt,
2936− enum rdma_remove_reason why,
2937− struct ib_uobject *uobj)
2938−{
2939− if (atomic_read(usecnt) && ib_is_destroy_retryable(-EBUSY, why, uobj))
2940− return -EBUSY;
2941− return 0;
2942−}
2943−
29442908 /**
29452909 * ib_modify_qp_is_ok - Check that the supplied attribute mask
29462910 * contains all required attributes and no attributes not allowed for
@@ -3431,6 +3395,17 @@ enum ib_pd_flags { · 484 unmodified lines
34313395 struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
34323396 const char *caller);
34333397
3398+/**
3399+ * ib_alloc_pd - Allocates an unused protection domain.
3400+ * @device: The device on which to allocate the protection domain.
3401+ * @flags: protection domain flags
3402+ *
3403+ * A protection domain object provides an association between QPs, shared
3404+ * receive queues, address handles, memory regions, and memory windows.
3405+ *
3406+ * Every PD has a local_dma_lkey which can be used as the lkey value for local
3407+ * memory operations.
3408+ */
34343409 #define ib_alloc_pd(device, flags) \
34353410 __ib_alloc_pd((device), (flags), KBUILD_MODNAME)
34363411
@@ -3656,8 +3631,14 @@ static inline int ib_post_srq_recv(struct ib_srq *srq, · 219 unmodified lines
36563631 bad_recv_wr ? : &dummy);
36573632 }
36583633
3659−struct ib_qp *ib_create_qp(struct ib_pd *pd,
3660− struct ib_qp_init_attr *qp_init_attr);
3634+struct ib_qp *ib_create_named_qp(struct ib_pd *pd,
3635+ struct ib_qp_init_attr *qp_init_attr,
3636+ const char *caller);
3637+static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,
3638+ struct ib_qp_init_attr *init_attr)
3639+{
3640+ return ib_create_named_qp(pd, init_attr, KBUILD_MODNAME);
3641+}
36613642
36623643 /**
36633644 * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
@@ -3944,6 +3925,16 @@ static inline int ib_req_ncomp_notif(struct ib_cq *cq, int wc_cnt) · 280 unmodified lines
39443925 -ENOSYS;
39453926 }
39463927
3928+/*
3929+ * Drivers that don't need a DMA mapping at the RDMA layer, set dma_device to
3930+ * NULL. This causes the ib_dma* helpers to just stash the kernel virtual
3931+ * address into the dma address.
3932+ */
3933+static inline bool ib_uses_virt_dma(struct ib_device *dev)
3934+{
3935+ return IS_ENABLED(CONFIG_INFINIBAND_VIRT_DMA) && !dev->dma_device;
3936+}
3937+
39473938 /**
39483939 * ib_dma_mapping_error - check a DMA addr for error
39493940 * @dev: The device for which the dma_addr was created
@@ -3951,6 +3942,8 @@ static inline int ib_req_ncomp_notif(struct ib_cq *cq, int wc_cnt) · 1 unmodified line
39513942 */
39523943 static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
39533944 {
3945+ if (ib_uses_virt_dma(dev))
3946+ return 0;
39543947 return dma_mapping_error(dev->dma_device, dma_addr);
39553948 }
39563949
@@ -3965,6 +3958,8 @@ static inline u64 ib_dma_map_single(struct ib_device *dev, · 8 unmodified lines
39653958 void *cpu_addr, size_t size,
39663959 enum dma_data_direction direction)
39673960 {
3961+ if (ib_uses_virt_dma(dev))
3962+ return (uintptr_t)cpu_addr;
39683963 return dma_map_single(dev->dma_device, cpu_addr, size, direction);
39693964 }
39703965
@@ -3979,7 +3974,8 @@ static inline void ib_dma_unmap_single(struct ib_device *dev, · 8 unmodified lines
39793974 u64 addr, size_t size,
39803975 enum dma_data_direction direction)
39813976 {
3982− dma_unmap_single(dev->dma_device, addr, size, direction);
3977+ if (!ib_uses_virt_dma(dev))
3978+ dma_unmap_single(dev->dma_device, addr, size, direction);
39833979 }
39843980
39853981 /**
@@ -3996,6 +3992,8 @@ static inline u64 ib_dma_map_page(struct ib_device *dev, · 10 unmodified lines
39963992 size_t size,
39973993 enum dma_data_direction direction)
39983994 {
3995+ if (ib_uses_virt_dma(dev))
3996+ return (uintptr_t)(page_address(page) + offset);
39993997 return dma_map_page(dev->dma_device, page, offset, size, direction);
40003998 }
40013999
@@ -4010,7 +4008,30 @@ static inline void ib_dma_unmap_page(struct ib_device *dev, · 8 unmodified lines
40104008 u64 addr, size_t size,
40114009 enum dma_data_direction direction)
40124010 {
4013− dma_unmap_page(dev->dma_device, addr, size, direction);
4011+ if (!ib_uses_virt_dma(dev))
4012+ dma_unmap_page(dev->dma_device, addr, size, direction);
4013+}
4014+
4015+int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents);
4016+static inline int ib_dma_map_sg_attrs(struct ib_device *dev,
4017+ struct scatterlist *sg, int nents,
4018+ enum dma_data_direction direction,
4019+ unsigned long dma_attrs)
4020+{
4021+ if (ib_uses_virt_dma(dev))
4022+ return ib_dma_virt_map_sg(dev, sg, nents);
4023+ return dma_map_sg_attrs(dev->dma_device, sg, nents, direction,
4024+ dma_attrs);
4025+}
4026+
4027+static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,
4028+ struct scatterlist *sg, int nents,
4029+ enum dma_data_direction direction,
4030+ unsigned long dma_attrs)
4031+{
4032+ if (!ib_uses_virt_dma(dev))
4033+ dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction,
4034+ dma_attrs);
40144035 }
40154036
40164037 /**
@@ -4024,7 +4045,7 @@ static inline int ib_dma_map_sg(struct ib_device *dev, · 7 unmodified lines
40244045 struct scatterlist *sg, int nents,
40254046 enum dma_data_direction direction)
40264047 {
4027− return dma_map_sg(dev->dma_device, sg, nents, direction);
4048+ return ib_dma_map_sg_attrs(dev, sg, nents, direction, 0);
40284049 }
40294050
40304051 /**
@@ -4038,24 +4059,7 @@ static inline void ib_dma_unmap_sg(struct ib_device *dev, · 7 unmodified lines
40384059 struct scatterlist *sg, int nents,
40394060 enum dma_data_direction direction)
40404061 {
4041− dma_unmap_sg(dev->dma_device, sg, nents, direction);
4042−}
4043−
4044−static inline int ib_dma_map_sg_attrs(struct ib_device *dev,
4045− struct scatterlist *sg, int nents,
4046− enum dma_data_direction direction,
4047− unsigned long dma_attrs)
4048−{
4049− return dma_map_sg_attrs(dev->dma_device, sg, nents, direction,
4050− dma_attrs);
4051−}
4052−
4053−static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,
4054− struct scatterlist *sg, int nents,
4055− enum dma_data_direction direction,
4056− unsigned long dma_attrs)
4057−{
4058− dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction, dma_attrs);
4062+ ib_dma_unmap_sg_attrs(dev, sg, nents, direction, 0);
40594063 }
40604064
40614065 /**
@@ -4066,6 +4070,8 @@ static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev, · 4 unmodified lines
40664070 */
40674071 static inline unsigned int ib_dma_max_seg_size(struct ib_device *dev)
40684072 {
4073+ if (ib_uses_virt_dma(dev))
4074+ return UINT_MAX;
40694075 return dma_get_max_seg_size(dev->dma_device);
40704076 }
40714077
@@ -4081,7 +4087,8 @@ static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev, · 9 unmodified lines
40814087 size_t size,
40824088 enum dma_data_direction dir)
40834089 {
4084− dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
4090+ if (!ib_uses_virt_dma(dev))
4091+ dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
40854092 }
40864093
40874094 /**
@@ -4096,36 +4103,8 @@ static inline void ib_dma_sync_single_for_device(struct ib_device *dev, · 8 unmodified lines
40964103 size_t size,
40974104 enum dma_data_direction dir)
40984105 {
4099− dma_sync_single_for_device(dev->dma_device, addr, size, dir);
4100−}
4101−
4102−/**
4103− * ib_dma_alloc_coherent - Allocate memory and map it for DMA
4104− * @dev: The device for which the DMA address is requested
4105− * @size: The size of the region to allocate in bytes
4106− * @dma_handle: A pointer for returning the DMA address of the region
4107− * @flag: memory allocator flags
4108− */
4109−static inline void *ib_dma_alloc_coherent(struct ib_device *dev,
4110− size_t size,
4111− dma_addr_t *dma_handle,
4112− gfp_t flag)
4113−{
4114− return dma_alloc_coherent(dev->dma_device, size, dma_handle, flag);
4115−}
4116−
4117−/**
4118− * ib_dma_free_coherent - Free memory allocated by ib_dma_alloc_coherent()
4119− * @dev: The device for which the DMA addresses were allocated
4120− * @size: The size of the region
4121− * @cpu_addr: the address returned by ib_dma_alloc_coherent()
4122− * @dma_handle: the DMA address returned by ib_dma_alloc_coherent()
4123− */
4124−static inline void ib_dma_free_coherent(struct ib_device *dev,
4125− size_t size, void *cpu_addr,
4126− dma_addr_t dma_handle)
4127−{
4128− dma_free_coherent(dev->dma_device, size, cpu_addr, dma_handle);
4106+ if (!ib_uses_virt_dma(dev))
4107+ dma_sync_single_for_device(dev->dma_device, addr, size, dir);
41294108 }
41304109
41314110 /* ib_reg_user_mr - register a memory region for virtual addresses from kernel
@@ -4217,7 +4196,8 @@ struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device, · 85 unmodified lines
42174196 struct inode *inode, struct ib_udata *udata);
42184197 int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata);
42194198
4220−static inline int ib_check_mr_access(int flags)
4199+static inline int ib_check_mr_access(struct ib_device *ib_dev,
4200+ unsigned int flags)
42214201 {
42224202 /*
42234203 * Local write permission is required if remote write or
@@ -4230,6 +4210,9 @@ static inline int ib_check_mr_access(int flags) · 6 unmodified lines
42304210 if (flags & ~IB_ACCESS_SUPPORTED)
42314211 return -EINVAL;
42324212
4213+ if (flags & IB_ACCESS_ON_DEMAND &&
4214+ !(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
4215+ return -EINVAL;
42334216 return 0;
42344217 }
42354218
@@ -4616,6 +4599,19 @@ static inline struct ib_device *rdma_device_to_ibdev(struct device *device) · 380 unmodified lines
46164599 return coredev->owner;
46174600 }
46184601
4602+/**
4603+ * ibdev_to_node - return the NUMA node for a given ib_device
4604+ * @dev: device to get the NUMA node for.
4605+ */
4606+static inline int ibdev_to_node(struct ib_device *ibdev)
4607+{
4608+ struct device *parent = ibdev->dev.parent;
4609+
4610+ if (!parent)
4611+ return NUMA_NO_NODE;
4612+ return dev_to_node(parent);
4613+}
4614+
46194615 /**
46204616 * rdma_device_to_drv_device - Helper macro to reach back to driver's
46214617 * ib_device holder structure from device pointer.
include/rdma/restrack.h+24−0
@@ -68,6 +68,14 @@ struct rdma_restrack_entry { · 67 unmodified lines
6868 * As an example for that, see mlx5 QPs with type MLX5_IB_QPT_HW_GSI
6969 */
7070 bool valid;
71+ /**
72+ * @no_track: don't add this entry to restrack DB
73+ *
74+ * This field is used to mark an entry that doesn't need to be added to
75+ * internal restrack DB and presented later to the users at the nldev
76+ * query stage.
77+ */
78+ u8 no_track : 1;
7179 /*
7280 * @kref: Protect destroy of the resource
7381 */
@@ -145,4 +153,20 @@ int rdma_nl_stat_hwcounter_entry(struct sk_buff *msg, const char *name, · 71 unmodified lines
145153 struct rdma_restrack_entry *rdma_restrack_get_byid(struct ib_device *dev,
146154 enum rdma_restrack_type type,
147155 u32 id);
156+
157+/**
158+ * rdma_restrack_no_track() - don't add resource to the DB
159+ * @res: resource entry
160+ *
161+ * Every user of thie API should be cross examined.
162+ * Probaby you don't need to use this function.
163+ */
164+static inline void rdma_restrack_no_track(struct rdma_restrack_entry *res)
165+{
166+ res->no_track = true;
167+}
168+static inline bool rdma_restrack_is_tracked(struct rdma_restrack_entry *res)
169+{
170+ return !res->no_track;
171+}
148172 #endif /* _RDMA_RESTRACK_H_ */
include/rdma/uverbs_ioctl.h+19−6
@@ -647,12 +647,15 @@ static inline bool uverbs_attr_is_valid(const struct uverbs_attr_bundle *attrs_b · 646 unmodified lines
647647 * 'ucontext'.
648648 *
649649 */
650−#define rdma_udata_to_drv_context(udata, drv_dev_struct, member) \
651− (udata ? container_of(container_of(udata, struct uverbs_attr_bundle, \
652− driver_udata) \
653− ->context, \
654− drv_dev_struct, member) : \
655− (drv_dev_struct *)NULL)
650+static inline struct uverbs_attr_bundle *
651+rdma_udata_to_uverbs_attr_bundle(struct ib_udata *udata)
652+{
653+ return container_of(udata, struct uverbs_attr_bundle, driver_udata);
654+}
655+
656+#define rdma_udata_to_drv_context(udata, drv_dev_struct, member) \
657+ (udata ? container_of(rdma_udata_to_uverbs_attr_bundle(udata)->context, \
658+ drv_dev_struct, member) : (drv_dev_struct *)NULL)
656659
657660 #define IS_UVERBS_COPY_ERR(_ret) ((_ret) && (_ret) != -ENOENT)
658661
@@ -862,6 +865,16 @@ static inline __malloc void *uverbs_zalloc(struct uverbs_attr_bundle *bundle, · 203 unmodified lines
862865 {
863866 return _uverbs_alloc(bundle, size, GFP_KERNEL | __GFP_ZERO);
864867 }
868+
869+static inline __malloc void *uverbs_kcalloc(struct uverbs_attr_bundle *bundle,
870+ size_t n, size_t size)
871+{
872+ size_t bytes;
873+
874+ if (unlikely(check_mul_overflow(n, size, &bytes)))
875+ return ERR_PTR(-EOVERFLOW);
876+ return uverbs_zalloc(bundle, bytes);
877+}
865878 int _uverbs_get_const(s64 *to, const struct uverbs_attr_bundle *attrs_bundle,
866879 size_t idx, s64 lower_bound, u64 upper_bound,
867880 s64 *def_val);
include/rdma/uverbs_types.h+7−2
@@ -71,6 +71,8 @@ struct uverbs_obj_type_class { · 70 unmodified lines
7171 enum rdma_remove_reason why,
7272 struct uverbs_attr_bundle *attrs);
7373 void (*remove_handle)(struct ib_uobject *uobj);
74+ void (*swap_uobjects)(struct ib_uobject *obj_old,
75+ struct ib_uobject *obj_new);
7476 };
7577
7678 struct uverbs_obj_type {
@@ -116,6 +118,9 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, · 39 unmodified lines
116118 bool hw_obj_valid);
117119 void rdma_alloc_commit_uobject(struct ib_uobject *uobj,
118120 struct uverbs_attr_bundle *attrs);
121+void rdma_assign_uobject(struct ib_uobject *to_uobj,
122+ struct ib_uobject *new_uobj,
123+ struct uverbs_attr_bundle *attrs);
119124
120125 /*
121126 * uverbs_uobject_get is called in order to increase the reference count on
@@ -138,8 +143,8 @@ struct uverbs_obj_fd_type { · 16 unmodified lines
138143 * because the driver is removed or the FD is closed.
139144 */
140145 struct uverbs_obj_type type;
141− int (*destroy_object)(struct ib_uobject *uobj,
142− enum rdma_remove_reason why);
146+ void (*destroy_object)(struct ib_uobject *uobj,
147+ enum rdma_remove_reason why);
143148 const struct file_operations *fops;
144149 const char *name;
145150 int flags;
include/uapi/rdma/hns-abi.h+10−0
@@ -43,6 +43,10 @@ struct hns_roce_ib_create_cq { · 42 unmodified lines
4343 __u32 reserved;
4444 };
4545
46+enum hns_roce_cq_cap_flags {
47+ HNS_ROCE_CQ_FLAG_RECORD_DB = 1 << 0,
48+};
49+
4650 struct hns_roce_ib_create_cq_resp {
4751 __aligned_u64 cqn; /* Only 32 bits used, 64 for compat */
4852 __aligned_u64 cap_flags;
@@ -69,6 +73,12 @@ struct hns_roce_ib_create_qp { · 20 unmodified lines
6973 __aligned_u64 sdb_addr;
7074 };
7175
76+enum hns_roce_qp_cap_flags {
77+ HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0,
78+ HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1,
79+ HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2,
80+};
81+
7282 struct hns_roce_ib_create_qp_resp {
7383 __aligned_u64 cap_flags;
7484 };
include/uapi/rdma/ib_user_verbs.h+0−14
@@ -596,20 +596,6 @@ enum { · 595 unmodified lines
596596 IB_UVERBS_CREATE_QP_SUP_COMP_MASK = IB_UVERBS_CREATE_QP_MASK_IND_TABLE,
597597 };
598598
599−enum {
600− /*
601− * This value is equal to IB_QP_DEST_QPN.
602− */
603− IB_USER_LEGACY_LAST_QP_ATTR_MASK = 1ULL << 20,
604−};
605−
606−enum {
607− /*
608− * This value is equal to IB_QP_RATE_LIMIT.
609− */
610− IB_USER_LAST_QP_ATTR_MASK = 1ULL << 25,
611−};
612−
613599 struct ib_uverbs_ex_create_qp {
614600 __aligned_u64 user_handle;
615601 __u32 pd_handle;
include/uapi/rdma/rdma_user_rxe.h+21−0
@@ -181,4 +181,25 @@ struct rxe_modify_srq_cmd { · 180 unmodified lines
181181 __aligned_u64 mmap_info_addr;
182182 };
183183
184+/* This data structure is stored at the base of work and
185+ * completion queues shared between user space and kernel space.
186+ * It contains the producer and consumer indices. Is also
187+ * contains a copy of the queue size parameters for user space
188+ * to use but the kernel must use the parameters in the
189+ * rxe_queue struct. For performance reasons arrange to have
190+ * producer and consumer indices in separate cache lines
191+ * the kernel should always mask the indices to avoid accessing
192+ * memory outside of the data area
193+ */
194+struct rxe_queue_buf {
195+ __u32 log2_elem_size;
196+ __u32 index_mask;
197+ __u32 pad_1[30];
198+ __u32 producer_index;
199+ __u32 pad_2[31];
200+ __u32 consumer_index;
201+ __u32 pad_3[31];
202+ __u8 data[];
203+};
204+
184205 #endif /* RDMA_USER_RXE_H */
kernel/dma/Kconfig+0−5
@@ -75,11 +75,6 @@ config ARCH_HAS_DMA_PREP_COHERENT · 74 unmodified lines
7575 config ARCH_HAS_FORCE_DMA_UNENCRYPTED
7676 bool
7777
78−config DMA_VIRT_OPS
79− bool
80− depends on HAS_DMA
81− select DMA_OPS
82−
8378 config SWIOTLB
8479 bool
8580 select NEED_DMA_MAP_STATE
kernel/dma/Makefile+0−1
@@ -5,7 +5,6 @@ obj-$(CONFIG_DMA_OPS) += ops_helpers.o · 4 unmodified lines
55 obj-$(CONFIG_DMA_OPS) += dummy.o
66 obj-$(CONFIG_DMA_CMA) += contiguous.o
77 obj-$(CONFIG_DMA_DECLARE_COHERENT) += coherent.o
8−obj-$(CONFIG_DMA_VIRT_OPS) += virt.o
98 obj-$(CONFIG_DMA_API_DEBUG) += debug.o
109 obj-$(CONFIG_SWIOTLB) += swiotlb.o
1110 obj-$(CONFIG_DMA_COHERENT_POOL) += pool.o
kernel/dma/virt.c+0−61
@@ -1,61 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−/*
3− * DMA operations that map to virtual addresses without flushing memory.
4− */
5−#include <linux/export.h>
6−#include <linux/mm.h>
7−#include <linux/dma-map-ops.h>
8−#include <linux/scatterlist.h>
9−
10−static void *dma_virt_alloc(struct device *dev, size_t size,
11− dma_addr_t *dma_handle, gfp_t gfp,
12− unsigned long attrs)
13−{
14− void *ret;
15−
16− ret = (void *)__get_free_pages(gfp | __GFP_ZERO, get_order(size));
17− if (ret)
18− *dma_handle = (uintptr_t)ret;
19− return ret;
20−}
21−
22−static void dma_virt_free(struct device *dev, size_t size,
23− void *cpu_addr, dma_addr_t dma_addr,
24− unsigned long attrs)
25−{
26− free_pages((unsigned long)cpu_addr, get_order(size));
27−}
28−
29−static dma_addr_t dma_virt_map_page(struct device *dev, struct page *page,
30− unsigned long offset, size_t size,
31− enum dma_data_direction dir,
32− unsigned long attrs)
33−{
34− return (uintptr_t)(page_address(page) + offset);
35−}
36−
37−static int dma_virt_map_sg(struct device *dev, struct scatterlist *sgl,
38− int nents, enum dma_data_direction dir,
39− unsigned long attrs)
40−{
41− int i;
42− struct scatterlist *sg;
43−
44− for_each_sg(sgl, sg, nents, i) {
45− BUG_ON(!sg_page(sg));
46− sg_dma_address(sg) = (uintptr_t)sg_virt(sg);
47− sg_dma_len(sg) = sg->length;
48− }
49−
50− return nents;
51−}
52−
53−const struct dma_map_ops dma_virt_ops = {
54− .alloc = dma_virt_alloc,
55− .free = dma_virt_free,
56− .map_page = dma_virt_map_page,
57− .map_sg = dma_virt_map_sg,
58− .alloc_pages = dma_common_alloc_pages,
59− .free_pages = dma_common_free_pages,
60−};
61−EXPORT_SYMBOL(dma_virt_ops);
net/rds/ib.c+0−10
@@ -30,7 +30,6 @@ · 29 unmodified lines
3030 * SOFTWARE.
3131 *
3232 */
33−#include <linux/dmapool.h>
3433 #include <linux/kernel.h>
3534 #include <linux/in.h>
3635 #include <linux/if.h>
@@ -108,7 +107,6 @@ static void rds_ib_dev_free(struct work_struct *work) · 71 unmodified lines
108107 rds_ib_destroy_mr_pool(rds_ibdev->mr_1m_pool);
109108 if (rds_ibdev->pd)
110109 ib_dealloc_pd(rds_ibdev->pd);
111− dma_pool_destroy(rds_ibdev->rid_hdrs_pool);
112110
113111 list_for_each_entry_safe(i_ipaddr, i_next, &rds_ibdev->ipaddr_list, list) {
114112 list_del(&i_ipaddr->list);
@@ -191,14 +189,6 @@ static int rds_ib_add_one(struct ib_device *device) · 76 unmodified lines
191189 rds_ibdev->pd = NULL;
192190 goto put_dev;
193191 }
194− rds_ibdev->rid_hdrs_pool = dma_pool_create(device->name,
195− device->dma_device,
196− sizeof(struct rds_header),
197− L1_CACHE_BYTES, 0);
198− if (!rds_ibdev->rid_hdrs_pool) {
199− ret = -ENOMEM;
200− goto put_dev;
201− }
202192
203193 rds_ibdev->mr_1m_pool =
204194 rds_ib_create_mr_pool(rds_ibdev, RDS_IB_MR_1M_POOL);
net/rds/ib.h+0−13
@@ -246,7 +246,6 @@ struct rds_ib_device { · 245 unmodified lines
246246 struct list_head conn_list;
247247 struct ib_device *dev;
248248 struct ib_pd *pd;
249− struct dma_pool *rid_hdrs_pool; /* RDS headers DMA pool */
250249 u8 odp_capable:1;
251250
252251 unsigned int max_mrs;
@@ -264,13 +263,6 @@ struct rds_ib_device { · 11 unmodified lines
264263 int *vector_load;
265264 };
266265
267−static inline int ibdev_to_node(struct ib_device *ibdev)
268−{
269− struct device *parent;
270−
271− parent = ibdev->dev.parent;
272− return parent ? dev_to_node(parent) : NUMA_NO_NODE;
273−}
274266 #define rdsibdev_to_node(rdsibdev) ibdev_to_node(rdsibdev->dev)
275267
276268 /* bits for i_ack_flags */
@@ -387,11 +379,6 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, · 110 unmodified lines
387379 int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id, bool isv6);
388380 void rds_ib_cm_connect_complete(struct rds_connection *conn,
389381 struct rdma_cm_event *event);
390−struct rds_header **rds_dma_hdrs_alloc(struct ib_device *ibdev,
391− struct dma_pool *pool,
392− dma_addr_t **dma_addrs, u32 num_hdrs);
393−void rds_dma_hdrs_free(struct dma_pool *pool, struct rds_header **hdrs,
394− dma_addr_t *dma_addrs, u32 num_hdrs);
395382
396383 #define rds_ib_conn_error(conn, fmt...) \
397384 __rds_ib_conn_error(conn, KERN_WARNING "RDS/IB: " fmt)
net/rds/ib_cm.c+78−50
@@ -30,7 +30,6 @@ · 29 unmodified lines
3030 * SOFTWARE.
3131 *
3232 */
33−#include <linux/dmapool.h>
3433 #include <linux/kernel.h>
3534 #include <linux/in.h>
3635 #include <linux/slab.h>
@@ -441,42 +440,87 @@ static inline void ibdev_put_vector(struct rds_ib_device *rds_ibdev, int index) · 404 unmodified lines
441440 rds_ibdev->vector_load[index]--;
442441 }
443442
443+static void rds_dma_hdr_free(struct ib_device *dev, struct rds_header *hdr,
444+ dma_addr_t dma_addr, enum dma_data_direction dir)
445+{
446+ ib_dma_unmap_single(dev, dma_addr, sizeof(*hdr), dir);
447+ kfree(hdr);
448+}
449+
450+static struct rds_header *rds_dma_hdr_alloc(struct ib_device *dev,
451+ dma_addr_t *dma_addr, enum dma_data_direction dir)
452+{
453+ struct rds_header *hdr;
454+
455+ hdr = kzalloc_node(sizeof(*hdr), GFP_KERNEL, ibdev_to_node(dev));
456+ if (!hdr)
457+ return NULL;
458+
459+ *dma_addr = ib_dma_map_single(dev, hdr, sizeof(*hdr),
460+ DMA_BIDIRECTIONAL);
461+ if (ib_dma_mapping_error(dev, *dma_addr)) {
462+ kfree(hdr);
463+ return NULL;
464+ }
465+
466+ return hdr;
467+}
468+
469+/* Free the DMA memory used to store struct rds_header.
470+ *
471+ * @dev: the RDS IB device
472+ * @hdrs: pointer to the array storing DMA memory pointers
473+ * @dma_addrs: pointer to the array storing DMA addresses
474+ * @num_hdars: number of headers to free.
475+ */
476+static void rds_dma_hdrs_free(struct rds_ib_device *dev,
477+ struct rds_header **hdrs, dma_addr_t *dma_addrs, u32 num_hdrs,
478+ enum dma_data_direction dir)
479+{
480+ u32 i;
481+
482+ for (i = 0; i < num_hdrs; i++)
483+ rds_dma_hdr_free(dev->dev, hdrs[i], dma_addrs[i], dir);
484+ kvfree(hdrs);
485+ kvfree(dma_addrs);
486+}
487+
488+
444489 /* Allocate DMA coherent memory to be used to store struct rds_header for
445490 * sending/receiving packets. The pointers to the DMA memory and the
446491 * associated DMA addresses are stored in two arrays.
447492 *
448− * @ibdev: the IB device
449− * @pool: the DMA memory pool
493+ * @dev: the RDS IB device
450494 * @dma_addrs: pointer to the array for storing DMA addresses
451495 * @num_hdrs: number of headers to allocate
452496 *
453497 * It returns the pointer to the array storing the DMA memory pointers. On
454498 * error, NULL pointer is returned.
455499 */
456−struct rds_header **rds_dma_hdrs_alloc(struct ib_device *ibdev,
457− struct dma_pool *pool,
458− dma_addr_t **dma_addrs, u32 num_hdrs)
500+static struct rds_header **rds_dma_hdrs_alloc(struct rds_ib_device *dev,
501+ dma_addr_t **dma_addrs, u32 num_hdrs,
502+ enum dma_data_direction dir)
459503 {
460504 struct rds_header **hdrs;
461505 dma_addr_t *hdr_daddrs;
462506 u32 i;
463507
464508 hdrs = kvmalloc_node(sizeof(*hdrs) * num_hdrs, GFP_KERNEL,
465− ibdev_to_node(ibdev));
509+ ibdev_to_node(dev->dev));
466510 if (!hdrs)
467511 return NULL;
468512
469513 hdr_daddrs = kvmalloc_node(sizeof(*hdr_daddrs) * num_hdrs, GFP_KERNEL,
470− ibdev_to_node(ibdev));
514+ ibdev_to_node(dev->dev));
471515 if (!hdr_daddrs) {
472516 kvfree(hdrs);
473517 return NULL;
474518 }
475519
476520 for (i = 0; i < num_hdrs; i++) {
477− hdrs[i] = dma_pool_zalloc(pool, GFP_KERNEL, &hdr_daddrs[i]);
521+ hdrs[i] = rds_dma_hdr_alloc(dev->dev, &hdr_daddrs[i], dir);
478522 if (!hdrs[i]) {
479− rds_dma_hdrs_free(pool, hdrs, hdr_daddrs, i);
523+ rds_dma_hdrs_free(dev, hdrs, hdr_daddrs, i, dir);
480524 return NULL;
481525 }
482526 }
@@ -485,24 +529,6 @@ struct rds_header **rds_dma_hdrs_alloc(struct ib_device *ibdev, · 2 unmodified lines
485529 return hdrs;
486530 }
487531
488−/* Free the DMA memory used to store struct rds_header.
489− *
490− * @pool: the DMA memory pool
491− * @hdrs: pointer to the array storing DMA memory pointers
492− * @dma_addrs: pointer to the array storing DMA addresses
493− * @num_hdars: number of headers to free.
494− */
495−void rds_dma_hdrs_free(struct dma_pool *pool, struct rds_header **hdrs,
496− dma_addr_t *dma_addrs, u32 num_hdrs)
497−{
498− u32 i;
499−
500− for (i = 0; i < num_hdrs; i++)
501− dma_pool_free(pool, hdrs[i], dma_addrs[i]);
502− kvfree(hdrs);
503− kvfree(dma_addrs);
504−}
505−
506532 /*
507533 * This needs to be very careful to not leave IS_ERR pointers around for
508534 * cleanup to trip over.
@@ -516,7 +542,6 @@ static int rds_ib_setup_qp(struct rds_connection *conn) · 7 unmodified lines
516542 struct rds_ib_device *rds_ibdev;
517543 unsigned long max_wrs;
518544 int ret, fr_queue_space;
519− struct dma_pool *pool;
520545
521546 /*
522547 * It's normal to see a null device if an incoming connection races
@@ -612,25 +637,26 @@ static int rds_ib_setup_qp(struct rds_connection *conn) · 89 unmodified lines
612637 goto recv_cq_out;
613638 }
614639
615− pool = rds_ibdev->rid_hdrs_pool;
616− ic->i_send_hdrs = rds_dma_hdrs_alloc(dev, pool, &ic->i_send_hdrs_dma,
617− ic->i_send_ring.w_nr);
640+ ic->i_send_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_send_hdrs_dma,
641+ ic->i_send_ring.w_nr,
642+ DMA_TO_DEVICE);
618643 if (!ic->i_send_hdrs) {
619644 ret = -ENOMEM;
620645 rdsdebug("DMA send hdrs alloc failed\n");
621646 goto qp_out;
622647 }
623648
624− ic->i_recv_hdrs = rds_dma_hdrs_alloc(dev, pool, &ic->i_recv_hdrs_dma,
625− ic->i_recv_ring.w_nr);
649+ ic->i_recv_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_recv_hdrs_dma,
650+ ic->i_recv_ring.w_nr,
651+ DMA_FROM_DEVICE);
626652 if (!ic->i_recv_hdrs) {
627653 ret = -ENOMEM;
628654 rdsdebug("DMA recv hdrs alloc failed\n");
629655 goto send_hdrs_dma_out;
630656 }
631657
632− ic->i_ack = dma_pool_zalloc(pool, GFP_KERNEL,
633− &ic->i_ack_dma);
658+ ic->i_ack = rds_dma_hdr_alloc(rds_ibdev->dev, &ic->i_ack_dma,
659+ DMA_TO_DEVICE);
634660 if (!ic->i_ack) {
635661 ret = -ENOMEM;
636662 rdsdebug("DMA ack header alloc failed\n");
@@ -666,18 +692,19 @@ sends_out: · 29 unmodified lines
666692 vfree(ic->i_sends);
667693
668694 ack_dma_out:
669− dma_pool_free(pool, ic->i_ack, ic->i_ack_dma);
695+ rds_dma_hdr_free(rds_ibdev->dev, ic->i_ack, ic->i_ack_dma,
696+ DMA_TO_DEVICE);
670697 ic->i_ack = NULL;
671698
672699 recv_hdrs_dma_out:
673− rds_dma_hdrs_free(pool, ic->i_recv_hdrs, ic->i_recv_hdrs_dma,
674− ic->i_recv_ring.w_nr);
700+ rds_dma_hdrs_free(rds_ibdev, ic->i_recv_hdrs, ic->i_recv_hdrs_dma,
701+ ic->i_recv_ring.w_nr, DMA_FROM_DEVICE);
675702 ic->i_recv_hdrs = NULL;
676703 ic->i_recv_hdrs_dma = NULL;
677704
678705 send_hdrs_dma_out:
679− rds_dma_hdrs_free(pool, ic->i_send_hdrs, ic->i_send_hdrs_dma,
680− ic->i_send_ring.w_nr);
706+ rds_dma_hdrs_free(rds_ibdev, ic->i_send_hdrs, ic->i_send_hdrs_dma,
707+ ic->i_send_ring.w_nr, DMA_TO_DEVICE);
681708 ic->i_send_hdrs = NULL;
682709 ic->i_send_hdrs_dma = NULL;
683710
@@ -1110,29 +1137,30 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp) · 426 unmodified lines
11101137 }
11111138
11121139 if (ic->rds_ibdev) {
1113− struct dma_pool *pool;
1114−
1115− pool = ic->rds_ibdev->rid_hdrs_pool;
1116−
11171140 /* then free the resources that ib callbacks use */
11181141 if (ic->i_send_hdrs) {
1119− rds_dma_hdrs_free(pool, ic->i_send_hdrs,
1142+ rds_dma_hdrs_free(ic->rds_ibdev,
1143+ ic->i_send_hdrs,
11201144 ic->i_send_hdrs_dma,
1121− ic->i_send_ring.w_nr);
1145+ ic->i_send_ring.w_nr,
1146+ DMA_TO_DEVICE);
11221147 ic->i_send_hdrs = NULL;
11231148 ic->i_send_hdrs_dma = NULL;
11241149 }
11251150
11261151 if (ic->i_recv_hdrs) {
1127− rds_dma_hdrs_free(pool, ic->i_recv_hdrs,
1152+ rds_dma_hdrs_free(ic->rds_ibdev,
1153+ ic->i_recv_hdrs,
11281154 ic->i_recv_hdrs_dma,
1129− ic->i_recv_ring.w_nr);
1155+ ic->i_recv_ring.w_nr,
1156+ DMA_FROM_DEVICE);
11301157 ic->i_recv_hdrs = NULL;
11311158 ic->i_recv_hdrs_dma = NULL;
11321159 }
11331160
11341161 if (ic->i_ack) {
1135− dma_pool_free(pool, ic->i_ack, ic->i_ack_dma);
1162+ rds_dma_hdr_free(ic->rds_ibdev->dev, ic->i_ack,
1163+ ic->i_ack_dma, DMA_TO_DEVICE);
11361164 ic->i_ack = NULL;
11371165 }
11381166 } else {
net/rds/ib_recv.c+15−3
@@ -662,10 +662,16 @@ static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credi · 661 unmodified lines
662662 seq = rds_ib_get_ack(ic);
663663
664664 rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq);
665+
666+ ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev, ic->i_ack_dma,
667+ sizeof(*hdr), DMA_TO_DEVICE);
665668 rds_message_populate_header(hdr, 0, 0, 0);
666669 hdr->h_ack = cpu_to_be64(seq);
667670 hdr->h_credit = adv_credits;
668671 rds_message_make_checksum(hdr);
672+ ib_dma_sync_single_for_device(ic->rds_ibdev->dev, ic->i_ack_dma,
673+ sizeof(*hdr), DMA_TO_DEVICE);
674+
669675 ic->i_ack_queued = jiffies;
670676
671677 ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, NULL);
@@ -845,6 +851,7 @@ static void rds_ib_process_recv(struct rds_connection *conn, · 173 unmodified lines
845851 struct rds_ib_connection *ic = conn->c_transport_data;
846852 struct rds_ib_incoming *ibinc = ic->i_ibinc;
847853 struct rds_header *ihdr, *hdr;
854+ dma_addr_t dma_addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs];
848855
849856 /* XXX shut down the connection if port 0,0 are seen? */
850857
@@ -863,6 +870,8 @@ static void rds_ib_process_recv(struct rds_connection *conn, · 12 unmodified lines
863870
864871 ihdr = ic->i_recv_hdrs[recv - ic->i_recvs];
865872
873+ ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev, dma_addr,
874+ sizeof(*ihdr), DMA_FROM_DEVICE);
866875 /* Validate the checksum. */
867876 if (!rds_message_verify_checksum(ihdr)) {
868877 rds_ib_conn_error(conn, "incoming message "
@@ -870,7 +879,7 @@ static void rds_ib_process_recv(struct rds_connection *conn, · 1 unmodified line
870879 "forcing a reconnect\n",
871880 &conn->c_faddr);
872881 rds_stats_inc(s_recv_drop_bad_checksum);
873− return;
882+ goto done;
874883 }
875884
876885 /* Process the ACK sequence which comes with every packet */
@@ -899,7 +908,7 @@ static void rds_ib_process_recv(struct rds_connection *conn, · 22 unmodified lines
899908 */
900909 rds_ib_frag_free(ic, recv->r_frag);
901910 recv->r_frag = NULL;
902− return;
911+ goto done;
903912 }
904913
905914 /*
@@ -933,7 +942,7 @@ static void rds_ib_process_recv(struct rds_connection *conn, · 27 unmodified lines
933942 hdr->h_dport != ihdr->h_dport) {
934943 rds_ib_conn_error(conn,
935944 "fragment header mismatch; forcing reconnect\n");
936− return;
945+ goto done;
937946 }
938947 }
939948
@@ -965,6 +974,9 @@ static void rds_ib_process_recv(struct rds_connection *conn, · 25 unmodified lines
965974
966975 rds_inc_put(&ibinc->ii_inc);
967976 }
977+done:
978+ ib_dma_sync_single_for_device(ic->rds_ibdev->dev, dma_addr,
979+ sizeof(*ihdr), DMA_FROM_DEVICE);
968980 }
969981
970982 void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic,
net/rds/ib_send.c+8−0
@@ -638,6 +638,10 @@ int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm, · 637 unmodified lines
638638 send->s_sge[0].length = sizeof(struct rds_header);
639639 send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;
640640
641+ ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev,
642+ ic->i_send_hdrs_dma[pos],
643+ sizeof(struct rds_header),
644+ DMA_TO_DEVICE);
641645 memcpy(ic->i_send_hdrs[pos], &rm->m_inc.i_hdr,
642646 sizeof(struct rds_header));
643647
@@ -688,6 +692,10 @@ int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm, · 44 unmodified lines
688692 adv_credits = 0;
689693 rds_ib_stats_inc(s_ib_tx_credit_updates);
690694 }
695+ ib_dma_sync_single_for_device(ic->rds_ibdev->dev,
696+ ic->i_send_hdrs_dma[pos],
697+ sizeof(struct rds_header),
698+ DMA_TO_DEVICE);
691699
692700 if (prev)
693701 prev->s_wr.next = &send->s_wr;
tools/testing/scatterlist/main.c+41−23
@@ -9,6 +9,7 @@ struct test { · 8 unmodified lines
99 int alloc_ret;
1010 unsigned num_pages;
1111 unsigned *pfn;
12+ unsigned *pfn_app;
1213 unsigned size;
1314 unsigned int max_seg;
1415 unsigned int expected_segments;
@@ -52,31 +53,39 @@ int main(void) · 37 unmodified lines
5253 {
5354 const unsigned int sgmax = UINT_MAX;
5455 struct test *test, tests[] = {
55− { -EINVAL, 1, pfn(0), PAGE_SIZE, 0, 1 },
56− { 0, 1, pfn(0), PAGE_SIZE, PAGE_SIZE + 1, 1 },
57− { 0, 1, pfn(0), PAGE_SIZE, sgmax + 1, 1 },
58− { 0, 1, pfn(0), PAGE_SIZE, sgmax, 1 },
59− { 0, 1, pfn(0), 1, sgmax, 1 },
60− { 0, 2, pfn(0, 1), 2 * PAGE_SIZE, sgmax, 1 },
61− { 0, 2, pfn(1, 0), 2 * PAGE_SIZE, sgmax, 2 },
62− { 0, 3, pfn(0, 1, 2), 3 * PAGE_SIZE, sgmax, 1 },
63− { 0, 3, pfn(0, 2, 1), 3 * PAGE_SIZE, sgmax, 3 },
64− { 0, 3, pfn(0, 1, 3), 3 * PAGE_SIZE, sgmax, 2 },
65− { 0, 3, pfn(1, 2, 4), 3 * PAGE_SIZE, sgmax, 2 },
66− { 0, 3, pfn(1, 3, 4), 3 * PAGE_SIZE, sgmax, 2 },
67− { 0, 4, pfn(0, 1, 3, 4), 4 * PAGE_SIZE, sgmax, 2 },
68− { 0, 5, pfn(0, 1, 3, 4, 5), 5 * PAGE_SIZE, sgmax, 2 },
69− { 0, 5, pfn(0, 1, 3, 4, 6), 5 * PAGE_SIZE, sgmax, 3 },
70− { 0, 5, pfn(0, 1, 2, 3, 4), 5 * PAGE_SIZE, sgmax, 1 },
71− { 0, 5, pfn(0, 1, 2, 3, 4), 5 * PAGE_SIZE, 2 * PAGE_SIZE, 3 },
72− { 0, 6, pfn(0, 1, 2, 3, 4, 5), 6 * PAGE_SIZE, 2 * PAGE_SIZE, 3 },
73− { 0, 6, pfn(0, 2, 3, 4, 5, 6), 6 * PAGE_SIZE, 2 * PAGE_SIZE, 4 },
74− { 0, 6, pfn(0, 1, 3, 4, 5, 6), 6 * PAGE_SIZE, 2 * PAGE_SIZE, 3 },
75− { 0, 0, NULL, 0, 0, 0 },
56+ { -EINVAL, 1, pfn(0), NULL, PAGE_SIZE, 0, 1 },
57+ { 0, 1, pfn(0), NULL, PAGE_SIZE, PAGE_SIZE + 1, 1 },
58+ { 0, 1, pfn(0), NULL, PAGE_SIZE, sgmax + 1, 1 },
59+ { 0, 1, pfn(0), NULL, PAGE_SIZE, sgmax, 1 },
60+ { 0, 1, pfn(0), NULL, 1, sgmax, 1 },
61+ { 0, 2, pfn(0, 1), NULL, 2 * PAGE_SIZE, sgmax, 1 },
62+ { 0, 2, pfn(1, 0), NULL, 2 * PAGE_SIZE, sgmax, 2 },
63+ { 0, 3, pfn(0, 1, 2), NULL, 3 * PAGE_SIZE, sgmax, 1 },
64+ { 0, 3, pfn(0, 1, 2), NULL, 3 * PAGE_SIZE, sgmax, 1 },
65+ { 0, 3, pfn(0, 1, 2), pfn(3, 4, 5), 3 * PAGE_SIZE, sgmax, 1 },
66+ { 0, 3, pfn(0, 1, 2), pfn(4, 5, 6), 3 * PAGE_SIZE, sgmax, 2 },
67+ { 0, 3, pfn(0, 2, 1), NULL, 3 * PAGE_SIZE, sgmax, 3 },
68+ { 0, 3, pfn(0, 1, 3), NULL, 3 * PAGE_SIZE, sgmax, 2 },
69+ { 0, 3, pfn(1, 2, 4), NULL, 3 * PAGE_SIZE, sgmax, 2 },
70+ { 0, 3, pfn(1, 3, 4), NULL, 3 * PAGE_SIZE, sgmax, 2 },
71+ { 0, 4, pfn(0, 1, 3, 4), NULL, 4 * PAGE_SIZE, sgmax, 2 },
72+ { 0, 5, pfn(0, 1, 3, 4, 5), NULL, 5 * PAGE_SIZE, sgmax, 2 },
73+ { 0, 5, pfn(0, 1, 3, 4, 6), NULL, 5 * PAGE_SIZE, sgmax, 3 },
74+ { 0, 5, pfn(0, 1, 2, 3, 4), NULL, 5 * PAGE_SIZE, sgmax, 1 },
75+ { 0, 5, pfn(0, 1, 2, 3, 4), NULL, 5 * PAGE_SIZE, 2 * PAGE_SIZE,
76+ 3 },
77+ { 0, 6, pfn(0, 1, 2, 3, 4, 5), NULL, 6 * PAGE_SIZE,
78+ 2 * PAGE_SIZE, 3 },
79+ { 0, 6, pfn(0, 2, 3, 4, 5, 6), NULL, 6 * PAGE_SIZE,
80+ 2 * PAGE_SIZE, 4 },
81+ { 0, 6, pfn(0, 1, 3, 4, 5, 6), pfn(7, 8, 9, 10, 11, 12),
82+ 6 * PAGE_SIZE, 12 * PAGE_SIZE, 2 },
83+ { 0, 0, NULL, NULL, 0, 0, 0 },
7684 };
7785 unsigned int i;
7886
7987 for (i = 0, test = tests; test->expected_segments; test++, i++) {
88+ int left_pages = test->pfn_app ? test->num_pages : 0;
8089 struct page *pages[MAX_PAGES];
8190 struct sg_table st;
8291 struct scatterlist *sg;
@@ -84,14 +93,23 @@ int main(void) · 1 unmodified line
8493 set_pages(pages, test->pfn, test->num_pages);
8594
8695 sg = __sg_alloc_table_from_pages(&st, pages, test->num_pages, 0,
87− test->size, test->max_seg, NULL, 0, GFP_KERNEL);
96+ test->size, test->max_seg, NULL, left_pages, GFP_KERNEL);
8897 assert(PTR_ERR_OR_ZERO(sg) == test->alloc_ret);
8998
9099 if (test->alloc_ret)
91100 continue;
92101
102+ if (test->pfn_app) {
103+ set_pages(pages, test->pfn_app, test->num_pages);
104+ sg = __sg_alloc_table_from_pages(&st, pages, test->num_pages, 0,
105+ test->size, test->max_seg, sg, 0, GFP_KERNEL);
106+
107+ assert(PTR_ERR_OR_ZERO(sg) == test->alloc_ret);
108+ }
109+
93110 VALIDATE(st.nents == test->expected_segments, &st, test);
94− VALIDATE(st.orig_nents == test->expected_segments, &st, test);
111+ if (!test->pfn_app)
112+ VALIDATE(st.orig_nents == test->expected_segments, &st, test);
95113
96114 sg_free_table(&st);
97115 }