History

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull rdma updates from Jason Gunthorpe: "Quite a small cycle this time, even with the rc8. I suppose everyone went to sleep over xmas. - Minor driver updates for hfi1, cxgb4, erdma, hns, irdma, mlx5, siw, mana - inline CQE support for hns - Have mlx5 display device error codes - Pinned DMABUF support for irdma - Continued rxe cleanups, particularly converting the MRs to use xarray - Improvements to what can be cached in the mlx5 mkey cache" * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (61 commits) IB/mlx5: Extend debug control for CC parameters IB/hfi1: Fix sdma.h tx->num_descs off-by-one errors IB/hfi1: Fix math bugs in hfi1_can_pin_pages() RDMA/irdma: Add support for dmabuf pin memory regions RDMA/mlx5: Use query_special_contexts for mkeys net/mlx5e: Use query_special_contexts for mkeys net/mlx5: Change define name for 0x100 lkey value net/mlx5: Expose bits for querying special mkeys RDMA/rxe: Fix missing memory barriers in rxe_queue.h RDMA/mana_ib: Fix a bug when the PF indicates more entries for registering memory on first packet RDMA/rxe: Remove rxe_alloc() RDMA/cma: Distinguish between sockaddr_in and sockaddr_in6 by size Subject: RDMA/rxe: Handle zero length rdma iw_cxgb4: Fix potential NULL dereference in c4iw_fill_res_cm_id_entry() RDMA/mlx5: Use rdma_umem_for_each_dma_block() RDMA/umem: Remove unused 'work' member from struct ib_umem RDMA/irdma: Cap MSIX used to online CPUs + 1 RDMA/mlx5: Check reg_create() create for errors RDMA/restrack: Correct spelling RDMA/cxgb4: Fix potential null-ptr-deref in pass_establish() ...

Linus Torvalds committed 3 years ago

Commit 8cbd9232 parents 143c7bc 66fb1d565 files · +2112 −1597Browse files

Unified · Split

65 files
  1. drivers/infiniband/core/cma.c +159 −141
  2. drivers/infiniband/core/sa_query.c +55 −116
  3. drivers/infiniband/hw/cxgb4/cm.c +7 −0
  4. drivers/infiniband/hw/cxgb4/cq.c +1 −1
  5. drivers/infiniband/hw/cxgb4/restrack.c +1 −1
  6. drivers/infiniband/hw/cxgb4/t4fw_ri_api.h +6 −20
  7. drivers/infiniband/hw/erdma/erdma_hw.h +2 −2
  8. drivers/infiniband/hw/erdma/erdma_verbs.c +3 −1
  9. drivers/infiniband/hw/hfi1/chip.c +32 −27
  10. drivers/infiniband/hw/hfi1/exp_rcv.h +3 −2
  11. drivers/infiniband/hw/hfi1/file_ops.c +57 −24
  12. drivers/infiniband/hw/hfi1/init.c +1 −1
  13. drivers/infiniband/hw/hfi1/sdma.c +2 −2
  14. drivers/infiniband/hw/hfi1/sdma.h +7 −8
  15. drivers/infiniband/hw/hfi1/user_exp_rcv.c +20 −26
  16. drivers/infiniband/hw/hfi1/user_pages.c +40 −21
  17. drivers/infiniband/hw/hfi1/verbs.c +32 −49
  18. drivers/infiniband/hw/hns/hns_roce_device.h +2 −17
  19. drivers/infiniband/hw/hns/hns_roce_hw_v2.c +33 −76
  20. drivers/infiniband/hw/hns/hns_roce_hw_v2.h +2 −1
  21. drivers/infiniband/hw/hns/hns_roce_main.c +15 −2
  22. drivers/infiniband/hw/hns/hns_roce_qp.c +1 −65
  23. drivers/infiniband/hw/irdma/hw.c +2 −0
  24. drivers/infiniband/hw/irdma/verbs.c +210 −103
  25. drivers/infiniband/hw/mana/main.c +12 −10
  26. drivers/infiniband/hw/mlx4/main.c +8 −0
  27. drivers/infiniband/hw/mlx4/mlx4_ib.h +3 −0
  28. drivers/infiniband/hw/mlx4/qp.c +85 −36
  29. drivers/infiniband/hw/mlx5/cmd.c +26 −19
  30. drivers/infiniband/hw/mlx5/cmd.h +1 −2
  31. drivers/infiniband/hw/mlx5/cong.c +25 −3
  32. drivers/infiniband/hw/mlx5/main.c +13 −11
  33. drivers/infiniband/hw/mlx5/mlx5_ib.h +39 −12
  34. drivers/infiniband/hw/mlx5/mr.c +379 −111
  35. drivers/infiniband/hw/mlx5/odp.c +27 −40
  36. drivers/infiniband/hw/mlx5/qp.c +128 −36
  37. drivers/infiniband/hw/mlx5/qp.h +3 −1
  38. drivers/infiniband/hw/mlx5/qpc.c +5 −2
  39. drivers/infiniband/hw/mlx5/srq.c +1 −1
  40. drivers/infiniband/hw/mlx5/umr.c +1 −3
  41. drivers/infiniband/hw/mlx5/wr.c +1 −1
  42. drivers/infiniband/sw/rxe/rxe.h +38 −0
  43. drivers/infiniband/sw/rxe/rxe_loc.h +8 −4
  44. drivers/infiniband/sw/rxe/rxe_mr.c +338 −268
  45. drivers/infiniband/sw/rxe/rxe_pool.c +0 −46
  46. drivers/infiniband/sw/rxe/rxe_pool.h +0 −3
  47. drivers/infiniband/sw/rxe/rxe_queue.h +66 −42
  48. drivers/infiniband/sw/rxe/rxe_resp.c +75 −127
  49. drivers/infiniband/sw/rxe/rxe_verbs.c +53 −62
  50. drivers/infiniband/sw/rxe/rxe_verbs.h +10 −22
  51. drivers/infiniband/sw/siw/siw_mem.c +12 −11
  52. drivers/infiniband/ulp/ipoib/ipoib_main.c +1 −1
  53. drivers/infiniband/ulp/srp/ib_srp.c +1 −1
  54. drivers/net/ethernet/mellanox/mlx4/qp.c +9 −5
  55. drivers/net/ethernet/mellanox/mlx5/core/en_main.c +21 −1
  56. include/linux/mlx4/qp.h +1 −0
  57. include/linux/mlx5/driver.h +0 −1
  58. include/linux/mlx5/mlx5_ifc.h +20 −2
  59. include/linux/mlx5/qp.h +1 −1
  60. include/rdma/ib_sa.h +1 −1
  61. include/rdma/ib_umem.h +0 −1
  62. include/rdma/ib_verbs.h +1 −1
  63. include/rdma/rdma_cm.h +0 −1
  64. include/rdma/restrack.h +2 −2
  65. include/uapi/rdma/hns-abi.h +4 −0
drivers/infiniband/core/cma.c+159−141
@@ -479,13 +479,20 @@ static int compare_netdev_and_ip(int ifindex_a, struct sockaddr *sa, · 478 unmodified lines
479479 if (sa->sa_family != sb->sa_family)
480480 return sa->sa_family - sb->sa_family;
481481
482− if (sa->sa_family == AF_INET)
483− return memcmp((char *)&((struct sockaddr_in *)sa)->sin_addr,
484− (char *)&((struct sockaddr_in *)sb)->sin_addr,
482+ if (sa->sa_family == AF_INET &&
483+ __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in)) {
484+ return memcmp(&((struct sockaddr_in *)sa)->sin_addr,
485+ &((struct sockaddr_in *)sb)->sin_addr,
485486 sizeof(((struct sockaddr_in *)sa)->sin_addr));
487+ }
488+
489+ if (sa->sa_family == AF_INET6 &&
490+ __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in6)) {
491+ return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr,
492+ &((struct sockaddr_in6 *)sb)->sin6_addr);
493+ }
486494
487− return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr,
488− &((struct sockaddr_in6 *)sb)->sin6_addr);
495+ return -1;
489496 }
490497
491498 static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv)
@@ -2819,8 +2826,8 @@ int rdma_set_min_rnr_timer(struct rdma_cm_id *id, u8 min_rnr_timer) · 2327 unmodified lines
28192826 }
28202827 EXPORT_SYMBOL(rdma_set_min_rnr_timer);
28212828
2822−static void route_set_path_rec_inbound(struct cma_work *work,
2823− struct sa_path_rec *path_rec)
2829+static int route_set_path_rec_inbound(struct cma_work *work,
2830+ struct sa_path_rec *path_rec)
28242831 {
28252832 struct rdma_route *route = &work->id->id.route;
28262833
@@ -2828,14 +2835,15 @@ static void route_set_path_rec_inbound(struct cma_work *work, · 1 unmodified line
28282835 route->path_rec_inbound =
28292836 kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL);
28302837 if (!route->path_rec_inbound)
2831− return;
2838+ return -ENOMEM;
28322839 }
28332840
28342841 *route->path_rec_inbound = *path_rec;
2842+ return 0;
28352843 }
28362844
2837−static void route_set_path_rec_outbound(struct cma_work *work,
2838− struct sa_path_rec *path_rec)
2845+static int route_set_path_rec_outbound(struct cma_work *work,
2846+ struct sa_path_rec *path_rec)
28392847 {
28402848 struct rdma_route *route = &work->id->id.route;
28412849
@@ -2843,14 +2851,15 @@ static void route_set_path_rec_outbound(struct cma_work *work, · 1 unmodified line
28432851 route->path_rec_outbound =
28442852 kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL);
28452853 if (!route->path_rec_outbound)
2846− return;
2854+ return -ENOMEM;
28472855 }
28482856
28492857 *route->path_rec_outbound = *path_rec;
2858+ return 0;
28502859 }
28512860
28522861 static void cma_query_handler(int status, struct sa_path_rec *path_rec,
2853− int num_prs, void *context)
2862+ unsigned int num_prs, void *context)
28542863 {
28552864 struct cma_work *work = context;
28562865 struct rdma_route *route;
@@ -2865,13 +2874,15 @@ static void cma_query_handler(int status, struct sa_path_rec *path_rec, · 8 unmodified lines
28652874 if (!path_rec[i].flags || (path_rec[i].flags & IB_PATH_GMP))
28662875 *route->path_rec = path_rec[i];
28672876 else if (path_rec[i].flags & IB_PATH_INBOUND)
2868− route_set_path_rec_inbound(work, &path_rec[i]);
2877+ status = route_set_path_rec_inbound(work, &path_rec[i]);
28692878 else if (path_rec[i].flags & IB_PATH_OUTBOUND)
2870− route_set_path_rec_outbound(work, &path_rec[i]);
2871− }
2872− if (!route->path_rec) {
2873− status = -EINVAL;
2874− goto fail;
2879+ status = route_set_path_rec_outbound(work,
2880+ &path_rec[i]);
2881+ else
2882+ status = -EINVAL;
2883+
2884+ if (status)
2885+ goto fail;
28752886 }
28762887
28772888 route->num_pri_alt_paths = 1;
@@ -3541,121 +3552,6 @@ err: · 663 unmodified lines
35413552 return ret;
35423553 }
35433554
3544−static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
3545− const struct sockaddr *dst_addr)
3546−{
3547− struct sockaddr_storage zero_sock = {};
3548−
3549− if (src_addr && src_addr->sa_family)
3550− return rdma_bind_addr(id, src_addr);
3551−
3552− /*
3553− * When the src_addr is not specified, automatically supply an any addr
3554− */
3555− zero_sock.ss_family = dst_addr->sa_family;
3556− if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) {
3557− struct sockaddr_in6 *src_addr6 =
3558− (struct sockaddr_in6 *)&zero_sock;
3559− struct sockaddr_in6 *dst_addr6 =
3560− (struct sockaddr_in6 *)dst_addr;
3561−
3562− src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
3563− if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
3564− id->route.addr.dev_addr.bound_dev_if =
3565− dst_addr6->sin6_scope_id;
3566− } else if (dst_addr->sa_family == AF_IB) {
3567− ((struct sockaddr_ib *)&zero_sock)->sib_pkey =
3568− ((struct sockaddr_ib *)dst_addr)->sib_pkey;
3569− }
3570− return rdma_bind_addr(id, (struct sockaddr *)&zero_sock);
3571−}
3572−
3573−/*
3574− * If required, resolve the source address for bind and leave the id_priv in
3575− * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior
3576− * calls made by ULP, a previously bound ID will not be re-bound and src_addr is
3577− * ignored.
3578− */
3579−static int resolve_prepare_src(struct rdma_id_private *id_priv,
3580− struct sockaddr *src_addr,
3581− const struct sockaddr *dst_addr)
3582−{
3583− int ret;
3584−
3585− memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
3586− if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
3587− /* For a well behaved ULP state will be RDMA_CM_IDLE */
3588− ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr);
3589− if (ret)
3590− goto err_dst;
3591− if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
3592− RDMA_CM_ADDR_QUERY))) {
3593− ret = -EINVAL;
3594− goto err_dst;
3595− }
3596− }
3597−
3598− if (cma_family(id_priv) != dst_addr->sa_family) {
3599− ret = -EINVAL;
3600− goto err_state;
3601− }
3602− return 0;
3603−
3604−err_state:
3605− cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
3606−err_dst:
3607− memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr));
3608− return ret;
3609−}
3610−
3611−int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
3612− const struct sockaddr *dst_addr, unsigned long timeout_ms)
3613−{
3614− struct rdma_id_private *id_priv =
3615− container_of(id, struct rdma_id_private, id);
3616− int ret;
3617−
3618− ret = resolve_prepare_src(id_priv, src_addr, dst_addr);
3619− if (ret)
3620− return ret;
3621−
3622− if (cma_any_addr(dst_addr)) {
3623− ret = cma_resolve_loopback(id_priv);
3624− } else {
3625− if (dst_addr->sa_family == AF_IB) {
3626− ret = cma_resolve_ib_addr(id_priv);
3627− } else {
3628− /*
3629− * The FSM can return back to RDMA_CM_ADDR_BOUND after
3630− * rdma_resolve_ip() is called, eg through the error
3631− * path in addr_handler(). If this happens the existing
3632− * request must be canceled before issuing a new one.
3633− * Since canceling a request is a bit slow and this
3634− * oddball path is rare, keep track once a request has
3635− * been issued. The track turns out to be a permanent
3636− * state since this is the only cancel as it is
3637− * immediately before rdma_resolve_ip().
3638− */
3639− if (id_priv->used_resolve_ip)
3640− rdma_addr_cancel(&id->route.addr.dev_addr);
3641− else
3642− id_priv->used_resolve_ip = 1;
3643− ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
3644− &id->route.addr.dev_addr,
3645− timeout_ms, addr_handler,
3646− false, id_priv);
3647− }
3648− }
3649− if (ret)
3650− goto err;
3651−
3652− return 0;
3653−err:
3654− cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
3655− return ret;
3656−}
3657−EXPORT_SYMBOL(rdma_resolve_addr);
3658−
36593555 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
36603556 {
36613557 struct rdma_id_private *id_priv;
@@ -4058,27 +3954,26 @@ err: · 396 unmodified lines
40583954 }
40593955 EXPORT_SYMBOL(rdma_listen);
40603956
4061−int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
3957+static int rdma_bind_addr_dst(struct rdma_id_private *id_priv,
3958+ struct sockaddr *addr, const struct sockaddr *daddr)
40623959 {
4063− struct rdma_id_private *id_priv;
3960+ struct sockaddr *id_daddr;
40643961 int ret;
4065− struct sockaddr *daddr;
40663962
40673963 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
40683964 addr->sa_family != AF_IB)
40693965 return -EAFNOSUPPORT;
40703966
4071− id_priv = container_of(id, struct rdma_id_private, id);
40723967 if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
40733968 return -EINVAL;
40743969
4075− ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
3970+ ret = cma_check_linklocal(&id_priv->id.route.addr.dev_addr, addr);
40763971 if (ret)
40773972 goto err1;
40783973
40793974 memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
40803975 if (!cma_any_addr(addr)) {
4081− ret = cma_translate_addr(addr, &id->route.addr.dev_addr);
3976+ ret = cma_translate_addr(addr, &id_priv->id.route.addr.dev_addr);
40823977 if (ret)
40833978 goto err1;
40843979
@@ -4098,8 +3993,10 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) · 13 unmodified lines
40983993 }
40993994 #endif
41003995 }
4101− daddr = cma_dst_addr(id_priv);
4102− daddr->sa_family = addr->sa_family;
3996+ id_daddr = cma_dst_addr(id_priv);
3997+ if (daddr != id_daddr)
3998+ memcpy(id_daddr, daddr, rdma_addr_size(addr));
3999+ id_daddr->sa_family = addr->sa_family;
41034000
41044001 ret = cma_get_port(id_priv);
41054002 if (ret)
@@ -4115,6 +4012,127 @@ err1: · 9 unmodified lines
41154012 cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
41164013 return ret;
41174014 }
4015+
4016+static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
4017+ const struct sockaddr *dst_addr)
4018+{
4019+ struct rdma_id_private *id_priv =
4020+ container_of(id, struct rdma_id_private, id);
4021+ struct sockaddr_storage zero_sock = {};
4022+
4023+ if (src_addr && src_addr->sa_family)
4024+ return rdma_bind_addr_dst(id_priv, src_addr, dst_addr);
4025+
4026+ /*
4027+ * When the src_addr is not specified, automatically supply an any addr
4028+ */
4029+ zero_sock.ss_family = dst_addr->sa_family;
4030+ if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) {
4031+ struct sockaddr_in6 *src_addr6 =
4032+ (struct sockaddr_in6 *)&zero_sock;
4033+ struct sockaddr_in6 *dst_addr6 =
4034+ (struct sockaddr_in6 *)dst_addr;
4035+
4036+ src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
4037+ if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
4038+ id->route.addr.dev_addr.bound_dev_if =
4039+ dst_addr6->sin6_scope_id;
4040+ } else if (dst_addr->sa_family == AF_IB) {
4041+ ((struct sockaddr_ib *)&zero_sock)->sib_pkey =
4042+ ((struct sockaddr_ib *)dst_addr)->sib_pkey;
4043+ }
4044+ return rdma_bind_addr_dst(id_priv, (struct sockaddr *)&zero_sock, dst_addr);
4045+}
4046+
4047+/*
4048+ * If required, resolve the source address for bind and leave the id_priv in
4049+ * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior
4050+ * calls made by ULP, a previously bound ID will not be re-bound and src_addr is
4051+ * ignored.
4052+ */
4053+static int resolve_prepare_src(struct rdma_id_private *id_priv,
4054+ struct sockaddr *src_addr,
4055+ const struct sockaddr *dst_addr)
4056+{
4057+ int ret;
4058+
4059+ if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
4060+ /* For a well behaved ULP state will be RDMA_CM_IDLE */
4061+ ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr);
4062+ if (ret)
4063+ return ret;
4064+ if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
4065+ RDMA_CM_ADDR_QUERY)))
4066+ return -EINVAL;
4067+
4068+ }
4069+
4070+ if (cma_family(id_priv) != dst_addr->sa_family) {
4071+ ret = -EINVAL;
4072+ goto err_state;
4073+ }
4074+ return 0;
4075+
4076+err_state:
4077+ cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
4078+ return ret;
4079+}
4080+
4081+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
4082+ const struct sockaddr *dst_addr, unsigned long timeout_ms)
4083+{
4084+ struct rdma_id_private *id_priv =
4085+ container_of(id, struct rdma_id_private, id);
4086+ int ret;
4087+
4088+ ret = resolve_prepare_src(id_priv, src_addr, dst_addr);
4089+ if (ret)
4090+ return ret;
4091+
4092+ if (cma_any_addr(dst_addr)) {
4093+ ret = cma_resolve_loopback(id_priv);
4094+ } else {
4095+ if (dst_addr->sa_family == AF_IB) {
4096+ ret = cma_resolve_ib_addr(id_priv);
4097+ } else {
4098+ /*
4099+ * The FSM can return back to RDMA_CM_ADDR_BOUND after
4100+ * rdma_resolve_ip() is called, eg through the error
4101+ * path in addr_handler(). If this happens the existing
4102+ * request must be canceled before issuing a new one.
4103+ * Since canceling a request is a bit slow and this
4104+ * oddball path is rare, keep track once a request has
4105+ * been issued. The track turns out to be a permanent
4106+ * state since this is the only cancel as it is
4107+ * immediately before rdma_resolve_ip().
4108+ */
4109+ if (id_priv->used_resolve_ip)
4110+ rdma_addr_cancel(&id->route.addr.dev_addr);
4111+ else
4112+ id_priv->used_resolve_ip = 1;
4113+ ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
4114+ &id->route.addr.dev_addr,
4115+ timeout_ms, addr_handler,
4116+ false, id_priv);
4117+ }
4118+ }
4119+ if (ret)
4120+ goto err;
4121+
4122+ return 0;
4123+err:
4124+ cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
4125+ return ret;
4126+}
4127+EXPORT_SYMBOL(rdma_resolve_addr);
4128+
4129+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
4130+{
4131+ struct rdma_id_private *id_priv =
4132+ container_of(id, struct rdma_id_private, id);
4133+
4134+ return rdma_bind_addr_dst(id_priv, addr, cma_dst_addr(id_priv));
4135+}
41184136 EXPORT_SYMBOL(rdma_bind_addr);
41194137
41204138 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
drivers/infiniband/core/sa_query.c+55−116
@@ -106,7 +106,7 @@ struct ib_sa_device { · 105 unmodified lines
106106
107107 struct ib_sa_query {
108108 void (*callback)(struct ib_sa_query *sa_query, int status,
109− int num_prs, struct ib_sa_mad *mad);
109+ struct ib_sa_mad *mad);
110110 void (*release)(struct ib_sa_query *);
111111 struct ib_sa_client *client;
112112 struct ib_sa_port *port;
@@ -118,12 +118,6 @@ struct ib_sa_query { · 5 unmodified lines
118118 u32 seq; /* Local svc request sequence number */
119119 unsigned long timeout; /* Local svc timeout */
120120 u8 path_use; /* How will the pathrecord be used */
121−
122− /* A separate buffer to save pathrecords of a response, as in cases
123− * like IB/netlink, mulptiple pathrecords are supported, so that
124− * mad->data is not large enough to hold them
125− */
126− void *resp_pr_data;
127121 };
128122
129123 #define IB_SA_ENABLE_LOCAL_SERVICE 0x00000001
@@ -132,7 +126,7 @@ struct ib_sa_query { · 2 unmodified lines
132126
133127 struct ib_sa_path_query {
134128 void (*callback)(int status, struct sa_path_rec *rec,
135− int num_paths, void *context);
129+ unsigned int num_paths, void *context);
136130 void *context;
137131 struct ib_sa_query sa_query;
138132 struct sa_path_rec *conv_pr;
@@ -690,6 +684,8 @@ static const struct ib_field guidinfo_rec_table[] = { · 551 unmodified lines
690684 .size_bits = 512 },
691685 };
692686
687+#define RDMA_PRIMARY_PATH_MAX_REC_NUM 3
688+
693689 static inline void ib_sa_disable_local_svc(struct ib_sa_query *query)
694690 {
695691 query->flags &= ~IB_SA_ENABLE_LOCAL_SERVICE;
@@ -874,30 +870,21 @@ static void send_handler(struct ib_mad_agent *agent, · 178 unmodified lines
874870 static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query,
875871 const struct nlmsghdr *nlh)
876872 {
877− struct ib_path_rec_data *srec, *drec;
873+ struct sa_path_rec recs[RDMA_PRIMARY_PATH_MAX_REC_NUM];
878874 struct ib_sa_path_query *path_query;
875+ struct ib_path_rec_data *rec_data;
879876 struct ib_mad_send_wc mad_send_wc;
880877 const struct nlattr *head, *curr;
881878 struct ib_sa_mad *mad = NULL;
882− int len, rem, num_prs = 0;
879+ int len, rem, status = -EIO;
880+ unsigned int num_prs = 0;
883881 u32 mask = 0;
884− int status = -EIO;
885882
886883 if (!query->callback)
887884 goto out;
888885
889886 path_query = container_of(query, struct ib_sa_path_query, sa_query);
890887 mad = query->mad_buf->mad;
891− if (!path_query->conv_pr &&
892− (be16_to_cpu(mad->mad_hdr.attr_id) == IB_SA_ATTR_PATH_REC)) {
893− /* Need a larger buffer for possible multiple PRs */
894− query->resp_pr_data = kvcalloc(RDMA_PRIMARY_PATH_MAX_REC_NUM,
895− sizeof(*drec), GFP_KERNEL);
896− if (!query->resp_pr_data) {
897− query->callback(query, -ENOMEM, 0, NULL);
898− return;
899− }
900− }
901888
902889 head = (const struct nlattr *) nlmsg_data(nlh);
903890 len = nlmsg_len(nlh);
@@ -917,36 +904,41 @@ static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query, · 13 unmodified lines
917904 break;
918905 }
919906
920− drec = (struct ib_path_rec_data *)query->resp_pr_data;
921907 nla_for_each_attr(curr, head, len, rem) {
922908 if (curr->nla_type != LS_NLA_TYPE_PATH_RECORD)
923909 continue;
924910
925− srec = nla_data(curr);
926− if ((srec->flags & mask) != mask)
911+ rec_data = nla_data(curr);
912+ if ((rec_data->flags & mask) != mask)
927913 continue;
928914
929− status = 0;
930− if (!drec) {
931− memcpy(mad->data, srec->path_rec,
932− sizeof(srec->path_rec));
933− num_prs = 1;
934− break;
915+ if ((query->flags & IB_SA_QUERY_OPA) ||
916+ path_query->conv_pr) {
917+ mad->mad_hdr.method |= IB_MGMT_METHOD_RESP;
918+ memcpy(mad->data, rec_data->path_rec,
919+ sizeof(rec_data->path_rec));
920+ query->callback(query, 0, mad);
921+ goto out;
935922 }
936923
937− memcpy(drec, srec, sizeof(*drec));
938− drec++;
924+ status = 0;
925+ ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),
926+ rec_data->path_rec, &recs[num_prs]);
927+ recs[num_prs].flags = rec_data->flags;
928+ recs[num_prs].rec_type = SA_PATH_REC_TYPE_IB;
929+ sa_path_set_dmac_zero(&recs[num_prs]);
930+
939931 num_prs++;
940932 if (num_prs >= RDMA_PRIMARY_PATH_MAX_REC_NUM)
941933 break;
942934 }
943935
944− if (!status)
936+ if (!status) {
945937 mad->mad_hdr.method |= IB_MGMT_METHOD_RESP;
946−
947− query->callback(query, status, num_prs, mad);
948− kvfree(query->resp_pr_data);
949− query->resp_pr_data = NULL;
938+ path_query->callback(status, recs, num_prs,
939+ path_query->context);
940+ } else
941+ query->callback(query, status, mad);
950942
951943 out:
952944 mad_send_wc.send_buf = query->mad_buf;
@@ -1451,11 +1443,26 @@ static int opa_pr_query_possible(struct ib_sa_client *client, · 498 unmodified lines
14511443 return PR_IB_SUPPORTED;
14521444 }
14531445
1454−static void ib_sa_pr_callback_single(struct ib_sa_path_query *query,
1455− int status, struct ib_sa_mad *mad)
1446+static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query,
1447+ int status, struct ib_sa_mad *mad)
14561448 {
1449+ struct ib_sa_path_query *query =
1450+ container_of(sa_query, struct ib_sa_path_query, sa_query);
14571451 struct sa_path_rec rec = {};
14581452
1453+ if (!mad) {
1454+ query->callback(status, NULL, 0, query->context);
1455+ return;
1456+ }
1457+
1458+ if (sa_query->flags & IB_SA_QUERY_OPA) {
1459+ ib_unpack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table),
1460+ mad->data, &rec);
1461+ rec.rec_type = SA_PATH_REC_TYPE_OPA;
1462+ query->callback(status, &rec, 1, query->context);
1463+ return;
1464+ }
1465+
14591466 ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),
14601467 mad->data, &rec);
14611468 rec.rec_type = SA_PATH_REC_TYPE_IB;
@@ -1472,71 +1479,6 @@ static void ib_sa_pr_callback_single(struct ib_sa_path_query *query, · 10 unmodified lines
14721479 }
14731480 }
14741481
1475−/**
1476− * ib_sa_pr_callback_multiple() - Parse path records then do callback.
1477− *
1478− * In a multiple-PR case the PRs are saved in "query->resp_pr_data"
1479− * (instead of"mad->data") and with "ib_path_rec_data" structure format,
1480− * so that rec->flags can be set to indicate the type of PR.
1481− * This is valid only in IB fabric.
1482− */
1483−static void ib_sa_pr_callback_multiple(struct ib_sa_path_query *query,
1484− int status, int num_prs,
1485− struct ib_path_rec_data *rec_data)
1486−{
1487− struct sa_path_rec *rec;
1488− int i;
1489−
1490− rec = kvcalloc(num_prs, sizeof(*rec), GFP_KERNEL);
1491− if (!rec) {
1492− query->callback(-ENOMEM, NULL, 0, query->context);
1493− return;
1494− }
1495−
1496− for (i = 0; i < num_prs; i++) {
1497− ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),
1498− rec_data[i].path_rec, rec + i);
1499− rec[i].rec_type = SA_PATH_REC_TYPE_IB;
1500− sa_path_set_dmac_zero(rec + i);
1501− rec[i].flags = rec_data[i].flags;
1502− }
1503−
1504− query->callback(status, rec, num_prs, query->context);
1505− kvfree(rec);
1506−}
1507−
1508−static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query,
1509− int status, int num_prs,
1510− struct ib_sa_mad *mad)
1511−{
1512− struct ib_sa_path_query *query =
1513− container_of(sa_query, struct ib_sa_path_query, sa_query);
1514− struct sa_path_rec rec;
1515−
1516− if (!mad || !num_prs) {
1517− query->callback(status, NULL, 0, query->context);
1518− return;
1519− }
1520−
1521− if (sa_query->flags & IB_SA_QUERY_OPA) {
1522− if (num_prs != 1) {
1523− query->callback(-EINVAL, NULL, 0, query->context);
1524− return;
1525− }
1526−
1527− ib_unpack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table),
1528− mad->data, &rec);
1529− rec.rec_type = SA_PATH_REC_TYPE_OPA;
1530− query->callback(status, &rec, num_prs, query->context);
1531− } else {
1532− if (!sa_query->resp_pr_data)
1533− ib_sa_pr_callback_single(query, status, mad);
1534− else
1535− ib_sa_pr_callback_multiple(query, status, num_prs,
1536− sa_query->resp_pr_data);
1537− }
1538−}
1539−
15401482 static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
15411483 {
15421484 struct ib_sa_path_query *query =
@@ -1578,7 +1520,7 @@ int ib_sa_path_rec_get(struct ib_sa_client *client, · 35 unmodified lines
15781520 unsigned long timeout_ms, gfp_t gfp_mask,
15791521 void (*callback)(int status,
15801522 struct sa_path_rec *resp,
1581− int num_paths, void *context),
1523+ unsigned int num_paths, void *context),
15821524 void *context,
15831525 struct ib_sa_query **sa_query)
15841526 {
@@ -1677,8 +1619,7 @@ err1: · 92 unmodified lines
16771619 EXPORT_SYMBOL(ib_sa_path_rec_get);
16781620
16791621 static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query,
1680− int status, int num_prs,
1681− struct ib_sa_mad *mad)
1622+ int status, struct ib_sa_mad *mad)
16821623 {
16831624 struct ib_sa_mcmember_query *query =
16841625 container_of(sa_query, struct ib_sa_mcmember_query, sa_query);
@@ -1769,8 +1710,7 @@ err1: · 84 unmodified lines
17691710
17701711 /* Support GuidInfoRecord */
17711712 static void ib_sa_guidinfo_rec_callback(struct ib_sa_query *sa_query,
1772− int status, int num_paths,
1773− struct ib_sa_mad *mad)
1713+ int status, struct ib_sa_mad *mad)
17741714 {
17751715 struct ib_sa_guidinfo_query *query =
17761716 container_of(sa_query, struct ib_sa_guidinfo_query, sa_query);
@@ -1879,8 +1819,7 @@ static void ib_classportinfo_cb(void *context) · 102 unmodified lines
18791819 }
18801820
18811821 static void ib_sa_classport_info_rec_callback(struct ib_sa_query *sa_query,
1882− int status, int num_prs,
1883− struct ib_sa_mad *mad)
1822+ int status, struct ib_sa_mad *mad)
18841823 {
18851824 unsigned long flags;
18861825 struct ib_sa_classport_info_query *query =
@@ -2055,13 +1994,13 @@ static void send_handler(struct ib_mad_agent *agent, · 168 unmodified lines
20551994 /* No callback -- already got recv */
20561995 break;
20571996 case IB_WC_RESP_TIMEOUT_ERR:
2058− query->callback(query, -ETIMEDOUT, 0, NULL);
1997+ query->callback(query, -ETIMEDOUT, NULL);
20591998 break;
20601999 case IB_WC_WR_FLUSH_ERR:
2061− query->callback(query, -EINTR, 0, NULL);
2000+ query->callback(query, -EINTR, NULL);
20622001 break;
20632002 default:
2064− query->callback(query, -EIO, 0, NULL);
2003+ query->callback(query, -EIO, NULL);
20652004 break;
20662005 }
20672006
@@ -2089,10 +2028,10 @@ static void recv_handler(struct ib_mad_agent *mad_agent, · 21 unmodified lines
20892028 if (mad_recv_wc->wc->status == IB_WC_SUCCESS)
20902029 query->callback(query,
20912030 mad_recv_wc->recv_buf.mad->mad_hdr.status ?
2092− -EINVAL : 0, 1,
2031+ -EINVAL : 0,
20932032 (struct ib_sa_mad *) mad_recv_wc->recv_buf.mad);
20942033 else
2095− query->callback(query, -EIO, 0, NULL);
2034+ query->callback(query, -EIO, NULL);
20962035 }
20972036
20982037 ib_free_recv_mad(mad_recv_wc);
drivers/infiniband/hw/cxgb4/cm.c+7−0
@@ -2676,6 +2676,9 @@ static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb) · 2675 unmodified lines
26762676 u16 tcp_opt = ntohs(req->tcp_opt);
26772677
26782678 ep = get_ep_from_tid(dev, tid);
2679+ if (!ep)
2680+ return 0;
2681+
26792682 pr_debug("ep %p tid %u\n", ep, ep->hwtid);
26802683 ep->snd_seq = be32_to_cpu(req->snd_isn);
26812684 ep->rcv_seq = be32_to_cpu(req->rcv_isn);
@@ -4144,6 +4147,10 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb) · 1462 unmodified lines
41444147
41454148 if (neigh->dev->flags & IFF_LOOPBACK) {
41464149 pdev = ip_dev_find(&init_net, iph->daddr);
4150+ if (!pdev) {
4151+ pr_err("%s - failed to find device!\n", __func__);
4152+ goto free_dst;
4153+ }
41474154 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh,
41484155 pdev, 0);
41494156 pi = (struct port_info *)netdev_priv(pdev);
drivers/infiniband/hw/cxgb4/cq.c+1−1
@@ -767,7 +767,7 @@ static int __c4iw_poll_cq_one(struct c4iw_cq *chp, struct c4iw_qp *qhp, · 766 unmodified lines
767767 goto out;
768768
769769 wc->wr_id = cookie;
770− wc->qp = qhp ? &qhp->ibqp : NULL;
770+ wc->qp = &qhp->ibqp;
771771 wc->vendor_err = CQE_STATUS(&cqe);
772772 wc->wc_flags = 0;
773773
drivers/infiniband/hw/cxgb4/restrack.c+1−1
@@ -238,7 +238,7 @@ int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, · 237 unmodified lines
238238 if (rdma_nl_put_driver_u64_hex(msg, "history", epcp->history))
239239 goto err_cancel_table;
240240
241− if (epcp->state == LISTEN) {
241+ if (listen_ep) {
242242 if (rdma_nl_put_driver_u32(msg, "stid", listen_ep->stid))
243243 goto err_cancel_table;
244244 if (rdma_nl_put_driver_u32(msg, "backlog", listen_ep->backlog))
drivers/infiniband/hw/cxgb4/t4fw_ri_api.h+6−20
@@ -122,9 +122,7 @@ struct fw_ri_dsgl { · 121 unmodified lines
122122 __be16 nsge;
123123 __be32 len0;
124124 __be64 addr0;
125−#ifndef C99_NOT_SUPPORTED
126125 struct fw_ri_dsge_pair sge[];
127−#endif
128126 };
129127
130128 struct fw_ri_sge {
@@ -138,9 +136,7 @@ struct fw_ri_isgl { · 7 unmodified lines
138136 __u8 r1;
139137 __be16 nsge;
140138 __be32 r2;
141−#ifndef C99_NOT_SUPPORTED
142139 struct fw_ri_sge sge[];
143−#endif
144140 };
145141
146142 struct fw_ri_immd {
@@ -148,9 +144,7 @@ struct fw_ri_immd { · 1 unmodified line
148144 __u8 r1;
149145 __be16 r2;
150146 __be32 immdlen;
151−#ifndef C99_NOT_SUPPORTED
152147 __u8 data[];
153−#endif
154148 };
155149
156150 struct fw_ri_tpte {
@@ -320,9 +314,7 @@ struct fw_ri_res_wr { · 163 unmodified lines
320314 __be32 op_nres;
321315 __be32 len16_pkd;
322316 __u64 cookie;
323−#ifndef C99_NOT_SUPPORTED
324317 struct fw_ri_res res[];
325−#endif
326318 };
327319
328320 #define FW_RI_RES_WR_NRES_S 0
@@ -562,12 +554,10 @@ struct fw_ri_rdma_write_wr { · 233 unmodified lines
562554 __be32 plen;
563555 __be32 stag_sink;
564556 __be64 to_sink;
565−#ifndef C99_NOT_SUPPORTED
566557 union {
567− struct fw_ri_immd immd_src[0];
568− struct fw_ri_isgl isgl_src[0];
558+ DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);
559+ DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);
569560 } u;
570−#endif
571561 };
572562
573563 struct fw_ri_send_wr {
@@ -581,12 +571,10 @@ struct fw_ri_send_wr { · 7 unmodified lines
581571 __be32 plen;
582572 __be32 r3;
583573 __be64 r4;
584−#ifndef C99_NOT_SUPPORTED
585574 union {
586− struct fw_ri_immd immd_src[0];
587− struct fw_ri_isgl isgl_src[0];
575+ DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);
576+ DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);
588577 } u;
589−#endif
590578 };
591579
592580 #define FW_RI_SEND_WR_SENDOP_S 0
@@ -618,12 +606,10 @@ struct fw_ri_rdma_write_cmpl_wr { · 25 unmodified lines
618606 struct fw_ri_isgl isgl_src;
619607 } u_cmpl;
620608 __be64 r3;
621−#ifndef C99_NOT_SUPPORTED
622609 union fw_ri_write {
623− struct fw_ri_immd immd_src[0];
624− struct fw_ri_isgl isgl_src[0];
610+ DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);
611+ DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);
625612 } u;
626−#endif
627613 };
628614
629615 struct fw_ri_rdma_read_wr {
drivers/infiniband/hw/erdma/erdma_hw.h+2−2
@@ -397,7 +397,7 @@ struct erdma_write_sqe { · 396 unmodified lines
397397
398398 __le32 rsvd;
399399
400− struct erdma_sge sgl[0];
400+ struct erdma_sge sgl[];
401401 };
402402
403403 struct erdma_send_sqe {
@@ -408,7 +408,7 @@ struct erdma_send_sqe { · 4 unmodified lines
408408 };
409409
410410 __le32 length;
411− struct erdma_sge sgl[0];
411+ struct erdma_sge sgl[];
412412 };
413413
414414 struct erdma_readreq_sqe {
drivers/infiniband/hw/erdma/erdma_verbs.c+3−1
@@ -1110,12 +1110,14 @@ int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma) · 1109 unmodified lines
11101110 prot = pgprot_device(vma->vm_page_prot);
11111111 break;
11121112 default:
1113− return -EINVAL;
1113+ err = -EINVAL;
1114+ goto put_entry;
11141115 }
11151116
11161117 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
11171118 prot, rdma_entry);
11181119
1120+put_entry:
11191121 rdma_user_mmap_entry_put(rdma_entry);
11201122 return err;
11211123 }
drivers/infiniband/hw/hfi1/chip.c+32−27
@@ -1056,7 +1056,7 @@ static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr); · 1055 unmodified lines
10561056 static void handle_temp_err(struct hfi1_devdata *dd);
10571057 static void dc_shutdown(struct hfi1_devdata *dd);
10581058 static void dc_start(struct hfi1_devdata *dd);
1059−static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
1059+static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp,
10601060 unsigned int *np);
10611061 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
10621062 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
@@ -13362,7 +13362,6 @@ static int set_up_context_variables(struct hfi1_devdata *dd) · 12299 unmodified lines
1336213362 int ret;
1336313363 unsigned ngroups;
1336413364 int rmt_count;
13365− int user_rmt_reduced;
1336613365 u32 n_usr_ctxts;
1336713366 u32 send_contexts = chip_send_contexts(dd);
1336813367 u32 rcv_contexts = chip_rcv_contexts(dd);
@@ -13421,28 +13420,34 @@ static int set_up_context_variables(struct hfi1_devdata *dd) · 52 unmodified lines
1342113420 (num_kernel_contexts + n_usr_ctxts),
1342213421 &node_affinity.real_cpu_mask);
1342313422 /*
13424− * The RMT entries are currently allocated as shown below:
13425− * 1. QOS (0 to 128 entries);
13426− * 2. FECN (num_kernel_context - 1 + num_user_contexts +
13427− * num_netdev_contexts);
13428− * 3. netdev (num_netdev_contexts).
13429− * It should be noted that FECN oversubscribe num_netdev_contexts
13430− * entries of RMT because both netdev and PSM could allocate any receive
13431− * context between dd->first_dyn_alloc_text and dd->num_rcv_contexts,
13432− * and PSM FECN must reserve an RMT entry for each possible PSM receive
13433− * context.
13423+ * RMT entries are allocated as follows:
13424+ * 1. QOS (0 to 128 entries)
13425+ * 2. FECN (num_kernel_context - 1 [a] + num_user_contexts +
13426+ * num_netdev_contexts [b])
13427+ * 3. netdev (NUM_NETDEV_MAP_ENTRIES)
13428+ *
13429+ * Notes:
13430+ * [a] Kernel contexts (except control) are included in FECN if kernel
13431+ * TID_RDMA is active.
13432+ * [b] Netdev and user contexts are randomly allocated from the same
13433+ * context pool, so FECN must cover all contexts in the pool.
1343413434 */
13435− rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_netdev_contexts * 2);
13436− if (HFI1_CAP_IS_KSET(TID_RDMA))
13437− rmt_count += num_kernel_contexts - 1;
13438− if (rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) {
13439− user_rmt_reduced = NUM_MAP_ENTRIES - rmt_count;
13440− dd_dev_err(dd,
13441− "RMT size is reducing the number of user receive contexts from %u to %d\n",
13442− n_usr_ctxts,
13443− user_rmt_reduced);
13444− /* recalculate */
13445− n_usr_ctxts = user_rmt_reduced;
13435+ rmt_count = qos_rmt_entries(num_kernel_contexts - 1, NULL, NULL)
13436+ + (HFI1_CAP_IS_KSET(TID_RDMA) ? num_kernel_contexts - 1
13437+ : 0)
13438+ + n_usr_ctxts
13439+ + num_netdev_contexts
13440+ + NUM_NETDEV_MAP_ENTRIES;
13441+ if (rmt_count > NUM_MAP_ENTRIES) {
13442+ int over = rmt_count - NUM_MAP_ENTRIES;
13443+ /* try to squish user contexts, minimum of 1 */
13444+ if (over >= n_usr_ctxts) {
13445+ dd_dev_err(dd, "RMT overflow: reduce the requested number of contexts\n");
13446+ return -EINVAL;
13447+ }
13448+ dd_dev_err(dd, "RMT overflow: reducing # user contexts from %u to %u\n",
13449+ n_usr_ctxts, n_usr_ctxts - over);
13450+ n_usr_ctxts -= over;
1344613451 }
1344713452
1344813453 /* the first N are kernel contexts, the rest are user/netdev contexts */
@@ -14299,15 +14304,15 @@ static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index) · 850 unmodified lines
1429914304 }
1430014305
1430114306 /* return the number of RSM map table entries that will be used for QOS */
14302−static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
14307+static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp,
1430314308 unsigned int *np)
1430414309 {
1430514310 int i;
1430614311 unsigned int m, n;
14307− u8 max_by_vl = 0;
14312+ uint max_by_vl = 0;
1430814313
1430914314 /* is QOS active at all? */
14310− if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
14315+ if (n_krcv_queues < MIN_KERNEL_KCTXTS ||
1431114316 num_vls == 1 ||
1431214317 krcvqsset <= 1)
1431314318 goto no_qos;
@@ -14365,7 +14370,7 @@ static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt) · 51 unmodified lines
1436514370
1436614371 if (!rmt)
1436714372 goto bail;
14368− rmt_entries = qos_rmt_entries(dd, &m, &n);
14373+ rmt_entries = qos_rmt_entries(dd->n_krcv_queues - 1, &m, &n);
1436914374 if (rmt_entries == 0)
1437014375 goto bail;
1437114376 qpns_per_vl = 1 << m;
drivers/infiniband/hw/hfi1/exp_rcv.h+3−2
@@ -133,12 +133,13 @@ static inline struct tid_group *tid_group_pop(struct exp_tid_set *set) · 132 unmodified lines
133133 return grp;
134134 }
135135
136−static inline u32 rcventry2tidinfo(u32 rcventry)
136+static inline u32 create_tid(u32 rcventry, u32 npages)
137137 {
138138 u32 pair = rcventry & ~0x1;
139139
140140 return EXP_TID_SET(IDX, pair >> 1) |
141− EXP_TID_SET(CTRL, 1 << (rcventry - pair));
141+ EXP_TID_SET(CTRL, 1 << (rcventry - pair)) |
142+ EXP_TID_SET(LEN, npages);
142143 }
143144
144145 /**
drivers/infiniband/hw/hfi1/file_ops.c+57−24
@@ -306,6 +306,17 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from) · 305 unmodified lines
306306 return reqs;
307307 }
308308
309+static inline void mmap_cdbg(u16 ctxt, u8 subctxt, u8 type, u8 mapio, u8 vmf,
310+ u64 memaddr, void *memvirt, dma_addr_t memdma,
311+ ssize_t memlen, struct vm_area_struct *vma)
312+{
313+ hfi1_cdbg(PROC,
314+ "%u:%u type:%u io/vf/dma:%d/%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx",
315+ ctxt, subctxt, type, mapio, vmf, !!memdma,
316+ memaddr ?: (u64)memvirt, memlen,
317+ vma->vm_end - vma->vm_start, vma->vm_flags);
318+}
319+
309320 static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
310321 {
311322 struct hfi1_filedata *fd = fp->private_data;
@@ -315,6 +326,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) · 3 unmodified lines
315326 u64 token = vma->vm_pgoff << PAGE_SHIFT,
316327 memaddr = 0;
317328 void *memvirt = NULL;
329+ dma_addr_t memdma = 0;
318330 u8 subctxt, mapio = 0, vmf = 0, type;
319331 ssize_t memlen = 0;
320332 int ret = 0;
@@ -334,6 +346,11 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) · 13 unmodified lines
334346 goto done;
335347 }
336348
349+ /*
350+ * vm_pgoff is used as a buffer selector cookie. Always mmap from
351+ * the beginning.
352+ */
353+ vma->vm_pgoff = 0;
337354 flags = vma->vm_flags;
338355
339356 switch (type) {
@@ -355,7 +372,8 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) · 15 unmodified lines
355372 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
356373 mapio = 1;
357374 break;
358− case PIO_CRED:
375+ case PIO_CRED: {
376+ u64 cr_page_offset;
359377 if (flags & VM_WRITE) {
360378 ret = -EPERM;
361379 goto done;
@@ -365,10 +383,11 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) · 3 unmodified lines
365383 * second or third page allocated for credit returns (if number
366384 * of enabled contexts > 64 and 128 respectively).
367385 */
368− memvirt = dd->cr_base[uctxt->numa_id].va;
369− memaddr = virt_to_phys(memvirt) +
370− (((u64)uctxt->sc->hw_free -
371− (u64)dd->cr_base[uctxt->numa_id].va) & PAGE_MASK);
386+ cr_page_offset = ((u64)uctxt->sc->hw_free -
387+ (u64)dd->cr_base[uctxt->numa_id].va) &
388+ PAGE_MASK;
389+ memvirt = dd->cr_base[uctxt->numa_id].va + cr_page_offset;
390+ memdma = dd->cr_base[uctxt->numa_id].dma + cr_page_offset;
372391 memlen = PAGE_SIZE;
373392 flags &= ~VM_MAYWRITE;
374393 flags |= VM_DONTCOPY | VM_DONTEXPAND;
@@ -378,14 +397,16 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) · 3 unmodified lines
378397 * memory been flagged as non-cached?
379398 */
380399 /* vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); */
381− mapio = 1;
382400 break;
401+ }
383402 case RCV_HDRQ:
384403 memlen = rcvhdrq_size(uctxt);
385404 memvirt = uctxt->rcvhdrq;
405+ memdma = uctxt->rcvhdrq_dma;
386406 break;
387407 case RCV_EGRBUF: {
388− unsigned long addr;
408+ unsigned long vm_start_save;
409+ unsigned long vm_end_save;
389410 int i;
390411 /*
391412 * The RcvEgr buffer need to be handled differently
@@ -404,24 +425,34 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) · 12 unmodified lines
404425 goto done;
405426 }
406427 vm_flags_clear(vma, VM_MAYWRITE);
407− addr = vma->vm_start;
428+ /*
429+ * Mmap multiple separate allocations into a single vma. From
430+ * here, dma_mmap_coherent() calls dma_direct_mmap(), which
431+ * requires the mmap to exactly fill the vma starting at
432+ * vma_start. Adjust the vma start and end for each eager
433+ * buffer segment mapped. Restore the originals when done.
434+ */
435+ vm_start_save = vma->vm_start;
436+ vm_end_save = vma->vm_end;
437+ vma->vm_end = vma->vm_start;
408438 for (i = 0 ; i < uctxt->egrbufs.numbufs; i++) {
409439 memlen = uctxt->egrbufs.buffers[i].len;
410440 memvirt = uctxt->egrbufs.buffers[i].addr;
411− ret = remap_pfn_range(
412− vma, addr,
413− /*
414− * virt_to_pfn() does the same, but
415− * it's not available on x86_64
416− * when CONFIG_MMU is enabled.
417− */
418− PFN_DOWN(__pa(memvirt)),
419− memlen,
420− vma->vm_page_prot);
421− if (ret < 0)
441+ memdma = uctxt->egrbufs.buffers[i].dma;
442+ vma->vm_end += memlen;
443+ mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr,
444+ memvirt, memdma, memlen, vma);
445+ ret = dma_mmap_coherent(&dd->pcidev->dev, vma,
446+ memvirt, memdma, memlen);
447+ if (ret < 0) {
448+ vma->vm_start = vm_start_save;
449+ vma->vm_end = vm_end_save;
422450 goto done;
423− addr += memlen;
451+ }
452+ vma->vm_start += memlen;
424453 }
454+ vma->vm_start = vm_start_save;
455+ vma->vm_end = vm_end_save;
425456 ret = 0;
426457 goto done;
427458 }
@@ -481,6 +512,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) · 53 unmodified lines
481512 }
482513 memlen = PAGE_SIZE;
483514 memvirt = (void *)hfi1_rcvhdrtail_kvaddr(uctxt);
515+ memdma = uctxt->rcvhdrqtailaddr_dma;
484516 flags &= ~VM_MAYWRITE;
485517 break;
486518 case SUBCTXT_UREGS:
@@ -529,14 +561,15 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) · 42 unmodified lines
529561 }
530562
531563 vm_flags_reset(vma, flags);
532− hfi1_cdbg(PROC,
533− "%u:%u type:%u io/vf:%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx\n",
534− ctxt, subctxt, type, mapio, vmf, memaddr, memlen,
535− vma->vm_end - vma->vm_start, vma->vm_flags);
564+ mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr, memvirt, memdma,
565+ memlen, vma);
536566 if (vmf) {
537567 vma->vm_pgoff = PFN_DOWN(memaddr);
538568 vma->vm_ops = &vm_ops;
539569 ret = 0;
570+ } else if (memdma) {
571+ ret = dma_mmap_coherent(&dd->pcidev->dev, vma,
572+ memvirt, memdma, memlen);
540573 } else if (mapio) {
541574 ret = io_remap_pfn_range(vma, vma->vm_start,
542575 PFN_DOWN(memaddr),
drivers/infiniband/hw/hfi1/init.c+1−1
@@ -464,7 +464,7 @@ bail: · 463 unmodified lines
464464 *
465465 * This wrapper is the free function that matches hfi1_create_ctxtdata().
466466 * When a context is done being used (kernel or user), this function is called
467− * for the "final" put to match the kref init from hf1i_create_ctxtdata().
467+ * for the "final" put to match the kref init from hfi1_create_ctxtdata().
468468 * Other users of the context do a get/put sequence to make sure that the
469469 * structure isn't removed while in use.
470470 */
drivers/infiniband/hw/hfi1/sdma.c+2−2
@@ -3160,8 +3160,7 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) · 3159 unmodified lines
31603160 {
31613161 int rval = 0;
31623162
3163− tx->num_desc++;
3164− if ((unlikely(tx->num_desc == tx->desc_limit))) {
3163+ if ((unlikely(tx->num_desc + 1 == tx->desc_limit))) {
31653164 rval = _extend_sdma_tx_descs(dd, tx);
31663165 if (rval) {
31673166 __sdma_txclean(dd, tx);
@@ -3174,6 +3173,7 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) · 6 unmodified lines
31743173 SDMA_MAP_NONE,
31753174 dd->sdma_pad_phys,
31763175 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)));
3176+ tx->num_desc++;
31773177 _sdma_close_tx(dd, tx);
31783178 return rval;
31793179 }
drivers/infiniband/hw/hfi1/sdma.h+7−8
@@ -631,14 +631,13 @@ static inline void sdma_txclean(struct hfi1_devdata *dd, struct sdma_txreq *tx) · 630 unmodified lines
631631 static inline void _sdma_close_tx(struct hfi1_devdata *dd,
632632 struct sdma_txreq *tx)
633633 {
634− tx->descp[tx->num_desc].qw[0] |=
635− SDMA_DESC0_LAST_DESC_FLAG;
636− tx->descp[tx->num_desc].qw[1] |=
637− dd->default_desc1;
634+ u16 last_desc = tx->num_desc - 1;
635+
636+ tx->descp[last_desc].qw[0] |= SDMA_DESC0_LAST_DESC_FLAG;
637+ tx->descp[last_desc].qw[1] |= dd->default_desc1;
638638 if (tx->flags & SDMA_TXREQ_F_URGENT)
639− tx->descp[tx->num_desc].qw[1] |=
640− (SDMA_DESC1_HEAD_TO_HOST_FLAG |
641− SDMA_DESC1_INT_REQ_FLAG);
639+ tx->descp[last_desc].qw[1] |= (SDMA_DESC1_HEAD_TO_HOST_FLAG |
640+ SDMA_DESC1_INT_REQ_FLAG);
642641 }
643642
644643 static inline int _sdma_txadd_daddr(
@@ -655,6 +654,7 @@ static inline int _sdma_txadd_daddr( · 10 unmodified lines
655654 type,
656655 addr, len);
657656 WARN_ON(len > tx->tlen);
657+ tx->num_desc++;
658658 tx->tlen -= len;
659659 /* special cases for last */
660660 if (!tx->tlen) {
@@ -666,7 +666,6 @@ static inline int _sdma_txadd_daddr( · 5 unmodified lines
666666 _sdma_close_tx(dd, tx);
667667 }
668668 }
669− tx->num_desc++;
670669 return rval;
671670 }
672671
drivers/infiniband/hw/hfi1/user_exp_rcv.c+20−26
@@ -27,8 +27,7 @@ static bool tid_cover_invalidate(struct mmu_interval_notifier *mni, · 26 unmodified lines
2727 const struct mmu_notifier_range *range,
2828 unsigned long cur_seq);
2929 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,
30− struct tid_group *grp,
31− unsigned int start, u16 count,
30+ struct tid_group *grp, u16 count,
3231 u32 *tidlist, unsigned int *tididx,
3332 unsigned int *pmapped);
3433 static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo);
@@ -250,7 +249,7 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, · 215 unmodified lines
250249 int ret = 0, need_group = 0, pinned;
251250 struct hfi1_ctxtdata *uctxt = fd->uctxt;
252251 struct hfi1_devdata *dd = uctxt->dd;
253− unsigned int ngroups, pageidx = 0, pageset_count,
252+ unsigned int ngroups, pageset_count,
254253 tididx = 0, mapped, mapped_pages = 0;
255254 u32 *tidlist = NULL;
256255 struct tid_user_buf *tidbuf;
@@ -332,7 +331,7 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, · 75 unmodified lines
332331 tid_group_pop(&uctxt->tid_group_list);
333332
334333 ret = program_rcvarray(fd, tidbuf, grp,
335− pageidx, dd->rcv_entries.group_size,
334+ dd->rcv_entries.group_size,
336335 tidlist, &tididx, &mapped);
337336 /*
338337 * If there was a failure to program the RcvArray
@@ -348,11 +347,10 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, · 9 unmodified lines
348347
349348 tid_group_add_tail(grp, &uctxt->tid_full_list);
350349 ngroups--;
351− pageidx += ret;
352350 mapped_pages += mapped;
353351 }
354352
355− while (pageidx < pageset_count) {
353+ while (tididx < pageset_count) {
356354 struct tid_group *grp, *ptr;
357355 /*
358356 * If we don't have any partially used tid groups, check
@@ -374,11 +372,11 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, · 15 unmodified lines
374372 */
375373 list_for_each_entry_safe(grp, ptr, &uctxt->tid_used_list.list,
376374 list) {
377− unsigned use = min_t(unsigned, pageset_count - pageidx,
375+ unsigned use = min_t(unsigned, pageset_count - tididx,
378376 grp->size - grp->used);
379377
380378 ret = program_rcvarray(fd, tidbuf, grp,
381− pageidx, use, tidlist,
379+ use, tidlist,
382380 &tididx, &mapped);
383381 if (ret < 0) {
384382 hfi1_cdbg(TID,
@@ -390,11 +388,10 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, · 5 unmodified lines
390388 tid_group_move(grp,
391389 &uctxt->tid_used_list,
392390 &uctxt->tid_full_list);
393− pageidx += ret;
394391 mapped_pages += mapped;
395392 need_group = 0;
396393 /* Check if we are done so we break out early */
397− if (pageidx >= pageset_count)
394+ if (tididx >= pageset_count)
398395 break;
399396 } else if (WARN_ON(ret == 0)) {
400397 /*
@@ -638,7 +635,6 @@ static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages) · 237 unmodified lines
638635 * struct tid_pageset holding information on physically contiguous
639636 * chunks from the user buffer), and other fields.
640637 * @grp: RcvArray group
641− * @start: starting index into sets array
642638 * @count: number of struct tid_pageset's to program
643639 * @tidlist: the array of u32 elements when the information about the
644640 * programmed RcvArray entries is to be encoded.
@@ -658,14 +654,14 @@ static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages) · 13 unmodified lines
658654 * number of RcvArray entries programmed.
659655 */
660656 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf,
661− struct tid_group *grp,
662− unsigned int start, u16 count,
657+ struct tid_group *grp, u16 count,
663658 u32 *tidlist, unsigned int *tididx,
664659 unsigned int *pmapped)
665660 {
666661 struct hfi1_ctxtdata *uctxt = fd->uctxt;
667662 struct hfi1_devdata *dd = uctxt->dd;
668663 u16 idx;
664+ unsigned int start = *tididx;
669665 u32 tidinfo = 0, rcventry, useidx = 0;
670666 int mapped = 0;
671667
@@ -710,8 +706,7 @@ static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf, · 38 unmodified lines
710706 return ret;
711707 mapped += npages;
712708
713− tidinfo = rcventry2tidinfo(rcventry - uctxt->expected_base) |
714− EXP_TID_SET(LEN, npages);
709+ tidinfo = create_tid(rcventry - uctxt->expected_base, npages);
715710 tidlist[(*tididx)++] = tidinfo;
716711 grp->used++;
717712 grp->map |= 1 << useidx++;
@@ -795,20 +790,20 @@ static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo) · 77 unmodified lines
795790 struct hfi1_ctxtdata *uctxt = fd->uctxt;
796791 struct hfi1_devdata *dd = uctxt->dd;
797792 struct tid_rb_node *node;
798− u8 tidctrl = EXP_TID_GET(tidinfo, CTRL);
793+ u32 tidctrl = EXP_TID_GET(tidinfo, CTRL);
799794 u32 tididx = EXP_TID_GET(tidinfo, IDX) << 1, rcventry;
800795
801− if (tididx >= uctxt->expected_count) {
802− dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n",
803− tididx, uctxt->ctxt);
804− return -EINVAL;
805− }
806−
807− if (tidctrl == 0x3)
796+ if (tidctrl == 0x3 || tidctrl == 0x0)
808797 return -EINVAL;
809798
810799 rcventry = tididx + (tidctrl - 1);
811800
801+ if (rcventry >= uctxt->expected_count) {
802+ dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n",
803+ rcventry, uctxt->ctxt);
804+ return -EINVAL;
805+ }
806+
812807 node = fd->entry_to_rb[rcventry];
813808 if (!node || node->rcventry != (uctxt->expected_base + rcventry))
814809 return -EBADF;
@@ -920,9 +915,8 @@ static bool tid_rb_invalidate(struct mmu_interval_notifier *mni, · 105 unmodified lines
920915 spin_lock(&fdata->invalid_lock);
921916 if (fdata->invalid_tid_idx < uctxt->expected_count) {
922917 fdata->invalid_tids[fdata->invalid_tid_idx] =
923− rcventry2tidinfo(node->rcventry - uctxt->expected_base);
924− fdata->invalid_tids[fdata->invalid_tid_idx] |=
925− EXP_TID_SET(LEN, node->npages);
918+ create_tid(node->rcventry - uctxt->expected_base,
919+ node->npages);
926920 if (!fdata->invalid_tid_idx) {
927921 unsigned long *ev;
928922
drivers/infiniband/hw/hfi1/user_pages.c+40−21
@@ -29,33 +29,52 @@ MODULE_PARM_DESC(cache_size, "Send and receive side cache size limit (in MB)"); · 28 unmodified lines
2929 bool hfi1_can_pin_pages(struct hfi1_devdata *dd, struct mm_struct *mm,
3030 u32 nlocked, u32 npages)
3131 {
32− unsigned long ulimit = rlimit(RLIMIT_MEMLOCK), pinned, cache_limit,
33− size = (cache_size * (1UL << 20)); /* convert to bytes */
34− unsigned int usr_ctxts =
35− dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt;
36− bool can_lock = capable(CAP_IPC_LOCK);
32+ unsigned long ulimit_pages;
33+ unsigned long cache_limit_pages;
34+ unsigned int usr_ctxts;
3735
3836 /*
39− * Calculate per-cache size. The calculation below uses only a quarter
40− * of the available per-context limit. This leaves space for other
41− * pinning. Should we worry about shared ctxts?
37+ * Perform RLIMIT_MEMLOCK based checks unless CAP_IPC_LOCK is present.
4238 */
43− cache_limit = (ulimit / usr_ctxts) / 4;
44−
45− /* If ulimit isn't set to "unlimited" and is smaller than cache_size. */
46− if (ulimit != (-1UL) && size > cache_limit)
47− size = cache_limit;
48−
49− /* Convert to number of pages */
50− size = DIV_ROUND_UP(size, PAGE_SIZE);
51−
52− pinned = atomic64_read(&mm->pinned_vm);
39+ if (!capable(CAP_IPC_LOCK)) {
40+ ulimit_pages =
41+ DIV_ROUND_DOWN_ULL(rlimit(RLIMIT_MEMLOCK), PAGE_SIZE);
42+
43+ /*
44+ * Pinning these pages would exceed this process's locked memory
45+ * limit.
46+ */
47+ if (atomic64_read(&mm->pinned_vm) + npages > ulimit_pages)
48+ return false;
49+
50+ /*
51+ * Only allow 1/4 of the user's RLIMIT_MEMLOCK to be used for HFI
52+ * caches. This fraction is then equally distributed among all
53+ * existing user contexts. Note that if RLIMIT_MEMLOCK is
54+ * 'unlimited' (-1), the value of this limit will be > 2^42 pages
55+ * (2^64 / 2^12 / 2^8 / 2^2).
56+ *
57+ * The effectiveness of this check may be reduced if I/O occurs on
58+ * some user contexts before all user contexts are created. This
59+ * check assumes that this process is the only one using this
60+ * context (e.g., the corresponding fd was not passed to another
61+ * process for concurrent access) as there is no per-context,
62+ * per-process tracking of pinned pages. It also assumes that each
63+ * user context has only one cache to limit.
64+ */
65+ usr_ctxts = dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt;
66+ if (nlocked + npages > (ulimit_pages / usr_ctxts / 4))
67+ return false;
68+ }
5369
54− /* First, check the absolute limit against all pinned pages. */
55− if (pinned + npages >= ulimit && !can_lock)
70+ /*
71+ * Pinning these pages would exceed the size limit for this cache.
72+ */
73+ cache_limit_pages = cache_size * (1024 * 1024) / PAGE_SIZE;
74+ if (nlocked + npages > cache_limit_pages)
5675 return false;
5776
58− return ((nlocked + npages) <= size) || can_lock;
77+ return true;
5978 }
6079
6180 int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t npages,
drivers/infiniband/hw/hfi1/verbs.c+32−49
@@ -1598,13 +1598,11 @@ static const char * const driver_cntr_names[] = { · 1597 unmodified lines
15981598 "DRIVER_EgrHdrFull"
15991599 };
16001600
1601−static DEFINE_MUTEX(cntr_names_lock); /* protects the *_cntr_names bufers */
16021601 static struct rdma_stat_desc *dev_cntr_descs;
16031602 static struct rdma_stat_desc *port_cntr_descs;
16041603 int num_driver_cntrs = ARRAY_SIZE(driver_cntr_names);
16051604 static int num_dev_cntrs;
16061605 static int num_port_cntrs;
1607−static int cntr_names_initialized;
16081606
16091607 /*
16101608 * Convert a list of names separated by '\n' into an array of NULL terminated
@@ -1615,8 +1613,8 @@ static int init_cntr_names(const char *names_in, const size_t names_len, · 4 unmodified lines
16151613 int num_extra_names, int *num_cntrs,
16161614 struct rdma_stat_desc **cntr_descs)
16171615 {
1618− struct rdma_stat_desc *q;
1619− char *names_out, *p;
1616+ struct rdma_stat_desc *names_out;
1617+ char *p;
16201618 int i, n;
16211619
16221620 n = 0;
@@ -1624,65 +1622,45 @@ static int init_cntr_names(const char *names_in, const size_t names_len, · 1 unmodified line
16241622 if (names_in[i] == '\n')
16251623 n++;
16261624
1627− names_out =
1628− kzalloc((n + num_extra_names) * sizeof(*q) + names_len,
1629− GFP_KERNEL);
1625+ names_out = kzalloc((n + num_extra_names) * sizeof(*names_out)
1626+ + names_len,
1627+ GFP_KERNEL);
16301628 if (!names_out) {
16311629 *num_cntrs = 0;
16321630 *cntr_descs = NULL;
16331631 return -ENOMEM;
16341632 }
16351633
1636− p = names_out + (n + num_extra_names) * sizeof(*q);
1634+ p = (char *)&names_out[n + num_extra_names];
16371635 memcpy(p, names_in, names_len);
16381636
1639− q = (struct rdma_stat_desc *)names_out;
16401637 for (i = 0; i < n; i++) {
1641− q[i].name = p;
1638+ names_out[i].name = p;
16421639 p = strchr(p, '\n');
16431640 *p++ = '\0';
16441641 }
16451642
16461643 *num_cntrs = n;
1647− *cntr_descs = (struct rdma_stat_desc *)names_out;
1644+ *cntr_descs = names_out;
16481645 return 0;
16491646 }
16501647
1651−static int init_counters(struct ib_device *ibdev)
1652−{
1653− struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
1654− int i, err = 0;
1655−
1656− mutex_lock(&cntr_names_lock);
1657− if (cntr_names_initialized)
1658− goto out_unlock;
1659−
1660− err = init_cntr_names(dd->cntrnames, dd->cntrnameslen, num_driver_cntrs,
1661− &num_dev_cntrs, &dev_cntr_descs);
1662− if (err)
1663− goto out_unlock;
1664−
1665− for (i = 0; i < num_driver_cntrs; i++)
1666− dev_cntr_descs[num_dev_cntrs + i].name = driver_cntr_names[i];
1667−
1668− err = init_cntr_names(dd->portcntrnames, dd->portcntrnameslen, 0,
1669− &num_port_cntrs, &port_cntr_descs);
1670− if (err) {
1671− kfree(dev_cntr_descs);
1672− dev_cntr_descs = NULL;
1673− goto out_unlock;
1674− }
1675− cntr_names_initialized = 1;
1676−
1677−out_unlock:
1678− mutex_unlock(&cntr_names_lock);
1679− return err;
1680−}
1681−
16821648 static struct rdma_hw_stats *hfi1_alloc_hw_device_stats(struct ib_device *ibdev)
16831649 {
1684− if (init_counters(ibdev))
1685− return NULL;
1650+ if (!dev_cntr_descs) {
1651+ struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
1652+ int i, err;
1653+
1654+ err = init_cntr_names(dd->cntrnames, dd->cntrnameslen,
1655+ num_driver_cntrs,
1656+ &num_dev_cntrs, &dev_cntr_descs);
1657+ if (err)
1658+ return NULL;
1659+
1660+ for (i = 0; i < num_driver_cntrs; i++)
1661+ dev_cntr_descs[num_dev_cntrs + i].name =
1662+ driver_cntr_names[i];
1663+ }
16861664 return rdma_alloc_hw_stats_struct(dev_cntr_descs,
16871665 num_dev_cntrs + num_driver_cntrs,
16881666 RDMA_HW_STATS_DEFAULT_LIFESPAN);
@@ -1691,8 +1669,16 @@ static struct rdma_hw_stats *hfi1_alloc_hw_device_stats(struct ib_device *ibdev) · 2 unmodified lines
16911669 static struct rdma_hw_stats *hfi_alloc_hw_port_stats(struct ib_device *ibdev,
16921670 u32 port_num)
16931671 {
1694− if (init_counters(ibdev))
1695− return NULL;
1672+ if (!port_cntr_descs) {
1673+ struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
1674+ int err;
1675+
1676+ err = init_cntr_names(dd->portcntrnames, dd->portcntrnameslen,
1677+ 0,
1678+ &num_port_cntrs, &port_cntr_descs);
1679+ if (err)
1680+ return NULL;
1681+ }
16961682 return rdma_alloc_hw_stats_struct(port_cntr_descs, num_port_cntrs,
16971683 RDMA_HW_STATS_DEFAULT_LIFESPAN);
16981684 }
@@ -1917,13 +1903,10 @@ void hfi1_unregister_ib_device(struct hfi1_devdata *dd) · 218 unmodified lines
19171903 del_timer_sync(&dev->mem_timer);
19181904 verbs_txreq_exit(dev);
19191905
1920− mutex_lock(&cntr_names_lock);
19211906 kfree(dev_cntr_descs);
19221907 kfree(port_cntr_descs);
19231908 dev_cntr_descs = NULL;
19241909 port_cntr_descs = NULL;
1925− cntr_names_initialized = 0;
1926− mutex_unlock(&cntr_names_lock);
19271910 }
19281911
19291912 void hfi1_cnp_rcv(struct hfi1_packet *packet)
drivers/infiniband/hw/hns/hns_roce_device.h+2−17
@@ -144,6 +144,7 @@ enum { · 143 unmodified lines
144144 HNS_ROCE_CAP_FLAG_DIRECT_WQE = BIT(12),
145145 HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14),
146146 HNS_ROCE_CAP_FLAG_STASH = BIT(17),
147+ HNS_ROCE_CAP_FLAG_CQE_INLINE = BIT(19),
147148 };
148149
149150 #define HNS_ROCE_DB_TYPE_COUNT 2
@@ -567,21 +568,6 @@ struct hns_roce_mbox_msg { · 417 unmodified lines
567568
568569 struct hns_roce_dev;
569570
570−struct hns_roce_rinl_sge {
571− void *addr;
572− u32 len;
573−};
574−
575−struct hns_roce_rinl_wqe {
576− struct hns_roce_rinl_sge *sg_list;
577− u32 sge_cnt;
578−};
579−
580−struct hns_roce_rinl_buf {
581− struct hns_roce_rinl_wqe *wqe_list;
582− u32 wqe_cnt;
583−};
584−
585571 enum {
586572 HNS_ROCE_FLUSH_FLAG = 0,
587573 };
@@ -632,7 +618,6 @@ struct hns_roce_qp { · 44 unmodified lines
632618 /* 0: flush needed, 1: unneeded */
633619 unsigned long flush_flag;
634620 struct hns_roce_work flush_work;
635− struct hns_roce_rinl_buf rq_inl_buf;
636621 struct list_head node; /* all qps are on a list */
637622 struct list_head rq_node; /* all recv qps are on a list */
638623 struct list_head sq_node; /* all send qps are on a list */
@@ -887,7 +872,7 @@ struct hns_roce_hw { · 248 unmodified lines
887872 u32 step_idx);
888873 int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
889874 int attr_mask, enum ib_qp_state cur_state,
890− enum ib_qp_state new_state);
875+ enum ib_qp_state new_state, struct ib_udata *udata);
891876 int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,
892877 struct hns_roce_qp *hr_qp);
893878 void (*dereg_mr)(struct hns_roce_dev *hr_dev);
drivers/infiniband/hw/hns/hns_roce_hw_v2.c+33−76
@@ -821,22 +821,10 @@ static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe, · 820 unmodified lines
821821 static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr,
822822 u32 wqe_idx, u32 max_sge)
823823 {
824− struct hns_roce_rinl_sge *sge_list;
825824 void *wqe = NULL;
826− u32 i;
827825
828826 wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
829827 fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge);
830−
831− /* rq support inline data */
832− if (hr_qp->rq_inl_buf.wqe_cnt) {
833− sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
834− hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = (u32)wr->num_sge;
835− for (i = 0; i < wr->num_sge; i++) {
836− sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr;
837− sge_list[i].len = wr->sg_list[i].length;
838− }
839− }
840828 }
841829
842830 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
@@ -2849,7 +2837,7 @@ static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev, · 2006 unmodified lines
28492837 attr->port_num = 1;
28502838 attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
28512839 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
2852− IB_QPS_INIT);
2840+ IB_QPS_INIT, NULL);
28532841 if (ret) {
28542842 ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n",
28552843 ret);
@@ -2871,7 +2859,7 @@ static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev, · 15 unmodified lines
28712859 rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num);
28722860
28732861 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
2874− IB_QPS_RTR);
2862+ IB_QPS_RTR, NULL);
28752863 hr_dev->loop_idc = loopback;
28762864 if (ret) {
28772865 ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",
@@ -2886,7 +2874,7 @@ static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev, · 8 unmodified lines
28862874 attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;
28872875 attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;
28882876 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR,
2889− IB_QPS_RTS);
2877+ IB_QPS_RTS, NULL);
28902878 if (ret)
28912879 ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",
28922880 ret);
@@ -3730,39 +3718,6 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, · 837 unmodified lines
37303718 return 0;
37313719 }
37323720
3733−static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
3734− struct hns_roce_qp *qp,
3735− struct ib_wc *wc)
3736−{
3737− struct hns_roce_rinl_sge *sge_list;
3738− u32 wr_num, wr_cnt, sge_num;
3739− u32 sge_cnt, data_len, size;
3740− void *wqe_buf;
3741−
3742− wr_num = hr_reg_read(cqe, CQE_WQE_IDX);
3743− wr_cnt = wr_num & (qp->rq.wqe_cnt - 1);
3744−
3745− sge_list = qp->rq_inl_buf.wqe_list[wr_cnt].sg_list;
3746− sge_num = qp->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
3747− wqe_buf = hns_roce_get_recv_wqe(qp, wr_cnt);
3748− data_len = wc->byte_len;
3749−
3750− for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
3751− size = min(sge_list[sge_cnt].len, data_len);
3752− memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
3753−
3754− data_len -= size;
3755− wqe_buf += size;
3756− }
3757−
3758− if (unlikely(data_len)) {
3759− wc->status = IB_WC_LOC_LEN_ERR;
3760− return -EAGAIN;
3761− }
3762−
3763− return 0;
3764−}
3765−
37663721 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,
37673722 int num_entries, struct ib_wc *wc)
37683723 {
@@ -3974,22 +3929,10 @@ static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) · 205 unmodified lines
39743929 wc->opcode = ib_opcode;
39753930 }
39763931
3977−static inline bool is_rq_inl_enabled(struct ib_wc *wc, u32 hr_opcode,
3978− struct hns_roce_v2_cqe *cqe)
3979−{
3980− return wc->qp->qp_type != IB_QPT_UD && wc->qp->qp_type != IB_QPT_GSI &&
3981− (hr_opcode == HNS_ROCE_V2_OPCODE_SEND ||
3982− hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
3983− hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
3984− hr_reg_read(cqe, CQE_RQ_INLINE);
3985−}
3986−
39873932 static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)
39883933 {
3989− struct hns_roce_qp *qp = to_hr_qp(wc->qp);
39903934 u32 hr_opcode;
39913935 int ib_opcode;
3992− int ret;
39933936
39943937 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
39953938
@@ -4014,12 +3957,6 @@ static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) · 18 unmodified lines
40143957 else
40153958 wc->opcode = ib_opcode;
40163959
4017− if (is_rq_inl_enabled(wc, hr_opcode, cqe)) {
4018− ret = hns_roce_handle_recv_inl_wqe(cqe, qp, wc);
4019− if (unlikely(ret))
4020− return ret;
4021− }
4022−
40233960 wc->sl = hr_reg_read(cqe, CQE_SL);
40243961 wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN);
40253962 wc->slid = 0;
@@ -4445,10 +4382,6 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, · 419 unmodified lines
44454382 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H,
44464383 upper_32_bits(hr_qp->rdb.dma));
44474384
4448− if (ibqp->qp_type != IB_QPT_UD && ibqp->qp_type != IB_QPT_GSI)
4449− hr_reg_write_bool(context, QPC_RQIE,
4450− hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE);
4451−
44524385 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));
44534386
44544387 if (ibqp->srq) {
@@ -4639,8 +4572,11 @@ static inline enum ib_mtu get_mtu(struct ib_qp *ibqp, · 184 unmodified lines
46394572 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
46404573 const struct ib_qp_attr *attr, int attr_mask,
46414574 struct hns_roce_v2_qp_context *context,
4642− struct hns_roce_v2_qp_context *qpc_mask)
4575+ struct hns_roce_v2_qp_context *qpc_mask,
4576+ struct ib_udata *udata)
46434577 {
4578+ struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,
4579+ struct hns_roce_ucontext, ibucontext);
46444580 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
46454581 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
46464582 struct ib_device *ibdev = &hr_dev->ib_dev;
@@ -4760,6 +4696,26 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, · 113 unmodified lines
47604696 hr_reg_write(context, QPC_LP_SGEN_INI, 3);
47614697 hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI);
47624698
4699+ if (udata && ibqp->qp_type == IB_QPT_RC &&
4700+ (uctx->config & HNS_ROCE_RQ_INLINE_FLAGS)) {
4701+ hr_reg_write_bool(context, QPC_RQIE,
4702+ hr_dev->caps.flags &
4703+ HNS_ROCE_CAP_FLAG_RQ_INLINE);
4704+ hr_reg_clear(qpc_mask, QPC_RQIE);
4705+ }
4706+
4707+ if (udata &&
4708+ (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_XRC_TGT) &&
4709+ (uctx->config & HNS_ROCE_CQE_INLINE_FLAGS)) {
4710+ hr_reg_write_bool(context, QPC_CQEIE,
4711+ hr_dev->caps.flags &
4712+ HNS_ROCE_CAP_FLAG_CQE_INLINE);
4713+ hr_reg_clear(qpc_mask, QPC_CQEIE);
4714+
4715+ hr_reg_write(context, QPC_CQEIS, 0);
4716+ hr_reg_clear(qpc_mask, QPC_CQEIS);
4717+ }
4718+
47634719 return 0;
47644720 }
47654721
@@ -5107,7 +5063,8 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, · 341 unmodified lines
51075063 enum ib_qp_state cur_state,
51085064 enum ib_qp_state new_state,
51095065 struct hns_roce_v2_qp_context *context,
5110− struct hns_roce_v2_qp_context *qpc_mask)
5066+ struct hns_roce_v2_qp_context *qpc_mask,
5067+ struct ib_udata *udata)
51115068 {
51125069 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
51135070 int ret = 0;
@@ -5124,7 +5081,7 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, · 10 unmodified lines
51245081 modify_qp_init_to_init(ibqp, attr, context, qpc_mask);
51255082 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
51265083 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
5127− qpc_mask);
5084+ qpc_mask, udata);
51285085 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
51295086 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
51305087 qpc_mask);
@@ -5329,7 +5286,7 @@ static void v2_set_flushed_fields(struct ib_qp *ibqp, · 198 unmodified lines
53295286 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
53305287 const struct ib_qp_attr *attr,
53315288 int attr_mask, enum ib_qp_state cur_state,
5332− enum ib_qp_state new_state)
5289+ enum ib_qp_state new_state, struct ib_udata *udata)
53335290 {
53345291 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
53355292 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
@@ -5352,7 +5309,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, · 16 unmodified lines
53525309 memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
53535310
53545311 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
5355− new_state, context, qpc_mask);
5312+ new_state, context, qpc_mask, udata);
53565313 if (ret)
53575314 goto out;
53585315
@@ -5555,7 +5512,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, · 196 unmodified lines
55555512 if (modify_qp_is_ok(hr_qp)) {
55565513 /* Modify qp to reset before destroying qp */
55575514 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
5558− hr_qp->state, IB_QPS_RESET);
5515+ hr_qp->state, IB_QPS_RESET, udata);
55595516 if (ret)
55605517 ibdev_err(ibdev,
55615518 "failed to modify QP to RST, ret = %d.\n",
drivers/infiniband/hw/hns/hns_roce_hw_v2.h+2−1
@@ -531,7 +531,8 @@ struct hns_roce_v2_qp_context { · 530 unmodified lines
531531 #define QPC_RQ_RTY_TX_ERR QPC_FIELD_LOC(607, 607)
532532 #define QPC_RX_CQN QPC_FIELD_LOC(631, 608)
533533 #define QPC_XRC_QP_TYPE QPC_FIELD_LOC(632, 632)
534−#define QPC_RSV3 QPC_FIELD_LOC(634, 633)
534+#define QPC_CQEIE QPC_FIELD_LOC(633, 633)
535+#define QPC_CQEIS QPC_FIELD_LOC(634, 634)
535536 #define QPC_MIN_RNR_TIME QPC_FIELD_LOC(639, 635)
536537 #define QPC_RQ_PRODUCER_IDX QPC_FIELD_LOC(655, 640)
537538 #define QPC_RQ_CONSUMER_IDX QPC_FIELD_LOC(671, 656)
drivers/infiniband/hw/hns/hns_roce_main.c+15−2
@@ -379,6 +379,18 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, · 378 unmodified lines
379379 resp.max_inline_data = hr_dev->caps.max_sq_inline;
380380 }
381381
382+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
383+ context->config |= ucmd.config & HNS_ROCE_RQ_INLINE_FLAGS;
384+ if (context->config & HNS_ROCE_RQ_INLINE_FLAGS)
385+ resp.config |= HNS_ROCE_RSP_RQ_INLINE_FLAGS;
386+ }
387+
388+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQE_INLINE) {
389+ context->config |= ucmd.config & HNS_ROCE_CQE_INLINE_FLAGS;
390+ if (context->config & HNS_ROCE_CQE_INLINE_FLAGS)
391+ resp.config |= HNS_ROCE_RSP_CQE_INLINE_FLAGS;
392+ }
393+
382394 ret = hns_roce_uar_alloc(hr_dev, &context->uar);
383395 if (ret)
384396 goto error_fail_uar_alloc;
@@ -443,14 +455,15 @@ static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) · 58 unmodified lines
443455 prot = pgprot_device(vma->vm_page_prot);
444456 break;
445457 default:
446− return -EINVAL;
458+ ret = -EINVAL;
459+ goto out;
447460 }
448461
449462 ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE,
450463 prot, rdma_entry);
451464
465+out:
452466 rdma_user_mmap_entry_put(rdma_entry);
453−
454467 return ret;
455468 }
456469
drivers/infiniband/hw/hns/hns_roce_qp.c+1−65
@@ -433,7 +433,6 @@ static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, · 432 unmodified lines
433433 if (!has_rq) {
434434 hr_qp->rq.wqe_cnt = 0;
435435 hr_qp->rq.max_gs = 0;
436− hr_qp->rq_inl_buf.wqe_cnt = 0;
437436 cap->max_recv_wr = 0;
438437 cap->max_recv_sge = 0;
439438
@@ -463,12 +462,6 @@ static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, · 23 unmodified lines
463462 hr_qp->rq.max_gs);
464463
465464 hr_qp->rq.wqe_cnt = cnt;
466− if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE &&
467− hr_qp->ibqp.qp_type != IB_QPT_UD &&
468− hr_qp->ibqp.qp_type != IB_QPT_GSI)
469− hr_qp->rq_inl_buf.wqe_cnt = cnt;
470− else
471− hr_qp->rq_inl_buf.wqe_cnt = 0;
472465
473466 cap->max_recv_wr = cnt;
474467 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
@@ -732,49 +725,6 @@ static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) · 257 unmodified lines
732725 return 1;
733726 }
734727
735−static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
736− struct ib_qp_init_attr *init_attr)
737−{
738− u32 max_recv_sge = init_attr->cap.max_recv_sge;
739− u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt;
740− struct hns_roce_rinl_wqe *wqe_list;
741− int i;
742−
743− /* allocate recv inline buf */
744− wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe),
745− GFP_KERNEL);
746− if (!wqe_list)
747− goto err;
748−
749− /* Allocate a continuous buffer for all inline sge we need */
750− wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge *
751− sizeof(struct hns_roce_rinl_sge)),
752− GFP_KERNEL);
753− if (!wqe_list[0].sg_list)
754− goto err_wqe_list;
755−
756− /* Assign buffers of sg_list to each inline wqe */
757− for (i = 1; i < wqe_cnt; i++)
758− wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge];
759−
760− hr_qp->rq_inl_buf.wqe_list = wqe_list;
761−
762− return 0;
763−
764−err_wqe_list:
765− kfree(wqe_list);
766−
767−err:
768− return -ENOMEM;
769−}
770−
771−static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
772−{
773− if (hr_qp->rq_inl_buf.wqe_list)
774− kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
775− kfree(hr_qp->rq_inl_buf.wqe_list);
776−}
777−
778728 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
779729 struct ib_qp_init_attr *init_attr,
780730 struct ib_udata *udata, unsigned long addr)
@@ -783,18 +733,6 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 2 unmodified lines
783733 struct hns_roce_buf_attr buf_attr = {};
784734 int ret;
785735
786− if (!udata && hr_qp->rq_inl_buf.wqe_cnt) {
787− ret = alloc_rq_inline_buf(hr_qp, init_attr);
788− if (ret) {
789− ibdev_err(ibdev,
790− "failed to alloc inline buf, ret = %d.\n",
791− ret);
792− return ret;
793− }
794− } else {
795− hr_qp->rq_inl_buf.wqe_list = NULL;
796− }
797−
798736 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
799737 if (ret) {
800738 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
@@ -814,7 +752,6 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 13 unmodified lines
814752 return 0;
815753
816754 err_inline:
817− free_rq_inline_buf(hr_qp);
818755
819756 return ret;
820757 }
@@ -822,7 +759,6 @@ err_inline: · 1 unmodified line
822759 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
823760 {
824761 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
825− free_rq_inline_buf(hr_qp);
826762 }
827763
828764 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,
@@ -1410,7 +1346,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 581 unmodified lines
14101346 goto out;
14111347
14121348 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1413− new_state);
1349+ new_state, udata);
14141350
14151351 out:
14161352 mutex_unlock(&hr_qp->mutex);
drivers/infiniband/hw/irdma/hw.c+2−0
@@ -483,6 +483,8 @@ static int irdma_save_msix_info(struct irdma_pci_f *rf) · 482 unmodified lines
483483 iw_qvlist->num_vectors = rf->msix_count;
484484 if (rf->msix_count <= num_online_cpus())
485485 rf->msix_shared = true;
486+ else if (rf->msix_count > num_online_cpus() + 1)
487+ rf->msix_count = num_online_cpus() + 1;
486488
487489 pmsix = rf->msix_entries;
488490 for (i = 0, ceq_idx = 0; i < rf->msix_count; i++, iw_qvinfo++) {
drivers/infiniband/hw/irdma/verbs.c+210−103
@@ -2745,6 +2745,162 @@ static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, · 2744 unmodified lines
27452745 return ret;
27462746 }
27472747
2748+static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
2749+{
2750+ struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
2751+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2752+ bool use_pbles;
2753+ u32 stag;
2754+ int err;
2755+
2756+ use_pbles = iwmr->page_cnt != 1;
2757+
2758+ err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
2759+ if (err)
2760+ return err;
2761+
2762+ if (use_pbles) {
2763+ err = irdma_check_mr_contiguous(&iwpbl->pble_alloc,
2764+ iwmr->page_size);
2765+ if (err) {
2766+ irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
2767+ iwpbl->pbl_allocated = false;
2768+ }
2769+ }
2770+
2771+ stag = irdma_create_stag(iwdev);
2772+ if (!stag) {
2773+ err = -ENOMEM;
2774+ goto free_pble;
2775+ }
2776+
2777+ iwmr->stag = stag;
2778+ iwmr->ibmr.rkey = stag;
2779+ iwmr->ibmr.lkey = stag;
2780+ err = irdma_hwreg_mr(iwdev, iwmr, access);
2781+ if (err)
2782+ goto err_hwreg;
2783+
2784+ return 0;
2785+
2786+err_hwreg:
2787+ irdma_free_stag(iwdev, stag);
2788+
2789+free_pble:
2790+ if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
2791+ irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
2792+
2793+ return err;
2794+}
2795+
2796+static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region,
2797+ struct ib_pd *pd, u64 virt,
2798+ enum irdma_memreg_type reg_type)
2799+{
2800+ struct irdma_device *iwdev = to_iwdev(pd->device);
2801+ struct irdma_pbl *iwpbl = NULL;
2802+ struct irdma_mr *iwmr = NULL;
2803+ unsigned long pgsz_bitmap;
2804+
2805+ iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
2806+ if (!iwmr)
2807+ return ERR_PTR(-ENOMEM);
2808+
2809+ iwpbl = &iwmr->iwpbl;
2810+ iwpbl->iwmr = iwmr;
2811+ iwmr->region = region;
2812+ iwmr->ibmr.pd = pd;
2813+ iwmr->ibmr.device = pd->device;
2814+ iwmr->ibmr.iova = virt;
2815+ iwmr->type = reg_type;
2816+
2817+ pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ?
2818+ iwdev->rf->sc_dev.hw_attrs.page_size_cap : PAGE_SIZE;
2819+
2820+ iwmr->page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt);
2821+ if (unlikely(!iwmr->page_size)) {
2822+ kfree(iwmr);
2823+ return ERR_PTR(-EOPNOTSUPP);
2824+ }
2825+
2826+ iwmr->len = region->length;
2827+ iwpbl->user_base = virt;
2828+ iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size);
2829+
2830+ return iwmr;
2831+}
2832+
2833+static void irdma_free_iwmr(struct irdma_mr *iwmr)
2834+{
2835+ kfree(iwmr);
2836+}
2837+
2838+static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req,
2839+ struct ib_udata *udata,
2840+ struct irdma_mr *iwmr)
2841+{
2842+ struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
2843+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2844+ struct irdma_ucontext *ucontext = NULL;
2845+ unsigned long flags;
2846+ bool use_pbles;
2847+ u32 total;
2848+ int err;
2849+
2850+ total = req.sq_pages + req.rq_pages + 1;
2851+ if (total > iwmr->page_cnt)
2852+ return -EINVAL;
2853+
2854+ total = req.sq_pages + req.rq_pages;
2855+ use_pbles = total > 2;
2856+ err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
2857+ if (err)
2858+ return err;
2859+
2860+ ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2861+ ibucontext);
2862+ spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
2863+ list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
2864+ iwpbl->on_list = true;
2865+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
2866+
2867+ return 0;
2868+}
2869+
2870+static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
2871+ struct ib_udata *udata,
2872+ struct irdma_mr *iwmr)
2873+{
2874+ struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
2875+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2876+ struct irdma_ucontext *ucontext = NULL;
2877+ u8 shadow_pgcnt = 1;
2878+ unsigned long flags;
2879+ bool use_pbles;
2880+ u32 total;
2881+ int err;
2882+
2883+ if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)
2884+ shadow_pgcnt = 0;
2885+ total = req.cq_pages + shadow_pgcnt;
2886+ if (total > iwmr->page_cnt)
2887+ return -EINVAL;
2888+
2889+ use_pbles = req.cq_pages > 1;
2890+ err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
2891+ if (err)
2892+ return err;
2893+
2894+ ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2895+ ibucontext);
2896+ spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
2897+ list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
2898+ iwpbl->on_list = true;
2899+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
2900+
2901+ return 0;
2902+}
2903+
27482904 /**
27492905 * irdma_reg_user_mr - Register a user memory region
27502906 * @pd: ptr of pd
@@ -2760,18 +2916,10 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · 9 unmodified lines
27602916 {
27612917 #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)
27622918 struct irdma_device *iwdev = to_iwdev(pd->device);
2763− struct irdma_ucontext *ucontext;
2764− struct irdma_pble_alloc *palloc;
2765− struct irdma_pbl *iwpbl;
2766− struct irdma_mr *iwmr;
2767− struct ib_umem *region;
2768− struct irdma_mem_reg_req req;
2769− u32 total, stag = 0;
2770− u8 shadow_pgcnt = 1;
2771− bool use_pbles = false;
2772− unsigned long flags;
2773− int err = -EINVAL;
2774− int ret;
2919+ struct irdma_mem_reg_req req = {};
2920+ struct ib_umem *region = NULL;
2921+ struct irdma_mr *iwmr = NULL;
2922+ int err;
27752923
27762924 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
27772925 return ERR_PTR(-EINVAL);
@@ -2792,122 +2940,80 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · 14 unmodified lines
27922940 return ERR_PTR(-EFAULT);
27932941 }
27942942
2795− iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
2796− if (!iwmr) {
2943+ iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type);
2944+ if (IS_ERR(iwmr)) {
27972945 ib_umem_release(region);
2798− return ERR_PTR(-ENOMEM);
2799− }
2800−
2801− iwpbl = &iwmr->iwpbl;
2802− iwpbl->iwmr = iwmr;
2803− iwmr->region = region;
2804− iwmr->ibmr.pd = pd;
2805− iwmr->ibmr.device = pd->device;
2806− iwmr->ibmr.iova = virt;
2807− iwmr->page_size = PAGE_SIZE;
2808−
2809− if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) {
2810− iwmr->page_size = ib_umem_find_best_pgsz(region,
2811− iwdev->rf->sc_dev.hw_attrs.page_size_cap,
2812− virt);
2813− if (unlikely(!iwmr->page_size)) {
2814− kfree(iwmr);
2815− ib_umem_release(region);
2816− return ERR_PTR(-EOPNOTSUPP);
2817− }
2946+ return (struct ib_mr *)iwmr;
28182947 }
2819− iwmr->len = region->length;
2820− iwpbl->user_base = virt;
2821− palloc = &iwpbl->pble_alloc;
2822− iwmr->type = req.reg_type;
2823− iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size);
28242948
28252949 switch (req.reg_type) {
28262950 case IRDMA_MEMREG_TYPE_QP:
2827− total = req.sq_pages + req.rq_pages + shadow_pgcnt;
2828− if (total > iwmr->page_cnt) {
2829− err = -EINVAL;
2830− goto error;
2831− }
2832− total = req.sq_pages + req.rq_pages;
2833− use_pbles = (total > 2);
2834− err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
2951+ err = irdma_reg_user_mr_type_qp(req, udata, iwmr);
28352952 if (err)
28362953 goto error;
28372954
2838− ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2839− ibucontext);
2840− spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
2841− list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
2842− iwpbl->on_list = true;
2843− spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
28442955 break;
28452956 case IRDMA_MEMREG_TYPE_CQ:
2846− if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)
2847− shadow_pgcnt = 0;
2848− total = req.cq_pages + shadow_pgcnt;
2849− if (total > iwmr->page_cnt) {
2850− err = -EINVAL;
2851− goto error;
2852− }
2853−
2854− use_pbles = (req.cq_pages > 1);
2855− err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
2957+ err = irdma_reg_user_mr_type_cq(req, udata, iwmr);
28562958 if (err)
28572959 goto error;
2858−
2859− ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2860− ibucontext);
2861− spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
2862− list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
2863− iwpbl->on_list = true;
2864− spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
28652960 break;
28662961 case IRDMA_MEMREG_TYPE_MEM:
2867− use_pbles = (iwmr->page_cnt != 1);
2868−
2869− err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
2962+ err = irdma_reg_user_mr_type_mem(iwmr, access);
28702963 if (err)
28712964 goto error;
28722965
2873− if (use_pbles) {
2874− ret = irdma_check_mr_contiguous(palloc,
2875− iwmr->page_size);
2876− if (ret) {
2877− irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
2878− iwpbl->pbl_allocated = false;
2879− }
2880− }
2881−
2882− stag = irdma_create_stag(iwdev);
2883− if (!stag) {
2884− err = -ENOMEM;
2885− goto error;
2886− }
2887−
2888− iwmr->stag = stag;
2889− iwmr->ibmr.rkey = stag;
2890− iwmr->ibmr.lkey = stag;
2891− err = irdma_hwreg_mr(iwdev, iwmr, access);
2892− if (err) {
2893− irdma_free_stag(iwdev, stag);
2894− goto error;
2895− }
2896−
28972966 break;
28982967 default:
2968+ err = -EINVAL;
28992969 goto error;
29002970 }
29012971
2902− iwmr->type = req.reg_type;
2903−
29042972 return &iwmr->ibmr;
2905−
29062973 error:
2907− if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
2908− irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
29092974 ib_umem_release(region);
2910− kfree(iwmr);
2975+ irdma_free_iwmr(iwmr);
2976+
2977+ return ERR_PTR(err);
2978+}
2979+
2980+static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
2981+ u64 len, u64 virt,
2982+ int fd, int access,
2983+ struct ib_udata *udata)
2984+{
2985+ struct irdma_device *iwdev = to_iwdev(pd->device);
2986+ struct ib_umem_dmabuf *umem_dmabuf;
2987+ struct irdma_mr *iwmr;
2988+ int err;
2989+
2990+ if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
2991+ return ERR_PTR(-EINVAL);
2992+
2993+ umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd, access);
2994+ if (IS_ERR(umem_dmabuf)) {
2995+ err = PTR_ERR(umem_dmabuf);
2996+ ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n", err);
2997+ return ERR_PTR(err);
2998+ }
2999+
3000+ iwmr = irdma_alloc_iwmr(&umem_dmabuf->umem, pd, virt, IRDMA_MEMREG_TYPE_MEM);
3001+ if (IS_ERR(iwmr)) {
3002+ err = PTR_ERR(iwmr);
3003+ goto err_release;
3004+ }
3005+
3006+ err = irdma_reg_user_mr_type_mem(iwmr, access);
3007+ if (err)
3008+ goto err_iwmr;
3009+
3010+ return &iwmr->ibmr;
3011+
3012+err_iwmr:
3013+ irdma_free_iwmr(iwmr);
3014+
3015+err_release:
3016+ ib_umem_release(&umem_dmabuf->umem);
29113017
29123018 return ERR_PTR(err);
29133019 }
@@ -4418,6 +4524,7 @@ static const struct ib_device_ops irdma_dev_ops = { · 1504 unmodified lines
44184524 .query_port = irdma_query_port,
44194525 .query_qp = irdma_query_qp,
44204526 .reg_user_mr = irdma_reg_user_mr,
4527+ .reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf,
44214528 .req_notify_cq = irdma_req_notify_cq,
44224529 .resize_cq = irdma_resize_cq,
44234530 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),
drivers/infiniband/hw/mana/main.c+12−10
@@ -249,7 +249,8 @@ static int · 248 unmodified lines
249249 mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
250250 struct gdma_context *gc,
251251 struct gdma_create_dma_region_req *create_req,
252− size_t num_pages, mana_handle_t *gdma_region)
252+ size_t num_pages, mana_handle_t *gdma_region,
253+ u32 expected_status)
253254 {
254255 struct gdma_create_dma_region_resp create_resp = {};
255256 unsigned int create_req_msg_size;
@@ -261,7 +262,7 @@ mana_ib_gd_first_dma_region(struct mana_ib_dev *dev, · 5 unmodified lines
261262
262263 err = mana_gd_send_request(gc, create_req_msg_size, create_req,
263264 sizeof(create_resp), &create_resp);
264− if (err || create_resp.hdr.status) {
265+ if (err || create_resp.hdr.status != expected_status) {
265266 ibdev_dbg(&dev->ib_dev,
266267 "Failed to create DMA region: %d, 0x%x\n",
267268 err, create_resp.hdr.status);
@@ -372,14 +373,21 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, · 104 unmodified lines
372373
373374 page_addr_list = create_req->page_addr_list;
374375 rdma_umem_for_each_dma_block(umem, &biter, page_sz) {
376+ u32 expected_status = 0;
377+
375378 page_addr_list[tail++] = rdma_block_iter_dma_address(&biter);
376379 if (tail < num_pages_to_handle)
377380 continue;
378381
382+ if (num_pages_processed + num_pages_to_handle <
383+ num_pages_total)
384+ expected_status = GDMA_STATUS_MORE_ENTRIES;
385+
379386 if (!num_pages_processed) {
380387 /* First create message */
381388 err = mana_ib_gd_first_dma_region(dev, gc, create_req,
382− tail, gdma_region);
389+ tail, gdma_region,
390+ expected_status);
383391 if (err)
384392 goto out;
385393
@@ -392,14 +400,8 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, · 6 unmodified lines
392400 page_addr_list = add_req->page_addr_list;
393401 } else {
394402 /* Subsequent create messages */
395− u32 expected_s = 0;
396−
397− if (num_pages_processed + num_pages_to_handle <
398− num_pages_total)
399− expected_s = GDMA_STATUS_MORE_ENTRIES;
400−
401403 err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail,
402− expected_s);
404+ expected_status);
403405 if (err)
404406 break;
405407 }
drivers/infiniband/hw/mlx4/main.c+8−0
@@ -3303,6 +3303,10 @@ static int __init mlx4_ib_init(void) · 3302 unmodified lines
33033303 if (!wq)
33043304 return -ENOMEM;
33053305
3306+ err = mlx4_ib_qp_event_init();
3307+ if (err)
3308+ goto clean_qp_event;
3309+
33063310 err = mlx4_ib_cm_init();
33073311 if (err)
33083312 goto clean_wq;
@@ -3324,6 +3328,9 @@ clean_cm: · 15 unmodified lines
33243328 mlx4_ib_cm_destroy();
33253329
33263330 clean_wq:
3331+ mlx4_ib_qp_event_cleanup();
3332+
3333+clean_qp_event:
33273334 destroy_workqueue(wq);
33283335 return err;
33293336 }
@@ -3333,6 +3340,7 @@ static void __exit mlx4_ib_cleanup(void) · 3 unmodified lines
33333340 mlx4_unregister_interface(&mlx4_ib_interface);
33343341 mlx4_ib_mcg_destroy();
33353342 mlx4_ib_cm_destroy();
3343+ mlx4_ib_qp_event_cleanup();
33363344 destroy_workqueue(wq);
33373345 }
33383346
drivers/infiniband/hw/mlx4/mlx4_ib.h+3−0
@@ -940,4 +940,7 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, · 939 unmodified lines
940940 int mlx4_ib_cm_init(void);
941941 void mlx4_ib_cm_destroy(void);
942942
943+int mlx4_ib_qp_event_init(void);
944+void mlx4_ib_qp_event_cleanup(void);
945+
943946 #endif /* MLX4_IB_H */
drivers/infiniband/hw/mlx4/qp.c+85−36
@@ -102,6 +102,14 @@ enum mlx4_ib_source_type { · 101 unmodified lines
102102 MLX4_IB_RWQ_SRC = 1,
103103 };
104104
105+struct mlx4_ib_qp_event_work {
106+ struct work_struct work;
107+ struct mlx4_qp *qp;
108+ enum mlx4_event type;
109+};
110+
111+static struct workqueue_struct *mlx4_ib_qp_event_wq;
112+
105113 static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
106114 {
107115 if (!mlx4_is_master(dev->dev))
@@ -200,50 +208,77 @@ static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n) · 92 unmodified lines
200208 }
201209 }
202210
211+static void mlx4_ib_handle_qp_event(struct work_struct *_work)
212+{
213+ struct mlx4_ib_qp_event_work *qpe_work =
214+ container_of(_work, struct mlx4_ib_qp_event_work, work);
215+ struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp;
216+ struct ib_event event = {};
217+
218+ event.device = ibqp->device;
219+ event.element.qp = ibqp;
220+
221+ switch (qpe_work->type) {
222+ case MLX4_EVENT_TYPE_PATH_MIG:
223+ event.event = IB_EVENT_PATH_MIG;
224+ break;
225+ case MLX4_EVENT_TYPE_COMM_EST:
226+ event.event = IB_EVENT_COMM_EST;
227+ break;
228+ case MLX4_EVENT_TYPE_SQ_DRAINED:
229+ event.event = IB_EVENT_SQ_DRAINED;
230+ break;
231+ case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
232+ event.event = IB_EVENT_QP_LAST_WQE_REACHED;
233+ break;
234+ case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
235+ event.event = IB_EVENT_QP_FATAL;
236+ break;
237+ case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
238+ event.event = IB_EVENT_PATH_MIG_ERR;
239+ break;
240+ case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
241+ event.event = IB_EVENT_QP_REQ_ERR;
242+ break;
243+ case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
244+ event.event = IB_EVENT_QP_ACCESS_ERR;
245+ break;
246+ default:
247+ pr_warn("Unexpected event type %d on QP %06x\n",
248+ qpe_work->type, qpe_work->qp->qpn);
249+ goto out;
250+ }
251+
252+ ibqp->event_handler(&event, ibqp->qp_context);
253+
254+out:
255+ mlx4_put_qp(qpe_work->qp);
256+ kfree(qpe_work);
257+}
258+
203259 static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
204260 {
205− struct ib_event event;
206261 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
262+ struct mlx4_ib_qp_event_work *qpe_work;
207263
208264 if (type == MLX4_EVENT_TYPE_PATH_MIG)
209265 to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
210266
211− if (ibqp->event_handler) {
212− event.device = ibqp->device;
213− event.element.qp = ibqp;
214− switch (type) {
215− case MLX4_EVENT_TYPE_PATH_MIG:
216− event.event = IB_EVENT_PATH_MIG;
217− break;
218− case MLX4_EVENT_TYPE_COMM_EST:
219− event.event = IB_EVENT_COMM_EST;
220− break;
221− case MLX4_EVENT_TYPE_SQ_DRAINED:
222− event.event = IB_EVENT_SQ_DRAINED;
223− break;
224− case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
225− event.event = IB_EVENT_QP_LAST_WQE_REACHED;
226− break;
227− case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
228− event.event = IB_EVENT_QP_FATAL;
229− break;
230− case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
231− event.event = IB_EVENT_PATH_MIG_ERR;
232− break;
233− case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
234− event.event = IB_EVENT_QP_REQ_ERR;
235− break;
236− case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
237− event.event = IB_EVENT_QP_ACCESS_ERR;
238− break;
239− default:
240− pr_warn("Unexpected event type %d "
241− "on QP %06x\n", type, qp->qpn);
242− return;
243− }
267+ if (!ibqp->event_handler)
268+ goto out_no_handler;
244269
245− ibqp->event_handler(&event, ibqp->qp_context);
246− }
270+ qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);
271+ if (!qpe_work)
272+ goto out_no_handler;
273+
274+ qpe_work->qp = qp;
275+ qpe_work->type = type;
276+ INIT_WORK(&qpe_work->work, mlx4_ib_handle_qp_event);
277+ queue_work(mlx4_ib_qp_event_wq, &qpe_work->work);
278+ return;
279+
280+out_no_handler:
281+ mlx4_put_qp(qp);
247282 }
248283
249284 static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)
@@ -4468,3 +4503,17 @@ void mlx4_ib_drain_rq(struct ib_qp *qp) · 4218 unmodified lines
44684503
44694504 handle_drain_completion(cq, &rdrain, dev);
44704505 }
4506+
4507+int mlx4_ib_qp_event_init(void)
4508+{
4509+ mlx4_ib_qp_event_wq = alloc_ordered_workqueue("mlx4_ib_qp_event_wq", 0);
4510+ if (!mlx4_ib_qp_event_wq)
4511+ return -ENOMEM;
4512+
4513+ return 0;
4514+}
4515+
4516+void mlx4_ib_qp_event_cleanup(void)
4517+{
4518+ destroy_workqueue(mlx4_ib_qp_event_wq);
4519+}
drivers/infiniband/hw/mlx5/cmd.c+26−19
@@ -5,34 +5,41 @@ · 4 unmodified lines
55
66 #include "cmd.h"
77
8−int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey)
8+int mlx5r_cmd_query_special_mkeys(struct mlx5_ib_dev *dev)
99 {
1010 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
1111 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
12+ bool is_terminate, is_dump, is_null;
1213 int err;
1314
14− MLX5_SET(query_special_contexts_in, in, opcode,
15− MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
16− err = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);
17− if (!err)
18− *mkey = MLX5_GET(query_special_contexts_out, out,
19− dump_fill_mkey);
20− return err;
21−}
15+ is_terminate = MLX5_CAP_GEN(dev->mdev, terminate_scatter_list_mkey);
16+ is_dump = MLX5_CAP_GEN(dev->mdev, dump_fill_mkey);
17+ is_null = MLX5_CAP_GEN(dev->mdev, null_mkey);
2218
23−int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey)
24−{
25− u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
26− u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
27− int err;
19+ dev->mkeys.terminate_scatter_list_mkey = MLX5_TERMINATE_SCATTER_LIST_LKEY;
20+ if (!is_terminate && !is_dump && !is_null)
21+ return 0;
2822
2923 MLX5_SET(query_special_contexts_in, in, opcode,
3024 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
31− err = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);
32− if (!err)
33− *null_mkey = MLX5_GET(query_special_contexts_out, out,
34− null_mkey);
35− return err;
25+ err = mlx5_cmd_exec_inout(dev->mdev, query_special_contexts, in, out);
26+ if (err)
27+ return err;
28+
29+ if (is_dump)
30+ dev->mkeys.dump_fill_mkey = MLX5_GET(query_special_contexts_out,
31+ out, dump_fill_mkey);
32+
33+ if (is_null)
34+ dev->mkeys.null_mkey = cpu_to_be32(
35+ MLX5_GET(query_special_contexts_out, out, null_mkey));
36+
37+ if (is_terminate)
38+ dev->mkeys.terminate_scatter_list_mkey =
39+ cpu_to_be32(MLX5_GET(query_special_contexts_out, out,
40+ terminate_scatter_list_mkey));
41+
42+ return 0;
3643 }
3744
3845 int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
drivers/infiniband/hw/mlx5/cmd.h+1−2
@@ -37,8 +37,7 @@ · 36 unmodified lines
3737 #include <linux/kernel.h>
3838 #include <linux/mlx5/driver.h>
3939
40−int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey);
41−int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey);
40+int mlx5r_cmd_query_special_mkeys(struct mlx5_ib_dev *dev);
4241 int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
4342 void *out);
4443 int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);
drivers/infiniband/hw/mlx5/cong.c+25−3
@@ -38,6 +38,7 @@ · 37 unmodified lines
3838 enum mlx5_ib_cong_node_type {
3939 MLX5_IB_RROCE_ECN_RP = 1,
4040 MLX5_IB_RROCE_ECN_NP = 2,
41+ MLX5_IB_RROCE_GENERAL = 3,
4142 };
4243
4344 static const char * const mlx5_ib_dbg_cc_name[] = {
@@ -61,6 +62,8 @@ static const char * const mlx5_ib_dbg_cc_name[] = { · 17 unmodified lines
6162 "np_cnp_dscp",
6263 "np_cnp_prio_mode",
6364 "np_cnp_prio",
65+ "rtt_resp_dscp_valid",
66+ "rtt_resp_dscp",
6467 };
6568
6669 #define MLX5_IB_RP_CLAMP_TGT_RATE_ATTR BIT(1)
@@ -84,14 +87,18 @@ static const char * const mlx5_ib_dbg_cc_name[] = { · 17 unmodified lines
8487 #define MLX5_IB_NP_CNP_DSCP_ATTR BIT(3)
8588 #define MLX5_IB_NP_CNP_PRIO_MODE_ATTR BIT(4)
8689
90+#define MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR BIT(0)
91+
8792 static enum mlx5_ib_cong_node_type
8893 mlx5_ib_param_to_node(enum mlx5_ib_dbg_cc_types param_offset)
8994 {
90− if (param_offset >= MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE &&
91− param_offset <= MLX5_IB_DBG_CC_RP_GD)
95+ if (param_offset <= MLX5_IB_DBG_CC_RP_GD)
9296 return MLX5_IB_RROCE_ECN_RP;
93− else
97+
98+ if (param_offset <= MLX5_IB_DBG_CC_NP_CNP_PRIO)
9499 return MLX5_IB_RROCE_ECN_NP;
100+
101+ return MLX5_IB_RROCE_GENERAL;
95102 }
96103
97104 static u32 mlx5_get_cc_param_val(void *field, int offset)
@@ -157,6 +164,12 @@ static u32 mlx5_get_cc_param_val(void *field, int offset) · 59 unmodified lines
157164 case MLX5_IB_DBG_CC_NP_CNP_PRIO:
158165 return MLX5_GET(cong_control_r_roce_ecn_np, field,
159166 cnp_802p_prio);
167+ case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID:
168+ return MLX5_GET(cong_control_r_roce_general, field,
169+ rtt_resp_dscp_valid);
170+ case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP:
171+ return MLX5_GET(cong_control_r_roce_general, field,
172+ rtt_resp_dscp);
160173 default:
161174 return 0;
162175 }
@@ -264,6 +277,15 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset, · 101 unmodified lines
264277 MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_prio_mode, 0);
265278 MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_802p_prio, var);
266279 break;
280+ case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID:
281+ *attr_mask |= MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR;
282+ MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp_valid, var);
283+ break;
284+ case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP:
285+ *attr_mask |= MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR;
286+ MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp_valid, 1);
287+ MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp, var);
288+ break;
267289 }
268290 }
269291
drivers/infiniband/hw/mlx5/main.c+13−11
@@ -1756,13 +1756,9 @@ static int set_ucontext_resp(struct ib_ucontext *uctx, · 1755 unmodified lines
17561756 struct mlx5_ib_dev *dev = to_mdev(ibdev);
17571757 struct mlx5_ib_ucontext *context = to_mucontext(uctx);
17581758 struct mlx5_bfreg_info *bfregi = &context->bfregi;
1759− int err;
17601759
17611760 if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1762− err = mlx5_cmd_dump_fill_mkey(dev->mdev,
1763− &resp->dump_fill_mkey);
1764− if (err)
1765− return err;
1761+ resp->dump_fill_mkey = dev->mkeys.dump_fill_mkey;
17661762 resp->comp_mask |=
17671763 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
17681764 }
@@ -3666,6 +3662,10 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) · 1897 unmodified lines
36663662 dev->port[i].roce.last_port_state = IB_PORT_DOWN;
36673663 }
36683664
3665+ err = mlx5r_cmd_query_special_mkeys(dev);
3666+ if (err)
3667+ return err;
3668+
36693669 err = mlx5_ib_init_multiport_master(dev);
36703670 if (err)
36713671 return err;
@@ -4030,12 +4030,7 @@ static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev) · 358 unmodified lines
40304030
40314031 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
40324032 {
4033− int err;
4034−
4035− err = mlx5_mkey_cache_cleanup(dev);
4036− if (err)
4037− mlx5_ib_warn(dev, "mr cache cleanup failed\n");
4038−
4033+ mlx5_mkey_cache_cleanup(dev);
40394034 mlx5r_umr_resource_cleanup(dev);
40404035 }
40414036
@@ -4433,6 +4428,10 @@ static int __init mlx5_ib_init(void) · 391 unmodified lines
44334428 return -ENOMEM;
44344429 }
44354430
4431+ ret = mlx5_ib_qp_event_init();
4432+ if (ret)
4433+ goto qp_event_err;
4434+
44364435 mlx5_ib_odp_init();
44374436 ret = mlx5r_rep_init();
44384437 if (ret)
@@ -4450,6 +4449,8 @@ drv_err: · 11 unmodified lines
44504449 mp_err:
44514450 mlx5r_rep_cleanup();
44524451 rep_err:
4452+ mlx5_ib_qp_event_cleanup();
4453+qp_event_err:
44534454 destroy_workqueue(mlx5_ib_event_wq);
44544455 free_page((unsigned long)xlt_emergency_page);
44554456 return ret;
@@ -4461,6 +4462,7 @@ static void __exit mlx5_ib_cleanup(void) · 5 unmodified lines
44614462 auxiliary_driver_unregister(&mlx5r_mp_driver);
44624463 mlx5r_rep_cleanup();
44634464
4465+ mlx5_ib_qp_event_cleanup();
44644466 destroy_workqueue(mlx5_ib_event_wq);
44654467 free_page((unsigned long)xlt_emergency_page);
44664468 }
drivers/infiniband/hw/mlx5/mlx5_ib.h+39−12
@@ -617,12 +617,21 @@ enum mlx5_mkey_type { · 616 unmodified lines
617617 MLX5_MKEY_INDIRECT_DEVX,
618618 };
619619
620+struct mlx5r_cache_rb_key {
621+ u8 ats:1;
622+ unsigned int access_mode;
623+ unsigned int access_flags;
624+ unsigned int ndescs;
625+};
626+
620627 struct mlx5_ib_mkey {
621628 u32 key;
622629 enum mlx5_mkey_type type;
623630 unsigned int ndescs;
624631 struct wait_queue_head wait;
625632 refcount_t usecount;
633+ /* User Mkey must hold either a rb_key or a cache_ent. */
634+ struct mlx5r_cache_rb_key rb_key;
626635 struct mlx5_cache_ent *cache_ent;
627636 };
628637
@@ -737,11 +746,11 @@ struct mlx5_cache_ent { · 108 unmodified lines
737746 unsigned long reserved;
738747
739748 char name[4];
740− u32 order;
741− u32 access_mode;
742− u32 page;
743− unsigned int ndescs;
744749
750+ struct rb_node node;
751+ struct mlx5r_cache_rb_key rb_key;
752+
753+ u8 is_tmp:1;
745754 u8 disabled:1;
746755 u8 fill_to_high_water:1;
747756
@@ -771,9 +780,11 @@ struct mlx5r_async_create_mkey { · 23 unmodified lines
771780
772781 struct mlx5_mkey_cache {
773782 struct workqueue_struct *wq;
774− struct mlx5_cache_ent ent[MAX_MKEY_CACHE_ENTRIES];
775− struct dentry *root;
783+ struct rb_root rb_root;
784+ struct mutex rb_lock;
785+ struct dentry *fs_root;
776786 unsigned long last_add;
787+ struct delayed_work remove_ent_dwork;
777788 };
778789
779790 struct mlx5_ib_port_resources {
@@ -877,6 +888,8 @@ enum mlx5_ib_dbg_cc_types { · 97 unmodified lines
877888 MLX5_IB_DBG_CC_NP_CNP_DSCP,
878889 MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE,
879890 MLX5_IB_DBG_CC_NP_CNP_PRIO,
891+ MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID,
892+ MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP,
880893 MLX5_IB_DBG_CC_MAX,
881894 };
882895
@@ -1054,6 +1067,13 @@ struct mlx5_port_caps { · 171 unmodified lines
10541067 u8 ext_port_cap;
10551068 };
10561069
1070+
1071+struct mlx5_special_mkeys {
1072+ u32 dump_fill_mkey;
1073+ __be32 null_mkey;
1074+ __be32 terminate_scatter_list_mkey;
1075+};
1076+
10571077 struct mlx5_ib_dev {
10581078 struct ib_device ib_dev;
10591079 struct mlx5_core_dev *mdev;
@@ -1084,7 +1104,6 @@ struct mlx5_ib_dev { · 24 unmodified lines
10841104
10851105 struct xarray odp_mkeys;
10861106
1087− u32 null_mkey;
10881107 struct mlx5_ib_flow_db *flow_db;
10891108 /* protect resources needed as part of reset flow */
10901109 spinlock_t reset_flow_resource_lock;
@@ -1113,6 +1132,7 @@ struct mlx5_ib_dev { · 22 unmodified lines
11131132 struct mlx5_port_caps port_caps[MLX5_MAX_PORTS];
11141133 u16 pkey_table_len;
11151134 u8 lag_ports;
1135+ struct mlx5_special_mkeys mkeys;
11161136 };
11171137
11181138 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
@@ -1319,11 +1339,15 @@ void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas, · 200 unmodified lines
13191339 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
13201340 int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);
13211341 int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev);
1322−int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev);
1342+void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev);
1343+struct mlx5_cache_ent *
1344+mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,
1345+ struct mlx5r_cache_rb_key rb_key,
1346+ bool persistent_entry);
13231347
13241348 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
1325− struct mlx5_cache_ent *ent,
1326− int access_flags);
1349+ int access_flags, int access_mode,
1350+ int ndescs);
13271351
13281352 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
13291353 struct ib_mr_status *mr_status);
@@ -1347,7 +1371,7 @@ int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq); · 17 unmodified lines
13471371 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev);
13481372 int __init mlx5_ib_odp_init(void);
13491373 void mlx5_ib_odp_cleanup(void);
1350−void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent);
1374+int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev);
13511375 void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,
13521376 struct mlx5_ib_mr *mr, int flags);
13531377
@@ -1366,7 +1390,10 @@ static inline int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, · 12 unmodified lines
13661390 static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev) {}
13671391 static inline int mlx5_ib_odp_init(void) { return 0; }
13681392 static inline void mlx5_ib_odp_cleanup(void) {}
1369−static inline void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent) {}
1393+static inline int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev)
1394+{
1395+ return 0;
1396+}
13701397 static inline void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,
13711398 struct mlx5_ib_mr *mr, int flags) {}
13721399
drivers/infiniband/hw/mlx5/mr.c+379−111
@@ -140,19 +140,16 @@ static void create_mkey_warn(struct mlx5_ib_dev *dev, int status, void *out) · 139 unmodified lines
140140 mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out);
141141 }
142142
143−
144−static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings,
145− void *to_store)
143+static int push_mkey_locked(struct mlx5_cache_ent *ent, bool limit_pendings,
144+ void *to_store)
146145 {
147146 XA_STATE(xas, &ent->mkeys, 0);
148147 void *curr;
149148
150− xa_lock_irq(&ent->mkeys);
151149 if (limit_pendings &&
152− (ent->reserved - ent->stored) > MAX_PENDING_REG_MR) {
153− xa_unlock_irq(&ent->mkeys);
150+ (ent->reserved - ent->stored) > MAX_PENDING_REG_MR)
154151 return -EAGAIN;
155− }
152+
156153 while (1) {
157154 /*
158155 * This is cmpxchg (NULL, XA_ZERO_ENTRY) however this version
@@ -191,6 +188,7 @@ static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings, · 32 unmodified lines
191188 break;
192189 xa_lock_irq(&ent->mkeys);
193190 }
191+ xa_lock_irq(&ent->mkeys);
194192 if (xas_error(&xas))
195193 return xas_error(&xas);
196194 if (WARN_ON(curr))
@@ -198,6 +196,17 @@ static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings, · 1 unmodified line
198196 return 0;
199197 }
200198
199+static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings,
200+ void *to_store)
201+{
202+ int ret;
203+
204+ xa_lock_irq(&ent->mkeys);
205+ ret = push_mkey_locked(ent, limit_pendings, to_store);
206+ xa_unlock_irq(&ent->mkeys);
207+ return ret;
208+}
209+
201210 static void undo_push_reserve_mkey(struct mlx5_cache_ent *ent)
202211 {
203212 void *old;
@@ -292,12 +301,14 @@ static void set_cache_mkc(struct mlx5_cache_ent *ent, void *mkc) · 88 unmodified lines
292301 set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd);
293302 MLX5_SET(mkc, mkc, free, 1);
294303 MLX5_SET(mkc, mkc, umr_en, 1);
295− MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
296− MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7);
304+ MLX5_SET(mkc, mkc, access_mode_1_0, ent->rb_key.access_mode & 0x3);
305+ MLX5_SET(mkc, mkc, access_mode_4_2,
306+ (ent->rb_key.access_mode >> 2) & 0x7);
297307
298308 MLX5_SET(mkc, mkc, translations_octword_size,
299− get_mkc_octo_size(ent->access_mode, ent->ndescs));
300− MLX5_SET(mkc, mkc, log_page_size, ent->page);
309+ get_mkc_octo_size(ent->rb_key.access_mode,
310+ ent->rb_key.ndescs));
311+ MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
301312 }
302313
303314 /* Asynchronously schedule new MRs to be populated in the cache. */
@@ -515,18 +526,22 @@ static const struct file_operations limit_fops = { · 211 unmodified lines
515526
516527 static bool someone_adding(struct mlx5_mkey_cache *cache)
517528 {
518− unsigned int i;
519−
520− for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
521− struct mlx5_cache_ent *ent = &cache->ent[i];
522− bool ret;
529+ struct mlx5_cache_ent *ent;
530+ struct rb_node *node;
531+ bool ret;
523532
533+ mutex_lock(&cache->rb_lock);
534+ for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) {
535+ ent = rb_entry(node, struct mlx5_cache_ent, node);
524536 xa_lock_irq(&ent->mkeys);
525537 ret = ent->stored < ent->limit;
526538 xa_unlock_irq(&ent->mkeys);
527− if (ret)
539+ if (ret) {
540+ mutex_unlock(&cache->rb_lock);
528541 return true;
542+ }
529543 }
544+ mutex_unlock(&cache->rb_lock);
530545 return false;
531546 }
532547
@@ -539,7 +554,7 @@ static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent) · 6 unmodified lines
539554 {
540555 lockdep_assert_held(&ent->mkeys.xa_lock);
541556
542− if (ent->disabled || READ_ONCE(ent->dev->fill_delay))
557+ if (ent->disabled || READ_ONCE(ent->dev->fill_delay) || ent->is_tmp)
543558 return;
544559 if (ent->stored < ent->limit) {
545560 ent->fill_to_high_water = true;
@@ -590,8 +605,8 @@ static void __cache_work_func(struct mlx5_cache_ent *ent) · 44 unmodified lines
590605 if (err != -EAGAIN) {
591606 mlx5_ib_warn(
592607 dev,
593− "command failed order %d, err %d\n",
594− ent->order, err);
608+ "add keys command failed, err %d\n",
609+ err);
595610 queue_delayed_work(cache->wq, &ent->dwork,
596611 msecs_to_jiffies(1000));
597612 }
@@ -637,16 +652,99 @@ static void delayed_cache_work_func(struct work_struct *work) · 39 unmodified lines
637652 __cache_work_func(ent);
638653 }
639654
640−struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
641− struct mlx5_cache_ent *ent,
642− int access_flags)
655+static int cache_ent_key_cmp(struct mlx5r_cache_rb_key key1,
656+ struct mlx5r_cache_rb_key key2)
657+{
658+ int res;
659+
660+ res = key1.ats - key2.ats;
661+ if (res)
662+ return res;
663+
664+ res = key1.access_mode - key2.access_mode;
665+ if (res)
666+ return res;
667+
668+ res = key1.access_flags - key2.access_flags;
669+ if (res)
670+ return res;
671+
672+ /*
673+ * keep ndescs the last in the compare table since the find function
674+ * searches for an exact match on all properties and only closest
675+ * match in size.
676+ */
677+ return key1.ndescs - key2.ndescs;
678+}
679+
680+static int mlx5_cache_ent_insert(struct mlx5_mkey_cache *cache,
681+ struct mlx5_cache_ent *ent)
682+{
683+ struct rb_node **new = &cache->rb_root.rb_node, *parent = NULL;
684+ struct mlx5_cache_ent *cur;
685+ int cmp;
686+
687+ /* Figure out where to put new node */
688+ while (*new) {
689+ cur = rb_entry(*new, struct mlx5_cache_ent, node);
690+ parent = *new;
691+ cmp = cache_ent_key_cmp(cur->rb_key, ent->rb_key);
692+ if (cmp > 0)
693+ new = &((*new)->rb_left);
694+ if (cmp < 0)
695+ new = &((*new)->rb_right);
696+ if (cmp == 0) {
697+ mutex_unlock(&cache->rb_lock);
698+ return -EEXIST;
699+ }
700+ }
701+
702+ /* Add new node and rebalance tree. */
703+ rb_link_node(&ent->node, parent, new);
704+ rb_insert_color(&ent->node, &cache->rb_root);
705+
706+ return 0;
707+}
708+
709+static struct mlx5_cache_ent *
710+mkey_cache_ent_from_rb_key(struct mlx5_ib_dev *dev,
711+ struct mlx5r_cache_rb_key rb_key)
712+{
713+ struct rb_node *node = dev->cache.rb_root.rb_node;
714+ struct mlx5_cache_ent *cur, *smallest = NULL;
715+ int cmp;
716+
717+ /*
718+ * Find the smallest ent with order >= requested_order.
719+ */
720+ while (node) {
721+ cur = rb_entry(node, struct mlx5_cache_ent, node);
722+ cmp = cache_ent_key_cmp(cur->rb_key, rb_key);
723+ if (cmp > 0) {
724+ smallest = cur;
725+ node = node->rb_left;
726+ }
727+ if (cmp < 0)
728+ node = node->rb_right;
729+ if (cmp == 0)
730+ return cur;
731+ }
732+
733+ return (smallest &&
734+ smallest->rb_key.access_mode == rb_key.access_mode &&
735+ smallest->rb_key.access_flags == rb_key.access_flags &&
736+ smallest->rb_key.ats == rb_key.ats) ?
737+ smallest :
738+ NULL;
739+}
740+
741+static struct mlx5_ib_mr *_mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
742+ struct mlx5_cache_ent *ent,
743+ int access_flags)
643744 {
644745 struct mlx5_ib_mr *mr;
645746 int err;
646747
647− if (!mlx5r_umr_can_reconfig(dev, 0, access_flags))
648− return ERR_PTR(-EOPNOTSUPP);
649−
650748 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
651749 if (!mr)
652750 return ERR_PTR(-ENOMEM);
@@ -677,10 +775,48 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, · 24 unmodified lines
677775 return mr;
678776 }
679777
680−static void clean_keys(struct mlx5_ib_dev *dev, int c)
778+static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev,
779+ int access_flags)
780+{
781+ int ret = 0;
782+
783+ if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
784+ MLX5_CAP_GEN(dev->mdev, atomic) &&
785+ MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
786+ ret |= IB_ACCESS_REMOTE_ATOMIC;
787+
788+ if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
789+ MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) &&
790+ !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
791+ ret |= IB_ACCESS_RELAXED_ORDERING;
792+
793+ if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
794+ MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) &&
795+ !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
796+ ret |= IB_ACCESS_RELAXED_ORDERING;
797+
798+ return ret;
799+}
800+
801+struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
802+ int access_flags, int access_mode,
803+ int ndescs)
804+{
805+ struct mlx5r_cache_rb_key rb_key = {
806+ .ndescs = ndescs,
807+ .access_mode = access_mode,
808+ .access_flags = get_unchangeable_access_flags(dev, access_flags)
809+ };
810+ struct mlx5_cache_ent *ent = mkey_cache_ent_from_rb_key(dev, rb_key);
811+
812+ if (!ent)
813+ return ERR_PTR(-EOPNOTSUPP);
814+
815+ return _mlx5_mr_cache_alloc(dev, ent, access_flags);
816+}
817+
818+static void clean_keys(struct mlx5_ib_dev *dev, struct mlx5_cache_ent *ent)
681819 {
682− struct mlx5_mkey_cache *cache = &dev->cache;
683− struct mlx5_cache_ent *ent = &cache->ent[c];
684820 u32 mkey;
685821
686822 cancel_delayed_work(&ent->dwork);
@@ -699,31 +835,39 @@ static void mlx5_mkey_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) · 12 unmodified lines
699835 if (!mlx5_debugfs_root || dev->is_rep)
700836 return;
701837
702− debugfs_remove_recursive(dev->cache.root);
703− dev->cache.root = NULL;
838+ debugfs_remove_recursive(dev->cache.fs_root);
839+ dev->cache.fs_root = NULL;
704840 }
705841
706−static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev)
842+static void mlx5_mkey_cache_debugfs_add_ent(struct mlx5_ib_dev *dev,
843+ struct mlx5_cache_ent *ent)
707844 {
708− struct mlx5_mkey_cache *cache = &dev->cache;
709− struct mlx5_cache_ent *ent;
845+ int order = order_base_2(ent->rb_key.ndescs);
710846 struct dentry *dir;
711− int i;
712847
713848 if (!mlx5_debugfs_root || dev->is_rep)
714849 return;
715850
716− cache->root = debugfs_create_dir("mr_cache", mlx5_debugfs_get_dev_root(dev->mdev));
851+ if (ent->rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
852+ order = MLX5_IMR_KSM_CACHE_ENTRY + 2;
717853
718− for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
719− ent = &cache->ent[i];
720− sprintf(ent->name, "%d", ent->order);
721− dir = debugfs_create_dir(ent->name, cache->root);
722− debugfs_create_file("size", 0600, dir, ent, &size_fops);
723− debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
724− debugfs_create_ulong("cur", 0400, dir, &ent->stored);
725− debugfs_create_u32("miss", 0600, dir, &ent->miss);
726− }
854+ sprintf(ent->name, "%d", order);
855+ dir = debugfs_create_dir(ent->name, dev->cache.fs_root);
856+ debugfs_create_file("size", 0600, dir, ent, &size_fops);
857+ debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
858+ debugfs_create_ulong("cur", 0400, dir, &ent->stored);
859+ debugfs_create_u32("miss", 0600, dir, &ent->miss);
860+}
861+
862+static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev)
863+{
864+ struct dentry *dbg_root = mlx5_debugfs_get_dev_root(dev->mdev);
865+ struct mlx5_mkey_cache *cache = &dev->cache;
866+
867+ if (!mlx5_debugfs_root || dev->is_rep)
868+ return;
869+
870+ cache->fs_root = debugfs_create_dir("mr_cache", dbg_root);
727871 }
728872
729873 static void delay_time_func(struct timer_list *t)
@@ -733,13 +877,100 @@ static void delay_time_func(struct timer_list *t) · 3 unmodified lines
733877 WRITE_ONCE(dev->fill_delay, 0);
734878 }
735879
880+struct mlx5_cache_ent *
881+mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,
882+ struct mlx5r_cache_rb_key rb_key,
883+ bool persistent_entry)
884+{
885+ struct mlx5_cache_ent *ent;
886+ int order;
887+ int ret;
888+
889+ ent = kzalloc(sizeof(*ent), GFP_KERNEL);
890+ if (!ent)
891+ return ERR_PTR(-ENOMEM);
892+
893+ xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);
894+ ent->rb_key = rb_key;
895+ ent->dev = dev;
896+ ent->is_tmp = !persistent_entry;
897+
898+ INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
899+
900+ ret = mlx5_cache_ent_insert(&dev->cache, ent);
901+ if (ret) {
902+ kfree(ent);
903+ return ERR_PTR(ret);
904+ }
905+
906+ if (persistent_entry) {
907+ if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
908+ order = MLX5_IMR_KSM_CACHE_ENTRY;
909+ else
910+ order = order_base_2(rb_key.ndescs) - 2;
911+
912+ if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
913+ !dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
914+ mlx5r_umr_can_load_pas(dev, 0))
915+ ent->limit = dev->mdev->profile.mr_cache[order].limit;
916+ else
917+ ent->limit = 0;
918+
919+ mlx5_mkey_cache_debugfs_add_ent(dev, ent);
920+ } else {
921+ mod_delayed_work(ent->dev->cache.wq,
922+ &ent->dev->cache.remove_ent_dwork,
923+ msecs_to_jiffies(30 * 1000));
924+ }
925+
926+ return ent;
927+}
928+
929+static void remove_ent_work_func(struct work_struct *work)
930+{
931+ struct mlx5_mkey_cache *cache;
932+ struct mlx5_cache_ent *ent;
933+ struct rb_node *cur;
934+
935+ cache = container_of(work, struct mlx5_mkey_cache,
936+ remove_ent_dwork.work);
937+ mutex_lock(&cache->rb_lock);
938+ cur = rb_last(&cache->rb_root);
939+ while (cur) {
940+ ent = rb_entry(cur, struct mlx5_cache_ent, node);
941+ cur = rb_prev(cur);
942+ mutex_unlock(&cache->rb_lock);
943+
944+ xa_lock_irq(&ent->mkeys);
945+ if (!ent->is_tmp) {
946+ xa_unlock_irq(&ent->mkeys);
947+ mutex_lock(&cache->rb_lock);
948+ continue;
949+ }
950+ xa_unlock_irq(&ent->mkeys);
951+
952+ clean_keys(ent->dev, ent);
953+ mutex_lock(&cache->rb_lock);
954+ }
955+ mutex_unlock(&cache->rb_lock);
956+}
957+
736958 int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
737959 {
738960 struct mlx5_mkey_cache *cache = &dev->cache;
961+ struct rb_root *root = &dev->cache.rb_root;
962+ struct mlx5r_cache_rb_key rb_key = {
963+ .access_mode = MLX5_MKC_ACCESS_MODE_MTT,
964+ };
739965 struct mlx5_cache_ent *ent;
966+ struct rb_node *node;
967+ int ret;
740968 int i;
741969
742970 mutex_init(&dev->slow_path_mutex);
971+ mutex_init(&dev->cache.rb_lock);
972+ dev->cache.rb_root = RB_ROOT;
973+ INIT_DELAYED_WORK(&dev->cache.remove_ent_dwork, remove_ent_work_func);
743974 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
744975 if (!cache->wq) {
745976 mlx5_ib_warn(dev, "failed to create work queue\n");
@@ -748,52 +979,51 @@ int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev) · 2 unmodified lines
748979
749980 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
750981 timer_setup(&dev->delay_timer, delay_time_func, 0);
751− for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
752− ent = &cache->ent[i];
753− xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);
754− ent->order = i + 2;
755− ent->dev = dev;
756− ent->limit = 0;
757−
758− INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
759−
760− if (i > MKEY_CACHE_LAST_STD_ENTRY) {
761− mlx5_odp_init_mkey_cache_entry(ent);
762− continue;
982+ mlx5_mkey_cache_debugfs_init(dev);
983+ mutex_lock(&cache->rb_lock);
984+ for (i = 0; i <= mkey_cache_max_order(dev); i++) {
985+ rb_key.ndescs = 1 << (i + 2);
986+ ent = mlx5r_cache_create_ent_locked(dev, rb_key, true);
987+ if (IS_ERR(ent)) {
988+ ret = PTR_ERR(ent);
989+ goto err;
763990 }
991+ }
764992
765− if (ent->order > mkey_cache_max_order(dev))
766− continue;
993+ ret = mlx5_odp_init_mkey_cache(dev);
994+ if (ret)
995+ goto err;
767996
768− ent->page = PAGE_SHIFT;
769− ent->ndescs = 1 << ent->order;
770− ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
771− if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
772− !dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
773− mlx5r_umr_can_load_pas(dev, 0))
774− ent->limit = dev->mdev->profile.mr_cache[i].limit;
775− else
776− ent->limit = 0;
997+ mutex_unlock(&cache->rb_lock);
998+ for (node = rb_first(root); node; node = rb_next(node)) {
999+ ent = rb_entry(node, struct mlx5_cache_ent, node);
7771000 xa_lock_irq(&ent->mkeys);
7781001 queue_adjust_cache_locked(ent);
7791002 xa_unlock_irq(&ent->mkeys);
7801003 }
7811004
782− mlx5_mkey_cache_debugfs_init(dev);
783−
7841005 return 0;
1006+
1007+err:
1008+ mutex_unlock(&cache->rb_lock);
1009+ mlx5_mkey_cache_debugfs_cleanup(dev);
1010+ mlx5_ib_warn(dev, "failed to create mkey cache entry\n");
1011+ return ret;
7851012 }
7861013
787−int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev)
1014+void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev)
7881015 {
789− unsigned int i;
1016+ struct rb_root *root = &dev->cache.rb_root;
1017+ struct mlx5_cache_ent *ent;
1018+ struct rb_node *node;
7901019
7911020 if (!dev->cache.wq)
792− return 0;
793−
794− for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
795− struct mlx5_cache_ent *ent = &dev->cache.ent[i];
1021+ return;
7961022
1023+ cancel_delayed_work_sync(&dev->cache.remove_ent_dwork);
1024+ mutex_lock(&dev->cache.rb_lock);
1025+ for (node = rb_first(root); node; node = rb_next(node)) {
1026+ ent = rb_entry(node, struct mlx5_cache_ent, node);
7971027 xa_lock_irq(&ent->mkeys);
7981028 ent->disabled = true;
7991029 xa_unlock_irq(&ent->mkeys);
@@ -803,13 +1033,18 @@ int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev) · 3 unmodified lines
8031033 mlx5_mkey_cache_debugfs_cleanup(dev);
8041034 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
8051035
806− for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++)
807− clean_keys(dev, i);
1036+ node = rb_first(root);
1037+ while (node) {
1038+ ent = rb_entry(node, struct mlx5_cache_ent, node);
1039+ node = rb_next(node);
1040+ clean_keys(dev, ent);
1041+ rb_erase(&ent->node, root);
1042+ kfree(ent);
1043+ }
1044+ mutex_unlock(&dev->cache.rb_lock);
8081045
8091046 destroy_workqueue(dev->cache.wq);
8101047 del_timer_sync(&dev->delay_timer);
811−
812− return 0;
8131048 }
8141049
8151050 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
@@ -873,23 +1108,10 @@ static int get_octo_len(u64 addr, u64 len, int page_shift) · 57 unmodified lines
8731108 static int mkey_cache_max_order(struct mlx5_ib_dev *dev)
8741109 {
8751110 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
876− return MKEY_CACHE_LAST_STD_ENTRY + 2;
1111+ return MKEY_CACHE_LAST_STD_ENTRY;
8771112 return MLX5_MAX_UMR_SHIFT;
8781113 }
8791114
880−static struct mlx5_cache_ent *mkey_cache_ent_from_order(struct mlx5_ib_dev *dev,
881− unsigned int order)
882−{
883− struct mlx5_mkey_cache *cache = &dev->cache;
884−
885− if (order < cache->ent[0].order)
886− return &cache->ent[0];
887− order = order - cache->ent[0].order;
888− if (order > MKEY_CACHE_LAST_STD_ENTRY)
889− return NULL;
890− return &cache->ent[order];
891−}
892−
8931115 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
8941116 u64 length, int access_flags, u64 iova)
8951117 {
@@ -916,6 +1138,9 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, · 20 unmodified lines
9161138 struct ib_umem *umem, u64 iova,
9171139 int access_flags)
9181140 {
1141+ struct mlx5r_cache_rb_key rb_key = {
1142+ .access_mode = MLX5_MKC_ACCESS_MODE_MTT,
1143+ };
9191144 struct mlx5_ib_dev *dev = to_mdev(pd->device);
9201145 struct mlx5_cache_ent *ent;
9211146 struct mlx5_ib_mr *mr;
@@ -928,22 +1153,26 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, · 6 unmodified lines
9281153 0, iova);
9291154 if (WARN_ON(!page_size))
9301155 return ERR_PTR(-EINVAL);
931− ent = mkey_cache_ent_from_order(
932− dev, order_base_2(ib_umem_num_dma_blocks(umem, page_size)));
1156+
1157+ rb_key.ndescs = ib_umem_num_dma_blocks(umem, page_size);
1158+ rb_key.ats = mlx5_umem_needs_ats(dev, umem, access_flags);
1159+ rb_key.access_flags = get_unchangeable_access_flags(dev, access_flags);
1160+ ent = mkey_cache_ent_from_rb_key(dev, rb_key);
9331161 /*
934− * Matches access in alloc_cache_mr(). If the MR can't come from the
935− * cache then synchronously create an uncached one.
1162+ * If the MR can't come from the cache then synchronously create an uncached
1163+ * one.
9361164 */
937− if (!ent || ent->limit == 0 ||
938− !mlx5r_umr_can_reconfig(dev, 0, access_flags) ||
939− mlx5_umem_needs_ats(dev, umem, access_flags)) {
1165+ if (!ent) {
9401166 mutex_lock(&dev->slow_path_mutex);
9411167 mr = reg_create(pd, umem, iova, access_flags, page_size, false);
9421168 mutex_unlock(&dev->slow_path_mutex);
1169+ if (IS_ERR(mr))
1170+ return mr;
1171+ mr->mmkey.rb_key = rb_key;
9431172 return mr;
9441173 }
9451174
946− mr = mlx5_mr_cache_alloc(dev, ent, access_flags);
1175+ mr = _mlx5_mr_cache_alloc(dev, ent, access_flags);
9471176 if (IS_ERR(mr))
9481177 return mr;
9491178
@@ -1030,6 +1259,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, · 80 unmodified lines
10301259 goto err_2;
10311260 }
10321261 mr->mmkey.type = MLX5_MKEY_MR;
1262+ mr->mmkey.ndescs = get_octo_len(iova, umem->length, mr->page_shift);
10331263 mr->umem = umem;
10341264 set_mr_fields(dev, mr, umem->length, access_flags, iova);
10351265 kvfree(in);
@@ -1372,7 +1602,7 @@ static bool can_use_umr_rereg_pas(struct mlx5_ib_mr *mr, · 336 unmodified lines
13721602 mlx5_umem_find_best_pgsz(new_umem, mkc, log_page_size, 0, iova);
13731603 if (WARN_ON(!*page_size))
13741604 return false;
1375− return (1ULL << mr->mmkey.cache_ent->order) >=
1605+ return (mr->mmkey.cache_ent->rb_key.ndescs) >=
13761606 ib_umem_num_dma_blocks(new_umem, *page_size);
13771607 }
13781608
@@ -1567,6 +1797,49 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr) · 188 unmodified lines
15671797 }
15681798 }
15691799
1800+static int cache_ent_find_and_store(struct mlx5_ib_dev *dev,
1801+ struct mlx5_ib_mr *mr)
1802+{
1803+ struct mlx5_mkey_cache *cache = &dev->cache;
1804+ struct mlx5_cache_ent *ent;
1805+ int ret;
1806+
1807+ if (mr->mmkey.cache_ent) {
1808+ xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
1809+ mr->mmkey.cache_ent->in_use--;
1810+ goto end;
1811+ }
1812+
1813+ mutex_lock(&cache->rb_lock);
1814+ ent = mkey_cache_ent_from_rb_key(dev, mr->mmkey.rb_key);
1815+ if (ent) {
1816+ if (ent->rb_key.ndescs == mr->mmkey.rb_key.ndescs) {
1817+ if (ent->disabled) {
1818+ mutex_unlock(&cache->rb_lock);
1819+ return -EOPNOTSUPP;
1820+ }
1821+ mr->mmkey.cache_ent = ent;
1822+ xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
1823+ mutex_unlock(&cache->rb_lock);
1824+ goto end;
1825+ }
1826+ }
1827+
1828+ ent = mlx5r_cache_create_ent_locked(dev, mr->mmkey.rb_key, false);
1829+ mutex_unlock(&cache->rb_lock);
1830+ if (IS_ERR(ent))
1831+ return PTR_ERR(ent);
1832+
1833+ mr->mmkey.cache_ent = ent;
1834+ xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
1835+
1836+end:
1837+ ret = push_mkey_locked(mr->mmkey.cache_ent, false,
1838+ xa_mk_value(mr->mmkey.key));
1839+ xa_unlock_irq(&mr->mmkey.cache_ent->mkeys);
1840+ return ret;
1841+}
1842+
15701843 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
15711844 {
15721845 struct mlx5_ib_mr *mr = to_mmr(ibmr);
@@ -1612,16 +1885,11 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) · 39 unmodified lines
16121885 }
16131886
16141887 /* Stop DMA */
1615− if (mr->mmkey.cache_ent) {
1616− xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
1617− mr->mmkey.cache_ent->in_use--;
1618− xa_unlock_irq(&mr->mmkey.cache_ent->mkeys);
1619−
1888+ if (mr->umem && mlx5r_umr_can_load_pas(dev, mr->umem->length))
16201889 if (mlx5r_umr_revoke_mr(mr) ||
1621− push_mkey(mr->mmkey.cache_ent, false,
1622− xa_mk_value(mr->mmkey.key)))
1890+ cache_ent_find_and_store(dev, mr))
16231891 mr->mmkey.cache_ent = NULL;
1624− }
1892+
16251893 if (!mr->mmkey.cache_ent) {
16261894 rc = destroy_mkey(to_mdev(mr->ibmr.device), mr);
16271895 if (rc)
drivers/infiniband/hw/mlx5/odp.c+27−40
@@ -104,7 +104,7 @@ static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, · 103 unmodified lines
104104 if (flags & MLX5_IB_UPD_XLT_ZAP) {
105105 for (; pklm != end; pklm++, idx++) {
106106 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
107− pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey);
107+ pklm->key = mr_to_mdev(imr)->mkeys.null_mkey;
108108 pklm->va = 0;
109109 }
110110 return;
@@ -137,7 +137,7 @@ static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, · 26 unmodified lines
137137 pklm->key = cpu_to_be32(mtt->ibmr.lkey);
138138 pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE);
139139 } else {
140− pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey);
140+ pklm->key = mr_to_mdev(imr)->mkeys.null_mkey;
141141 pklm->va = 0;
142142 }
143143 }
@@ -417,8 +417,9 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, · 273 unmodified lines
417417 if (IS_ERR(odp))
418418 return ERR_CAST(odp);
419419
420− mr = mlx5_mr_cache_alloc(dev, &dev->cache.ent[MLX5_IMR_MTT_CACHE_ENTRY],
421− imr->access_flags);
420+ mr = mlx5_mr_cache_alloc(dev, imr->access_flags,
421+ MLX5_MKC_ACCESS_MODE_MTT,
422+ MLX5_IMR_MTT_ENTRIES);
422423 if (IS_ERR(mr)) {
423424 ib_umem_odp_release(odp);
424425 return mr;
@@ -492,9 +493,8 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, · 67 unmodified lines
492493 if (IS_ERR(umem_odp))
493494 return ERR_CAST(umem_odp);
494495
495− imr = mlx5_mr_cache_alloc(dev,
496− &dev->cache.ent[MLX5_IMR_KSM_CACHE_ENTRY],
497− access_flags);
496+ imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM,
497+ mlx5_imr_ksm_entries);
498498 if (IS_ERR(imr)) {
499499 ib_umem_odp_release(umem_odp);
500500 return imr;
@@ -986,7 +986,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 485 unmodified lines
986986 {
987987 int ret = 0, npages = 0;
988988 u64 io_virt;
989− u32 key;
989+ __be32 key;
990990 u32 byte_count;
991991 size_t bcnt;
992992 int inline_segment;
@@ -1000,7 +1000,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 7 unmodified lines
10001000 struct mlx5_wqe_data_seg *dseg = wqe;
10011001
10021002 io_virt = be64_to_cpu(dseg->addr);
1003− key = be32_to_cpu(dseg->lkey);
1003+ key = dseg->lkey;
10041004 byte_count = be32_to_cpu(dseg->byte_count);
10051005 inline_segment = !!(byte_count & MLX5_INLINE_SEG);
10061006 bcnt = byte_count & ~MLX5_INLINE_SEG;
@@ -1014,7 +1014,8 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 7 unmodified lines
10141014 }
10151015
10161016 /* receive WQE end of sg list. */
1017− if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY &&
1017+ if (receive_queue && bcnt == 0 &&
1018+ key == dev->mkeys.terminate_scatter_list_mkey &&
10181019 io_virt == 0)
10191020 break;
10201021
@@ -1034,7 +1035,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 13 unmodified lines
10341035 continue;
10351036 }
10361037
1037− ret = pagefault_single_data_segment(dev, NULL, key,
1038+ ret = pagefault_single_data_segment(dev, NULL, be32_to_cpu(key),
10381039 io_virt, bcnt,
10391040 &pfault->bytes_committed,
10401041 bytes_mapped);
@@ -1587,26 +1588,22 @@ mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) · 546 unmodified lines
15871588 return err;
15881589 }
15891590
1590−void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent)
1591+int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev)
15911592 {
1592− if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1593− return;
1593+ struct mlx5r_cache_rb_key rb_key = {
1594+ .access_mode = MLX5_MKC_ACCESS_MODE_KSM,
1595+ .ndescs = mlx5_imr_ksm_entries,
1596+ };
1597+ struct mlx5_cache_ent *ent;
15941598
1595− switch (ent->order - 2) {
1596− case MLX5_IMR_MTT_CACHE_ENTRY:
1597− ent->page = PAGE_SHIFT;
1598− ent->ndescs = MLX5_IMR_MTT_ENTRIES;
1599− ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
1600− ent->limit = 0;
1601− break;
1599+ if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1600+ return 0;
16021601
1603− case MLX5_IMR_KSM_CACHE_ENTRY:
1604− ent->page = MLX5_KSM_PAGE_SHIFT;
1605− ent->ndescs = mlx5_imr_ksm_entries;
1606− ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM;
1607− ent->limit = 0;
1608− break;
1609− }
1602+ ent = mlx5r_cache_create_ent_locked(dev, rb_key, true);
1603+ if (IS_ERR(ent))
1604+ return PTR_ERR(ent);
1605+
1606+ return 0;
16101607 }
16111608
16121609 static const struct ib_device_ops mlx5_ib_dev_odp_ops = {
@@ -1615,25 +1612,15 @@ static const struct ib_device_ops mlx5_ib_dev_odp_ops = { · 2 unmodified lines
16151612
16161613 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)
16171614 {
1618− int ret = 0;
1619−
16201615 internal_fill_odp_caps(dev);
16211616
16221617 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT))
1623− return ret;
1618+ return 0;
16241619
16251620 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops);
16261621
1627− if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) {
1628− ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey);
1629− if (ret) {
1630− mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret);
1631− return ret;
1632− }
1633− }
1634−
16351622 mutex_init(&dev->odp_eq_mutex);
1636− return ret;
1623+ return 0;
16371624 }
16381625
16391626 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev)
drivers/infiniband/hw/mlx5/qp.c+128−36
@@ -71,6 +71,14 @@ struct mlx5_modify_raw_qp_param { · 70 unmodified lines
7171 u32 port;
7272 };
7373
74+struct mlx5_ib_qp_event_work {
75+ struct work_struct work;
76+ struct mlx5_core_qp *qp;
77+ int type;
78+};
79+
80+static struct workqueue_struct *mlx5_ib_qp_event_wq;
81+
7482 static void get_cqs(enum ib_qp_type qp_type,
7583 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
7684 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
@@ -302,51 +310,120 @@ int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer, · 225 unmodified lines
302310 return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);
303311 }
304312
313+static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp)
314+{
315+ struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
316+ int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
317+ struct mlx5_ib_qp *qp = to_mqp(ibqp);
318+ void *pas_ext_union, *err_syn;
319+ u32 *outb;
320+ int err;
321+
322+ if (!MLX5_CAP_GEN(dev->mdev, qpc_extension) ||
323+ !MLX5_CAP_GEN(dev->mdev, qp_error_syndrome))
324+ return;
325+
326+ outb = kzalloc(outlen, GFP_KERNEL);
327+ if (!outb)
328+ return;
329+
330+ err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
331+ true);
332+ if (err)
333+ goto out;
334+
335+ pas_ext_union =
336+ MLX5_ADDR_OF(query_qp_out, outb, qp_pas_or_qpc_ext_and_pas);
337+ err_syn = MLX5_ADDR_OF(qpc_extension_and_pas_list_in, pas_ext_union,
338+ qpc_data_extension.error_syndrome);
339+
340+ pr_err("%s/%d: QP %d error: %s (0x%x 0x%x 0x%x)\n",
341+ ibqp->device->name, ibqp->port, ibqp->qp_num,
342+ ib_wc_status_msg(
343+ MLX5_GET(cqe_error_syndrome, err_syn, syndrome)),
344+ MLX5_GET(cqe_error_syndrome, err_syn, vendor_error_syndrome),
345+ MLX5_GET(cqe_error_syndrome, err_syn, hw_syndrome_type),
346+ MLX5_GET(cqe_error_syndrome, err_syn, hw_error_syndrome));
347+out:
348+ kfree(outb);
349+}
350+
351+static void mlx5_ib_handle_qp_event(struct work_struct *_work)
352+{
353+ struct mlx5_ib_qp_event_work *qpe_work =
354+ container_of(_work, struct mlx5_ib_qp_event_work, work);
355+ struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp;
356+ struct ib_event event = {};
357+
358+ event.device = ibqp->device;
359+ event.element.qp = ibqp;
360+ switch (qpe_work->type) {
361+ case MLX5_EVENT_TYPE_PATH_MIG:
362+ event.event = IB_EVENT_PATH_MIG;
363+ break;
364+ case MLX5_EVENT_TYPE_COMM_EST:
365+ event.event = IB_EVENT_COMM_EST;
366+ break;
367+ case MLX5_EVENT_TYPE_SQ_DRAINED:
368+ event.event = IB_EVENT_SQ_DRAINED;
369+ break;
370+ case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
371+ event.event = IB_EVENT_QP_LAST_WQE_REACHED;
372+ break;
373+ case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
374+ event.event = IB_EVENT_QP_FATAL;
375+ break;
376+ case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
377+ event.event = IB_EVENT_PATH_MIG_ERR;
378+ break;
379+ case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
380+ event.event = IB_EVENT_QP_REQ_ERR;
381+ break;
382+ case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
383+ event.event = IB_EVENT_QP_ACCESS_ERR;
384+ break;
385+ default:
386+ pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n",
387+ qpe_work->type, qpe_work->qp->qpn);
388+ goto out;
389+ }
390+
391+ if ((event.event == IB_EVENT_QP_FATAL) ||
392+ (event.event == IB_EVENT_QP_ACCESS_ERR))
393+ mlx5_ib_qp_err_syndrome(ibqp);
394+
395+ ibqp->event_handler(&event, ibqp->qp_context);
396+
397+out:
398+ mlx5_core_res_put(&qpe_work->qp->common);
399+ kfree(qpe_work);
400+}
401+
305402 static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
306403 {
307404 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
308− struct ib_event event;
405+ struct mlx5_ib_qp_event_work *qpe_work;
309406
310407 if (type == MLX5_EVENT_TYPE_PATH_MIG) {
311408 /* This event is only valid for trans_qps */
312409 to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
313410 }
314411
315− if (ibqp->event_handler) {
316− event.device = ibqp->device;
317− event.element.qp = ibqp;
318− switch (type) {
319− case MLX5_EVENT_TYPE_PATH_MIG:
320− event.event = IB_EVENT_PATH_MIG;
321− break;
322− case MLX5_EVENT_TYPE_COMM_EST:
323− event.event = IB_EVENT_COMM_EST;
324− break;
325− case MLX5_EVENT_TYPE_SQ_DRAINED:
326− event.event = IB_EVENT_SQ_DRAINED;
327− break;
328− case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
329− event.event = IB_EVENT_QP_LAST_WQE_REACHED;
330− break;
331− case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
332− event.event = IB_EVENT_QP_FATAL;
333− break;
334− case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
335− event.event = IB_EVENT_PATH_MIG_ERR;
336− break;
337− case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
338− event.event = IB_EVENT_QP_REQ_ERR;
339− break;
340− case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
341− event.event = IB_EVENT_QP_ACCESS_ERR;
342− break;
343− default:
344− pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn);
345− return;
346− }
412+ if (!ibqp->event_handler)
413+ goto out_no_handler;
347414
348− ibqp->event_handler(&event, ibqp->qp_context);
349− }
415+ qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);
416+ if (!qpe_work)
417+ goto out_no_handler;
418+
419+ qpe_work->qp = qp;
420+ qpe_work->type = type;
421+ INIT_WORK(&qpe_work->work, mlx5_ib_handle_qp_event);
422+ queue_work(mlx5_ib_qp_event_wq, &qpe_work->work);
423+ return;
424+
425+out_no_handler:
426+ mlx5_core_res_put(&qp->common);
350427 }
351428
352429 static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
@@ -4827,7 +4904,8 @@ static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, · 4474 unmodified lines
48274904 if (!outb)
48284905 return -ENOMEM;
48294906
4830− err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen);
4907+ err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
4908+ false);
48314909 if (err)
48324910 goto out;
48334911
@@ -5720,3 +5798,17 @@ out: · 886 unmodified lines
57205798 mutex_unlock(&mqp->mutex);
57215799 return err;
57225800 }
5801+
5802+int mlx5_ib_qp_event_init(void)
5803+{
5804+ mlx5_ib_qp_event_wq = alloc_ordered_workqueue("mlx5_ib_qp_event_wq", 0);
5805+ if (!mlx5_ib_qp_event_wq)
5806+ return -ENOMEM;
5807+
5808+ return 0;
5809+}
5810+
5811+void mlx5_ib_qp_event_cleanup(void)
5812+{
5813+ destroy_workqueue(mlx5_ib_qp_event_wq);
5814+}
drivers/infiniband/hw/mlx5/qp.h+3−1
@@ -20,7 +20,7 @@ int mlx5_core_qp_modify(struct mlx5_ib_dev *dev, u16 opcode, u32 opt_param_mask, · 19 unmodified lines
2020 int mlx5_core_destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp);
2121 int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct);
2222 int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
23− u32 *out, int outlen);
23+ u32 *out, int outlen, bool qpc_ext);
2424 int mlx5_core_dct_query(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,
2525 u32 *out, int outlen);
2626
@@ -44,4 +44,6 @@ void mlx5_core_res_put(struct mlx5_core_rsc_common *res); · 17 unmodified lines
4444 int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn);
4545 int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn);
4646 int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter);
47+int mlx5_ib_qp_event_init(void);
48+void mlx5_ib_qp_event_cleanup(void);
4749 #endif /* _MLX5_IB_QP_H */
drivers/infiniband/hw/mlx5/qpc.c+5−2
@@ -135,7 +135,8 @@ static int rsc_event_notifier(struct notifier_block *nb, · 134 unmodified lines
135135 case MLX5_RES_SQ:
136136 qp = (struct mlx5_core_qp *)common;
137137 qp->event(qp, event_type);
138− break;
138+ /* Need to put resource in event handler */
139+ return NOTIFY_OK;
139140 case MLX5_RES_DCT:
140141 dct = (struct mlx5_core_dct *)common;
141142 if (event_type == MLX5_EVENT_TYPE_DCT_DRAINED)
@@ -504,12 +505,14 @@ void mlx5_cleanup_qp_table(struct mlx5_ib_dev *dev) · 362 unmodified lines
504505 }
505506
506507 int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
507− u32 *out, int outlen)
508+ u32 *out, int outlen, bool qpc_ext)
508509 {
509510 u32 in[MLX5_ST_SZ_DW(query_qp_in)] = {};
510511
511512 MLX5_SET(query_qp_in, in, opcode, MLX5_CMD_OP_QUERY_QP);
512513 MLX5_SET(query_qp_in, in, qpn, qp->qpn);
514+ MLX5_SET(query_qp_in, in, qpc_ext, qpc_ext);
515+
513516 return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, outlen);
514517 }
515518
drivers/infiniband/hw/mlx5/srq.c+1−1
@@ -447,7 +447,7 @@ int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, · 446 unmodified lines
447447
448448 if (i < srq->msrq.max_avail_gather) {
449449 scat[i].byte_count = 0;
450− scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
450+ scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey;
451451 scat[i].addr = 0;
452452 }
453453 }
drivers/infiniband/hw/mlx5/umr.c+1−3
@@ -636,9 +636,7 @@ int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) · 635 unmodified lines
636636 mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg);
637637
638638 cur_mtt = mtt;
639− rdma_for_each_block(mr->umem->sgt_append.sgt.sgl, &biter,
640− mr->umem->sgt_append.sgt.nents,
641− BIT(mr->page_shift)) {
639+ rdma_umem_for_each_dma_block(mr->umem, &biter, BIT(mr->page_shift)) {
642640 if (cur_mtt == (void *)mtt + sg.length) {
643641 dma_sync_single_for_device(ddev, sg.addr, sg.length,
644642 DMA_TO_DEVICE);
drivers/infiniband/hw/mlx5/wr.c+1−1
@@ -1252,7 +1252,7 @@ int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, · 1251 unmodified lines
12521252
12531253 if (i < qp->rq.max_gs) {
12541254 scat[i].byte_count = 0;
1255− scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
1255+ scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey;
12561256 scat[i].addr = 0;
12571257 }
12581258
drivers/infiniband/sw/rxe/rxe.h+38−0
@@ -57,6 +57,44 @@ · 56 unmodified lines
5757 #define rxe_dbg_mw(mw, fmt, ...) ibdev_dbg((mw)->ibmw.device, \
5858 "mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
5959
60+/* responder states */
61+enum resp_states {
62+ RESPST_NONE,
63+ RESPST_GET_REQ,
64+ RESPST_CHK_PSN,
65+ RESPST_CHK_OP_SEQ,
66+ RESPST_CHK_OP_VALID,
67+ RESPST_CHK_RESOURCE,
68+ RESPST_CHK_LENGTH,
69+ RESPST_CHK_RKEY,
70+ RESPST_EXECUTE,
71+ RESPST_READ_REPLY,
72+ RESPST_ATOMIC_REPLY,
73+ RESPST_ATOMIC_WRITE_REPLY,
74+ RESPST_PROCESS_FLUSH,
75+ RESPST_COMPLETE,
76+ RESPST_ACKNOWLEDGE,
77+ RESPST_CLEANUP,
78+ RESPST_DUPLICATE_REQUEST,
79+ RESPST_ERR_MALFORMED_WQE,
80+ RESPST_ERR_UNSUPPORTED_OPCODE,
81+ RESPST_ERR_MISALIGNED_ATOMIC,
82+ RESPST_ERR_PSN_OUT_OF_SEQ,
83+ RESPST_ERR_MISSING_OPCODE_FIRST,
84+ RESPST_ERR_MISSING_OPCODE_LAST_C,
85+ RESPST_ERR_MISSING_OPCODE_LAST_D1E,
86+ RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
87+ RESPST_ERR_RNR,
88+ RESPST_ERR_RKEY_VIOLATION,
89+ RESPST_ERR_INVALIDATE_RKEY,
90+ RESPST_ERR_LENGTH,
91+ RESPST_ERR_CQ_OVERFLOW,
92+ RESPST_ERROR,
93+ RESPST_RESET,
94+ RESPST_DONE,
95+ RESPST_EXIT,
96+};
97+
6098 void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
6199
62100 int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name);
drivers/infiniband/sw/rxe/rxe_loc.h+8−4
@@ -64,12 +64,16 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr); · 63 unmodified lines
6464 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
6565 int access, struct rxe_mr *mr);
6666 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
67−int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length);
68−int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
69− enum rxe_mr_copy_dir dir);
67+int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length);
68+int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
69+ unsigned int length, enum rxe_mr_copy_dir dir);
7070 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
7171 void *addr, int length, enum rxe_mr_copy_dir dir);
72−void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
72+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
73+ int sg_nents, unsigned int *sg_offset);
74+int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
75+ u64 compare, u64 swap_add, u64 *orig_val);
76+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value);
7377 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
7478 enum rxe_mr_lookup_type type);
7579 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c+338−268
@@ -26,22 +26,22 @@ u8 rxe_get_next_key(u32 last_key) · 25 unmodified lines
2626
2727 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
2828 {
29−
30−
3129 switch (mr->ibmr.type) {
3230 case IB_MR_TYPE_DMA:
3331 return 0;
3432
3533 case IB_MR_TYPE_USER:
3634 case IB_MR_TYPE_MEM_REG:
37− if (iova < mr->ibmr.iova || length > mr->ibmr.length ||
38− iova > mr->ibmr.iova + mr->ibmr.length - length)
39− return -EFAULT;
35+ if (iova < mr->ibmr.iova ||
36+ iova + length > mr->ibmr.iova + mr->ibmr.length) {
37+ rxe_dbg_mr(mr, "iova/length out of range");
38+ return -EINVAL;
39+ }
4040 return 0;
4141
4242 default:
43− rxe_dbg_mr(mr, "type (%d) not supported\n", mr->ibmr.type);
44− return -EFAULT;
43+ rxe_dbg_mr(mr, "mr type not supported\n");
44+ return -EINVAL;
4545 }
4646 }
4747
@@ -62,57 +62,31 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) · 14 unmodified lines
6262 mr->lkey = mr->ibmr.lkey = lkey;
6363 mr->rkey = mr->ibmr.rkey = rkey;
6464
65+ mr->access = access;
66+ mr->ibmr.page_size = PAGE_SIZE;
67+ mr->page_mask = PAGE_MASK;
68+ mr->page_shift = PAGE_SHIFT;
6569 mr->state = RXE_MR_STATE_INVALID;
6670 }
6771
68−static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf)
69−{
70− int i;
71− int num_map;
72− struct rxe_map **map = mr->map;
73−
74− num_map = (num_buf + RXE_BUF_PER_MAP - 1) / RXE_BUF_PER_MAP;
75−
76− mr->map = kmalloc_array(num_map, sizeof(*map), GFP_KERNEL);
77− if (!mr->map)
78− goto err1;
79−
80− for (i = 0; i < num_map; i++) {
81− mr->map[i] = kmalloc(sizeof(**map), GFP_KERNEL);
82− if (!mr->map[i])
83− goto err2;
84− }
85−
86− BUILD_BUG_ON(!is_power_of_2(RXE_BUF_PER_MAP));
87−
88− mr->map_shift = ilog2(RXE_BUF_PER_MAP);
89− mr->map_mask = RXE_BUF_PER_MAP - 1;
90−
91− mr->num_buf = num_buf;
92− mr->num_map = num_map;
93− mr->max_buf = num_map * RXE_BUF_PER_MAP;
94−
95− return 0;
96−
97−err2:
98− for (i--; i >= 0; i--)
99− kfree(mr->map[i]);
100−
101− kfree(mr->map);
102− mr->map = NULL;
103−err1:
104− return -ENOMEM;
105−}
106−
10772 void rxe_mr_init_dma(int access, struct rxe_mr *mr)
10873 {
10974 rxe_mr_init(access, mr);
11075
111− mr->access = access;
11276 mr->state = RXE_MR_STATE_VALID;
11377 mr->ibmr.type = IB_MR_TYPE_DMA;
11478 }
11579
80+static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
81+{
82+ return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);
83+}
84+
85+static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)
86+{
87+ return iova & (mr_page_size(mr) - 1);
88+}
89+
11690 static bool is_pmem_page(struct page *pg)
11791 {
11892 unsigned long paddr = page_to_phys(pg);
@@ -122,86 +96,98 @@ static bool is_pmem_page(struct page *pg) · 3 unmodified lines
12296 IORES_DESC_PERSISTENT_MEMORY);
12397 }
12498
99+static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
100+{
101+ XA_STATE(xas, &mr->page_list, 0);
102+ struct sg_page_iter sg_iter;
103+ struct page *page;
104+ bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
105+
106+ __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0);
107+ if (!__sg_page_iter_next(&sg_iter))
108+ return 0;
109+
110+ do {
111+ xas_lock(&xas);
112+ while (true) {
113+ page = sg_page_iter_page(&sg_iter);
114+
115+ if (persistent && !is_pmem_page(page)) {
116+ rxe_dbg_mr(mr, "Page can't be persistent\n");
117+ xas_set_err(&xas, -EINVAL);
118+ break;
119+ }
120+
121+ xas_store(&xas, page);
122+ if (xas_error(&xas))
123+ break;
124+ xas_next(&xas);
125+ if (!__sg_page_iter_next(&sg_iter))
126+ break;
127+ }
128+ xas_unlock(&xas);
129+ } while (xas_nomem(&xas, GFP_KERNEL));
130+
131+ return xas_error(&xas);
132+}
133+
125134 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
126135 int access, struct rxe_mr *mr)
127136 {
128− struct rxe_map **map;
129− struct rxe_phys_buf *buf = NULL;
130− struct ib_umem *umem;
131− struct sg_page_iter sg_iter;
132− int num_buf;
133− void *vaddr;
137+ struct ib_umem *umem;
134138 int err;
135139
140+ rxe_mr_init(access, mr);
141+
142+ xa_init(&mr->page_list);
143+
136144 umem = ib_umem_get(&rxe->ib_dev, start, length, access);
137145 if (IS_ERR(umem)) {
138146 rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
139147 (int)PTR_ERR(umem));
140− err = PTR_ERR(umem);
141− goto err_out;
148+ return PTR_ERR(umem);
142149 }
143150
144− num_buf = ib_umem_num_pages(umem);
145−
146− rxe_mr_init(access, mr);
147−
148− err = rxe_mr_alloc(mr, num_buf);
151+ err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);
149152 if (err) {
150− rxe_dbg_mr(mr, "Unable to allocate memory for map\n");
151− goto err_release_umem;
153+ ib_umem_release(umem);
154+ return err;
152155 }
153156
154− mr->page_shift = PAGE_SHIFT;
155− mr->page_mask = PAGE_SIZE - 1;
156−
157− num_buf = 0;
158− map = mr->map;
159− if (length > 0) {
160− bool persistent_access = access & IB_ACCESS_FLUSH_PERSISTENT;
161−
162− buf = map[0]->buf;
163− for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {
164− struct page *pg = sg_page_iter_page(&sg_iter);
157+ mr->umem = umem;
158+ mr->ibmr.type = IB_MR_TYPE_USER;
159+ mr->state = RXE_MR_STATE_VALID;
165160
166− if (persistent_access && !is_pmem_page(pg)) {
167− rxe_dbg_mr(mr, "Unable to register persistent access to non-pmem device\n");
168− err = -EINVAL;
169− goto err_release_umem;
170− }
161+ return 0;
162+}
171163
172− if (num_buf >= RXE_BUF_PER_MAP) {
173− map++;
174− buf = map[0]->buf;
175− num_buf = 0;
176− }
164+static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf)
165+{
166+ XA_STATE(xas, &mr->page_list, 0);
167+ int i = 0;
168+ int err;
177169
178− vaddr = page_address(pg);
179− if (!vaddr) {
180− rxe_dbg_mr(mr, "Unable to get virtual address\n");
181− err = -ENOMEM;
182− goto err_release_umem;
183− }
184− buf->addr = (uintptr_t)vaddr;
185− buf->size = PAGE_SIZE;
186− num_buf++;
187− buf++;
170+ xa_init(&mr->page_list);
188171
172+ do {
173+ xas_lock(&xas);
174+ while (i != num_buf) {
175+ xas_store(&xas, XA_ZERO_ENTRY);
176+ if (xas_error(&xas))
177+ break;
178+ xas_next(&xas);
179+ i++;
189180 }
190− }
181+ xas_unlock(&xas);
182+ } while (xas_nomem(&xas, GFP_KERNEL));
191183
192− mr->umem = umem;
193− mr->access = access;
194− mr->offset = ib_umem_offset(umem);
195− mr->state = RXE_MR_STATE_VALID;
196− mr->ibmr.type = IB_MR_TYPE_USER;
197− mr->ibmr.page_size = PAGE_SIZE;
184+ err = xas_error(&xas);
185+ if (err)
186+ return err;
198187
199− return 0;
188+ mr->num_buf = num_buf;
200189
201−err_release_umem:
202− ib_umem_release(umem);
203−err_out:
204− return err;
190+ return 0;
205191 }
206192
207193 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)
@@ -215,7 +201,6 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) · 7 unmodified lines
215201 if (err)
216202 goto err1;
217203
218− mr->max_buf = max_pages;
219204 mr->state = RXE_MR_STATE_FREE;
220205 mr->ibmr.type = IB_MR_TYPE_MEM_REG;
221206
@@ -225,187 +210,125 @@ err1: · 3 unmodified lines
225210 return err;
226211 }
227212
228−static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
229− size_t *offset_out)
213+static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
230214 {
231− size_t offset = iova - mr->ibmr.iova + mr->offset;
232− int map_index;
233− int buf_index;
234− u64 length;
235−
236− if (likely(mr->page_shift)) {
237− *offset_out = offset & mr->page_mask;
238− offset >>= mr->page_shift;
239− *n_out = offset & mr->map_mask;
240− *m_out = offset >> mr->map_shift;
241− } else {
242− map_index = 0;
243− buf_index = 0;
215+ struct rxe_mr *mr = to_rmr(ibmr);
216+ struct page *page = virt_to_page(iova & mr->page_mask);
217+ bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
218+ int err;
244219
245− length = mr->map[map_index]->buf[buf_index].size;
220+ if (persistent && !is_pmem_page(page)) {
221+ rxe_dbg_mr(mr, "Page cannot be persistent\n");
222+ return -EINVAL;
223+ }
246224
247− while (offset >= length) {
248− offset -= length;
249− buf_index++;
225+ if (unlikely(mr->nbuf == mr->num_buf))
226+ return -ENOMEM;
250227
251− if (buf_index == RXE_BUF_PER_MAP) {
252− map_index++;
253− buf_index = 0;
254− }
255− length = mr->map[map_index]->buf[buf_index].size;
256− }
228+ err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL));
229+ if (err)
230+ return err;
257231
258− *m_out = map_index;
259− *n_out = buf_index;
260− *offset_out = offset;
261− }
232+ mr->nbuf++;
233+ return 0;
262234 }
263235
264−void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length)
236+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl,
237+ int sg_nents, unsigned int *sg_offset)
265238 {
266− size_t offset;
267− int m, n;
268− void *addr;
269−
270− if (mr->state != RXE_MR_STATE_VALID) {
271− rxe_dbg_mr(mr, "Not in valid state\n");
272− addr = NULL;
273− goto out;
274− }
275−
276− if (!mr->map) {
277− addr = (void *)(uintptr_t)iova;
278− goto out;
279− }
280−
281− if (mr_check_range(mr, iova, length)) {
282− rxe_dbg_mr(mr, "Range violation\n");
283− addr = NULL;
284− goto out;
285− }
286−
287− lookup_iova(mr, iova, &m, &n, &offset);
288−
289− if (offset + length > mr->map[m]->buf[n].size) {
290− rxe_dbg_mr(mr, "Crosses page boundary\n");
291− addr = NULL;
292− goto out;
293− }
239+ struct rxe_mr *mr = to_rmr(ibmr);
240+ unsigned int page_size = mr_page_size(mr);
294241
295− addr = (void *)(uintptr_t)mr->map[m]->buf[n].addr + offset;
242+ mr->nbuf = 0;
243+ mr->page_shift = ilog2(page_size);
244+ mr->page_mask = ~((u64)page_size - 1);
245+ mr->page_offset = mr->ibmr.iova & (page_size - 1);
296246
297−out:
298− return addr;
247+ return ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page);
299248 }
300249
301−int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)
250+static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
251+ unsigned int length, enum rxe_mr_copy_dir dir)
302252 {
303− size_t offset;
304−
305− if (length == 0)
306− return 0;
307−
308− if (mr->ibmr.type == IB_MR_TYPE_DMA)
309− return -EFAULT;
253+ unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);
254+ unsigned long index = rxe_mr_iova_to_index(mr, iova);
255+ unsigned int bytes;
256+ struct page *page;
257+ void *va;
310258
311− offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask;
312− while (length > 0) {
313− u8 *va;
314− int bytes;
315−
316− bytes = mr->ibmr.page_size - offset;
317− if (bytes > length)
318− bytes = length;
319−
320− va = iova_to_vaddr(mr, iova, length);
321− if (!va)
259+ while (length) {
260+ page = xa_load(&mr->page_list, index);
261+ if (!page)
322262 return -EFAULT;
323263
324− arch_wb_cache_pmem(va, bytes);
325−
264+ bytes = min_t(unsigned int, length,
265+ mr_page_size(mr) - page_offset);
266+ va = kmap_local_page(page);
267+ if (dir == RXE_FROM_MR_OBJ)
268+ memcpy(addr, va + page_offset, bytes);
269+ else
270+ memcpy(va + page_offset, addr, bytes);
271+ kunmap_local(va);
272+
273+ page_offset = 0;
274+ addr += bytes;
326275 length -= bytes;
327− iova += bytes;
328− offset = 0;
276+ index++;
329277 }
330278
331279 return 0;
332280 }
333281
334−/* copy data from a range (vaddr, vaddr+length-1) to or from
335− * a mr object starting at iova.
336− */
337−int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
338− enum rxe_mr_copy_dir dir)
282+static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 iova, void *addr,
283+ unsigned int length, enum rxe_mr_copy_dir dir)
339284 {
340− int err;
341− int bytes;
342− u8 *va;
343− struct rxe_map **map;
344− struct rxe_phys_buf *buf;
345− int m;
346− int i;
347− size_t offset;
285+ unsigned int page_offset = iova & (PAGE_SIZE - 1);
286+ unsigned int bytes;
287+ struct page *page;
288+ u8 *va;
348289
349− if (length == 0)
350− return 0;
351−
352− if (mr->ibmr.type == IB_MR_TYPE_DMA) {
353− u8 *src, *dest;
290+ while (length) {
291+ page = virt_to_page(iova & mr->page_mask);
292+ bytes = min_t(unsigned int, length,
293+ PAGE_SIZE - page_offset);
294+ va = kmap_local_page(page);
295+
296+ if (dir == RXE_TO_MR_OBJ)
297+ memcpy(va + page_offset, addr, bytes);
298+ else
299+ memcpy(addr, va + page_offset, bytes);
300+
301+ kunmap_local(va);
302+ page_offset = 0;
303+ iova += bytes;
304+ addr += bytes;
305+ length -= bytes;
306+ }
307+}
354308
355− src = (dir == RXE_TO_MR_OBJ) ? addr : ((void *)(uintptr_t)iova);
309+int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
310+ unsigned int length, enum rxe_mr_copy_dir dir)
311+{
312+ int err;
356313
357− dest = (dir == RXE_TO_MR_OBJ) ? ((void *)(uintptr_t)iova) : addr;
314+ if (length == 0)
315+ return 0;
358316
359− memcpy(dest, src, length);
317+ if (WARN_ON(!mr))
318+ return -EINVAL;
360319
320+ if (mr->ibmr.type == IB_MR_TYPE_DMA) {
321+ rxe_mr_copy_dma(mr, iova, addr, length, dir);
361322 return 0;
362323 }
363324
364− WARN_ON_ONCE(!mr->map);
365−
366325 err = mr_check_range(mr, iova, length);
367− if (err) {
368− err = -EFAULT;
369− goto err1;
370− }
371−
372− lookup_iova(mr, iova, &m, &i, &offset);
373−
374− map = mr->map + m;
375− buf = map[0]->buf + i;
376−
377− while (length > 0) {
378− u8 *src, *dest;
379−
380− va = (u8 *)(uintptr_t)buf->addr + offset;
381− src = (dir == RXE_TO_MR_OBJ) ? addr : va;
382− dest = (dir == RXE_TO_MR_OBJ) ? va : addr;
383−
384− bytes = buf->size - offset;
385−
386− if (bytes > length)
387− bytes = length;
388−
389− memcpy(dest, src, bytes);
390−
391− length -= bytes;
392− addr += bytes;
393−
394− offset = 0;
395− buf++;
396− i++;
397−
398− if (i == RXE_BUF_PER_MAP) {
399− i = 0;
400− map++;
401− buf = map[0]->buf;
402− }
326+ if (unlikely(err)) {
327+ rxe_dbg_mr(mr, "iova out of range");
328+ return err;
403329 }
404330
405− return 0;
406−
407−err1:
408− return err;
331+ return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
409332 }
410333
411334 /* copy data in or out of a wqe, i.e. sg list
@@ -477,7 +400,6 @@ int copy_data( · 65 unmodified lines
477400
478401 if (bytes > 0) {
479402 iova = sge->addr + offset;
480−
481403 err = rxe_mr_copy(mr, iova, addr, bytes, dir);
482404 if (err)
483405 goto err2;
@@ -504,6 +426,165 @@ err1: · 20 unmodified lines
504426 return err;
505427 }
506428
429+int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)
430+{
431+ unsigned int page_offset;
432+ unsigned long index;
433+ struct page *page;
434+ unsigned int bytes;
435+ int err;
436+ u8 *va;
437+
438+ /* mr must be valid even if length is zero */
439+ if (WARN_ON(!mr))
440+ return -EINVAL;
441+
442+ if (length == 0)
443+ return 0;
444+
445+ if (mr->ibmr.type == IB_MR_TYPE_DMA)
446+ return -EFAULT;
447+
448+ err = mr_check_range(mr, iova, length);
449+ if (err)
450+ return err;
451+
452+ while (length > 0) {
453+ index = rxe_mr_iova_to_index(mr, iova);
454+ page = xa_load(&mr->page_list, index);
455+ page_offset = rxe_mr_iova_to_page_offset(mr, iova);
456+ if (!page)
457+ return -EFAULT;
458+ bytes = min_t(unsigned int, length,
459+ mr_page_size(mr) - page_offset);
460+
461+ va = kmap_local_page(page);
462+ arch_wb_cache_pmem(va + page_offset, bytes);
463+ kunmap_local(va);
464+
465+ length -= bytes;
466+ iova += bytes;
467+ page_offset = 0;
468+ }
469+
470+ return 0;
471+}
472+
473+/* Guarantee atomicity of atomic operations at the machine level. */
474+static DEFINE_SPINLOCK(atomic_ops_lock);
475+
476+int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
477+ u64 compare, u64 swap_add, u64 *orig_val)
478+{
479+ unsigned int page_offset;
480+ struct page *page;
481+ u64 value;
482+ u64 *va;
483+
484+ if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
485+ rxe_dbg_mr(mr, "mr not in valid state");
486+ return RESPST_ERR_RKEY_VIOLATION;
487+ }
488+
489+ if (mr->ibmr.type == IB_MR_TYPE_DMA) {
490+ page_offset = iova & (PAGE_SIZE - 1);
491+ page = virt_to_page(iova & PAGE_MASK);
492+ } else {
493+ unsigned long index;
494+ int err;
495+
496+ err = mr_check_range(mr, iova, sizeof(value));
497+ if (err) {
498+ rxe_dbg_mr(mr, "iova out of range");
499+ return RESPST_ERR_RKEY_VIOLATION;
500+ }
501+ page_offset = rxe_mr_iova_to_page_offset(mr, iova);
502+ index = rxe_mr_iova_to_index(mr, iova);
503+ page = xa_load(&mr->page_list, index);
504+ if (!page)
505+ return RESPST_ERR_RKEY_VIOLATION;
506+ }
507+
508+ if (unlikely(page_offset & 0x7)) {
509+ rxe_dbg_mr(mr, "iova not aligned");
510+ return RESPST_ERR_MISALIGNED_ATOMIC;
511+ }
512+
513+ va = kmap_local_page(page);
514+
515+ spin_lock_bh(&atomic_ops_lock);
516+ value = *orig_val = va[page_offset >> 3];
517+
518+ if (opcode == IB_OPCODE_RC_COMPARE_SWAP) {
519+ if (value == compare)
520+ va[page_offset >> 3] = swap_add;
521+ } else {
522+ value += swap_add;
523+ va[page_offset >> 3] = value;
524+ }
525+ spin_unlock_bh(&atomic_ops_lock);
526+
527+ kunmap_local(va);
528+
529+ return 0;
530+}
531+
532+#if defined CONFIG_64BIT
533+/* only implemented or called for 64 bit architectures */
534+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
535+{
536+ unsigned int page_offset;
537+ struct page *page;
538+ u64 *va;
539+
540+ /* See IBA oA19-28 */
541+ if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
542+ rxe_dbg_mr(mr, "mr not in valid state");
543+ return RESPST_ERR_RKEY_VIOLATION;
544+ }
545+
546+ if (mr->ibmr.type == IB_MR_TYPE_DMA) {
547+ page_offset = iova & (PAGE_SIZE - 1);
548+ page = virt_to_page(iova & PAGE_MASK);
549+ } else {
550+ unsigned long index;
551+ int err;
552+
553+ /* See IBA oA19-28 */
554+ err = mr_check_range(mr, iova, sizeof(value));
555+ if (unlikely(err)) {
556+ rxe_dbg_mr(mr, "iova out of range");
557+ return RESPST_ERR_RKEY_VIOLATION;
558+ }
559+ page_offset = rxe_mr_iova_to_page_offset(mr, iova);
560+ index = rxe_mr_iova_to_index(mr, iova);
561+ page = xa_load(&mr->page_list, index);
562+ if (!page)
563+ return RESPST_ERR_RKEY_VIOLATION;
564+ }
565+
566+ /* See IBA A19.4.2 */
567+ if (unlikely(page_offset & 0x7)) {
568+ rxe_dbg_mr(mr, "misaligned address");
569+ return RESPST_ERR_MISALIGNED_ATOMIC;
570+ }
571+
572+ va = kmap_local_page(page);
573+
574+ /* Do atomic write after all prior operations have completed */
575+ smp_store_release(&va[page_offset >> 3], value);
576+
577+ kunmap_local(va);
578+
579+ return 0;
580+}
581+#else
582+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
583+{
584+ return RESPST_ERR_UNSUPPORTED_OPCODE;
585+}
586+#endif
587+
507588 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
508589 {
509590 struct rxe_sge *sge = &dma->sge[dma->cur_sge];
@@ -537,12 +618,6 @@ int advance_dma_data(struct rxe_dma_info *dma, unsigned int length) · 27 unmodified lines
537618 return 0;
538619 }
539620
540−/* (1) find the mr corresponding to lkey/rkey
541− * depending on lookup_type
542− * (2) verify that the (qp) pd matches the mr pd
543− * (3) verify that the mr can support the requested access
544− * (4) verify that mr state is valid
545− */
546621 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
547622 enum rxe_mr_lookup_type type)
548623 {
@@ -656,22 +731,17 @@ int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) · 107 unmodified lines
656731 return -EINVAL;
657732
658733 rxe_cleanup(mr);
659−
734+ kfree_rcu(mr);
660735 return 0;
661736 }
662737
663738 void rxe_mr_cleanup(struct rxe_pool_elem *elem)
664739 {
665740 struct rxe_mr *mr = container_of(elem, typeof(*mr), elem);
666− int i;
667741
668742 rxe_put(mr_pd(mr));
669743 ib_umem_release(mr->umem);
670744
671− if (mr->map) {
672− for (i = 0; i < mr->num_map; i++)
673− kfree(mr->map[i]);
674−
675− kfree(mr->map);
676− }
745+ if (mr->ibmr.type != IB_MR_TYPE_DMA)
746+ xa_destroy(&mr->page_list);
677747 }
drivers/infiniband/sw/rxe/rxe_pool.c+0−46
@@ -116,55 +116,12 @@ void rxe_pool_cleanup(struct rxe_pool *pool) · 115 unmodified lines
116116 WARN_ON(!xa_empty(&pool->xa));
117117 }
118118
119−void *rxe_alloc(struct rxe_pool *pool)
120−{
121− struct rxe_pool_elem *elem;
122− void *obj;
123− int err;
124−
125− if (WARN_ON(!(pool->type == RXE_TYPE_MR)))
126− return NULL;
127−
128− if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
129− goto err_cnt;
130−
131− obj = kzalloc(pool->elem_size, GFP_KERNEL);
132− if (!obj)
133− goto err_cnt;
134−
135− elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset);
136−
137− elem->pool = pool;
138− elem->obj = obj;
139− kref_init(&elem->ref_cnt);
140− init_completion(&elem->complete);
141−
142− /* allocate index in array but leave pointer as NULL so it
143− * can't be looked up until rxe_finalize() is called
144− */
145− err = xa_alloc_cyclic(&pool->xa, &elem->index, NULL, pool->limit,
146− &pool->next, GFP_KERNEL);
147− if (err < 0)
148− goto err_free;
149−
150− return obj;
151−
152−err_free:
153− kfree(obj);
154−err_cnt:
155− atomic_dec(&pool->num_elem);
156− return NULL;
157−}
158−
159119 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem,
160120 bool sleepable)
161121 {
162122 int err;
163123 gfp_t gfp_flags;
164124
165− if (WARN_ON(pool->type == RXE_TYPE_MR))
166− return -EINVAL;
167−
168125 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
169126 goto err_cnt;
170127
@@ -275,9 +232,6 @@ int __rxe_cleanup(struct rxe_pool_elem *elem, bool sleepable) · 104 unmodified lines
275232 if (pool->cleanup)
276233 pool->cleanup(elem);
277234
278− if (pool->type == RXE_TYPE_MR)
279− kfree_rcu(elem->obj);
280−
281235 atomic_dec(&pool->num_elem);
282236
283237 return err;
drivers/infiniband/sw/rxe/rxe_pool.h+0−3
@@ -54,9 +54,6 @@ void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool, · 53 unmodified lines
5454 /* free resources from object pool */
5555 void rxe_pool_cleanup(struct rxe_pool *pool);
5656
57−/* allocate an object from pool */
58−void *rxe_alloc(struct rxe_pool *pool);
59−
6057 /* connect already allocated object to pool */
6158 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem,
6259 bool sleepable);
drivers/infiniband/sw/rxe/rxe_queue.h+66−42
@@ -35,19 +35,26 @@ · 34 unmodified lines
3535 /**
3636 * enum queue_type - type of queue
3737 * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and
38− * read by client. Used by rxe driver only.
38+ * read by client which may be a user space
39+ * application or a kernel ulp.
40+ * Used by rxe internals only.
3941 * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and
40− * read by rxe driver. Used by rxe driver only.
41− * @QUEUE_TYPE_TO_DRIVER: Queue is written by client and
42− * read by rxe driver. Used by kernel client only.
43− * @QUEUE_TYPE_FROM_DRIVER: Queue is written by rxe driver and
44− * read by client. Used by kernel client only.
42+ * read by rxe driver.
43+ * Used by rxe internals only.
44+ * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and
45+ * read by rxe driver.
46+ * Used by kernel verbs APIs only on
47+ * behalf of ulps.
48+ * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and
49+ * read by kernel ulp.
50+ * Used by kernel verbs APIs only on
51+ * behalf of ulps.
4552 */
4653 enum queue_type {
4754 QUEUE_TYPE_TO_CLIENT,
4855 QUEUE_TYPE_FROM_CLIENT,
49− QUEUE_TYPE_TO_DRIVER,
50− QUEUE_TYPE_FROM_DRIVER,
56+ QUEUE_TYPE_FROM_ULP,
57+ QUEUE_TYPE_TO_ULP,
5158 };
5259
5360 struct rxe_queue_buf;
@@ -62,9 +69,9 @@ struct rxe_queue { · 8 unmodified lines
6269 u32 index_mask;
6370 enum queue_type type;
6471 /* private copy of index for shared queues between
65− * kernel space and user space. Kernel reads and writes
72+ * driver and clients. Driver reads and writes
6673 * this copy and then replicates to rxe_queue_buf
67− * for read access by user space.
74+ * for read access by clients.
6875 */
6976 u32 index;
7077 };
@@ -97,19 +104,21 @@ static inline u32 queue_get_producer(const struct rxe_queue *q, · 26 unmodified lines
97104
98105 switch (type) {
99106 case QUEUE_TYPE_FROM_CLIENT:
100− /* protect user index */
107+ /* used by rxe, client owns the index */
101108 prod = smp_load_acquire(&q->buf->producer_index);
102109 break;
103110 case QUEUE_TYPE_TO_CLIENT:
111+ /* used by rxe which owns the index */
104112 prod = q->index;
105113 break;
106− case QUEUE_TYPE_FROM_DRIVER:
107− /* protect driver index */
108− prod = smp_load_acquire(&q->buf->producer_index);
109− break;
110− case QUEUE_TYPE_TO_DRIVER:
114+ case QUEUE_TYPE_FROM_ULP:
115+ /* used by ulp which owns the index */
111116 prod = q->buf->producer_index;
112117 break;
118+ case QUEUE_TYPE_TO_ULP:
119+ /* used by ulp, rxe owns the index */
120+ prod = smp_load_acquire(&q->buf->producer_index);
121+ break;
113122 }
114123
115124 return prod;
@@ -122,19 +131,21 @@ static inline u32 queue_get_consumer(const struct rxe_queue *q, · 6 unmodified lines
122131
123132 switch (type) {
124133 case QUEUE_TYPE_FROM_CLIENT:
134+ /* used by rxe which owns the index */
125135 cons = q->index;
126136 break;
127137 case QUEUE_TYPE_TO_CLIENT:
128− /* protect user index */
138+ /* used by rxe, client owns the index */
129139 cons = smp_load_acquire(&q->buf->consumer_index);
130140 break;
131− case QUEUE_TYPE_FROM_DRIVER:
132− cons = q->buf->consumer_index;
133− break;
134− case QUEUE_TYPE_TO_DRIVER:
135− /* protect driver index */
141+ case QUEUE_TYPE_FROM_ULP:
142+ /* used by ulp, rxe owns the index */
136143 cons = smp_load_acquire(&q->buf->consumer_index);
137144 break;
145+ case QUEUE_TYPE_TO_ULP:
146+ /* used by ulp which owns the index */
147+ cons = q->buf->consumer_index;
148+ break;
138149 }
139150
140151 return cons;
@@ -172,24 +183,31 @@ static inline void queue_advance_producer(struct rxe_queue *q, · 31 unmodified lines
172183
173184 switch (type) {
174185 case QUEUE_TYPE_FROM_CLIENT:
175− pr_warn("%s: attempt to advance client index\n",
176− __func__);
186+ /* used by rxe, client owns the index */
187+ if (WARN_ON(1))
188+ pr_warn("%s: attempt to advance client index\n",
189+ __func__);
177190 break;
178191 case QUEUE_TYPE_TO_CLIENT:
192+ /* used by rxe which owns the index */
179193 prod = q->index;
180194 prod = (prod + 1) & q->index_mask;
181195 q->index = prod;
182− /* protect user index */
196+ /* release so client can read it safely */
183197 smp_store_release(&q->buf->producer_index, prod);
184198 break;
185− case QUEUE_TYPE_FROM_DRIVER:
186− pr_warn("%s: attempt to advance driver index\n",
187− __func__);
188− break;
189− case QUEUE_TYPE_TO_DRIVER:
199+ case QUEUE_TYPE_FROM_ULP:
200+ /* used by ulp which owns the index */
190201 prod = q->buf->producer_index;
191202 prod = (prod + 1) & q->index_mask;
192− q->buf->producer_index = prod;
203+ /* release so rxe can read it safely */
204+ smp_store_release(&q->buf->producer_index, prod);
205+ break;
206+ case QUEUE_TYPE_TO_ULP:
207+ /* used by ulp, rxe owns the index */
208+ if (WARN_ON(1))
209+ pr_warn("%s: attempt to advance driver index\n",
210+ __func__);
193211 break;
194212 }
195213 }
@@ -201,24 +219,30 @@ static inline void queue_advance_consumer(struct rxe_queue *q, · 5 unmodified lines
201219
202220 switch (type) {
203221 case QUEUE_TYPE_FROM_CLIENT:
204− cons = q->index;
205− cons = (cons + 1) & q->index_mask;
222+ /* used by rxe which owns the index */
223+ cons = (q->index + 1) & q->index_mask;
206224 q->index = cons;
207− /* protect user index */
225+ /* release so client can read it safely */
208226 smp_store_release(&q->buf->consumer_index, cons);
209227 break;
210228 case QUEUE_TYPE_TO_CLIENT:
211− pr_warn("%s: attempt to advance client index\n",
212− __func__);
229+ /* used by rxe, client owns the index */
230+ if (WARN_ON(1))
231+ pr_warn("%s: attempt to advance client index\n",
232+ __func__);
233+ break;
234+ case QUEUE_TYPE_FROM_ULP:
235+ /* used by ulp, rxe owns the index */
236+ if (WARN_ON(1))
237+ pr_warn("%s: attempt to advance driver index\n",
238+ __func__);
213239 break;
214− case QUEUE_TYPE_FROM_DRIVER:
240+ case QUEUE_TYPE_TO_ULP:
241+ /* used by ulp which owns the index */
215242 cons = q->buf->consumer_index;
216243 cons = (cons + 1) & q->index_mask;
217− q->buf->consumer_index = cons;
218− break;
219− case QUEUE_TYPE_TO_DRIVER:
220− pr_warn("%s: attempt to advance driver index\n",
221− __func__);
244+ /* release so rxe can read it safely */
245+ smp_store_release(&q->buf->consumer_index, cons);
222246 break;
223247 }
224248 }
drivers/infiniband/sw/rxe/rxe_resp.c+75−127
@@ -10,43 +10,6 @@ · 9 unmodified lines
1010 #include "rxe_loc.h"
1111 #include "rxe_queue.h"
1212
13−enum resp_states {
14− RESPST_NONE,
15− RESPST_GET_REQ,
16− RESPST_CHK_PSN,
17− RESPST_CHK_OP_SEQ,
18− RESPST_CHK_OP_VALID,
19− RESPST_CHK_RESOURCE,
20− RESPST_CHK_LENGTH,
21− RESPST_CHK_RKEY,
22− RESPST_EXECUTE,
23− RESPST_READ_REPLY,
24− RESPST_ATOMIC_REPLY,
25− RESPST_ATOMIC_WRITE_REPLY,
26− RESPST_PROCESS_FLUSH,
27− RESPST_COMPLETE,
28− RESPST_ACKNOWLEDGE,
29− RESPST_CLEANUP,
30− RESPST_DUPLICATE_REQUEST,
31− RESPST_ERR_MALFORMED_WQE,
32− RESPST_ERR_UNSUPPORTED_OPCODE,
33− RESPST_ERR_MISALIGNED_ATOMIC,
34− RESPST_ERR_PSN_OUT_OF_SEQ,
35− RESPST_ERR_MISSING_OPCODE_FIRST,
36− RESPST_ERR_MISSING_OPCODE_LAST_C,
37− RESPST_ERR_MISSING_OPCODE_LAST_D1E,
38− RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
39− RESPST_ERR_RNR,
40− RESPST_ERR_RKEY_VIOLATION,
41− RESPST_ERR_INVALIDATE_RKEY,
42− RESPST_ERR_LENGTH,
43− RESPST_ERR_CQ_OVERFLOW,
44− RESPST_ERROR,
45− RESPST_RESET,
46− RESPST_DONE,
47− RESPST_EXIT,
48−};
49−
5013 static char *resp_state_name[] = {
5114 [RESPST_NONE] = "NONE",
5215 [RESPST_GET_REQ] = "GET_REQ",
@@ -457,13 +420,23 @@ static enum resp_states rxe_resp_check_length(struct rxe_qp *qp, · 404 unmodified lines
457420 return RESPST_CHK_RKEY;
458421 }
459422
423+/* if the reth length field is zero we can assume nothing
424+ * about the rkey value and should not validate or use it.
425+ * Instead set qp->resp.rkey to 0 which is an invalid rkey
426+ * value since the minimum index part is 1.
427+ */
460428 static void qp_resp_from_reth(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
461429 {
430+ unsigned int length = reth_len(pkt);
431+
462432 qp->resp.va = reth_va(pkt);
463433 qp->resp.offset = 0;
464− qp->resp.rkey = reth_rkey(pkt);
465− qp->resp.resid = reth_len(pkt);
466− qp->resp.length = reth_len(pkt);
434+ qp->resp.resid = length;
435+ qp->resp.length = length;
436+ if (pkt->mask & RXE_READ_OR_WRITE_MASK && length == 0)
437+ qp->resp.rkey = 0;
438+ else
439+ qp->resp.rkey = reth_rkey(pkt);
467440 }
468441
469442 static void qp_resp_from_atmeth(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
@@ -474,6 +447,10 @@ static void qp_resp_from_atmeth(struct rxe_qp *qp, struct rxe_pkt_info *pkt) · 4 unmodified lines
474447 qp->resp.resid = sizeof(u64);
475448 }
476449
450+/* resolve the packet rkey to qp->resp.mr or set qp->resp.mr to NULL
451+ * if an invalid rkey is received or the rdma length is zero. For middle
452+ * or last packets use the stored value of mr.
453+ */
477454 static enum resp_states check_rkey(struct rxe_qp *qp,
478455 struct rxe_pkt_info *pkt)
479456 {
@@ -510,10 +487,12 @@ static enum resp_states check_rkey(struct rxe_qp *qp, · 30 unmodified lines
510487 return RESPST_EXECUTE;
511488 }
512489
513− /* A zero-byte op is not required to set an addr or rkey. See C9-88 */
490+ /* A zero-byte read or write op is not required to
491+ * set an addr or rkey. See C9-88
492+ */
514493 if ((pkt->mask & RXE_READ_OR_WRITE_MASK) &&
515− (pkt->mask & RXE_RETH_MASK) &&
516− reth_len(pkt) == 0) {
494+ (pkt->mask & RXE_RETH_MASK) && reth_len(pkt) == 0) {
495+ qp->resp.mr = NULL;
517496 return RESPST_EXECUTE;
518497 }
519498
@@ -592,6 +571,7 @@ skip_check_range: · 72 unmodified lines
592571 return RESPST_EXECUTE;
593572
594573 err:
574+ qp->resp.mr = NULL;
595575 if (mr)
596576 rxe_put(mr);
597577 if (mw)
@@ -725,17 +705,12 @@ static enum resp_states process_flush(struct rxe_qp *qp, · 127 unmodified lines
725705 return RESPST_ACKNOWLEDGE;
726706 }
727707
728−/* Guarantee atomicity of atomic operations at the machine level. */
729−static DEFINE_SPINLOCK(atomic_ops_lock);
730−
731708 static enum resp_states atomic_reply(struct rxe_qp *qp,
732− struct rxe_pkt_info *pkt)
709+ struct rxe_pkt_info *pkt)
733710 {
734− u64 *vaddr;
735− enum resp_states ret;
736711 struct rxe_mr *mr = qp->resp.mr;
737712 struct resp_res *res = qp->resp.res;
738− u64 value;
713+ int err;
739714
740715 if (!res) {
741716 res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_MASK);
@@ -743,32 +718,14 @@ static enum resp_states atomic_reply(struct rxe_qp *qp, · 1 unmodified line
743718 }
744719
745720 if (!res->replay) {
746− if (mr->state != RXE_MR_STATE_VALID) {
747− ret = RESPST_ERR_RKEY_VIOLATION;
748− goto out;
749− }
750−
751− vaddr = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset,
752− sizeof(u64));
753−
754− /* check vaddr is 8 bytes aligned. */
755− if (!vaddr || (uintptr_t)vaddr & 7) {
756− ret = RESPST_ERR_MISALIGNED_ATOMIC;
757− goto out;
758− }
721+ u64 iova = qp->resp.va + qp->resp.offset;
759722
760− spin_lock_bh(&atomic_ops_lock);
761− res->atomic.orig_val = value = *vaddr;
762−
763− if (pkt->opcode == IB_OPCODE_RC_COMPARE_SWAP) {
764− if (value == atmeth_comp(pkt))
765− value = atmeth_swap_add(pkt);
766− } else {
767− value += atmeth_swap_add(pkt);
768− }
769−
770− *vaddr = value;
771− spin_unlock_bh(&atomic_ops_lock);
723+ err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode,
724+ atmeth_comp(pkt),
725+ atmeth_swap_add(pkt),
726+ &res->atomic.orig_val);
727+ if (err)
728+ return err;
772729
773730 qp->resp.msn++;
774731
@@ -780,35 +737,35 @@ static enum resp_states atomic_reply(struct rxe_qp *qp, · 5 unmodified lines
780737 qp->resp.status = IB_WC_SUCCESS;
781738 }
782739
783− ret = RESPST_ACKNOWLEDGE;
784−out:
785− return ret;
740+ return RESPST_ACKNOWLEDGE;
786741 }
787742
788−#ifdef CONFIG_64BIT
789−static enum resp_states do_atomic_write(struct rxe_qp *qp,
790− struct rxe_pkt_info *pkt)
743+static enum resp_states atomic_write_reply(struct rxe_qp *qp,
744+ struct rxe_pkt_info *pkt)
791745 {
792− struct rxe_mr *mr = qp->resp.mr;
793− int payload = payload_size(pkt);
794− u64 src, *dst;
795−
796− if (mr->state != RXE_MR_STATE_VALID)
797− return RESPST_ERR_RKEY_VIOLATION;
746+ struct resp_res *res = qp->resp.res;
747+ struct rxe_mr *mr;
748+ u64 value;
749+ u64 iova;
750+ int err;
798751
799− memcpy(&src, payload_addr(pkt), payload);
752+ if (!res) {
753+ res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
754+ qp->resp.res = res;
755+ }
800756
801− dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
802− /* check vaddr is 8 bytes aligned. */
803− if (!dst || (uintptr_t)dst & 7)
804− return RESPST_ERR_MISALIGNED_ATOMIC;
757+ if (res->replay)
758+ return RESPST_ACKNOWLEDGE;
805759
806− /* Do atomic write after all prior operations have completed */
807− smp_store_release(dst, src);
760+ mr = qp->resp.mr;
761+ value = *(u64 *)payload_addr(pkt);
762+ iova = qp->resp.va + qp->resp.offset;
808763
809− /* decrease resp.resid to zero */
810− qp->resp.resid -= sizeof(payload);
764+ err = rxe_mr_do_atomic_write(mr, iova, value);
765+ if (err)
766+ return err;
811767
768+ qp->resp.resid = 0;
812769 qp->resp.msn++;
813770
814771 /* next expected psn, read handles this separately */
@@ -817,29 +774,8 @@ static enum resp_states do_atomic_write(struct rxe_qp *qp, · 2 unmodified lines
817774
818775 qp->resp.opcode = pkt->opcode;
819776 qp->resp.status = IB_WC_SUCCESS;
820− return RESPST_ACKNOWLEDGE;
821−}
822−#else
823−static enum resp_states do_atomic_write(struct rxe_qp *qp,
824− struct rxe_pkt_info *pkt)
825−{
826− return RESPST_ERR_UNSUPPORTED_OPCODE;
827−}
828−#endif /* CONFIG_64BIT */
829777
830−static enum resp_states atomic_write_reply(struct rxe_qp *qp,
831− struct rxe_pkt_info *pkt)
832−{
833− struct resp_res *res = qp->resp.res;
834−
835− if (!res) {
836− res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
837− qp->resp.res = res;
838− }
839−
840− if (res->replay)
841− return RESPST_ACKNOWLEDGE;
842− return do_atomic_write(qp, pkt);
778+ return RESPST_ACKNOWLEDGE;
843779 }
844780
845781 static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,
@@ -966,7 +902,11 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 120 unmodified lines
966902 }
967903
968904 if (res->state == rdatm_res_state_new) {
969− if (!res->replay) {
905+ if (!res->replay || qp->resp.length == 0) {
906+ /* if length == 0 mr will be NULL (is ok)
907+ * otherwise qp->resp.mr holds a ref on mr
908+ * which we transfer to mr and drop below.
909+ */
970910 mr = qp->resp.mr;
971911 qp->resp.mr = NULL;
972912 } else {
@@ -980,6 +920,10 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 7 unmodified lines
980920 else
981921 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
982922 } else {
923+ /* re-lookup mr from rkey on all later packets.
924+ * length will be non-zero. This can fail if someone
925+ * modifies or destroys the mr since the first packet.
926+ */
983927 mr = rxe_recheck_mr(qp, res->read.rkey);
984928 if (!mr)
985929 return RESPST_ERR_RKEY_VIOLATION;
@@ -997,18 +941,16 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 11 unmodified lines
997941 skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload,
998942 res->cur_psn, AETH_ACK_UNLIMITED);
999943 if (!skb) {
1000− if (mr)
1001− rxe_put(mr);
1002− return RESPST_ERR_RNR;
944+ state = RESPST_ERR_RNR;
945+ goto err_out;
1003946 }
1004947
1005948 err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
1006949 payload, RXE_FROM_MR_OBJ);
1007− if (mr)
1008− rxe_put(mr);
1009950 if (err) {
1010951 kfree_skb(skb);
1011− return RESPST_ERR_RKEY_VIOLATION;
952+ state = RESPST_ERR_RKEY_VIOLATION;
953+ goto err_out;
1012954 }
1013955
1014956 if (bth_pad(&ack_pkt)) {
@@ -1017,9 +959,12 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 2 unmodified lines
1017959 memset(pad, 0, bth_pad(&ack_pkt));
1018960 }
1019961
962+ /* rxe_xmit_packet always consumes the skb */
1020963 err = rxe_xmit_packet(qp, &ack_pkt, skb);
1021− if (err)
1022− return RESPST_ERR_RNR;
964+ if (err) {
965+ state = RESPST_ERR_RNR;
966+ goto err_out;
967+ }
1023968
1024969 res->read.va += payload;
1025970 res->read.resid -= payload;
@@ -1036,6 +981,9 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 10 unmodified lines
1036981 state = RESPST_CLEANUP;
1037982 }
1038983
984+err_out:
985+ if (mr)
986+ rxe_put(mr);
1039987 return state;
1040988 }
1041989
drivers/infiniband/sw/rxe/rxe_verbs.c+53−62
@@ -245,7 +245,7 @@ static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr) · 244 unmodified lines
245245 int num_sge = ibwr->num_sge;
246246 int full;
247247
248− full = queue_full(rq->queue, QUEUE_TYPE_TO_DRIVER);
248+ full = queue_full(rq->queue, QUEUE_TYPE_FROM_ULP);
249249 if (unlikely(full))
250250 return -ENOMEM;
251251
@@ -256,7 +256,7 @@ static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr) · 4 unmodified lines
256256 for (i = 0; i < num_sge; i++)
257257 length += ibwr->sg_list[i].length;
258258
259− recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_TO_DRIVER);
259+ recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_FROM_ULP);
260260 recv_wqe->wr_id = ibwr->wr_id;
261261
262262 memcpy(recv_wqe->dma.sge, ibwr->sg_list,
@@ -268,7 +268,7 @@ static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr) · 5 unmodified lines
268268 recv_wqe->dma.cur_sge = 0;
269269 recv_wqe->dma.sge_offset = 0;
270270
271− queue_advance_producer(rq->queue, QUEUE_TYPE_TO_DRIVER);
271+ queue_advance_producer(rq->queue, QUEUE_TYPE_FROM_ULP);
272272
273273 return 0;
274274 }
@@ -623,17 +623,17 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr, · 348 unmodified lines
623623
624624 spin_lock_irqsave(&qp->sq.sq_lock, flags);
625625
626− full = queue_full(sq->queue, QUEUE_TYPE_TO_DRIVER);
626+ full = queue_full(sq->queue, QUEUE_TYPE_FROM_ULP);
627627
628628 if (unlikely(full)) {
629629 spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
630630 return -ENOMEM;
631631 }
632632
633− send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_TO_DRIVER);
633+ send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_FROM_ULP);
634634 init_send_wqe(qp, ibwr, mask, length, send_wqe);
635635
636− queue_advance_producer(sq->queue, QUEUE_TYPE_TO_DRIVER);
636+ queue_advance_producer(sq->queue, QUEUE_TYPE_FROM_ULP);
637637
638638 spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
639639
@@ -821,12 +821,12 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) · 181 unmodified lines
821821
822822 spin_lock_irqsave(&cq->cq_lock, flags);
823823 for (i = 0; i < num_entries; i++) {
824− cqe = queue_head(cq->queue, QUEUE_TYPE_FROM_DRIVER);
824+ cqe = queue_head(cq->queue, QUEUE_TYPE_TO_ULP);
825825 if (!cqe)
826826 break;
827827
828828 memcpy(wc++, &cqe->ibwc, sizeof(*wc));
829− queue_advance_consumer(cq->queue, QUEUE_TYPE_FROM_DRIVER);
829+ queue_advance_consumer(cq->queue, QUEUE_TYPE_TO_ULP);
830830 }
831831 spin_unlock_irqrestore(&cq->cq_lock, flags);
832832
@@ -838,7 +838,7 @@ static int rxe_peek_cq(struct ib_cq *ibcq, int wc_cnt) · 5 unmodified lines
838838 struct rxe_cq *cq = to_rcq(ibcq);
839839 int count;
840840
841− count = queue_count(cq->queue, QUEUE_TYPE_FROM_DRIVER);
841+ count = queue_count(cq->queue, QUEUE_TYPE_TO_ULP);
842842
843843 return (count > wc_cnt) ? wc_cnt : count;
844844 }
@@ -854,7 +854,7 @@ static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) · 9 unmodified lines
854854 if (cq->notify != IB_CQ_NEXT_COMP)
855855 cq->notify = flags & IB_CQ_SOLICITED_MASK;
856856
857− empty = queue_empty(cq->queue, QUEUE_TYPE_FROM_DRIVER);
857+ empty = queue_empty(cq->queue, QUEUE_TYPE_TO_ULP);
858858
859859 if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty)
860860 ret = 1;
@@ -869,10 +869,17 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access) · 8 unmodified lines
869869 struct rxe_dev *rxe = to_rdev(ibpd->device);
870870 struct rxe_pd *pd = to_rpd(ibpd);
871871 struct rxe_mr *mr;
872+ int err;
872873
873− mr = rxe_alloc(&rxe->mr_pool);
874− if (!mr)
875− return ERR_PTR(-ENOMEM);
874+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
875+ if (!mr) {
876+ err = -ENOMEM;
877+ goto err_out;
878+ }
879+
880+ err = rxe_add_to_pool(&rxe->mr_pool, mr);
881+ if (err)
882+ goto err_free;
876883
877884 rxe_get(pd);
878885 mr->ibmr.pd = ibpd;
@@ -880,8 +887,12 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access) · 1 unmodified line
880887
881888 rxe_mr_init_dma(access, mr);
882889 rxe_finalize(mr);
883−
884890 return &mr->ibmr;
891+
892+err_free:
893+ kfree(mr);
894+err_out:
895+ return ERR_PTR(err);
885896 }
886897
887898 static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
@@ -895,9 +906,15 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, · 7 unmodified lines
895906 struct rxe_pd *pd = to_rpd(ibpd);
896907 struct rxe_mr *mr;
897908
898− mr = rxe_alloc(&rxe->mr_pool);
899− if (!mr)
900− return ERR_PTR(-ENOMEM);
909+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
910+ if (!mr) {
911+ err = -ENOMEM;
912+ goto err_out;
913+ }
914+
915+ err = rxe_add_to_pool(&rxe->mr_pool, mr);
916+ if (err)
917+ goto err_free;
901918
902919 rxe_get(pd);
903920 mr->ibmr.pd = ibpd;
@@ -905,14 +922,16 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, · 1 unmodified line
905922
906923 err = rxe_mr_init_user(rxe, start, length, iova, access, mr);
907924 if (err)
908− goto err1;
925+ goto err_cleanup;
909926
910927 rxe_finalize(mr);
911−
912928 return &mr->ibmr;
913929
914−err1:
930+err_cleanup:
915931 rxe_cleanup(mr);
932+err_free:
933+ kfree(mr);
934+err_out:
916935 return ERR_PTR(err);
917936 }
918937
@@ -927,9 +946,15 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, · 8 unmodified lines
927946 if (mr_type != IB_MR_TYPE_MEM_REG)
928947 return ERR_PTR(-EINVAL);
929948
930− mr = rxe_alloc(&rxe->mr_pool);
931− if (!mr)
932− return ERR_PTR(-ENOMEM);
949+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
950+ if (!mr) {
951+ err = -ENOMEM;
952+ goto err_out;
953+ }
954+
955+ err = rxe_add_to_pool(&rxe->mr_pool, mr);
956+ if (err)
957+ goto err_free;
933958
934959 rxe_get(pd);
935960 mr->ibmr.pd = ibpd;
@@ -937,53 +962,19 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, · 1 unmodified line
937962
938963 err = rxe_mr_init_fast(max_num_sg, mr);
939964 if (err)
940− goto err1;
965+ goto err_cleanup;
941966
942967 rxe_finalize(mr);
943−
944968 return &mr->ibmr;
945969
946−err1:
970+err_cleanup:
947971 rxe_cleanup(mr);
972+err_free:
973+ kfree(mr);
974+err_out:
948975 return ERR_PTR(err);
949976 }
950977
951−static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
952−{
953− struct rxe_mr *mr = to_rmr(ibmr);
954− struct rxe_map *map;
955− struct rxe_phys_buf *buf;
956−
957− if (unlikely(mr->nbuf == mr->num_buf))
958− return -ENOMEM;
959−
960− map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
961− buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
962−
963− buf->addr = addr;
964− buf->size = ibmr->page_size;
965− mr->nbuf++;
966−
967− return 0;
968−}
969−
970−static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
971− int sg_nents, unsigned int *sg_offset)
972−{
973− struct rxe_mr *mr = to_rmr(ibmr);
974− int n;
975−
976− mr->nbuf = 0;
977−
978− n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
979−
980− mr->page_shift = ilog2(ibmr->page_size);
981− mr->page_mask = ibmr->page_size - 1;
982− mr->offset = ibmr->iova & mr->page_mask;
983−
984− return n;
985−}
986−
987978 static ssize_t parent_show(struct device *device,
988979 struct device_attribute *attr, char *buf)
989980 {
drivers/infiniband/sw/rxe/rxe_verbs.h+10−22
@@ -283,17 +283,6 @@ enum rxe_mr_lookup_type { · 282 unmodified lines
283283 RXE_LOOKUP_REMOTE,
284284 };
285285
286−#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
287−
288−struct rxe_phys_buf {
289− u64 addr;
290− u64 size;
291−};
292−
293−struct rxe_map {
294− struct rxe_phys_buf buf[RXE_BUF_PER_MAP];
295−};
296−
297286 static inline int rkey_is_mw(u32 rkey)
298287 {
299288 u32 index = rkey >> 8;
@@ -310,25 +299,24 @@ struct rxe_mr { · 10 unmodified lines
310299 u32 lkey;
311300 u32 rkey;
312301 enum rxe_mr_state state;
313− u32 offset;
314302 int access;
303+ atomic_t num_mw;
315304
316− int page_shift;
317− int page_mask;
318− int map_shift;
319− int map_mask;
305+ unsigned int page_offset;
306+ unsigned int page_shift;
307+ u64 page_mask;
320308
321309 u32 num_buf;
322310 u32 nbuf;
323311
324− u32 max_buf;
325− u32 num_map;
326−
327− atomic_t num_mw;
328−
329− struct rxe_map **map;
312+ struct xarray page_list;
330313 };
331314
315+static inline unsigned int mr_page_size(struct rxe_mr *mr)
316+{
317+ return mr ? mr->ibmr.page_size : PAGE_SIZE;
318+}
319+
332320 enum rxe_mw_state {
333321 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
334322 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
drivers/infiniband/sw/siw/siw_mem.c+12−11
@@ -398,7 +398,7 @@ struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable) · 397 unmodified lines
398398
399399 mlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
400400
401− if (num_pages + atomic64_read(&mm_s->pinned_vm) > mlock_limit) {
401+ if (atomic64_add_return(num_pages, &mm_s->pinned_vm) > mlock_limit) {
402402 rv = -ENOMEM;
403403 goto out_sem_up;
404404 }
@@ -411,30 +411,27 @@ struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable) · 6 unmodified lines
411411 goto out_sem_up;
412412 }
413413 for (i = 0; num_pages; i++) {
414− int got, nents = min_t(int, num_pages, PAGES_PER_CHUNK);
415−
416− umem->page_chunk[i].plist =
414+ int nents = min_t(int, num_pages, PAGES_PER_CHUNK);
415+ struct page **plist =
417416 kcalloc(nents, sizeof(struct page *), GFP_KERNEL);
418− if (!umem->page_chunk[i].plist) {
417+
418+ if (!plist) {
419419 rv = -ENOMEM;
420420 goto out_sem_up;
421421 }
422− got = 0;
422+ umem->page_chunk[i].plist = plist;
423423 while (nents) {
424− struct page **plist = &umem->page_chunk[i].plist[got];
425−
426424 rv = pin_user_pages(first_page_va, nents, foll_flags,
427425 plist, NULL);
428426 if (rv < 0)
429427 goto out_sem_up;
430428
431429 umem->num_pages += rv;
432− atomic64_add(rv, &mm_s->pinned_vm);
433430 first_page_va += rv * PAGE_SIZE;
431+ plist += rv;
434432 nents -= rv;
435− got += rv;
433+ num_pages -= rv;
436434 }
437− num_pages -= got;
438435 }
439436 out_sem_up:
440437 mmap_read_unlock(mm_s);
@@ -442,6 +439,10 @@ out_sem_up: · 1 unmodified line
442439 if (rv > 0)
443440 return umem;
444441
442+ /* Adjust accounting for pages not pinned */
443+ if (num_pages)
444+ atomic64_sub(num_pages, &mm_s->pinned_vm);
445+
445446 siw_umem_release(umem, false);
446447
447448 return ERR_PTR(rv);
drivers/infiniband/ulp/ipoib/ipoib_main.c+1−1
@@ -742,7 +742,7 @@ void ipoib_flush_paths(struct net_device *dev) · 741 unmodified lines
742742
743743 static void path_rec_completion(int status,
744744 struct sa_path_rec *pathrec,
745− int num_prs, void *path_ptr)
745+ unsigned int num_prs, void *path_ptr)
746746 {
747747 struct ipoib_path *path = path_ptr;
748748 struct net_device *dev = path->dev;
drivers/infiniband/ulp/srp/ib_srp.c+1−1
@@ -699,7 +699,7 @@ static void srp_free_ch_ib(struct srp_target_port *target, · 698 unmodified lines
699699
700700 static void srp_path_rec_completion(int status,
701701 struct sa_path_rec *pathrec,
702− int num_paths, void *ch_ptr)
702+ unsigned int num_paths, void *ch_ptr)
703703 {
704704 struct srp_rdma_ch *ch = ch_ptr;
705705 struct srp_target_port *target = ch->target;
drivers/net/ethernet/mellanox/mlx4/qp.c+9−5
@@ -46,6 +46,13 @@ · 45 unmodified lines
4646 #define MLX4_BF_QP_SKIP_MASK 0xc0
4747 #define MLX4_MAX_BF_QP_RANGE 0x40
4848
49+void mlx4_put_qp(struct mlx4_qp *qp)
50+{
51+ if (refcount_dec_and_test(&qp->refcount))
52+ complete(&qp->free);
53+}
54+EXPORT_SYMBOL_GPL(mlx4_put_qp);
55+
4956 void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type)
5057 {
5158 struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
@@ -64,10 +71,8 @@ void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type) · 12 unmodified lines
6471 return;
6572 }
6673
74+ /* Need to call mlx4_put_qp() in event handler */
6775 qp->event(qp, event_type);
68−
69− if (refcount_dec_and_test(&qp->refcount))
70− complete(&qp->free);
7176 }
7277
7378 /* used for INIT/CLOSE port logic */
@@ -523,8 +528,7 @@ EXPORT_SYMBOL_GPL(mlx4_qp_remove); · 449 unmodified lines
523528
524529 void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp)
525530 {
526− if (refcount_dec_and_test(&qp->refcount))
527− complete(&qp->free);
531+ mlx4_put_qp(qp);
528532 wait_for_completion(&qp->free);
529533
530534 mlx4_qp_free_icm(dev, qp->qpn);
drivers/net/ethernet/mellanox/mlx5/core/en_main.c+21−1
@@ -668,6 +668,26 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) · 667 unmodified lines
668668 mlx5e_rq_shampo_hd_free(rq);
669669 }
670670
671+static __be32 mlx5e_get_terminate_scatter_list_mkey(struct mlx5_core_dev *dev)
672+{
673+ u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
674+ u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
675+ int res;
676+
677+ if (!MLX5_CAP_GEN(dev, terminate_scatter_list_mkey))
678+ return MLX5_TERMINATE_SCATTER_LIST_LKEY;
679+
680+ MLX5_SET(query_special_contexts_in, in, opcode,
681+ MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
682+ res = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);
683+ if (res)
684+ return MLX5_TERMINATE_SCATTER_LIST_LKEY;
685+
686+ res = MLX5_GET(query_special_contexts_out, out,
687+ terminate_scatter_list_mkey);
688+ return cpu_to_be32(res);
689+}
690+
671691 static int mlx5e_alloc_rq(struct mlx5e_params *params,
672692 struct mlx5e_xsk_param *xsk,
673693 struct mlx5e_rq_param *rqp,
@@ -832,7 +852,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, · 158 unmodified lines
832852 /* check if num_frags is not a pow of two */
833853 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
834854 wqe->data[f].byte_count = 0;
835− wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
855+ wqe->data[f].lkey = mlx5e_get_terminate_scatter_list_mkey(mdev);
836856 wqe->data[f].addr = 0;
837857 }
838858 }
include/linux/mlx4/qp.h+1−0
@@ -504,4 +504,5 @@ static inline u16 folded_qp(u32 q) · 503 unmodified lines
504504
505505 u16 mlx4_qp_roce_entropy(struct mlx4_dev *dev, u32 qpn);
506506
507+void mlx4_put_qp(struct mlx4_qp *qp);
507508 #endif /* MLX4_QP_H */
include/linux/mlx5/driver.h+0−1
@@ -742,7 +742,6 @@ enum { · 741 unmodified lines
742742
743743 enum {
744744 MKEY_CACHE_LAST_STD_ENTRY = 20,
745− MLX5_IMR_MTT_CACHE_ENTRY,
746745 MLX5_IMR_KSM_CACHE_ENTRY,
747746 MAX_MKEY_CACHE_ENTRIES
748747 };
include/linux/mlx5/mlx5_ifc.h+20−2
@@ -1510,7 +1510,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { · 1509 unmodified lines
15101510 u8 relaxed_ordering_write[0x1];
15111511 u8 relaxed_ordering_read[0x1];
15121512 u8 log_max_mkey[0x6];
1513− u8 reserved_at_f0[0x8];
1513+ u8 reserved_at_f0[0x6];
1514+ u8 terminate_scatter_list_mkey[0x1];
1515+ u8 repeated_mkey[0x1];
15141516 u8 dump_fill_mkey[0x1];
15151517 u8 reserved_at_f9[0x2];
15161518 u8 fast_teardown[0x1];
@@ -2183,6 +2185,17 @@ struct mlx5_ifc_cong_control_r_roce_ecn_rp_bits { · 666 unmodified lines
21832185 u8 reserved_at_360[0x4a0];
21842186 };
21852187
2188+struct mlx5_ifc_cong_control_r_roce_general_bits {
2189+ u8 reserved_at_0[0x80];
2190+
2191+ u8 reserved_at_80[0x10];
2192+ u8 rtt_resp_dscp_valid[0x1];
2193+ u8 reserved_at_91[0x9];
2194+ u8 rtt_resp_dscp[0x6];
2195+
2196+ u8 reserved_at_a0[0x760];
2197+};
2198+
21862199 struct mlx5_ifc_cong_control_802_1qau_rp_bits {
21872200 u8 reserved_at_0[0x80];
21882201
@@ -4353,6 +4366,7 @@ union mlx5_ifc_cong_control_roce_ecn_auto_bits { · 2164 unmodified lines
43534366 struct mlx5_ifc_cong_control_802_1qau_rp_bits cong_control_802_1qau_rp;
43544367 struct mlx5_ifc_cong_control_r_roce_ecn_rp_bits cong_control_r_roce_ecn_rp;
43554368 struct mlx5_ifc_cong_control_r_roce_ecn_np_bits cong_control_r_roce_ecn_np;
4369+ struct mlx5_ifc_cong_control_r_roce_general_bits cong_control_r_roce_general;
43564370 u8 reserved_at_0[0x800];
43574371 };
43584372
@@ -5261,7 +5275,11 @@ struct mlx5_ifc_query_special_contexts_out_bits { · 902 unmodified lines
52615275
52625276 u8 null_mkey[0x20];
52635277
5264− u8 reserved_at_a0[0x60];
5278+ u8 terminate_scatter_list_mkey[0x20];
5279+
5280+ u8 repeated_mkey[0x20];
5281+
5282+ u8 reserved_at_a0[0x20];
52655283 };
52665284
52675285 struct mlx5_ifc_query_special_contexts_in_bits {
include/linux/mlx5/qp.h+1−1
@@ -36,7 +36,7 @@ · 35 unmodified lines
3636 #include <linux/mlx5/device.h>
3737 #include <linux/mlx5/driver.h>
3838
39−#define MLX5_INVALID_LKEY 0x100
39+#define MLX5_TERMINATE_SCATTER_LIST_LKEY cpu_to_be32(0x100)
4040 /* UMR (3 WQE_BB's) + SIG (3 WQE_BB's) + PSV (mem) + PSV (wire) */
4141 #define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 8)
4242 #define MLX5_DIF_SIZE 8
include/rdma/ib_sa.h+1−1
@@ -414,7 +414,7 @@ int ib_sa_path_rec_get(struct ib_sa_client *client, struct ib_device *device, · 413 unmodified lines
414414 ib_sa_comp_mask comp_mask, unsigned long timeout_ms,
415415 gfp_t gfp_mask,
416416 void (*callback)(int status, struct sa_path_rec *resp,
417− int num_prs, void *context),
417+ unsigned int num_prs, void *context),
418418 void *context, struct ib_sa_query **query);
419419
420420 struct ib_sa_multicast {
include/rdma/ib_umem.h+0−1
@@ -25,7 +25,6 @@ struct ib_umem { · 24 unmodified lines
2525 u32 writable : 1;
2626 u32 is_odp : 1;
2727 u32 is_dmabuf : 1;
28− struct work_struct work;
2928 struct sg_append_table sgt_append;
3029 };
3130
include/rdma/ib_verbs.h+1−1
@@ -1168,7 +1168,7 @@ enum ib_qp_create_flags { · 1167 unmodified lines
11681168 */
11691169
11701170 struct ib_qp_init_attr {
1171− /* Consumer's event_handler callback must not block */
1171+ /* This callback occurs in workqueue context */
11721172 void (*event_handler)(struct ib_event *, void *);
11731173
11741174 void *qp_context;
include/rdma/rdma_cm.h+0−1
@@ -49,7 +49,6 @@ struct rdma_addr { · 48 unmodified lines
4949 struct rdma_dev_addr dev_addr;
5050 };
5151
52−#define RDMA_PRIMARY_PATH_MAX_REC_NUM 3
5352 struct rdma_route {
5453 struct rdma_addr addr;
5554 struct sa_path_rec *path_rec;
include/rdma/restrack.h+2−2
@@ -162,8 +162,8 @@ struct rdma_restrack_entry *rdma_restrack_get_byid(struct ib_device *dev, · 161 unmodified lines
162162 * rdma_restrack_no_track() - don't add resource to the DB
163163 * @res: resource entry
164164 *
165− * Every user of thie API should be cross examined.
166− * Probaby you don't need to use this function.
165+ * Every user of this API should be cross examined.
166+ * Probably you don't need to use this function.
167167 */
168168 static inline void rdma_restrack_no_track(struct rdma_restrack_entry *res)
169169 {
include/uapi/rdma/hns-abi.h+4−0
@@ -87,10 +87,14 @@ struct hns_roce_ib_create_qp_resp { · 86 unmodified lines
8787
8888 enum {
8989 HNS_ROCE_EXSGE_FLAGS = 1 << 0,
90+ HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1,
91+ HNS_ROCE_CQE_INLINE_FLAGS = 1 << 2,
9092 };
9193
9294 enum {
9395 HNS_ROCE_RSP_EXSGE_FLAGS = 1 << 0,
96+ HNS_ROCE_RSP_RQ_INLINE_FLAGS = 1 << 1,
97+ HNS_ROCE_RSP_CQE_INLINE_FLAGS = 1 << 2,
9498 };
9599
96100 struct hns_roce_ib_alloc_ucontext_resp {