History

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull rdma updates from Jason Gunthorpe: "Another small cycle. Mostly cleanups and bug fixes, quite a bit assisted from bots. There are a few new syzkaller splats that haven't been solved yet but they should get into the rcs in a few weeks, I think. Summary: - Update drivers to use common helpers for GUIDs, pkeys, bitmaps, memset_startat, and others - General code cleanups from bots - Simplify some of the rxe pool code in preparation for a larger rework - Clean out old stuff from hns, including all support for hip06 devices - Fix a bug where GID table entries could be missed if the table had holes in it - Rename paths and sessions in rtrs for better understandability - Consolidate the roce source port selection code - NDR speed support in mlx5" * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (83 commits) RDMA/irdma: Remove the redundant return RDMA/rxe: Use the standard method to produce udp source port RDMA/irdma: Make the source udp port vary RDMA/hns: Replace get_udp_sport with rdma_get_udp_sport RDMA/core: Calculate UDP source port based on flow label or lqpn/rqpn IB/qib: Fix typos RDMA/rtrs-clt: Rename rtrs_clt to rtrs_clt_sess RDMA/rtrs-srv: Rename rtrs_srv to rtrs_srv_sess RDMA/rtrs-clt: Rename rtrs_clt_sess to rtrs_clt_path RDMA/rtrs-srv: Rename rtrs_srv_sess to rtrs_srv_path RDMA/rtrs: Rename rtrs_sess to rtrs_path RDMA/hns: Modify the hop num of HIP09 EQ to 1 IB/iser: Align coding style across driver IB/iser: Remove un-needed casting to/from void pointer IB/iser: Don't suppress send completions IB/iser: Rename ib_ret local variable IB/iser: Fix RNR errors IB/iser: Remove deprecated pi_guard module param IB/mlx5: Expose NDR speed through MAD RDMA/cxgb4: Set queue pair state when being queried ...

Linus Torvalds committed 4 years ago

Commit 747c19e2 parents feb7a43 c0fe82b105 files · +2232 −8797Browse files

Unified · Split

105 files
  1. drivers/block/rnbd/rnbd-clt.c +2 −2
  2. drivers/block/rnbd/rnbd-clt.h +1 −1
  3. drivers/block/rnbd/rnbd-srv.c +8 −8
  4. drivers/block/rnbd/rnbd-srv.h +1 −1
  5. drivers/infiniband/core/cache.c +9 −3
  6. drivers/infiniband/core/cma.c +11 −7
  7. drivers/infiniband/core/device.c +2 −1
  8. drivers/infiniband/core/sysfs.c +2 −1
  9. drivers/infiniband/core/umem_odp.c +1 −2
  10. drivers/infiniband/core/uverbs_cmd.c +0 −1
  11. drivers/infiniband/hw/bnxt_re/ib_verbs.c +4 −5
  12. drivers/infiniband/hw/bnxt_re/main.c +1 −2
  13. drivers/infiniband/hw/bnxt_re/qplib_fp.c +5 −6
  14. drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +4 −8
  15. drivers/infiniband/hw/bnxt_re/qplib_rcfw.h +0 −1
  16. drivers/infiniband/hw/bnxt_re/qplib_res.c +0 −50
  17. drivers/infiniband/hw/bnxt_re/qplib_res.h +0 −7
  18. drivers/infiniband/hw/bnxt_re/qplib_sp.c +1 −98
  19. drivers/infiniband/hw/bnxt_re/qplib_sp.h +0 −9
  20. drivers/infiniband/hw/cxgb4/cm.c +3 −2
  21. drivers/infiniband/hw/cxgb4/id_table.c +6 −11
  22. drivers/infiniband/hw/cxgb4/provider.c +5 −3
  23. drivers/infiniband/hw/cxgb4/qp.c +1 −0
  24. drivers/infiniband/hw/hfi1/user_sdma.c +3 −5
  25. drivers/infiniband/hw/hns/Kconfig +2 −15
  26. drivers/infiniband/hw/hns/Makefile +0 −5
  27. drivers/infiniband/hw/hns/hns_roce_ah.c +2 −3
  28. drivers/infiniband/hw/hns/hns_roce_alloc.c +1 −2
  29. drivers/infiniband/hw/hns/hns_roce_cmd.c +5 −6
  30. drivers/infiniband/hw/hns/hns_roce_common.h +0 −202
  31. drivers/infiniband/hw/hns/hns_roce_cq.c +0 −13
  32. drivers/infiniband/hw/hns/hns_roce_db.c +0 −1
  33. drivers/infiniband/hw/hns/hns_roce_device.h +24 −84
  34. drivers/infiniband/hw/hns/hns_roce_hem.c +0 −1
  35. drivers/infiniband/hw/hns/hns_roce_hw_v1.c +0 −4675
  36. drivers/infiniband/hw/hns/hns_roce_hw_v1.h +0 −1147
  37. drivers/infiniband/hw/hns/hns_roce_hw_v2.c +23 −26
  38. drivers/infiniband/hw/hns/hns_roce_hw_v2.h +4 −18
  39. drivers/infiniband/hw/hns/hns_roce_main.c +30 −55
  40. drivers/infiniband/hw/hns/hns_roce_mr.c +10 −22
  41. drivers/infiniband/hw/hns/hns_roce_pd.c +3 −14
  42. drivers/infiniband/hw/hns/hns_roce_qp.c +61 −32
  43. drivers/infiniband/hw/irdma/i40iw_if.c +1 −1
  44. drivers/infiniband/hw/irdma/pble.h +1 −1
  45. drivers/infiniband/hw/irdma/verbs.c +10 −21
  46. drivers/infiniband/hw/mlx4/main.c +12 −22
  47. drivers/infiniband/hw/mlx5/cq.c +4 −1
  48. drivers/infiniband/hw/mlx5/mad.c +14 −9
  49. drivers/infiniband/hw/mlx5/mlx5_ib.h +2 −10
  50. drivers/infiniband/hw/mthca/mthca_allocator.c +5 −10
  51. drivers/infiniband/hw/mthca/mthca_mr.c +11 −14
  52. drivers/infiniband/hw/mthca/mthca_provider.c +6 −14
  53. drivers/infiniband/hw/ocrdma/ocrdma_hw.c +6 −10
  54. drivers/infiniband/hw/ocrdma/ocrdma_main.c +2 −15
  55. drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +8 −10
  56. drivers/infiniband/hw/ocrdma/ocrdma_verbs.h +0 −1
  57. drivers/infiniband/hw/qedr/verbs.c +2 −0
  58. drivers/infiniband/hw/qib/qib_iba6120.c +1 −1
  59. drivers/infiniband/hw/qib/qib_iba7220.c +1 −1
  60. drivers/infiniband/hw/qib/qib_iba7322.c +1 −1
  61. drivers/infiniband/hw/usnic/usnic_ib_sysfs.c +2 −1
  62. drivers/infiniband/hw/usnic/usnic_ib_verbs.c +3 −5
  63. drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c +5 −5
  64. drivers/infiniband/sw/rxe/Makefile +0 −1
  65. drivers/infiniband/sw/rxe/rxe.c +0 −4
  66. drivers/infiniband/sw/rxe/rxe.h +0 −2
  67. drivers/infiniband/sw/rxe/rxe_comp.c +3 −5
  68. drivers/infiniband/sw/rxe/rxe_cq.c +10 −14
  69. drivers/infiniband/sw/rxe/rxe_loc.h +5 −5
  70. drivers/infiniband/sw/rxe/rxe_mcast.c +5 −6
  71. drivers/infiniband/sw/rxe/rxe_mr.c +3 −3
  72. drivers/infiniband/sw/rxe/rxe_mw.c +9 −12
  73. drivers/infiniband/sw/rxe/rxe_net.c +2 −7
  74. drivers/infiniband/sw/rxe/rxe_opcode.c +370 −369
  75. drivers/infiniband/sw/rxe/rxe_pool.c +84 −93
  76. drivers/infiniband/sw/rxe/rxe_pool.h +26 −28
  77. drivers/infiniband/sw/rxe/rxe_qp.c +3 −3
  78. drivers/infiniband/sw/rxe/rxe_queue.c +4 −5
  79. drivers/infiniband/sw/rxe/rxe_req.c +5 −11
  80. drivers/infiniband/sw/rxe/rxe_srq.c +1 −1
  81. drivers/infiniband/sw/rxe/rxe_sysfs.c +0 −119
  82. drivers/infiniband/sw/rxe/rxe_task.c +8 −10
  83. drivers/infiniband/sw/rxe/rxe_verbs.c +17 −17
  84. drivers/infiniband/sw/rxe/rxe_verbs.h +11 −13
  85. drivers/infiniband/sw/siw/siw_verbs.c +4 −2
  86. drivers/infiniband/ulp/iser/iscsi_iser.c +28 −48
  87. drivers/infiniband/ulp/iser/iscsi_iser.h +3 −20
  88. drivers/infiniband/ulp/iser/iser_initiator.c +41 −65
  89. drivers/infiniband/ulp/iser/iser_memory.c +24 −34
  90. drivers/infiniband/ulp/iser/iser_verbs.c +57 −81
  91. drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c +4 −4
  92. drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c +74 −71
  93. drivers/infiniband/ulp/rtrs/rtrs-clt.c +554 −535
  94. drivers/infiniband/ulp/rtrs/rtrs-clt.h +21 −20
  95. drivers/infiniband/ulp/rtrs/rtrs-pri.h +9 −9
  96. drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c +61 −60
  97. drivers/infiniband/ulp/rtrs/rtrs-srv.c +347 −337
  98. drivers/infiniband/ulp/rtrs/rtrs-srv.h +8 −8
  99. drivers/infiniband/ulp/rtrs/rtrs.c +49 −49
  100. drivers/infiniband/ulp/rtrs/rtrs.h +18 −16
  101. drivers/net/ethernet/chelsio/cxgb4/t4_msg.h +1 −1
  102. include/rdma/ib_mad.h +1 −0
  103. include/rdma/ib_smi.h +11 −1
  104. include/rdma/ib_verbs.h +17 −0
  105. include/uapi/rdma/hns-abi.h +2 −0
drivers/block/rnbd/rnbd-clt.c+2−2
@@ -433,7 +433,7 @@ static void msg_conf(void *priv, int errno) · 432 unmodified lines
433 schedule_work(&iu->work);433 schedule_work(&iu->work);
434}434}
435 435
436static int send_usr_msg(struct rtrs_clt *rtrs, int dir,436static int send_usr_msg(struct rtrs_clt_sess *rtrs, int dir,
437 struct rnbd_iu *iu, struct kvec *vec,437 struct rnbd_iu *iu, struct kvec *vec,
438 size_t len, struct scatterlist *sg, unsigned int sg_len,438 size_t len, struct scatterlist *sg, unsigned int sg_len,
439 void (*conf)(struct work_struct *work),439 void (*conf)(struct work_struct *work),
@@ -1010,7 +1010,7 @@ static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev, · 570 unmodified lines
1010 struct request *rq,1010 struct request *rq,
1011 struct rnbd_iu *iu)1011 struct rnbd_iu *iu)
1012{1012{
1013 struct rtrs_clt *rtrs = dev->sess->rtrs;1013 struct rtrs_clt_sess *rtrs = dev->sess->rtrs;
1014 struct rtrs_permit *permit = iu->permit;1014 struct rtrs_permit *permit = iu->permit;
1015 struct rnbd_msg_io msg;1015 struct rnbd_msg_io msg;
1016 struct rtrs_clt_req_ops req_ops;1016 struct rtrs_clt_req_ops req_ops;
drivers/block/rnbd/rnbd-clt.h+1−1
@@ -75,7 +75,7 @@ struct rnbd_cpu_qlist { · 74 unmodified lines
75 75
76struct rnbd_clt_session {76struct rnbd_clt_session {
77 struct list_head list;77 struct list_head list;
78 struct rtrs_clt *rtrs;78 struct rtrs_clt_sess *rtrs;
79 wait_queue_head_t rtrs_waitq;79 wait_queue_head_t rtrs_waitq;
80 bool rtrs_ready;80 bool rtrs_ready;
81 struct rnbd_cpu_qlist __percpu81 struct rnbd_cpu_qlist __percpu
drivers/block/rnbd/rnbd-srv.c+8−8
@@ -263,15 +263,15 @@ out: · 262 unmodified lines
263 kfree(srv_sess);263 kfree(srv_sess);
264}264}
265 265
266static int create_sess(struct rtrs_srv *rtrs)266static int create_sess(struct rtrs_srv_sess *rtrs)
267{267{
268 struct rnbd_srv_session *srv_sess;268 struct rnbd_srv_session *srv_sess;
269 char sessname[NAME_MAX];269 char pathname[NAME_MAX];
270 int err;270 int err;
271 271
272 err = rtrs_srv_get_sess_name(rtrs, sessname, sizeof(sessname));272 err = rtrs_srv_get_path_name(rtrs, pathname, sizeof(pathname));
273 if (err) {273 if (err) {
274 pr_err("rtrs_srv_get_sess_name(%s): %d\n", sessname, err);274 pr_err("rtrs_srv_get_path_name(%s): %d\n", pathname, err);
275 275
276 return err;276 return err;
277 }277 }
@@ -284,8 +284,8 @@ static int create_sess(struct rtrs_srv *rtrs) · 6 unmodified lines
284 offsetof(struct rnbd_dev_blk_io, bio),284 offsetof(struct rnbd_dev_blk_io, bio),
285 BIOSET_NEED_BVECS);285 BIOSET_NEED_BVECS);
286 if (err) {286 if (err) {
287 pr_err("Allocating srv_session for session %s failed\n",287 pr_err("Allocating srv_session for path %s failed\n",
288 sessname);288 pathname);
289 kfree(srv_sess);289 kfree(srv_sess);
290 return err;290 return err;
291 }291 }
@@ -298,14 +298,14 @@ static int create_sess(struct rtrs_srv *rtrs) · 6 unmodified lines
298 mutex_unlock(&sess_lock);298 mutex_unlock(&sess_lock);
299 299
300 srv_sess->rtrs = rtrs;300 srv_sess->rtrs = rtrs;
301 strscpy(srv_sess->sessname, sessname, sizeof(srv_sess->sessname));301 strscpy(srv_sess->sessname, pathname, sizeof(srv_sess->sessname));
302 302
303 rtrs_srv_set_sess_priv(rtrs, srv_sess);303 rtrs_srv_set_sess_priv(rtrs, srv_sess);
304 304
305 return 0;305 return 0;
306}306}
307 307
308static int rnbd_srv_link_ev(struct rtrs_srv *rtrs,308static int rnbd_srv_link_ev(struct rtrs_srv_sess *rtrs,
309 enum rtrs_srv_link_ev ev, void *priv)309 enum rtrs_srv_link_ev ev, void *priv)
310{310{
311 struct rnbd_srv_session *srv_sess = priv;311 struct rnbd_srv_session *srv_sess = priv;
drivers/block/rnbd/rnbd-srv.h+1−1
@@ -20,7 +20,7 @@ · 19 unmodified lines
20struct rnbd_srv_session {20struct rnbd_srv_session {
21 /* Entry inside global sess_list */21 /* Entry inside global sess_list */
22 struct list_head list;22 struct list_head list;
23 struct rtrs_srv *rtrs;23 struct rtrs_srv_sess *rtrs;
24 char sessname[NAME_MAX];24 char sessname[NAME_MAX];
25 int queue_depth;25 int queue_depth;
26 struct bio_set sess_bio_set;26 struct bio_set sess_bio_set;
drivers/infiniband/core/cache.c+9−3
@@ -956,7 +956,7 @@ int rdma_query_gid(struct ib_device *device, u32 port_num, · 955 unmodified lines
956{956{
957 struct ib_gid_table *table;957 struct ib_gid_table *table;
958 unsigned long flags;958 unsigned long flags;
959 int res = -EINVAL;959 int res;
960 960
961 if (!rdma_is_port_valid(device, port_num))961 if (!rdma_is_port_valid(device, port_num))
962 return -EINVAL;962 return -EINVAL;
@@ -964,9 +964,15 @@ int rdma_query_gid(struct ib_device *device, u32 port_num, · 1 unmodified line
964 table = rdma_gid_table(device, port_num);964 table = rdma_gid_table(device, port_num);
965 read_lock_irqsave(&table->rwlock, flags);965 read_lock_irqsave(&table->rwlock, flags);
966 966
967 if (index < 0 || index >= table->sz ||967 if (index < 0 || index >= table->sz) {
968 !is_gid_entry_valid(table->data_vec[index]))968 res = -EINVAL;
969 goto done;969 goto done;
970 }
971
972 if (!is_gid_entry_valid(table->data_vec[index])) {
973 res = -ENOENT;
974 goto done;
975 }
970 976
971 memcpy(gid, &table->data_vec[index]->attr.gid, sizeof(*gid));977 memcpy(gid, &table->data_vec[index]->attr.gid, sizeof(*gid));
972 res = 0;978 res = 0;
drivers/infiniband/core/cma.c+11−7
@@ -766,6 +766,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) · 765 unmodified lines
766 unsigned int p;766 unsigned int p;
767 u16 pkey, index;767 u16 pkey, index;
768 enum ib_port_state port_state;768 enum ib_port_state port_state;
769 int ret;
769 int i;770 int i;
770 771
771 cma_dev = NULL;772 cma_dev = NULL;
@@ -784,9 +785,14 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) · 12 unmodified lines
784 785
785 if (ib_get_cached_port_state(cur_dev->device, p, &port_state))786 if (ib_get_cached_port_state(cur_dev->device, p, &port_state))
786 continue;787 continue;
787 for (i = 0; !rdma_query_gid(cur_dev->device,788
788 p, i, &gid);789 for (i = 0; i < cur_dev->device->port_data[p].immutable.gid_tbl_len;
789 i++) {790 ++i) {
791 ret = rdma_query_gid(cur_dev->device, p, i,
792 &gid);
793 if (ret)
794 continue;
795
790 if (!memcmp(&gid, dgid, sizeof(gid))) {796 if (!memcmp(&gid, dgid, sizeof(gid))) {
791 cma_dev = cur_dev;797 cma_dev = cur_dev;
792 sgid = gid;798 sgid = gid;
@@ -4033,8 +4039,7 @@ static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, · 3240 unmodified lines
4033 4039
4034 memset(&req, 0, sizeof req);4040 memset(&req, 0, sizeof req);
4035 offset = cma_user_data_offset(id_priv);4041 offset = cma_user_data_offset(id_priv);
4036 req.private_data_len = offset + conn_param->private_data_len;4042 if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
4037 if (req.private_data_len < conn_param->private_data_len)
4038 return -EINVAL;4043 return -EINVAL;
4039 4044
4040 if (req.private_data_len) {4045 if (req.private_data_len) {
@@ -4093,8 +4098,7 @@ static int cma_connect_ib(struct rdma_id_private *id_priv, · 52 unmodified lines
4093 4098
4094 memset(&req, 0, sizeof req);4099 memset(&req, 0, sizeof req);
4095 offset = cma_user_data_offset(id_priv);4100 offset = cma_user_data_offset(id_priv);
4096 req.private_data_len = offset + conn_param->private_data_len;4101 if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
4097 if (req.private_data_len < conn_param->private_data_len)
4098 return -EINVAL;4102 return -EINVAL;
4099 4103
4100 if (req.private_data_len) {4104 if (req.private_data_len) {
drivers/infiniband/core/device.c+2−1
@@ -2461,7 +2461,8 @@ int ib_find_gid(struct ib_device *device, union ib_gid *gid, · 2460 unmodified lines
2461 ++i) {2461 ++i) {
2462 ret = rdma_query_gid(device, port, i, &tmp_gid);2462 ret = rdma_query_gid(device, port, i, &tmp_gid);
2463 if (ret)2463 if (ret)
2464 return ret;2464 continue;
2465
2465 if (!memcmp(&tmp_gid, gid, sizeof *gid)) {2466 if (!memcmp(&tmp_gid, gid, sizeof *gid)) {
2466 *port_num = port;2467 *port_num = port;
2467 if (index)2468 if (index)
drivers/infiniband/core/sysfs.c+2−1
@@ -433,6 +433,7 @@ static struct attribute *port_default_attrs[] = { · 432 unmodified lines
433 &ib_port_attr_link_layer.attr,433 &ib_port_attr_link_layer.attr,
434 NULL434 NULL
435};435};
436ATTRIBUTE_GROUPS(port_default);
436 437
437static ssize_t print_ndev(const struct ib_gid_attr *gid_attr, char *buf)438static ssize_t print_ndev(const struct ib_gid_attr *gid_attr, char *buf)
438{439{
@@ -774,7 +775,7 @@ static void ib_port_gid_attr_release(struct kobject *kobj) · 335 unmodified lines
774static struct kobj_type port_type = {775static struct kobj_type port_type = {
775 .release = ib_port_release,776 .release = ib_port_release,
776 .sysfs_ops = &port_sysfs_ops,777 .sysfs_ops = &port_sysfs_ops,
777 .default_attrs = port_default_attrs778 .default_groups = port_default_groups,
778};779};
779 780
780static struct kobj_type gid_attr_type = {781static struct kobj_type gid_attr_type = {
drivers/infiniband/core/umem_odp.c+1−2
@@ -227,7 +227,6 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_device *device, · 226 unmodified lines
227 const struct mmu_interval_notifier_ops *ops)227 const struct mmu_interval_notifier_ops *ops)
228{228{
229 struct ib_umem_odp *umem_odp;229 struct ib_umem_odp *umem_odp;
230 struct mm_struct *mm;
231 int ret;230 int ret;
232 231
233 if (WARN_ON_ONCE(!(access & IB_ACCESS_ON_DEMAND)))232 if (WARN_ON_ONCE(!(access & IB_ACCESS_ON_DEMAND)))
@@ -241,7 +240,7 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_device *device, · 7 unmodified lines
241 umem_odp->umem.length = size;240 umem_odp->umem.length = size;
242 umem_odp->umem.address = addr;241 umem_odp->umem.address = addr;
243 umem_odp->umem.writable = ib_access_writable(access);242 umem_odp->umem.writable = ib_access_writable(access);
244 umem_odp->umem.owning_mm = mm = current->mm;243 umem_odp->umem.owning_mm = current->mm;
245 umem_odp->notifier.ops = ops;244 umem_odp->notifier.ops = ops;
246 245
247 umem_odp->page_shift = PAGE_SHIFT;246 umem_odp->page_shift = PAGE_SHIFT;
drivers/infiniband/core/uverbs_cmd.c+0−1
@@ -1399,7 +1399,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, · 1398 unmodified lines
1399 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :1399 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1400 IB_SIGNAL_REQ_WR;1400 IB_SIGNAL_REQ_WR;
1401 attr.qp_type = cmd->qp_type;1401 attr.qp_type = cmd->qp_type;
1402 attr.create_flags = 0;
1403 1402
1404 attr.cap.max_send_wr = cmd->max_send_wr;1403 attr.cap.max_send_wr = cmd->max_send_wr;
1405 attr.cap.max_recv_wr = cmd->max_recv_wr;1404 attr.cap.max_recv_wr = cmd->max_recv_wr;
drivers/infiniband/hw/bnxt_re/ib_verbs.c+4−5
@@ -262,13 +262,12 @@ void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) · 261 unmodified lines
262int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,262int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
263 u16 index, u16 *pkey)263 u16 index, u16 *pkey)
264{264{
265 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);265 if (index > 0)
266 return -EINVAL;
266 267
267 /* Ignore port_num */268 *pkey = IB_DEFAULT_PKEY_FULL;
268 269
269 memset(pkey, 0, sizeof(*pkey));270 return 0;
270 return bnxt_qplib_get_pkey(&rdev->qplib_res,
271 &rdev->qplib_res.pkey_tbl, index, pkey);
272}271}
273 272
274int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,273int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
drivers/infiniband/hw/bnxt_re/main.c+1−2
@@ -893,7 +893,6 @@ static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, · 892 unmodified lines
893 struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq,893 struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq,
894 qplib_srq);894 qplib_srq);
895 struct ib_event ib_event;895 struct ib_event ib_event;
896 int rc = 0;
897 896
898 ib_event.device = &srq->rdev->ibdev;897 ib_event.device = &srq->rdev->ibdev;
899 ib_event.element.srq = &srq->ib_srq;898 ib_event.element.srq = &srq->ib_srq;
@@ -907,7 +906,7 @@ static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, · 7 unmodified lines
907 (*srq->ib_srq.event_handler)(&ib_event,906 (*srq->ib_srq.event_handler)(&ib_event,
908 srq->ib_srq.srq_context);907 srq->ib_srq.srq_context);
909 }908 }
910 return rc;909 return 0;
911}910}
912 911
913static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,912static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
drivers/infiniband/hw/bnxt_re/qplib_fp.c+5−6
@@ -46,6 +46,7 @@ · 45 unmodified lines
46#include <linux/delay.h>46#include <linux/delay.h>
47#include <linux/prefetch.h>47#include <linux/prefetch.h>
48#include <linux/if_ether.h>48#include <linux/if_ether.h>
49#include <rdma/ib_mad.h>
49 50
50#include "roce_hsi.h"51#include "roce_hsi.h"
51 52
@@ -1232,7 +1233,7 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 1180 unmodified lines
1232 struct bnxt_qplib_rcfw *rcfw = res->rcfw;1233 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1233 struct cmdq_modify_qp req;1234 struct cmdq_modify_qp req;
1234 struct creq_modify_qp_resp resp;1235 struct creq_modify_qp_resp resp;
1235 u16 cmd_flags = 0, pkey;1236 u16 cmd_flags = 0;
1236 u32 temp32[4];1237 u32 temp32[4];
1237 u32 bmask;1238 u32 bmask;
1238 int rc;1239 int rc;
@@ -1255,11 +1256,9 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) · 16 unmodified lines
1255 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)1256 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
1256 req.access = qp->access;1257 req.access = qp->access;
1257 1258
1258 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {1259 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY)
1259 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,1260 req.pkey = cpu_to_le16(IB_DEFAULT_PKEY_FULL);
1260 qp->pkey_index, &pkey))1261
1261 req.pkey = cpu_to_le16(pkey);
1262 }
1263 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)1262 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
1264 req.qkey = cpu_to_le32(qp->qkey);1263 req.qkey = cpu_to_le32(qp->qkey);
1265 1264
drivers/infiniband/hw/bnxt_re/qplib_rcfw.c+4−8
@@ -555,7 +555,7 @@ skip_ctx_setup: · 554 unmodified lines
555 555
556void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)556void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
557{557{
558 kfree(rcfw->cmdq.cmdq_bitmap);558 bitmap_free(rcfw->cmdq.cmdq_bitmap);
559 kfree(rcfw->qp_tbl);559 kfree(rcfw->qp_tbl);
560 kfree(rcfw->crsqe_tbl);560 kfree(rcfw->crsqe_tbl);
561 bnxt_qplib_free_hwq(rcfw->res, &rcfw->cmdq.hwq);561 bnxt_qplib_free_hwq(rcfw->res, &rcfw->cmdq.hwq);
@@ -572,7 +572,6 @@ int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res, · 10 unmodified lines
572 struct bnxt_qplib_sg_info sginfo = {};572 struct bnxt_qplib_sg_info sginfo = {};
573 struct bnxt_qplib_cmdq_ctx *cmdq;573 struct bnxt_qplib_cmdq_ctx *cmdq;
574 struct bnxt_qplib_creq_ctx *creq;574 struct bnxt_qplib_creq_ctx *creq;
575 u32 bmap_size = 0;
576 575
577 rcfw->pdev = res->pdev;576 rcfw->pdev = res->pdev;
578 cmdq = &rcfw->cmdq;577 cmdq = &rcfw->cmdq;
@@ -613,13 +612,10 @@ int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res, · 34 unmodified lines
613 if (!rcfw->crsqe_tbl)612 if (!rcfw->crsqe_tbl)
614 goto fail;613 goto fail;
615 614
616 bmap_size = BITS_TO_LONGS(rcfw->cmdq_depth) * sizeof(unsigned long);615 cmdq->cmdq_bitmap = bitmap_zalloc(rcfw->cmdq_depth, GFP_KERNEL);
617 cmdq->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL);
618 if (!cmdq->cmdq_bitmap)616 if (!cmdq->cmdq_bitmap)
619 goto fail;617 goto fail;
620 618
621 cmdq->bmap_size = bmap_size;
622
623 /* Allocate one extra to hold the QP1 entries */619 /* Allocate one extra to hold the QP1 entries */
624 rcfw->qp_tbl_size = qp_tbl_sz + 1;620 rcfw->qp_tbl_size = qp_tbl_sz + 1;
625 rcfw->qp_tbl = kcalloc(rcfw->qp_tbl_size, sizeof(struct bnxt_qplib_qp_node),621 rcfw->qp_tbl = kcalloc(rcfw->qp_tbl_size, sizeof(struct bnxt_qplib_qp_node),
@@ -667,8 +663,8 @@ void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) · 41 unmodified lines
667 iounmap(cmdq->cmdq_mbox.reg.bar_reg);663 iounmap(cmdq->cmdq_mbox.reg.bar_reg);
668 iounmap(creq->creq_db.reg.bar_reg);664 iounmap(creq->creq_db.reg.bar_reg);
669 665
670 indx = find_first_bit(cmdq->cmdq_bitmap, cmdq->bmap_size);666 indx = find_first_bit(cmdq->cmdq_bitmap, rcfw->cmdq_depth);
671 if (indx != cmdq->bmap_size)667 if (indx != rcfw->cmdq_depth)
672 dev_err(&rcfw->pdev->dev,668 dev_err(&rcfw->pdev->dev,
673 "disabling RCFW with pending cmd-bit %lx\n", indx);669 "disabling RCFW with pending cmd-bit %lx\n", indx);
674 670
drivers/infiniband/hw/bnxt_re/qplib_rcfw.h+0−1
@@ -152,7 +152,6 @@ struct bnxt_qplib_cmdq_ctx { · 151 unmodified lines
152 wait_queue_head_t waitq;152 wait_queue_head_t waitq;
153 unsigned long flags;153 unsigned long flags;
154 unsigned long *cmdq_bitmap;154 unsigned long *cmdq_bitmap;
155 u32 bmap_size;
156 u32 seq_num;155 u32 seq_num;
157};156};
158 157
drivers/infiniband/hw/bnxt_re/qplib_res.c+0−50
@@ -649,31 +649,6 @@ static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, · 648 unmodified lines
649 memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);649 memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
650}650}
651 651
652static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res,
653 struct bnxt_qplib_pkey_tbl *pkey_tbl)
654{
655 if (!pkey_tbl->tbl)
656 dev_dbg(&res->pdev->dev, "PKEY tbl not present\n");
657 else
658 kfree(pkey_tbl->tbl);
659
660 pkey_tbl->tbl = NULL;
661 pkey_tbl->max = 0;
662 pkey_tbl->active = 0;
663}
664
665static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res,
666 struct bnxt_qplib_pkey_tbl *pkey_tbl,
667 u16 max)
668{
669 pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL);
670 if (!pkey_tbl->tbl)
671 return -ENOMEM;
672
673 pkey_tbl->max = max;
674 return 0;
675};
676
677/* PDs */652/* PDs */
678int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd)653int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd)
679{654{
@@ -843,24 +818,6 @@ unmap_io: · 163 unmodified lines
843 return -ENOMEM;818 return -ENOMEM;
844}819}
845 820
846/* PKEYs */
847static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl)
848{
849 memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
850 pkey_tbl->active = 0;
851}
852
853static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res,
854 struct bnxt_qplib_pkey_tbl *pkey_tbl)
855{
856 u16 pkey = 0xFFFF;
857
858 memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
859
860 /* pkey default = 0xFFFF */
861 bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false);
862}
863
864/* Stats */821/* Stats */
865static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,822static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
866 struct bnxt_qplib_stats *stats)823 struct bnxt_qplib_stats *stats)
@@ -891,21 +848,18 @@ static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev, · 24 unmodified lines
891 848
892void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)849void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)
893{850{
894 bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl);
895 bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);851 bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
896}852}
897 853
898int bnxt_qplib_init_res(struct bnxt_qplib_res *res)854int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
899{855{
900 bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);856 bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
901 bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl);
902 857
903 return 0;858 return 0;
904}859}
905 860
906void bnxt_qplib_free_res(struct bnxt_qplib_res *res)861void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
907{862{
908 bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl);
909 bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);863 bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
910 bnxt_qplib_free_pd_tbl(&res->pd_tbl);864 bnxt_qplib_free_pd_tbl(&res->pd_tbl);
911 bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);865 bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
@@ -924,10 +878,6 @@ int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, · 12 unmodified lines
924 if (rc)878 if (rc)
925 goto fail;879 goto fail;
926 880
927 rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey);
928 if (rc)
929 goto fail;
930
931 rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);881 rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);
932 if (rc)882 if (rc)
933 goto fail;883 goto fail;
drivers/infiniband/hw/bnxt_re/qplib_res.h+0−7
@@ -185,12 +185,6 @@ struct bnxt_qplib_sgid_tbl { · 184 unmodified lines
185 u8 *vlan;185 u8 *vlan;
186};186};
187 187
188struct bnxt_qplib_pkey_tbl {
189 u16 *tbl;
190 u16 max;
191 u16 active;
192};
193
194struct bnxt_qplib_dpi {188struct bnxt_qplib_dpi {
195 u32 dpi;189 u32 dpi;
196 void __iomem *dbr;190 void __iomem *dbr;
@@ -258,7 +252,6 @@ struct bnxt_qplib_res { · 61 unmodified lines
258 struct bnxt_qplib_rcfw *rcfw;252 struct bnxt_qplib_rcfw *rcfw;
259 struct bnxt_qplib_pd_tbl pd_tbl;253 struct bnxt_qplib_pd_tbl pd_tbl;
260 struct bnxt_qplib_sgid_tbl sgid_tbl;254 struct bnxt_qplib_sgid_tbl sgid_tbl;
261 struct bnxt_qplib_pkey_tbl pkey_tbl;
262 struct bnxt_qplib_dpi_tbl dpi_tbl;255 struct bnxt_qplib_dpi_tbl dpi_tbl;
263 bool prio;256 bool prio;
264 bool is_vf;257 bool is_vf;
drivers/infiniband/hw/bnxt_re/qplib_sp.c+1−98
@@ -146,17 +146,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, · 145 unmodified lines
146 attr->max_srq = le16_to_cpu(sb->max_srq);146 attr->max_srq = le16_to_cpu(sb->max_srq);
147 attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;147 attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
148 attr->max_srq_sges = sb->max_srq_sge;148 attr->max_srq_sges = sb->max_srq_sge;
149 attr->max_pkey = le32_to_cpu(sb->max_pkeys);149 attr->max_pkey = 1;
150 /*
151 * Some versions of FW reports more than 0xFFFF.
152 * Restrict it for now to 0xFFFF to avoid
153 * reporting trucated value
154 */
155 if (attr->max_pkey > 0xFFFF) {
156 /* ib_port_attr::pkey_tbl_len is u16 */
157 attr->max_pkey = 0xFFFF;
158 }
159
160 attr->max_inline_data = le32_to_cpu(sb->max_inline_data);150 attr->max_inline_data = le32_to_cpu(sb->max_inline_data);
161 attr->l2_db_size = (sb->l2_db_space_size + 1) *151 attr->l2_db_size = (sb->l2_db_space_size + 1) *
162 (0x01 << RCFW_DBR_BASE_PAGE_SHIFT);152 (0x01 << RCFW_DBR_BASE_PAGE_SHIFT);
@@ -414,93 +404,6 @@ int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, · 251 unmodified lines
414 return rc;404 return rc;
415}405}
416 406
417/* pkeys */
418int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,
419 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,
420 u16 *pkey)
421{
422 if (index == 0xFFFF) {
423 *pkey = 0xFFFF;
424 return 0;
425 }
426 if (index >= pkey_tbl->max) {
427 dev_err(&res->pdev->dev,
428 "Index %d exceeded PKEY table max (%d)\n",
429 index, pkey_tbl->max);
430 return -EINVAL;
431 }
432 memcpy(pkey, &pkey_tbl->tbl[index], sizeof(*pkey));
433 return 0;
434}
435
436int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res,
437 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
438 bool update)
439{
440 int i, rc = 0;
441
442 if (!pkey_tbl) {
443 dev_err(&res->pdev->dev, "PKEY table not allocated\n");
444 return -EINVAL;
445 }
446
447 /* Do we need a pkey_lock here? */
448 if (!pkey_tbl->active) {
449 dev_err(&res->pdev->dev, "PKEY table has no active entries\n");
450 return -ENOMEM;
451 }
452 for (i = 0; i < pkey_tbl->max; i++) {
453 if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey)))
454 break;
455 }
456 if (i == pkey_tbl->max) {
457 dev_err(&res->pdev->dev,
458 "PKEY 0x%04x not found in the pkey table\n", *pkey);
459 return -ENOMEM;
460 }
461 memset(&pkey_tbl->tbl[i], 0, sizeof(*pkey));
462 pkey_tbl->active--;
463
464 /* unlock */
465 return rc;
466}
467
468int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,
469 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
470 bool update)
471{
472 int i, free_idx, rc = 0;
473
474 if (!pkey_tbl) {
475 dev_err(&res->pdev->dev, "PKEY table not allocated\n");
476 return -EINVAL;
477 }
478
479 /* Do we need a pkey_lock here? */
480 if (pkey_tbl->active == pkey_tbl->max) {
481 dev_err(&res->pdev->dev, "PKEY table is full\n");
482 return -ENOMEM;
483 }
484 free_idx = pkey_tbl->max;
485 for (i = 0; i < pkey_tbl->max; i++) {
486 if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey)))
487 return -EALREADY;
488 else if (!pkey_tbl->tbl[i] && free_idx == pkey_tbl->max)
489 free_idx = i;
490 }
491 if (free_idx == pkey_tbl->max) {
492 dev_err(&res->pdev->dev,
493 "PKEY table is FULL but count is not MAX??\n");
494 return -ENOMEM;
495 }
496 /* Add PKEY to the pkey_tbl */
497 memcpy(&pkey_tbl->tbl[free_idx], pkey, sizeof(*pkey));
498 pkey_tbl->active++;
499
500 /* unlock */
501 return rc;
502}
503
504/* AH */407/* AH */
505int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,408int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
506 bool block)409 bool block)
drivers/infiniband/hw/bnxt_re/qplib_sp.h+0−9
@@ -255,15 +255,6 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, · 254 unmodified lines
255int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,255int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
256 struct bnxt_qplib_gid *gid, u16 gid_idx,256 struct bnxt_qplib_gid *gid, u16 gid_idx,
257 const u8 *smac);257 const u8 *smac);
258int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,
259 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,
260 u16 *pkey);
261int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res,
262 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
263 bool update);
264int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,
265 struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
266 bool update);
267int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,258int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
268 struct bnxt_qplib_dev_attr *attr, bool vf);259 struct bnxt_qplib_dev_attr *attr, bool vf);
269int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,260int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,
drivers/infiniband/hw/cxgb4/cm.c+3−2
@@ -2471,7 +2471,8 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb, · 2470 unmodified lines
2471 skb_get(skb);2471 skb_get(skb);
2472 rpl = cplhdr(skb);2472 rpl = cplhdr(skb);
2473 if (!is_t4(adapter_type)) {2473 if (!is_t4(adapter_type)) {
2474 skb_trim(skb, roundup(sizeof(*rpl5), 16));2474 BUILD_BUG_ON(sizeof(*rpl5) != roundup(sizeof(*rpl5), 16));
2475 skb_trim(skb, sizeof(*rpl5));
2475 rpl5 = (void *)rpl;2476 rpl5 = (void *)rpl;
2476 INIT_TP_WR(rpl5, ep->hwtid);2477 INIT_TP_WR(rpl5, ep->hwtid);
2477 } else {2478 } else {
@@ -2487,7 +2488,7 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb, · 9 unmodified lines
2487 opt2 |= CONG_CNTRL_V(CONG_ALG_TAHOE);2488 opt2 |= CONG_CNTRL_V(CONG_ALG_TAHOE);
2488 opt2 |= T5_ISS_F;2489 opt2 |= T5_ISS_F;
2489 rpl5 = (void *)rpl;2490 rpl5 = (void *)rpl;
2490 memset(&rpl5->iss, 0, roundup(sizeof(*rpl5)-sizeof(*rpl), 16));2491 memset_after(rpl5, 0, iss);
2491 if (peer2peer)2492 if (peer2peer)
2492 isn += 4;2493 isn += 4;
2493 rpl5->iss = cpu_to_be32(isn);2494 rpl5->iss = cpu_to_be32(isn);
drivers/infiniband/hw/cxgb4/id_table.c+6−11
@@ -59,7 +59,7 @@ u32 c4iw_id_alloc(struct c4iw_id_table *alloc) · 58 unmodified lines
59 alloc->last = obj + 1;59 alloc->last = obj + 1;
60 if (alloc->last >= alloc->max)60 if (alloc->last >= alloc->max)
61 alloc->last = 0;61 alloc->last = 0;
62 set_bit(obj, alloc->table);62 __set_bit(obj, alloc->table);
63 obj += alloc->start;63 obj += alloc->start;
64 } else64 } else
65 obj = -1;65 obj = -1;
@@ -75,37 +75,32 @@ void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj) · 9 unmodified lines
75 obj -= alloc->start;75 obj -= alloc->start;
76 76
77 spin_lock_irqsave(&alloc->lock, flags);77 spin_lock_irqsave(&alloc->lock, flags);
78 clear_bit(obj, alloc->table);78 __clear_bit(obj, alloc->table);
79 spin_unlock_irqrestore(&alloc->lock, flags);79 spin_unlock_irqrestore(&alloc->lock, flags);
80}80}
81 81
82int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,82int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,
83 u32 reserved, u32 flags)83 u32 reserved, u32 flags)
84{84{
85 int i;
86
87 alloc->start = start;85 alloc->start = start;
88 alloc->flags = flags;86 alloc->flags = flags;
89 if (flags & C4IW_ID_TABLE_F_RANDOM)87 if (flags & C4IW_ID_TABLE_F_RANDOM)
90 alloc->last = prandom_u32() % RANDOM_SKIP;88 alloc->last = prandom_u32() % RANDOM_SKIP;
91 else89 else
92 alloc->last = 0;90 alloc->last = 0;
93 alloc->max = num;91 alloc->max = num;
94 spin_lock_init(&alloc->lock);92 spin_lock_init(&alloc->lock);
95 alloc->table = kmalloc_array(BITS_TO_LONGS(num), sizeof(long),93 alloc->table = bitmap_zalloc(num, GFP_KERNEL);
96 GFP_KERNEL);
97 if (!alloc->table)94 if (!alloc->table)
98 return -ENOMEM;95 return -ENOMEM;
99 96
100 bitmap_zero(alloc->table, num);
101 if (!(alloc->flags & C4IW_ID_TABLE_F_EMPTY))97 if (!(alloc->flags & C4IW_ID_TABLE_F_EMPTY))
102 for (i = 0; i < reserved; ++i)98 bitmap_set(alloc->table, 0, reserved);
103 set_bit(i, alloc->table);
104 99
105 return 0;100 return 0;
106}101}
107 102
108void c4iw_id_table_free(struct c4iw_id_table *alloc)103void c4iw_id_table_free(struct c4iw_id_table *alloc)
109{104{
110 kfree(alloc->table);105 bitmap_free(alloc->table);
111}106}
drivers/infiniband/hw/cxgb4/provider.c+5−3
@@ -41,6 +41,7 @@ · 40 unmodified lines
41#include <linux/ethtool.h>41#include <linux/ethtool.h>
42#include <linux/rtnetlink.h>42#include <linux/rtnetlink.h>
43#include <linux/inetdevice.h>43#include <linux/inetdevice.h>
44#include <net/addrconf.h>
44#include <linux/io.h>45#include <linux/io.h>
45 46
46#include <asm/irq.h>47#include <asm/irq.h>
@@ -264,7 +265,8 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro · 217 unmodified lines
264 return -EINVAL;265 return -EINVAL;
265 266
266 dev = to_c4iw_dev(ibdev);267 dev = to_c4iw_dev(ibdev);
267 memcpy(&props->sys_image_guid, dev->rdev.lldi.ports[0]->dev_addr, 6);268 addrconf_addr_eui48((u8 *)&props->sys_image_guid,
269 dev->rdev.lldi.ports[0]->dev_addr);
268 props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type);270 props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type);
269 props->fw_ver = dev->rdev.lldi.fw_vers;271 props->fw_ver = dev->rdev.lldi.fw_vers;
270 props->device_cap_flags = dev->device_cap_flags;272 props->device_cap_flags = dev->device_cap_flags;
@@ -525,8 +527,8 @@ void c4iw_register_device(struct work_struct *work) · 254 unmodified lines
525 struct c4iw_dev *dev = ctx->dev;527 struct c4iw_dev *dev = ctx->dev;
526 528
527 pr_debug("c4iw_dev %p\n", dev);529 pr_debug("c4iw_dev %p\n", dev);
528 memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));530 addrconf_addr_eui48((u8 *)&dev->ibdev.node_guid,
529 memcpy(&dev->ibdev.node_guid, dev->rdev.lldi.ports[0]->dev_addr, 6);531 dev->rdev.lldi.ports[0]->dev_addr);
530 dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;532 dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
531 if (fastreg_support)533 if (fastreg_support)
532 dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;534 dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
drivers/infiniband/hw/cxgb4/qp.c+1−0
@@ -2460,6 +2460,7 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 2459 unmodified lines
2460 memset(attr, 0, sizeof(*attr));2460 memset(attr, 0, sizeof(*attr));
2461 memset(init_attr, 0, sizeof(*init_attr));2461 memset(init_attr, 0, sizeof(*init_attr));
2462 attr->qp_state = to_ib_qp_state(qhp->attr.state);2462 attr->qp_state = to_ib_qp_state(qhp->attr.state);
2463 attr->cur_qp_state = to_ib_qp_state(qhp->attr.state);
2463 init_attr->cap.max_send_wr = qhp->attr.sq_num_entries;2464 init_attr->cap.max_send_wr = qhp->attr.sq_num_entries;
2464 init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries;2465 init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries;
2465 init_attr->cap.max_send_sge = qhp->attr.sq_max_sges;2466 init_attr->cap.max_send_sge = qhp->attr.sq_max_sges;
drivers/infiniband/hw/hfi1/user_sdma.c+3−5
@@ -161,9 +161,7 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt, · 160 unmodified lines
161 if (!pq->reqs)161 if (!pq->reqs)
162 goto pq_reqs_nomem;162 goto pq_reqs_nomem;
163 163
164 pq->req_in_use = kcalloc(BITS_TO_LONGS(hfi1_sdma_comp_ring_size),164 pq->req_in_use = bitmap_zalloc(hfi1_sdma_comp_ring_size, GFP_KERNEL);
165 sizeof(*pq->req_in_use),
166 GFP_KERNEL);
167 if (!pq->req_in_use)165 if (!pq->req_in_use)
168 goto pq_reqs_no_in_use;166 goto pq_reqs_no_in_use;
169 167
@@ -210,7 +208,7 @@ cq_comps_nomem: · 40 unmodified lines
210cq_nomem:208cq_nomem:
211 kmem_cache_destroy(pq->txreq_cache);209 kmem_cache_destroy(pq->txreq_cache);
212pq_txreq_nomem:210pq_txreq_nomem:
213 kfree(pq->req_in_use);211 bitmap_free(pq->req_in_use);
214pq_reqs_no_in_use:212pq_reqs_no_in_use:
215 kfree(pq->reqs);213 kfree(pq->reqs);
216pq_reqs_nomem:214pq_reqs_nomem:
@@ -257,7 +255,7 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd, · 40 unmodified lines
257 pq->wait,255 pq->wait,
258 !atomic_read(&pq->n_reqs));256 !atomic_read(&pq->n_reqs));
259 kfree(pq->reqs);257 kfree(pq->reqs);
260 kfree(pq->req_in_use);258 bitmap_free(pq->req_in_use);
261 kmem_cache_destroy(pq->txreq_cache);259 kmem_cache_destroy(pq->txreq_cache);
262 flush_pq_iowait(pq);260 flush_pq_iowait(pq);
263 kfree(pq);261 kfree(pq);
drivers/infiniband/hw/hns/Kconfig+2−15
@@ -5,22 +5,9 @@ config INFINIBAND_HNS · 4 unmodified lines
5 depends on ARM64 || (COMPILE_TEST && 64BIT)5 depends on ARM64 || (COMPILE_TEST && 64BIT)
6 depends on (HNS_DSAF && HNS_ENET) || HNS36 depends on (HNS_DSAF && HNS_ENET) || HNS3
7 help7 help
8 This is a RoCE/RDMA driver for the Hisilicon RoCE engine. The engine8 This is a RoCE/RDMA driver for the Hisilicon RoCE engine.
9 is used in Hisilicon Hip06 and more further ICT SoC based on
10 platform device.
11 9
12 To compile HIP06 or HIP08 driver as module, choose M here.10 To compile HIP08 driver as module, choose M here.
13
14config INFINIBAND_HNS_HIP06
15 bool "Hisilicon Hip06 Family RoCE support"
16 depends on INFINIBAND_HNS && HNS && HNS_DSAF && HNS_ENET
17 depends on INFINIBAND_HNS=m || (HNS_DSAF=y && HNS_ENET=y)
18 help
19 RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and
20 Hip07 SoC. These RoCE engines are platform devices.
21
22 To compile this driver, choose Y here: if INFINIBAND_HNS is m, this
23 module will be called hns-roce-hw-v1
24 11
25config INFINIBAND_HNS_HIP0812config INFINIBAND_HNS_HIP08
26 bool "Hisilicon Hip08 Family RoCE support"13 bool "Hisilicon Hip08 Family RoCE support"
drivers/infiniband/hw/hns/Makefile+0−5
@@ -9,11 +9,6 @@ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ · 8 unmodified lines
9 hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \9 hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
10 hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o10 hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
11 11
12ifdef CONFIG_INFINIBAND_HNS_HIP06
13hns-roce-hw-v1-objs := hns_roce_hw_v1.o $(hns-roce-objs)
14obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v1.o
15endif
16
17ifdef CONFIG_INFINIBAND_HNS_HIP0812ifdef CONFIG_INFINIBAND_HNS_HIP08
18hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs)13hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs)
19obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o14obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o
drivers/infiniband/hw/hns/hns_roce_ah.c+2−3
@@ -30,7 +30,6 @@ · 29 unmodified lines
30 * SOFTWARE.30 * SOFTWARE.
31 */31 */
32 32
33#include <linux/platform_device.h>
34#include <linux/pci.h>33#include <linux/pci.h>
35#include <rdma/ib_addr.h>34#include <rdma/ib_addr.h>
36#include <rdma/ib_cache.h>35#include <rdma/ib_cache.h>
@@ -61,7 +60,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, · 24 unmodified lines
61 struct hns_roce_ah *ah = to_hr_ah(ibah);60 struct hns_roce_ah *ah = to_hr_ah(ibah);
62 int ret = 0;61 int ret = 0;
63 62
64 if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && udata)63 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && udata)
65 return -EOPNOTSUPP;64 return -EOPNOTSUPP;
66 65
67 ah->av.port = rdma_ah_get_port_num(ah_attr);66 ah->av.port = rdma_ah_get_port_num(ah_attr);
@@ -80,7 +79,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, · 12 unmodified lines
80 memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN);79 memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN);
81 80
82 /* HIP08 needs to record vlan info in Address Vector */81 /* HIP08 needs to record vlan info in Address Vector */
83 if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) {82 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
84 ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr,83 ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr,
85 &ah->av.vlan_id, NULL);84 &ah->av.vlan_id, NULL);
86 if (ret)85 if (ret)
drivers/infiniband/hw/hns/hns_roce_alloc.c+1−2
@@ -31,10 +31,9 @@ · 30 unmodified lines
31 * SOFTWARE.31 * SOFTWARE.
32 */32 */
33 33
34#include <linux/platform_device.h>
35#include <linux/vmalloc.h>34#include <linux/vmalloc.h>
36#include "hns_roce_device.h"
37#include <rdma/ib_umem.h>35#include <rdma/ib_umem.h>
36#include "hns_roce_device.h"
38 37
39void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf)38void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf)
40{39{
drivers/infiniband/hw/hns/hns_roce_cmd.c+5−6
@@ -31,7 +31,6 @@ · 30 unmodified lines
31 */31 */
32 32
33#include <linux/dmapool.h>33#include <linux/dmapool.h>
34#include <linux/platform_device.h>
35#include "hns_roce_common.h"34#include "hns_roce_common.h"
36#include "hns_roce_device.h"35#include "hns_roce_device.h"
37#include "hns_roce_cmd.h"36#include "hns_roce_cmd.h"
@@ -61,7 +60,7 @@ static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, · 23 unmodified lines
61 CMD_POLL_TOKEN, 0);60 CMD_POLL_TOKEN, 0);
62 if (ret) {61 if (ret) {
63 dev_err_ratelimited(hr_dev->dev,62 dev_err_ratelimited(hr_dev->dev,
64 "failed to post mailbox %x in poll mode, ret = %d.\n",63 "failed to post mailbox 0x%x in poll mode, ret = %d.\n",
65 op, ret);64 op, ret);
66 return ret;65 return ret;
67 }66 }
@@ -91,7 +90,7 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, · 23 unmodified lines
91 90
92 if (unlikely(token != context->token)) {91 if (unlikely(token != context->token)) {
93 dev_err_ratelimited(hr_dev->dev,92 dev_err_ratelimited(hr_dev->dev,
94 "[cmd] invalid ae token %x,context token is %x!\n",93 "[cmd] invalid ae token 0x%x, context token is 0x%x.\n",
95 token, context->token);94 token, context->token);
96 return;95 return;
97 }96 }
@@ -130,14 +129,14 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, · 32 unmodified lines
130 context->token, 1);129 context->token, 1);
131 if (ret) {130 if (ret) {
132 dev_err_ratelimited(dev,131 dev_err_ratelimited(dev,
133 "failed to post mailbox %x in event mode, ret = %d.\n",132 "failed to post mailbox 0x%x in event mode, ret = %d.\n",
134 op, ret);133 op, ret);
135 goto out;134 goto out;
136 }135 }
137 136
138 if (!wait_for_completion_timeout(&context->done,137 if (!wait_for_completion_timeout(&context->done,
139 msecs_to_jiffies(timeout))) {138 msecs_to_jiffies(timeout))) {
140 dev_err_ratelimited(dev, "[cmd] token %x mailbox %x timeout.\n",139 dev_err_ratelimited(dev, "[cmd] token 0x%x mailbox 0x%x timeout.\n",
141 context->token, op);140 context->token, op);
142 ret = -EBUSY;141 ret = -EBUSY;
143 goto out;142 goto out;
@@ -145,7 +144,7 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, · 1 unmodified line
145 144
146 ret = context->result;145 ret = context->result;
147 if (ret)146 if (ret)
148 dev_err_ratelimited(dev, "[cmd] token %x mailbox %x error %d\n",147 dev_err_ratelimited(dev, "[cmd] token 0x%x mailbox 0x%x error %d.\n",
149 context->token, op, ret);148 context->token, op, ret);
150 149
151out:150out:
drivers/infiniband/hw/hns/hns_roce_common.h+0−202
@@ -104,208 +104,6 @@ · 103 unmodified lines
104 104
105#define hr_reg_read(ptr, field) _hr_reg_read(ptr, field)105#define hr_reg_read(ptr, field) _hr_reg_read(ptr, field)
106 106
107#define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3
108#define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4
109
110#define ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S 5
111
112#define ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S 6
113
114#define ROCEE_GLB_CFG_ROCEE_PORT_ST_S 10
115#define ROCEE_GLB_CFG_ROCEE_PORT_ST_M \
116 (((1UL << 6) - 1) << ROCEE_GLB_CFG_ROCEE_PORT_ST_S)
117
118#define ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S 16
119
120#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S 0
121#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M \
122 (((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S)
123
124#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S 24
125#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M \
126 (((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S)
127
128#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S 0
129#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M \
130 (((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S)
131
132#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S 24
133#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M \
134 (((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S)
135
136#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S 0
137#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M \
138 (((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S)
139
140#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S 16
141#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M \
142 (((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S)
143
144#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S 0
145#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M \
146 (((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S)
147
148#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S 16
149#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M \
150 (((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S)
151
152#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_S 0
153#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_M \
154 (((1UL << 8) - 1) << ROCEE_RAQ_WL_ROCEE_RAQ_WL_S)
155
156#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S 0
157#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M \
158 (((1UL << 15) - 1) << \
159 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S)
160
161#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S 16
162#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M \
163 (((1UL << 4) - 1) << \
164 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S)
165
166#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S 20
167
168#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE 21
169
170#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S 0
171#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M \
172 (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S)
173
174#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S 5
175#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M \
176 (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S)
177
178#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S 0
179#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M \
180 (((1UL << 5) - 1) << ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S)
181
182#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S 5
183#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M \
184 (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S)
185
186#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S 0
187#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M \
188 (((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S)
189
190#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S 8
191#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M \
192 (((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S)
193
194#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S 0
195#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M \
196 (((1UL << 19) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S)
197
198#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_S 19
199
200#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S 20
201#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M \
202 (((1UL << 2) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S)
203
204#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S 22
205#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M \
206 (((1UL << 5) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S)
207
208#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S 31
209
210#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S 0
211#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M \
212 (((1UL << 3) - 1) << ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S)
213
214#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S 0
215#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M \
216 (((1UL << 15) - 1) << ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S)
217
218#define ROCEE_MB6_ROCEE_MB_CMD_S 0
219#define ROCEE_MB6_ROCEE_MB_CMD_M \
220 (((1UL << 8) - 1) << ROCEE_MB6_ROCEE_MB_CMD_S)
221
222#define ROCEE_MB6_ROCEE_MB_CMD_MDF_S 8
223#define ROCEE_MB6_ROCEE_MB_CMD_MDF_M \
224 (((1UL << 4) - 1) << ROCEE_MB6_ROCEE_MB_CMD_MDF_S)
225
226#define ROCEE_MB6_ROCEE_MB_EVENT_S 14
227
228#define ROCEE_MB6_ROCEE_MB_HW_RUN_S 15
229
230#define ROCEE_MB6_ROCEE_MB_TOKEN_S 16
231#define ROCEE_MB6_ROCEE_MB_TOKEN_M \
232 (((1UL << 16) - 1) << ROCEE_MB6_ROCEE_MB_TOKEN_S)
233
234#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S 0
235#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M \
236 (((1UL << 24) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S)
237
238#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S 24
239#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M \
240 (((1UL << 4) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S)
241
242#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S 28
243#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M \
244 (((1UL << 3) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S)
245
246#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S 31
247
248#define ROCEE_SMAC_H_ROCEE_SMAC_H_S 0
249#define ROCEE_SMAC_H_ROCEE_SMAC_H_M \
250 (((1UL << 16) - 1) << ROCEE_SMAC_H_ROCEE_SMAC_H_S)
251
252#define ROCEE_SMAC_H_ROCEE_PORT_MTU_S 16
253#define ROCEE_SMAC_H_ROCEE_PORT_MTU_M \
254 (((1UL << 4) - 1) << ROCEE_SMAC_H_ROCEE_PORT_MTU_S)
255
256#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S 0
257#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M \
258 (((1UL << 2) - 1) << ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S)
259
260#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S 8
261#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M \
262 (((1UL << 4) - 1) << ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S)
263
264#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S 17
265
266#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S 0
267#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M \
268 (((1UL << 5) - 1) << ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S)
269
270#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S 16
271#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M \
272 (((1UL << 16) - 1) << ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S)
273
274#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S 0
275#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M \
276 (((1UL << 16) - 1) << ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S)
277
278#define ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S 16
279#define ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S 1
280#define ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S 0
281
282#define ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S 0
283#define ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S 1
284
285#define ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S 0
286
287#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S 0
288#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M \
289 (((1UL << 28) - 1) << ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S)
290
291#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S 0
292#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M \
293 (((1UL << 28) - 1) << ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S)
294
295#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_S 0
296#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_M \
297 (((1UL << 16) - 1) << ROCEE_SDB_INV_CNT_SDB_INV_CNT_S)
298
299#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S 0
300#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M \
301 (((1UL << 16) - 1) << ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S)
302
303#define ROCEE_SDB_CNT_CMP_BITS 16
304
305#define ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S 20
306
307#define ROCEE_CNT_CLR_CE_CNT_CLR_CE_S 0
308
309/*************ROCEE_REG DEFINITION****************/107/*************ROCEE_REG DEFINITION****************/
310#define ROCEE_VENDOR_ID_REG 0x0108#define ROCEE_VENDOR_ID_REG 0x0
311#define ROCEE_VENDOR_PART_ID_REG 0x4109#define ROCEE_VENDOR_PART_ID_REG 0x4
drivers/infiniband/hw/hns/hns_roce_cq.c+0−13
@@ -30,7 +30,6 @@ · 29 unmodified lines
30 * SOFTWARE.30 * SOFTWARE.
31 */31 */
32 32
33#include <linux/platform_device.h>
34#include <rdma/ib_umem.h>33#include <rdma/ib_umem.h>
35#include <rdma/uverbs_ioctl.h>34#include <rdma/uverbs_ioctl.h>
36#include "hns_roce_device.h"35#include "hns_roce_device.h"
@@ -406,15 +405,6 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, · 369 unmodified lines
406 goto err_cqn;405 goto err_cqn;
407 }406 }
408 407
409 /*
410 * For the QP created by kernel space, tptr value should be initialized
411 * to zero; For the QP created by user space, it will cause synchronous
412 * problems if tptr is set to zero here, so we initialize it in user
413 * space.
414 */
415 if (!udata && hr_cq->tptr_addr)
416 *hr_cq->tptr_addr = 0;
417
418 if (udata) {408 if (udata) {
419 resp.cqn = hr_cq->cqn;409 resp.cqn = hr_cq->cqn;
420 ret = ib_copy_to_udata(udata, &resp,410 ret = ib_copy_to_udata(udata, &resp,
@@ -441,9 +431,6 @@ int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) · 20 unmodified lines
441 struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);431 struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
442 struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);432 struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
443 433
444 if (hr_dev->hw->destroy_cq)
445 hr_dev->hw->destroy_cq(ib_cq, udata);
446
447 free_cqc(hr_dev, hr_cq);434 free_cqc(hr_dev, hr_cq);
448 free_cqn(hr_dev, hr_cq->cqn);435 free_cqn(hr_dev, hr_cq->cqn);
449 free_cq_db(hr_dev, hr_cq, udata);436 free_cq_db(hr_dev, hr_cq, udata);
drivers/infiniband/hw/hns/hns_roce_db.c+0−1
@@ -4,7 +4,6 @@ · 3 unmodified lines
4 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.4 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 */5 */
6 6
7#include <linux/platform_device.h>
8#include <rdma/ib_umem.h>7#include <rdma/ib_umem.h>
9#include "hns_roce_device.h"8#include "hns_roce_device.h"
10 9
drivers/infiniband/hw/hns/hns_roce_device.h+24−84
@@ -36,36 +36,18 @@ · 35 unmodified lines
36#include <rdma/ib_verbs.h>36#include <rdma/ib_verbs.h>
37#include <rdma/hns-abi.h>37#include <rdma/hns-abi.h>
38 38
39#define DRV_NAME "hns_roce"
40
41#define PCI_REVISION_ID_HIP08 0x2139#define PCI_REVISION_ID_HIP08 0x21
42#define PCI_REVISION_ID_HIP09 0x3040#define PCI_REVISION_ID_HIP09 0x30
43 41
44#define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6')
45
46#define HNS_ROCE_MAX_MSG_LEN 0x8000000042#define HNS_ROCE_MAX_MSG_LEN 0x80000000
47 43
48#define HNS_ROCE_IB_MIN_SQ_STRIDE 644#define HNS_ROCE_IB_MIN_SQ_STRIDE 6
49 45
50#define BA_BYTE_LEN 846#define BA_BYTE_LEN 8
51 47
52/* Hardware specification only for v1 engine */
53#define HNS_ROCE_MIN_CQE_NUM 0x4048#define HNS_ROCE_MIN_CQE_NUM 0x40
54#define HNS_ROCE_MIN_WQE_NUM 0x20
55#define HNS_ROCE_MIN_SRQ_WQE_NUM 149#define HNS_ROCE_MIN_SRQ_WQE_NUM 1
56 50
57/* Hardware specification only for v1 engine */
58#define HNS_ROCE_MAX_INNER_MTPT_NUM 0x7
59#define HNS_ROCE_MAX_MTPT_PBL_NUM 0x100000
60
61#define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS 20
62#define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT \
63 (5000 / HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS)
64#define HNS_ROCE_CQE_WCMD_EMPTY_BIT 0x2
65#define HNS_ROCE_MIN_CQE_CNT 16
66
67#define HNS_ROCE_RESERVED_SGE 1
68
69#define HNS_ROCE_MAX_IRQ_NUM 12851#define HNS_ROCE_MAX_IRQ_NUM 128
70 52
71#define HNS_ROCE_SGE_IN_WQE 253#define HNS_ROCE_SGE_IN_WQE 2
@@ -102,18 +84,12 @@ · 30 unmodified lines
102#define HNS_ROCE_FRMR_MAX_PA 51284#define HNS_ROCE_FRMR_MAX_PA 512
103 85
104#define PKEY_ID 0xffff86#define PKEY_ID 0xffff
105#define GUID_LEN 8
106#define NODE_DESC_SIZE 6487#define NODE_DESC_SIZE 64
107#define DB_REG_OFFSET 0x100088#define DB_REG_OFFSET 0x1000
108 89
109/* Configure to HW for PAGE_SIZE larger than 4KB */90/* Configure to HW for PAGE_SIZE larger than 4KB */
110#define PG_SHIFT_OFFSET (PAGE_SHIFT - 12)91#define PG_SHIFT_OFFSET (PAGE_SHIFT - 12)
111 92
112#define PAGES_SHIFT_8 8
113#define PAGES_SHIFT_16 16
114#define PAGES_SHIFT_24 24
115#define PAGES_SHIFT_32 32
116
117#define HNS_ROCE_IDX_QUE_ENTRY_SZ 493#define HNS_ROCE_IDX_QUE_ENTRY_SZ 4
118#define SRQ_DB_REG 0x23094#define SRQ_DB_REG 0x230
119 95
@@ -122,11 +98,6 @@ · 2 unmodified lines
122 98
123#define CQ_BANKID_SHIFT 299#define CQ_BANKID_SHIFT 2
124 100
125/* The chip implementation of the consumer index is calculated
126 * according to twice the actual EQ depth
127 */
128#define EQ_DEPTH_COEFF 2
129
130enum {101enum {
131 SERV_TYPE_RC,102 SERV_TYPE_RC,
132 SERV_TYPE_UC,103 SERV_TYPE_UC,
@@ -182,6 +153,7 @@ enum { · 49 unmodified lines
182 HNS_ROCE_CAP_FLAG_FRMR = BIT(8),153 HNS_ROCE_CAP_FLAG_FRMR = BIT(8),
183 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),154 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),
184 HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10),155 HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10),
156 HNS_ROCE_CAP_FLAG_DIRECT_WQE = BIT(12),
185 HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14),157 HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14),
186 HNS_ROCE_CAP_FLAG_STASH = BIT(17),158 HNS_ROCE_CAP_FLAG_STASH = BIT(17),
187};159};
@@ -227,7 +199,7 @@ struct hns_roce_uar { · 39 unmodified lines
227 199
228enum hns_roce_mmap_type {200enum hns_roce_mmap_type {
229 HNS_ROCE_MMAP_TYPE_DB = 1,201 HNS_ROCE_MMAP_TYPE_DB = 1,
230 HNS_ROCE_MMAP_TYPE_TPTR,202 HNS_ROCE_MMAP_TYPE_DWQE,
231};203};
232 204
233struct hns_user_mmap_entry {205struct hns_user_mmap_entry {
@@ -242,7 +214,6 @@ struct hns_roce_ucontext { · 8 unmodified lines
242 struct list_head page_list;214 struct list_head page_list;
243 struct mutex page_mutex;215 struct mutex page_mutex;
244 struct hns_user_mmap_entry *db_mmap_entry;216 struct hns_user_mmap_entry *db_mmap_entry;
245 struct hns_user_mmap_entry *tptr_mmap_entry;
246};217};
247 218
248struct hns_roce_pd {219struct hns_roce_pd {
@@ -345,19 +316,16 @@ struct hns_roce_mw { · 96 unmodified lines
345 u32 pbl_buf_pg_sz;316 u32 pbl_buf_pg_sz;
346};317};
347 318
348/* Only support 4K page size for mr register */
349#define MR_SIZE_4K 0
350
351struct hns_roce_mr {319struct hns_roce_mr {
352 struct ib_mr ibmr;320 struct ib_mr ibmr;
353 u64 iova; /* MR's virtual original addr */321 u64 iova; /* MR's virtual original addr */
354 u64 size; /* Address range of MR */322 u64 size; /* Address range of MR */
355 u32 key; /* Key of MR */323 u32 key; /* Key of MR */
356 u32 pd; /* PD num of MR */324 u32 pd; /* PD num of MR */
357 u32 access; /* Access permission of MR */325 u32 access; /* Access permission of MR */
358 int enabled; /* MR's active status */326 int enabled; /* MR's active status */
359 int type; /* MR's register type */327 int type; /* MR's register type */
360 u32 pbl_hop_num; /* multi-hop number */328 u32 pbl_hop_num; /* multi-hop number */
361 struct hns_roce_mtr pbl_mtr;329 struct hns_roce_mtr pbl_mtr;
362 u32 npages;330 u32 npages;
363 dma_addr_t *page_list;331 dma_addr_t *page_list;
@@ -374,17 +342,17 @@ struct hns_roce_wq { · 10 unmodified lines
374 u32 wqe_cnt; /* WQE num */342 u32 wqe_cnt; /* WQE num */
375 u32 max_gs;343 u32 max_gs;
376 u32 rsv_sge;344 u32 rsv_sge;
377 int offset;345 u32 offset;
378 int wqe_shift; /* WQE size */346 u32 wqe_shift; /* WQE size */
379 u32 head;347 u32 head;
380 u32 tail;348 u32 tail;
381 void __iomem *db_reg;349 void __iomem *db_reg;
382};350};
383 351
384struct hns_roce_sge {352struct hns_roce_sge {
385 unsigned int sge_cnt; /* SGE num */353 unsigned int sge_cnt; /* SGE num */
386 int offset;354 u32 offset;
387 int sge_shift; /* SGE size */355 u32 sge_shift; /* SGE size */
388};356};
389 357
390struct hns_roce_buf_list {358struct hns_roce_buf_list {
@@ -453,7 +421,6 @@ struct hns_roce_cq { · 62 unmodified lines
453 u32 cons_index;421 u32 cons_index;
454 u32 *set_ci_db;422 u32 *set_ci_db;
455 void __iomem *db_reg;423 void __iomem *db_reg;
456 u16 *tptr_addr;
457 int arm_sn;424 int arm_sn;
458 int cqe_size;425 int cqe_size;
459 unsigned long cqn;426 unsigned long cqn;
@@ -468,7 +435,7 @@ struct hns_roce_cq { · 8 unmodified lines
468 435
469struct hns_roce_idx_que {436struct hns_roce_idx_que {
470 struct hns_roce_mtr mtr;437 struct hns_roce_mtr mtr;
471 int entry_shift;438 u32 entry_shift;
472 unsigned long *bitmap;439 unsigned long *bitmap;
473 u32 head;440 u32 head;
474 u32 tail;441 u32 tail;
@@ -480,7 +447,7 @@ struct hns_roce_srq { · 5 unmodified lines
480 u32 wqe_cnt;447 u32 wqe_cnt;
481 int max_gs;448 int max_gs;
482 u32 rsv_sge;449 u32 rsv_sge;
483 int wqe_shift;450 u32 wqe_shift;
484 u32 cqn;451 u32 cqn;
485 u32 xrcdn;452 u32 xrcdn;
486 void __iomem *db_reg;453 void __iomem *db_reg;
@@ -539,10 +506,6 @@ struct hns_roce_srq_table { · 52 unmodified lines
539 struct hns_roce_hem_table table;506 struct hns_roce_hem_table table;
540};507};
541 508
542struct hns_roce_raq_table {
543 struct hns_roce_buf_list *e_raq_buf;
544};
545
546struct hns_roce_av {509struct hns_roce_av {
547 u8 port;510 u8 port;
548 u8 gid_index;511 u8 gid_index;
@@ -627,10 +590,6 @@ struct hns_roce_work { · 78 unmodified lines
627 u32 queue_num;590 u32 queue_num;
628};591};
629 592
630enum {
631 HNS_ROCE_QP_CAP_DIRECT_WQE = BIT(5),
632};
633
634struct hns_roce_qp {593struct hns_roce_qp {
635 struct ib_qp ibqp;594 struct ib_qp ibqp;
636 struct hns_roce_wq rq;595 struct hns_roce_wq rq;
@@ -650,9 +609,7 @@ struct hns_roce_qp { · 13 unmodified lines
650 u8 sl;609 u8 sl;
651 u8 resp_depth;610 u8 resp_depth;
652 u8 state;611 u8 state;
653 u32 access_flags;
654 u32 atomic_rd_en;612 u32 atomic_rd_en;
655 u32 pkey_index;
656 u32 qkey;613 u32 qkey;
657 void (*event)(struct hns_roce_qp *qp,614 void (*event)(struct hns_roce_qp *qp,
658 enum hns_roce_event event_type);615 enum hns_roce_event event_type);
@@ -672,9 +629,10 @@ struct hns_roce_qp { · 13 unmodified lines
672 unsigned long flush_flag;629 unsigned long flush_flag;
673 struct hns_roce_work flush_work;630 struct hns_roce_work flush_work;
674 struct hns_roce_rinl_buf rq_inl_buf;631 struct hns_roce_rinl_buf rq_inl_buf;
675 struct list_head node; /* all qps are on a list */632 struct list_head node; /* all qps are on a list */
676 struct list_head rq_node; /* all recv qps are on a list */633 struct list_head rq_node; /* all recv qps are on a list */
677 struct list_head sq_node; /* all send qps are on a list */634 struct list_head sq_node; /* all send qps are on a list */
635 struct hns_user_mmap_entry *dwqe_mmap_entry;
678};636};
679 637
680struct hns_roce_ib_iboe {638struct hns_roce_ib_iboe {
@@ -684,11 +642,6 @@ struct hns_roce_ib_iboe { · 3 unmodified lines
684 u8 phy_port[HNS_ROCE_MAX_PORTS];642 u8 phy_port[HNS_ROCE_MAX_PORTS];
685};643};
686 644
687enum {
688 HNS_ROCE_EQ_STAT_INVALID = 0,
689 HNS_ROCE_EQ_STAT_VALID = 2,
690};
691
692struct hns_roce_ceqe {645struct hns_roce_ceqe {
693 __le32 comp;646 __le32 comp;
694 __le32 rsv[15];647 __le32 rsv[15];
@@ -720,12 +673,9 @@ struct hns_roce_eq { · 25 unmodified lines
720 int type_flag; /* Aeq:1 ceq:0 */673 int type_flag; /* Aeq:1 ceq:0 */
721 int eqn;674 int eqn;
722 u32 entries;675 u32 entries;
723 u32 log_entries;
724 int eqe_size;676 int eqe_size;
725 int irq;677 int irq;
726 int log_page_size;
727 u32 cons_index;678 u32 cons_index;
728 struct hns_roce_buf_list *buf_list;
729 int over_ignore;679 int over_ignore;
730 int coalesce;680 int coalesce;
731 int arm_st;681 int arm_st;
@@ -740,7 +690,6 @@ struct hns_roce_eq { · 8 unmodified lines
740 690
741struct hns_roce_eq_table {691struct hns_roce_eq_table {
742 struct hns_roce_eq *eq;692 struct hns_roce_eq *eq;
743 void __iomem **eqc_base; /* only for hw v1 */
744};693};
745 694
746enum cong_type {695enum cong_type {
@@ -767,7 +716,7 @@ struct hns_roce_caps { · 20 unmodified lines
767 u32 reserved_qps;716 u32 reserved_qps;
768 int num_qpc_timer;717 int num_qpc_timer;
769 int num_cqc_timer;718 int num_cqc_timer;
770 int num_srqs;719 u32 num_srqs;
771 u32 max_wqes;720 u32 max_wqes;
772 u32 max_srq_wrs;721 u32 max_srq_wrs;
773 u32 max_srq_sges;722 u32 max_srq_sges;
@@ -781,7 +730,7 @@ struct hns_roce_caps { · 7 unmodified lines
781 u32 min_cqes;730 u32 min_cqes;
782 u32 min_wqes;731 u32 min_wqes;
783 u32 reserved_cqs;732 u32 reserved_cqs;
784 int reserved_srqs;733 u32 reserved_srqs;
785 int num_aeq_vectors;734 int num_aeq_vectors;
786 int num_comp_vectors;735 int num_comp_vectors;
787 int num_other_vectors;736 int num_other_vectors;
@@ -855,7 +804,7 @@ struct hns_roce_caps { · 67 unmodified lines
855 u32 cqc_timer_ba_pg_sz;804 u32 cqc_timer_ba_pg_sz;
856 u32 cqc_timer_buf_pg_sz;805 u32 cqc_timer_buf_pg_sz;
857 u32 cqc_timer_hop_num;806 u32 cqc_timer_hop_num;
858 u32 cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */807 u32 cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */
859 u32 cqe_buf_pg_sz;808 u32 cqe_buf_pg_sz;
860 u32 cqe_hop_num;809 u32 cqe_hop_num;
861 u32 srqwqe_ba_pg_sz;810 u32 srqwqe_ba_pg_sz;
@@ -874,7 +823,7 @@ struct hns_roce_caps { · 12 unmodified lines
874 u32 gmv_hop_num;823 u32 gmv_hop_num;
875 u32 sl_num;824 u32 sl_num;
876 u32 llm_buf_pg_sz;825 u32 llm_buf_pg_sz;
877 u32 chunk_sz; /* chunk size in non multihop mode */826 u32 chunk_sz; /* chunk size in non multihop mode */
878 u64 flags;827 u64 flags;
879 u16 default_ceq_max_cnt;828 u16 default_ceq_max_cnt;
880 u16 default_ceq_period;829 u16 default_ceq_period;
@@ -897,7 +846,6 @@ enum hns_roce_device_state { · 16 unmodified lines
897};846};
898 847
899struct hns_roce_hw {848struct hns_roce_hw {
900 int (*reset)(struct hns_roce_dev *hr_dev, bool enable);
901 int (*cmq_init)(struct hns_roce_dev *hr_dev);849 int (*cmq_init)(struct hns_roce_dev *hr_dev);
902 void (*cmq_exit)(struct hns_roce_dev *hr_dev);850 void (*cmq_exit)(struct hns_roce_dev *hr_dev);
903 int (*hw_profile)(struct hns_roce_dev *hr_dev);851 int (*hw_profile)(struct hns_roce_dev *hr_dev);
@@ -909,14 +857,12 @@ struct hns_roce_hw { · 5 unmodified lines
909 int (*poll_mbox_done)(struct hns_roce_dev *hr_dev,857 int (*poll_mbox_done)(struct hns_roce_dev *hr_dev,
910 unsigned int timeout);858 unsigned int timeout);
911 bool (*chk_mbox_avail)(struct hns_roce_dev *hr_dev, bool *is_busy);859 bool (*chk_mbox_avail)(struct hns_roce_dev *hr_dev, bool *is_busy);
912 int (*set_gid)(struct hns_roce_dev *hr_dev, u32 port, int gid_index,860 int (*set_gid)(struct hns_roce_dev *hr_dev, int gid_index,
913 const union ib_gid *gid, const struct ib_gid_attr *attr);861 const union ib_gid *gid, const struct ib_gid_attr *attr);
914 int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port,862 int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port,
915 const u8 *addr);863 const u8 *addr);
916 void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port,
917 enum ib_mtu mtu);
918 int (*write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf,864 int (*write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf,
919 struct hns_roce_mr *mr, unsigned long mtpt_idx);865 struct hns_roce_mr *mr);
920 int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev,866 int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev,
921 struct hns_roce_mr *mr, int flags,867 struct hns_roce_mr *mr, int flags,
922 void *mb_buf);868 void *mb_buf);
@@ -936,9 +882,6 @@ struct hns_roce_hw { · 13 unmodified lines
936 enum ib_qp_state new_state);882 enum ib_qp_state new_state);
937 int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,883 int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,
938 struct hns_roce_qp *hr_qp);884 struct hns_roce_qp *hr_qp);
939 int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
940 struct ib_udata *udata);
941 int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata);
942 int (*init_eq)(struct hns_roce_dev *hr_dev);885 int (*init_eq)(struct hns_roce_dev *hr_dev);
943 void (*cleanup_eq)(struct hns_roce_dev *hr_dev);886 void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
944 int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);887 int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
@@ -948,13 +891,11 @@ struct hns_roce_hw { · 3 unmodified lines
948 891
949struct hns_roce_dev {892struct hns_roce_dev {
950 struct ib_device ib_dev;893 struct ib_device ib_dev;
951 struct platform_device *pdev;
952 struct pci_dev *pci_dev;894 struct pci_dev *pci_dev;
953 struct device *dev;895 struct device *dev;
954 struct hns_roce_uar priv_uar;896 struct hns_roce_uar priv_uar;
955 const char *irq_names[HNS_ROCE_MAX_IRQ_NUM];897 const char *irq_names[HNS_ROCE_MAX_IRQ_NUM];
956 spinlock_t sm_lock;898 spinlock_t sm_lock;
957 spinlock_t bt_cmd_lock;
958 bool active;899 bool active;
959 bool is_reset;900 bool is_reset;
960 bool dis_db;901 bool dis_db;
@@ -1001,8 +942,6 @@ struct hns_roce_dev { · 40 unmodified lines
1001 int loop_idc;942 int loop_idc;
1002 u32 sdb_offset;943 u32 sdb_offset;
1003 u32 odb_offset;944 u32 odb_offset;
1004 dma_addr_t tptr_dma_addr; /* only for hw v1 */
1005 u32 tptr_size; /* only for hw v1 */
1006 const struct hns_roce_hw *hw;945 const struct hns_roce_hw *hw;
1007 void *priv;946 void *priv;
1008 struct workqueue_struct *irq_workq;947 struct workqueue_struct *irq_workq;
@@ -1010,6 +949,7 @@ struct hns_roce_dev { · 1 unmodified line
1010 u32 func_num;949 u32 func_num;
1011 u32 is_vf;950 u32 is_vf;
1012 u32 cong_algo_tmpl_id;951 u32 cong_algo_tmpl_id;
952 u64 dwqe_page;
1013};953};
1014 954
1015static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)955static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
@@ -1158,7 +1098,7 @@ void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev); · 142 unmodified lines
1158/* hns roce hw need current block and next block addr from mtt */1098/* hns roce hw need current block and next block addr from mtt */
1159#define MTT_MIN_COUNT 21099#define MTT_MIN_COUNT 2
1160int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,1100int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
1161 int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr);1101 u32 offset, u64 *mtt_buf, int mtt_max, u64 *base_addr);
1162int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,1102int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
1163 struct hns_roce_buf_attr *buf_attr,1103 struct hns_roce_buf_attr *buf_attr,
1164 unsigned int page_shift, struct ib_udata *udata,1104 unsigned int page_shift, struct ib_udata *udata,
drivers/infiniband/hw/hns/hns_roce_hem.c+0−1
@@ -31,7 +31,6 @@ · 30 unmodified lines
31 * SOFTWARE.31 * SOFTWARE.
32 */32 */
33 33
34#include <linux/platform_device.h>
35#include "hns_roce_device.h"34#include "hns_roce_device.h"
36#include "hns_roce_hem.h"35#include "hns_roce_hem.h"
37#include "hns_roce_common.h"36#include "hns_roce_common.h"
drivers/infiniband/hw/hns/hns_roce_hw_v1.c+0−4675
@@ -1,4675 +0,0 @@
1/*
2 * Copyright (c) 2016 Hisilicon Limited.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/platform_device.h>
34#include <linux/acpi.h>
35#include <linux/etherdevice.h>
36#include <linux/interrupt.h>
37#include <linux/of.h>
38#include <linux/of_platform.h>
39#include <rdma/ib_umem.h>
40#include "hns_roce_common.h"
41#include "hns_roce_device.h"
42#include "hns_roce_cmd.h"
43#include "hns_roce_hem.h"
44#include "hns_roce_hw_v1.h"
45
46/**
47 * hns_get_gid_index - Get gid index.
48 * @hr_dev: pointer to structure hns_roce_dev.
49 * @port: port, value range: 0 ~ MAX
50 * @gid_index: gid_index, value range: 0 ~ MAX
51 * Description:
52 * N ports shared gids, allocation method as follow:
53 * GID[0][0], GID[1][0],.....GID[N - 1][0],
54 * GID[0][0], GID[1][0],.....GID[N - 1][0],
55 * And so on
56 */
57u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index)
58{
59 return gid_index * hr_dev->caps.num_ports + port;
60}
61
62static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg)
63{
64 dseg->lkey = cpu_to_le32(sg->lkey);
65 dseg->addr = cpu_to_le64(sg->addr);
66 dseg->len = cpu_to_le32(sg->length);
67}
68
69static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr,
70 u32 rkey)
71{
72 rseg->raddr = cpu_to_le64(remote_addr);
73 rseg->rkey = cpu_to_le32(rkey);
74 rseg->len = 0;
75}
76
77static int hns_roce_v1_post_send(struct ib_qp *ibqp,
78 const struct ib_send_wr *wr,
79 const struct ib_send_wr **bad_wr)
80{
81 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
82 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
83 struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL;
84 struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
85 struct hns_roce_wqe_data_seg *dseg = NULL;
86 struct hns_roce_qp *qp = to_hr_qp(ibqp);
87 struct device *dev = &hr_dev->pdev->dev;
88 struct hns_roce_sq_db sq_db = {};
89 int ps_opcode, i;
90 unsigned long flags = 0;
91 void *wqe = NULL;
92 __le32 doorbell[2];
93 const u8 *smac;
94 int ret = 0;
95 int loopback;
96 u32 wqe_idx;
97 int nreq;
98
99 if (unlikely(ibqp->qp_type != IB_QPT_GSI &&
100 ibqp->qp_type != IB_QPT_RC)) {
101 dev_err(dev, "un-supported QP type\n");
102 *bad_wr = NULL;
103 return -EOPNOTSUPP;
104 }
105
106 spin_lock_irqsave(&qp->sq.lock, flags);
107
108 for (nreq = 0; wr; ++nreq, wr = wr->next) {
109 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
110 ret = -ENOMEM;
111 *bad_wr = wr;
112 goto out;
113 }
114
115 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
116
117 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
118 dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
119 wr->num_sge, qp->sq.max_gs);
120 ret = -EINVAL;
121 *bad_wr = wr;
122 goto out;
123 }
124
125 wqe = hns_roce_get_send_wqe(qp, wqe_idx);
126 qp->sq.wrid[wqe_idx] = wr->wr_id;
127
128 /* Corresponding to the RC and RD type wqe process separately */
129 if (ibqp->qp_type == IB_QPT_GSI) {
130 ud_sq_wqe = wqe;
131 roce_set_field(ud_sq_wqe->dmac_h,
132 UD_SEND_WQE_U32_4_DMAC_0_M,
133 UD_SEND_WQE_U32_4_DMAC_0_S,
134 ah->av.mac[0]);
135 roce_set_field(ud_sq_wqe->dmac_h,
136 UD_SEND_WQE_U32_4_DMAC_1_M,
137 UD_SEND_WQE_U32_4_DMAC_1_S,
138 ah->av.mac[1]);
139 roce_set_field(ud_sq_wqe->dmac_h,
140 UD_SEND_WQE_U32_4_DMAC_2_M,
141 UD_SEND_WQE_U32_4_DMAC_2_S,
142 ah->av.mac[2]);
143 roce_set_field(ud_sq_wqe->dmac_h,
144 UD_SEND_WQE_U32_4_DMAC_3_M,
145 UD_SEND_WQE_U32_4_DMAC_3_S,
146 ah->av.mac[3]);
147
148 roce_set_field(ud_sq_wqe->u32_8,
149 UD_SEND_WQE_U32_8_DMAC_4_M,
150 UD_SEND_WQE_U32_8_DMAC_4_S,
151 ah->av.mac[4]);
152 roce_set_field(ud_sq_wqe->u32_8,
153 UD_SEND_WQE_U32_8_DMAC_5_M,
154 UD_SEND_WQE_U32_8_DMAC_5_S,
155 ah->av.mac[5]);
156
157 smac = (const u8 *)hr_dev->dev_addr[qp->port];
158 loopback = ether_addr_equal_unaligned(ah->av.mac,
159 smac) ? 1 : 0;
160 roce_set_bit(ud_sq_wqe->u32_8,
161 UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S,
162 loopback);
163
164 roce_set_field(ud_sq_wqe->u32_8,
165 UD_SEND_WQE_U32_8_OPERATION_TYPE_M,
166 UD_SEND_WQE_U32_8_OPERATION_TYPE_S,
167 HNS_ROCE_WQE_OPCODE_SEND);
168 roce_set_field(ud_sq_wqe->u32_8,
169 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M,
170 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S,
171 2);
172 roce_set_bit(ud_sq_wqe->u32_8,
173 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S,
174 1);
175
176 ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ?
177 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
178 (wr->send_flags & IB_SEND_SOLICITED ?
179 cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
180 ((wr->opcode == IB_WR_SEND_WITH_IMM) ?
181 cpu_to_le32(HNS_ROCE_WQE_IMM) : 0);
182
183 roce_set_field(ud_sq_wqe->u32_16,
184 UD_SEND_WQE_U32_16_DEST_QP_M,
185 UD_SEND_WQE_U32_16_DEST_QP_S,
186 ud_wr(wr)->remote_qpn);
187 roce_set_field(ud_sq_wqe->u32_16,
188 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M,
189 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S,
190 ah->av.stat_rate);
191
192 roce_set_field(ud_sq_wqe->u32_36,
193 UD_SEND_WQE_U32_36_FLOW_LABEL_M,
194 UD_SEND_WQE_U32_36_FLOW_LABEL_S,
195 ah->av.flowlabel);
196 roce_set_field(ud_sq_wqe->u32_36,
197 UD_SEND_WQE_U32_36_PRIORITY_M,
198 UD_SEND_WQE_U32_36_PRIORITY_S,
199 ah->av.sl);
200 roce_set_field(ud_sq_wqe->u32_36,
201 UD_SEND_WQE_U32_36_SGID_INDEX_M,
202 UD_SEND_WQE_U32_36_SGID_INDEX_S,
203 hns_get_gid_index(hr_dev, qp->phy_port,
204 ah->av.gid_index));
205
206 roce_set_field(ud_sq_wqe->u32_40,
207 UD_SEND_WQE_U32_40_HOP_LIMIT_M,
208 UD_SEND_WQE_U32_40_HOP_LIMIT_S,
209 ah->av.hop_limit);
210 roce_set_field(ud_sq_wqe->u32_40,
211 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M,
212 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S,
213 ah->av.tclass);
214
215 memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN);
216
217 ud_sq_wqe->va0_l =
218 cpu_to_le32((u32)wr->sg_list[0].addr);
219 ud_sq_wqe->va0_h =
220 cpu_to_le32((wr->sg_list[0].addr) >> 32);
221 ud_sq_wqe->l_key0 =
222 cpu_to_le32(wr->sg_list[0].lkey);
223
224 ud_sq_wqe->va1_l =
225 cpu_to_le32((u32)wr->sg_list[1].addr);
226 ud_sq_wqe->va1_h =
227 cpu_to_le32((wr->sg_list[1].addr) >> 32);
228 ud_sq_wqe->l_key1 =
229 cpu_to_le32(wr->sg_list[1].lkey);
230 } else if (ibqp->qp_type == IB_QPT_RC) {
231 u32 tmp_len = 0;
232
233 ctrl = wqe;
234 memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
235 for (i = 0; i < wr->num_sge; i++)
236 tmp_len += wr->sg_list[i].length;
237
238 ctrl->msg_length =
239 cpu_to_le32(le32_to_cpu(ctrl->msg_length) + tmp_len);
240
241 ctrl->sgl_pa_h = 0;
242 ctrl->flag = 0;
243
244 switch (wr->opcode) {
245 case IB_WR_SEND_WITH_IMM:
246 case IB_WR_RDMA_WRITE_WITH_IMM:
247 ctrl->imm_data = wr->ex.imm_data;
248 break;
249 case IB_WR_SEND_WITH_INV:
250 ctrl->inv_key =
251 cpu_to_le32(wr->ex.invalidate_rkey);
252 break;
253 default:
254 ctrl->imm_data = 0;
255 break;
256 }
257
258 /* Ctrl field, ctrl set type: sig, solic, imm, fence */
259 /* SO wait for conforming application scenarios */
260 ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ?
261 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
262 (wr->send_flags & IB_SEND_SOLICITED ?
263 cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
264 ((wr->opcode == IB_WR_SEND_WITH_IMM ||
265 wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ?
266 cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) |
267 (wr->send_flags & IB_SEND_FENCE ?
268 (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0);
269
270 wqe += sizeof(struct hns_roce_wqe_ctrl_seg);
271
272 switch (wr->opcode) {
273 case IB_WR_RDMA_READ:
274 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
275 set_raddr_seg(wqe, rdma_wr(wr)->remote_addr,
276 rdma_wr(wr)->rkey);
277 break;
278 case IB_WR_RDMA_WRITE:
279 case IB_WR_RDMA_WRITE_WITH_IMM:
280 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
281 set_raddr_seg(wqe, rdma_wr(wr)->remote_addr,
282 rdma_wr(wr)->rkey);
283 break;
284 case IB_WR_SEND:
285 case IB_WR_SEND_WITH_INV:
286 case IB_WR_SEND_WITH_IMM:
287 ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
288 break;
289 case IB_WR_LOCAL_INV:
290 case IB_WR_ATOMIC_CMP_AND_SWP:
291 case IB_WR_ATOMIC_FETCH_AND_ADD:
292 case IB_WR_LSO:
293 default:
294 ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
295 break;
296 }
297 ctrl->flag |= cpu_to_le32(ps_opcode);
298 wqe += sizeof(struct hns_roce_wqe_raddr_seg);
299
300 dseg = wqe;
301 if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
302 if (le32_to_cpu(ctrl->msg_length) >
303 hr_dev->caps.max_sq_inline) {
304 ret = -EINVAL;
305 *bad_wr = wr;
306 dev_err(dev, "inline len(1-%d)=%d, illegal",
307 le32_to_cpu(ctrl->msg_length),
308 hr_dev->caps.max_sq_inline);
309 goto out;
310 }
311 for (i = 0; i < wr->num_sge; i++) {
312 memcpy(wqe, ((void *) (uintptr_t)
313 wr->sg_list[i].addr),
314 wr->sg_list[i].length);
315 wqe += wr->sg_list[i].length;
316 }
317 ctrl->flag |= cpu_to_le32(HNS_ROCE_WQE_INLINE);
318 } else {
319 /* sqe num is two */
320 for (i = 0; i < wr->num_sge; i++)
321 set_data_seg(dseg + i, wr->sg_list + i);
322
323 ctrl->flag |= cpu_to_le32(wr->num_sge <<
324 HNS_ROCE_WQE_SGE_NUM_BIT);
325 }
326 }
327 }
328
329out:
330 /* Set DB return */
331 if (likely(nreq)) {
332 qp->sq.head += nreq;
333
334 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
335 SQ_DOORBELL_U32_4_SQ_HEAD_S,
336 (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
337 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SL_M,
338 SQ_DOORBELL_U32_4_SL_S, qp->sl);
339 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,
340 SQ_DOORBELL_U32_4_PORT_S, qp->phy_port);
341 roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,
342 SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn);
343 roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);
344
345 doorbell[0] = sq_db.u32_4;
346 doorbell[1] = sq_db.u32_8;
347
348 hns_roce_write64_k(doorbell, qp->sq.db_reg);
349 }
350
351 spin_unlock_irqrestore(&qp->sq.lock, flags);
352
353 return ret;
354}
355
356static int hns_roce_v1_post_recv(struct ib_qp *ibqp,
357 const struct ib_recv_wr *wr,
358 const struct ib_recv_wr **bad_wr)
359{
360 struct hns_roce_rq_wqe_ctrl *ctrl = NULL;
361 struct hns_roce_wqe_data_seg *scat = NULL;
362 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
363 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
364 struct device *dev = &hr_dev->pdev->dev;
365 struct hns_roce_rq_db rq_db = {};
366 __le32 doorbell[2] = {0};
367 unsigned long flags = 0;
368 unsigned int wqe_idx;
369 int ret = 0;
370 int nreq;
371 int i;
372 u32 reg_val;
373
374 spin_lock_irqsave(&hr_qp->rq.lock, flags);
375
376 for (nreq = 0; wr; ++nreq, wr = wr->next) {
377 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
378 hr_qp->ibqp.recv_cq)) {
379 ret = -ENOMEM;
380 *bad_wr = wr;
381 goto out;
382 }
383
384 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
385
386 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
387 dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
388 wr->num_sge, hr_qp->rq.max_gs);
389 ret = -EINVAL;
390 *bad_wr = wr;
391 goto out;
392 }
393
394 ctrl = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
395
396 roce_set_field(ctrl->rwqe_byte_12,
397 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,
398 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S,
399 wr->num_sge);
400
401 scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1);
402
403 for (i = 0; i < wr->num_sge; i++)
404 set_data_seg(scat + i, wr->sg_list + i);
405
406 hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
407 }
408
409out:
410 if (likely(nreq)) {
411 hr_qp->rq.head += nreq;
412
413 if (ibqp->qp_type == IB_QPT_GSI) {
414 __le32 tmp;
415
416 /* SW update GSI rq header */
417 reg_val = roce_read(to_hr_dev(ibqp->device),
418 ROCEE_QP1C_CFG3_0_REG +
419 QP1C_CFGN_OFFSET * hr_qp->phy_port);
420 tmp = cpu_to_le32(reg_val);
421 roce_set_field(tmp,
422 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
423 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
424 hr_qp->rq.head);
425 reg_val = le32_to_cpu(tmp);
426 roce_write(to_hr_dev(ibqp->device),
427 ROCEE_QP1C_CFG3_0_REG +
428 QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val);
429 } else {
430 roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
431 RQ_DOORBELL_U32_4_RQ_HEAD_S,
432 hr_qp->rq.head);
433 roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
434 RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
435 roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
436 RQ_DOORBELL_U32_8_CMD_S, 1);
437 roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
438 1);
439
440 doorbell[0] = rq_db.u32_4;
441 doorbell[1] = rq_db.u32_8;
442
443 hns_roce_write64_k(doorbell, hr_qp->rq.db_reg);
444 }
445 }
446 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
447
448 return ret;
449}
450
451static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev,
452 int sdb_mode, int odb_mode)
453{
454 __le32 tmp;
455 u32 val;
456
457 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
458 tmp = cpu_to_le32(val);
459 roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);
460 roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);
461 val = le32_to_cpu(tmp);
462 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
463}
464
465static int hns_roce_v1_set_hem(struct hns_roce_dev *hr_dev,
466 struct hns_roce_hem_table *table, int obj,
467 int step_idx)
468{
469 spinlock_t *lock = &hr_dev->bt_cmd_lock;
470 struct device *dev = hr_dev->dev;
471 struct hns_roce_hem_iter iter;
472 void __iomem *bt_cmd;
473 __le32 bt_cmd_val[2];
474 __le32 bt_cmd_h = 0;
475 unsigned long flags;
476 __le32 bt_cmd_l;
477 int ret = 0;
478 u64 bt_ba;
479 long end;
480
481 /* Find the HEM(Hardware Entry Memory) entry */
482 unsigned long i = obj / (table->table_chunk_size / table->obj_size);
483
484 switch (table->type) {
485 case HEM_TYPE_QPC:
486 case HEM_TYPE_MTPT:
487 case HEM_TYPE_CQC:
488 case HEM_TYPE_SRQC:
489 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
490 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type);
491 break;
492 default:
493 return ret;
494 }
495
496 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
497 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
498 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
499 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
500
501 /* Currently iter only a chunk */
502 for (hns_roce_hem_first(table->hem[i], &iter);
503 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
504 bt_ba = hns_roce_hem_addr(&iter) >> HNS_HW_PAGE_SHIFT;
505
506 spin_lock_irqsave(lock, flags);
507
508 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
509
510 end = HW_SYNC_TIMEOUT_MSECS;
511 while (end > 0) {
512 if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT))
513 break;
514
515 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
516 end -= HW_SYNC_SLEEP_TIME_INTERVAL;
517 }
518
519 if (end <= 0) {
520 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
521 spin_unlock_irqrestore(lock, flags);
522 return -EBUSY;
523 }
524
525 bt_cmd_l = cpu_to_le32(bt_ba);
526 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
527 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
528 upper_32_bits(bt_ba));
529
530 bt_cmd_val[0] = bt_cmd_l;
531 bt_cmd_val[1] = bt_cmd_h;
532 hns_roce_write64_k(bt_cmd_val,
533 hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
534 spin_unlock_irqrestore(lock, flags);
535 }
536
537 return ret;
538}
539
540static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode,
541 u32 odb_mode)
542{
543 __le32 tmp;
544 u32 val;
545
546 /* Configure SDB/ODB extend mode */
547 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
548 tmp = cpu_to_le32(val);
549 roce_set_bit(tmp, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);
550 roce_set_bit(tmp, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);
551 val = le32_to_cpu(tmp);
552 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
553}
554
555static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept,
556 u32 sdb_alful)
557{
558 __le32 tmp;
559 u32 val;
560
561 /* Configure SDB */
562 val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG);
563 tmp = cpu_to_le32(val);
564 roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,
565 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful);
566 roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,
567 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept);
568 val = le32_to_cpu(tmp);
569 roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val);
570}
571
572static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,
573 u32 odb_alful)
574{
575 __le32 tmp;
576 u32 val;
577
578 /* Configure ODB */
579 val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG);
580 tmp = cpu_to_le32(val);
581 roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,
582 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful);
583 roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,
584 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept);
585 val = le32_to_cpu(tmp);
586 roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val);
587}
588
589static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
590 u32 ext_sdb_alful)
591{
592 struct hns_roce_v1_priv *priv = hr_dev->priv;
593 struct hns_roce_db_table *db = &priv->db_table;
594 struct device *dev = &hr_dev->pdev->dev;
595 dma_addr_t sdb_dma_addr;
596 __le32 tmp;
597 u32 val;
598
599 /* Configure extend SDB threshold */
600 roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);
601 roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);
602
603 /* Configure extend SDB base addr */
604 sdb_dma_addr = db->ext_db->sdb_buf_list->map;
605 roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12));
606
607 /* Configure extend SDB depth */
608 val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG);
609 tmp = cpu_to_le32(val);
610 roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,
611 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,
612 db->ext_db->esdb_dep);
613 /*
614 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
615 * using 4K page, and shift more 32 because of
616 * calculating the high 32 bit value evaluated to hardware.
617 */
618 roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,
619 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44);
620 val = le32_to_cpu(tmp);
621 roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val);
622
623 dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep);
624 dev_dbg(dev, "ext SDB threshold: empty: 0x%x, ful: 0x%x\n",
625 ext_sdb_alept, ext_sdb_alful);
626}
627
628static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
629 u32 ext_odb_alful)
630{
631 struct hns_roce_v1_priv *priv = hr_dev->priv;
632 struct hns_roce_db_table *db = &priv->db_table;
633 struct device *dev = &hr_dev->pdev->dev;
634 dma_addr_t odb_dma_addr;
635 __le32 tmp;
636 u32 val;
637
638 /* Configure extend ODB threshold */
639 roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);
640 roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);
641
642 /* Configure extend ODB base addr */
643 odb_dma_addr = db->ext_db->odb_buf_list->map;
644 roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12));
645
646 /* Configure extend ODB depth */
647 val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG);
648 tmp = cpu_to_le32(val);
649 roce_set_field(tmp, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,
650 ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,
651 db->ext_db->eodb_dep);
652 roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,
653 ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,
654 db->ext_db->eodb_dep);
655 val = le32_to_cpu(tmp);
656 roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val);
657
658 dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep);
659 dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
660 ext_odb_alept, ext_odb_alful);
661}
662
663static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
664 u32 odb_ext_mod)
665{
666 struct hns_roce_v1_priv *priv = hr_dev->priv;
667 struct hns_roce_db_table *db = &priv->db_table;
668 struct device *dev = &hr_dev->pdev->dev;
669 dma_addr_t sdb_dma_addr;
670 dma_addr_t odb_dma_addr;
671 int ret = 0;
672
673 db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
674 if (!db->ext_db)
675 return -ENOMEM;
676
677 if (sdb_ext_mod) {
678 db->ext_db->sdb_buf_list = kmalloc(
679 sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL);
680 if (!db->ext_db->sdb_buf_list) {
681 ret = -ENOMEM;
682 goto ext_sdb_buf_fail_out;
683 }
684
685 db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev,
686 HNS_ROCE_V1_EXT_SDB_SIZE,
687 &sdb_dma_addr, GFP_KERNEL);
688 if (!db->ext_db->sdb_buf_list->buf) {
689 ret = -ENOMEM;
690 goto alloc_sq_db_buf_fail;
691 }
692 db->ext_db->sdb_buf_list->map = sdb_dma_addr;
693
694 db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);
695 hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,
696 HNS_ROCE_V1_EXT_SDB_ALFUL);
697 } else
698 hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,
699 HNS_ROCE_V1_SDB_ALFUL);
700
701 if (odb_ext_mod) {
702 db->ext_db->odb_buf_list = kmalloc(
703 sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL);
704 if (!db->ext_db->odb_buf_list) {
705 ret = -ENOMEM;
706 goto ext_odb_buf_fail_out;
707 }
708
709 db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev,
710 HNS_ROCE_V1_EXT_ODB_SIZE,
711 &odb_dma_addr, GFP_KERNEL);
712 if (!db->ext_db->odb_buf_list->buf) {
713 ret = -ENOMEM;
714 goto alloc_otr_db_buf_fail;
715 }
716 db->ext_db->odb_buf_list->map = odb_dma_addr;
717
718 db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);
719 hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,
720 HNS_ROCE_V1_EXT_ODB_ALFUL);
721 } else
722 hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,
723 HNS_ROCE_V1_ODB_ALFUL);
724
725 hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);
726
727 return 0;
728
729alloc_otr_db_buf_fail:
730 kfree(db->ext_db->odb_buf_list);
731
732ext_odb_buf_fail_out:
733 if (sdb_ext_mod) {
734 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
735 db->ext_db->sdb_buf_list->buf,
736 db->ext_db->sdb_buf_list->map);
737 }
738
739alloc_sq_db_buf_fail:
740 if (sdb_ext_mod)
741 kfree(db->ext_db->sdb_buf_list);
742
743ext_sdb_buf_fail_out:
744 kfree(db->ext_db);
745 return ret;
746}
747
748static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev,
749 struct ib_pd *pd)
750{
751 struct device *dev = &hr_dev->pdev->dev;
752 struct ib_qp_init_attr init_attr;
753 struct ib_qp *qp;
754
755 memset(&init_attr, 0, sizeof(struct ib_qp_init_attr));
756 init_attr.qp_type = IB_QPT_RC;
757 init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
758 init_attr.cap.max_recv_wr = HNS_ROCE_MIN_WQE_NUM;
759 init_attr.cap.max_send_wr = HNS_ROCE_MIN_WQE_NUM;
760
761 qp = ib_create_qp(pd, &init_attr);
762 if (IS_ERR(qp)) {
763 dev_err(dev, "Create loop qp for mr free failed!");
764 return NULL;
765 }
766
767 return to_hr_qp(qp);
768}
769
770static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
771{
772 struct hns_roce_v1_priv *priv = hr_dev->priv;
773 struct hns_roce_free_mr *free_mr = &priv->free_mr;
774 struct hns_roce_caps *caps = &hr_dev->caps;
775 struct ib_device *ibdev = &hr_dev->ib_dev;
776 struct device *dev = &hr_dev->pdev->dev;
777 struct ib_cq_init_attr cq_init_attr;
778 struct ib_qp_attr attr = { 0 };
779 struct hns_roce_qp *hr_qp;
780 struct ib_cq *cq;
781 struct ib_pd *pd;
782 union ib_gid dgid;
783 __be64 subnet_prefix;
784 int attr_mask = 0;
785 int ret;
786 int i, j;
787 u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 };
788 u8 phy_port;
789 u32 port = 0;
790 u8 sl;
791
792 /* Reserved cq for loop qp */
793 cq_init_attr.cqe = HNS_ROCE_MIN_WQE_NUM * 2;
794 cq_init_attr.comp_vector = 0;
795
796 cq = rdma_zalloc_drv_obj(ibdev, ib_cq);
797 if (!cq)
798 return -ENOMEM;
799
800 ret = hns_roce_create_cq(cq, &cq_init_attr, NULL);
801 if (ret) {
802 dev_err(dev, "Create cq for reserved loop qp failed!");
803 goto alloc_cq_failed;
804 }
805 free_mr->mr_free_cq = to_hr_cq(cq);
806 free_mr->mr_free_cq->ib_cq.device = &hr_dev->ib_dev;
807 free_mr->mr_free_cq->ib_cq.uobject = NULL;
808 free_mr->mr_free_cq->ib_cq.comp_handler = NULL;
809 free_mr->mr_free_cq->ib_cq.event_handler = NULL;
810 free_mr->mr_free_cq->ib_cq.cq_context = NULL;
811 atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0);
812
813 pd = rdma_zalloc_drv_obj(ibdev, ib_pd);
814 if (!pd) {
815 ret = -ENOMEM;
816 goto alloc_mem_failed;
817 }
818
819 pd->device = ibdev;
820 ret = hns_roce_alloc_pd(pd, NULL);
821 if (ret)
822 goto alloc_pd_failed;
823
824 free_mr->mr_free_pd = to_hr_pd(pd);
825 free_mr->mr_free_pd->ibpd.device = &hr_dev->ib_dev;
826 free_mr->mr_free_pd->ibpd.uobject = NULL;
827 free_mr->mr_free_pd->ibpd.__internal_mr = NULL;
828 atomic_set(&free_mr->mr_free_pd->ibpd.usecnt, 0);
829
830 attr.qp_access_flags = IB_ACCESS_REMOTE_WRITE;
831 attr.pkey_index = 0;
832 attr.min_rnr_timer = 0;
833 /* Disable read ability */
834 attr.max_dest_rd_atomic = 0;
835 attr.max_rd_atomic = 0;
836 /* Use arbitrary values as rq_psn and sq_psn */
837 attr.rq_psn = 0x0808;
838 attr.sq_psn = 0x0808;
839 attr.retry_cnt = 7;
840 attr.rnr_retry = 7;
841 attr.timeout = 0x12;
842 attr.path_mtu = IB_MTU_256;
843 attr.ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
844 rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0);
845 rdma_ah_set_static_rate(&attr.ah_attr, 3);
846
847 subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
848 for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
849 phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) :
850 (i % HNS_ROCE_MAX_PORTS);
851 sl = i / HNS_ROCE_MAX_PORTS;
852
853 for (j = 0; j < caps->num_ports; j++) {
854 if (hr_dev->iboe.phy_port[j] == phy_port) {
855 queue_en[i] = 1;
856 port = j;
857 break;
858 }
859 }
860
861 if (!queue_en[i])
862 continue;
863
864 free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd);
865 if (!free_mr->mr_free_qp[i]) {
866 dev_err(dev, "Create loop qp failed!\n");
867 ret = -ENOMEM;
868 goto create_lp_qp_failed;
869 }
870 hr_qp = free_mr->mr_free_qp[i];
871
872 hr_qp->port = port;
873 hr_qp->phy_port = phy_port;
874 hr_qp->ibqp.qp_type = IB_QPT_RC;
875 hr_qp->ibqp.device = &hr_dev->ib_dev;
876 hr_qp->ibqp.uobject = NULL;
877 atomic_set(&hr_qp->ibqp.usecnt, 0);
878 hr_qp->ibqp.pd = pd;
879 hr_qp->ibqp.recv_cq = cq;
880 hr_qp->ibqp.send_cq = cq;
881
882 rdma_ah_set_port_num(&attr.ah_attr, port + 1);
883 rdma_ah_set_sl(&attr.ah_attr, sl);
884 attr.port_num = port + 1;
885
886 attr.dest_qp_num = hr_qp->qpn;
887 memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr),
888 hr_dev->dev_addr[port],
889 ETH_ALEN);
890
891 memcpy(&dgid.raw, &subnet_prefix, sizeof(u64));
892 memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3);
893 memcpy(&dgid.raw[13], hr_dev->dev_addr[port] + 3, 3);
894 dgid.raw[11] = 0xff;
895 dgid.raw[12] = 0xfe;
896 dgid.raw[8] ^= 2;
897 rdma_ah_set_dgid_raw(&attr.ah_attr, dgid.raw);
898
899 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
900 IB_QPS_RESET, IB_QPS_INIT);
901 if (ret) {
902 dev_err(dev, "modify qp failed(%d)!\n", ret);
903 goto create_lp_qp_failed;
904 }
905
906 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, IB_QP_DEST_QPN,
907 IB_QPS_INIT, IB_QPS_RTR);
908 if (ret) {
909 dev_err(dev, "modify qp failed(%d)!\n", ret);
910 goto create_lp_qp_failed;
911 }
912
913 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
914 IB_QPS_RTR, IB_QPS_RTS);
915 if (ret) {
916 dev_err(dev, "modify qp failed(%d)!\n", ret);
917 goto create_lp_qp_failed;
918 }
919 }
920
921 return 0;
922
923create_lp_qp_failed:
924 for (i -= 1; i >= 0; i--) {
925 hr_qp = free_mr->mr_free_qp[i];
926 if (ib_destroy_qp(&hr_qp->ibqp))
927 dev_err(dev, "Destroy qp %d for mr free failed!\n", i);
928 }
929
930 hns_roce_dealloc_pd(pd, NULL);
931
932alloc_pd_failed:
933 kfree(pd);
934
935alloc_mem_failed:
936 hns_roce_destroy_cq(cq, NULL);
937alloc_cq_failed:
938 kfree(cq);
939 return ret;
940}
941
942static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
943{
944 struct hns_roce_v1_priv *priv = hr_dev->priv;
945 struct hns_roce_free_mr *free_mr = &priv->free_mr;
946 struct device *dev = &hr_dev->pdev->dev;
947 struct hns_roce_qp *hr_qp;
948 int ret;
949 int i;
950
951 for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
952 hr_qp = free_mr->mr_free_qp[i];
953 if (!hr_qp)
954 continue;
955
956 ret = ib_destroy_qp(&hr_qp->ibqp);
957 if (ret)
958 dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n",
959 i, ret);
960 }
961
962 hns_roce_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL);
963 kfree(&free_mr->mr_free_cq->ib_cq);
964 hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL);
965 kfree(&free_mr->mr_free_pd->ibpd);
966}
967
968static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
969{
970 struct hns_roce_v1_priv *priv = hr_dev->priv;
971 struct hns_roce_db_table *db = &priv->db_table;
972 struct device *dev = &hr_dev->pdev->dev;
973 u32 sdb_ext_mod;
974 u32 odb_ext_mod;
975 u32 sdb_evt_mod;
976 u32 odb_evt_mod;
977 int ret;
978
979 memset(db, 0, sizeof(*db));
980
981 /* Default DB mode */
982 sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;
983 odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;
984 sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;
985 odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;
986
987 db->sdb_ext_mod = sdb_ext_mod;
988 db->odb_ext_mod = odb_ext_mod;
989
990 /* Init extend DB */
991 ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);
992 if (ret) {
993 dev_err(dev, "Failed in extend DB configuration.\n");
994 return ret;
995 }
996
997 hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);
998
999 return 0;
1000}
1001
1002static void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work)
1003{
1004 struct hns_roce_recreate_lp_qp_work *lp_qp_work;
1005 struct hns_roce_dev *hr_dev;
1006
1007 lp_qp_work = container_of(work, struct hns_roce_recreate_lp_qp_work,
1008 work);
1009 hr_dev = to_hr_dev(lp_qp_work->ib_dev);
1010
1011 hns_roce_v1_release_lp_qp(hr_dev);
1012
1013 if (hns_roce_v1_rsv_lp_qp(hr_dev))
1014 dev_err(&hr_dev->pdev->dev, "create reserver qp failed\n");
1015
1016 if (lp_qp_work->comp_flag)
1017 complete(lp_qp_work->comp);
1018
1019 kfree(lp_qp_work);
1020}
1021
1022static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev)
1023{
1024 long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS;
1025 struct hns_roce_v1_priv *priv = hr_dev->priv;
1026 struct hns_roce_free_mr *free_mr = &priv->free_mr;
1027 struct hns_roce_recreate_lp_qp_work *lp_qp_work;
1028 struct device *dev = &hr_dev->pdev->dev;
1029 struct completion comp;
1030
1031 lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work),
1032 GFP_KERNEL);
1033 if (!lp_qp_work)
1034 return -ENOMEM;
1035
1036 INIT_WORK(&(lp_qp_work->work), hns_roce_v1_recreate_lp_qp_work_fn);
1037
1038 lp_qp_work->ib_dev = &(hr_dev->ib_dev);
1039 lp_qp_work->comp = &comp;
1040 lp_qp_work->comp_flag = 1;
1041
1042 init_completion(lp_qp_work->comp);
1043
1044 queue_work(free_mr->free_mr_wq, &(lp_qp_work->work));
1045
1046 while (end > 0) {
1047 if (try_wait_for_completion(&comp))
1048 return 0;
1049 msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE);
1050 end -= HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE;
1051 }
1052
1053 lp_qp_work->comp_flag = 0;
1054 if (try_wait_for_completion(&comp))
1055 return 0;
1056
1057 dev_warn(dev, "recreate lp qp failed 20s timeout and return failed!\n");
1058 return -ETIMEDOUT;
1059}
1060
1061static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp *hr_qp)
1062{
1063 struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
1064 struct device *dev = &hr_dev->pdev->dev;
1065 struct ib_send_wr send_wr;
1066 const struct ib_send_wr *bad_wr;
1067 int ret;
1068
1069 memset(&send_wr, 0, sizeof(send_wr));
1070 send_wr.next = NULL;
1071 send_wr.num_sge = 0;
1072 send_wr.send_flags = 0;
1073 send_wr.sg_list = NULL;
1074 send_wr.wr_id = (unsigned long long)&send_wr;
1075 send_wr.opcode = IB_WR_RDMA_WRITE;
1076
1077 ret = hns_roce_v1_post_send(&hr_qp->ibqp, &send_wr, &bad_wr);
1078 if (ret) {
1079 dev_err(dev, "Post write wqe for mr free failed(%d)!", ret);
1080 return ret;
1081 }
1082
1083 return 0;
1084}
1085
1086static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
1087{
1088 unsigned long end =
1089 msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
1090 struct hns_roce_mr_free_work *mr_work =
1091 container_of(work, struct hns_roce_mr_free_work, work);
1092 struct hns_roce_dev *hr_dev = to_hr_dev(mr_work->ib_dev);
1093 struct hns_roce_v1_priv *priv = hr_dev->priv;
1094 struct hns_roce_free_mr *free_mr = &priv->free_mr;
1095 struct hns_roce_cq *mr_free_cq = free_mr->mr_free_cq;
1096 struct hns_roce_mr *hr_mr = mr_work->mr;
1097 struct device *dev = &hr_dev->pdev->dev;
1098 struct ib_wc wc[HNS_ROCE_V1_RESV_QP];
1099 struct hns_roce_qp *hr_qp;
1100 int ne = 0;
1101 int ret;
1102 int i;
1103
1104 for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
1105 hr_qp = free_mr->mr_free_qp[i];
1106 if (!hr_qp)
1107 continue;
1108 ne++;
1109
1110 ret = hns_roce_v1_send_lp_wqe(hr_qp);
1111 if (ret) {
1112 dev_err(dev,
1113 "Send wqe (qp:0x%lx) for mr free failed(%d)!\n",
1114 hr_qp->qpn, ret);
1115 goto free_work;
1116 }
1117 }
1118
1119 if (!ne) {
1120 dev_err(dev, "Reserved loop qp is absent!\n");
1121 goto free_work;
1122 }
1123
1124 do {
1125 ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc);
1126 if (ret < 0 && hr_qp) {
1127 dev_err(dev,
1128 "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n",
1129 hr_qp->qpn, ret, hr_mr->key, ne);
1130 goto free_work;
1131 }
1132 ne -= ret;
1133 usleep_range(HNS_ROCE_V1_FREE_MR_WAIT_VALUE * 1000,
1134 (1 + HNS_ROCE_V1_FREE_MR_WAIT_VALUE) * 1000);
1135 } while (ne && time_before_eq(jiffies, end));
1136
1137 if (ne != 0)
1138 dev_err(dev,
1139 "Poll cqe for mr 0x%x free timeout! Remain %d cqe\n",
1140 hr_mr->key, ne);
1141
1142free_work:
1143 if (mr_work->comp_flag)
1144 complete(mr_work->comp);
1145 kfree(mr_work);
1146}
1147
1148static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev,
1149 struct hns_roce_mr *mr, struct ib_udata *udata)
1150{
1151 struct hns_roce_v1_priv *priv = hr_dev->priv;
1152 struct hns_roce_free_mr *free_mr = &priv->free_mr;
1153 long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS;
1154 struct device *dev = &hr_dev->pdev->dev;
1155 struct hns_roce_mr_free_work *mr_work;
1156 unsigned long start = jiffies;
1157 struct completion comp;
1158 int ret = 0;
1159
1160 if (mr->enabled) {
1161 if (hns_roce_hw_destroy_mpt(hr_dev, NULL,
1162 key_to_hw_index(mr->key) &
1163 (hr_dev->caps.num_mtpts - 1)))
1164 dev_warn(dev, "DESTROY_MPT failed!\n");
1165 }
1166
1167 mr_work = kzalloc(sizeof(*mr_work), GFP_KERNEL);
1168 if (!mr_work) {
1169 ret = -ENOMEM;
1170 goto free_mr;
1171 }
1172
1173 INIT_WORK(&(mr_work->work), hns_roce_v1_mr_free_work_fn);
1174
1175 mr_work->ib_dev = &(hr_dev->ib_dev);
1176 mr_work->comp = &comp;
1177 mr_work->comp_flag = 1;
1178 mr_work->mr = (void *)mr;
1179 init_completion(mr_work->comp);
1180
1181 queue_work(free_mr->free_mr_wq, &(mr_work->work));
1182
1183 while (end > 0) {
1184 if (try_wait_for_completion(&comp))
1185 goto free_mr;
1186 msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE);
1187 end -= HNS_ROCE_V1_FREE_MR_WAIT_VALUE;
1188 }
1189
1190 mr_work->comp_flag = 0;
1191 if (try_wait_for_completion(&comp))
1192 goto free_mr;
1193
1194 dev_warn(dev, "Free mr work 0x%x over 50s and failed!\n", mr->key);
1195 ret = -ETIMEDOUT;
1196
1197free_mr:
1198 dev_dbg(dev, "Free mr 0x%x use 0x%x us.\n",
1199 mr->key, jiffies_to_usecs(jiffies) - jiffies_to_usecs(start));
1200
1201 ida_free(&hr_dev->mr_table.mtpt_ida.ida, (int)key_to_hw_index(mr->key));
1202 hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr);
1203 kfree(mr);
1204
1205 return ret;
1206}
1207
1208static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
1209{
1210 struct hns_roce_v1_priv *priv = hr_dev->priv;
1211 struct hns_roce_db_table *db = &priv->db_table;
1212 struct device *dev = &hr_dev->pdev->dev;
1213
1214 if (db->sdb_ext_mod) {
1215 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
1216 db->ext_db->sdb_buf_list->buf,
1217 db->ext_db->sdb_buf_list->map);
1218 kfree(db->ext_db->sdb_buf_list);
1219 }
1220
1221 if (db->odb_ext_mod) {
1222 dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,
1223 db->ext_db->odb_buf_list->buf,
1224 db->ext_db->odb_buf_list->map);
1225 kfree(db->ext_db->odb_buf_list);
1226 }
1227
1228 kfree(db->ext_db);
1229}
1230
1231static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
1232{
1233 struct hns_roce_v1_priv *priv = hr_dev->priv;
1234 struct hns_roce_raq_table *raq = &priv->raq_table;
1235 struct device *dev = &hr_dev->pdev->dev;
1236 dma_addr_t addr;
1237 int raq_shift;
1238 __le32 tmp;
1239 u32 val;
1240 int ret;
1241
1242 raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
1243 if (!raq->e_raq_buf)
1244 return -ENOMEM;
1245
1246 raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,
1247 &addr, GFP_KERNEL);
1248 if (!raq->e_raq_buf->buf) {
1249 ret = -ENOMEM;
1250 goto err_dma_alloc_raq;
1251 }
1252 raq->e_raq_buf->map = addr;
1253
1254 /* Configure raq extended address. 48bit 4K align */
1255 roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12);
1256
1257 /* Configure raq_shift */
1258 raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);
1259 val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG);
1260 tmp = cpu_to_le32(val);
1261 roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,
1262 ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift);
1263 /*
1264 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1265 * using 4K page, and shift more 32 because of
1266 * calculating the high 32 bit value evaluated to hardware.
1267 */
1268 roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,
1269 ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,
1270 raq->e_raq_buf->map >> 44);
1271 val = le32_to_cpu(tmp);
1272 roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val);
1273 dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);
1274
1275 /* Configure raq threshold */
1276 val = roce_read(hr_dev, ROCEE_RAQ_WL_REG);
1277 tmp = cpu_to_le32(val);
1278 roce_set_field(tmp, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,
1279 ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,
1280 HNS_ROCE_V1_EXT_RAQ_WF);
1281 val = le32_to_cpu(tmp);
1282 roce_write(hr_dev, ROCEE_RAQ_WL_REG, val);
1283 dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);
1284
1285 /* Enable extend raq */
1286 val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG);
1287 tmp = cpu_to_le32(val);
1288 roce_set_field(tmp,
1289 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,
1290 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,
1291 POL_TIME_INTERVAL_VAL);
1292 roce_set_bit(tmp, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);
1293 roce_set_field(tmp,
1294 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,
1295 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,
1296 2);
1297 roce_set_bit(tmp,
1298 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);
1299 val = le32_to_cpu(tmp);
1300 roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val);
1301 dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);
1302
1303 /* Enable raq drop */
1304 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1305 tmp = cpu_to_le32(val);
1306 roce_set_bit(tmp, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);
1307 val = le32_to_cpu(tmp);
1308 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1309 dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);
1310
1311 return 0;
1312
1313err_dma_alloc_raq:
1314 kfree(raq->e_raq_buf);
1315 return ret;
1316}
1317
1318static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
1319{
1320 struct hns_roce_v1_priv *priv = hr_dev->priv;
1321 struct hns_roce_raq_table *raq = &priv->raq_table;
1322 struct device *dev = &hr_dev->pdev->dev;
1323
1324 dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
1325 raq->e_raq_buf->map);
1326 kfree(raq->e_raq_buf);
1327}
1328
1329static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
1330{
1331 __le32 tmp;
1332 u32 val;
1333
1334 if (enable_flag) {
1335 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1336 /* Open all ports */
1337 tmp = cpu_to_le32(val);
1338 roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
1339 ROCEE_GLB_CFG_ROCEE_PORT_ST_S,
1340 ALL_PORT_VAL_OPEN);
1341 val = le32_to_cpu(tmp);
1342 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1343 } else {
1344 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1345 /* Close all ports */
1346 tmp = cpu_to_le32(val);
1347 roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
1348 ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);
1349 val = le32_to_cpu(tmp);
1350 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1351 }
1352}
1353
1354static int hns_roce_bt_init(struct hns_roce_dev *hr_dev)
1355{
1356 struct hns_roce_v1_priv *priv = hr_dev->priv;
1357 struct device *dev = &hr_dev->pdev->dev;
1358 int ret;
1359
1360 priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev,
1361 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map,
1362 GFP_KERNEL);
1363 if (!priv->bt_table.qpc_buf.buf)
1364 return -ENOMEM;
1365
1366 priv->bt_table.mtpt_buf.buf = dma_alloc_coherent(dev,
1367 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.mtpt_buf.map,
1368 GFP_KERNEL);
1369 if (!priv->bt_table.mtpt_buf.buf) {
1370 ret = -ENOMEM;
1371 goto err_failed_alloc_mtpt_buf;
1372 }
1373
1374 priv->bt_table.cqc_buf.buf = dma_alloc_coherent(dev,
1375 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.cqc_buf.map,
1376 GFP_KERNEL);
1377 if (!priv->bt_table.cqc_buf.buf) {
1378 ret = -ENOMEM;
1379 goto err_failed_alloc_cqc_buf;
1380 }
1381
1382 return 0;
1383
1384err_failed_alloc_cqc_buf:
1385 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1386 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
1387
1388err_failed_alloc_mtpt_buf:
1389 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1390 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
1391
1392 return ret;
1393}
1394
1395static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)
1396{
1397 struct hns_roce_v1_priv *priv = hr_dev->priv;
1398 struct device *dev = &hr_dev->pdev->dev;
1399
1400 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1401 priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map);
1402
1403 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1404 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
1405
1406 dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1407 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
1408}
1409
1410static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev)
1411{
1412 struct hns_roce_v1_priv *priv = hr_dev->priv;
1413 struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf;
1414 struct device *dev = &hr_dev->pdev->dev;
1415
1416 /*
1417 * This buffer will be used for CQ's tptr(tail pointer), also
1418 * named ci(customer index). Every CQ will use 2 bytes to save
1419 * cqe ci in hip06. Hardware will read this area to get new ci
1420 * when the queue is almost full.
1421 */
1422 tptr_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
1423 &tptr_buf->map, GFP_KERNEL);
1424 if (!tptr_buf->buf)
1425 return -ENOMEM;
1426
1427 hr_dev->tptr_dma_addr = tptr_buf->map;
1428 hr_dev->tptr_size = HNS_ROCE_V1_TPTR_BUF_SIZE;
1429
1430 return 0;
1431}
1432
1433static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev)
1434{
1435 struct hns_roce_v1_priv *priv = hr_dev->priv;
1436 struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf;
1437 struct device *dev = &hr_dev->pdev->dev;
1438
1439 dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
1440 tptr_buf->buf, tptr_buf->map);
1441}
1442
1443static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev)
1444{
1445 struct hns_roce_v1_priv *priv = hr_dev->priv;
1446 struct hns_roce_free_mr *free_mr = &priv->free_mr;
1447 struct device *dev = &hr_dev->pdev->dev;
1448 int ret;
1449
1450 free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr");
1451 if (!free_mr->free_mr_wq) {
1452 dev_err(dev, "Create free mr workqueue failed!\n");
1453 return -ENOMEM;
1454 }
1455
1456 ret = hns_roce_v1_rsv_lp_qp(hr_dev);
1457 if (ret) {
1458 dev_err(dev, "Reserved loop qp failed(%d)!\n", ret);
1459 destroy_workqueue(free_mr->free_mr_wq);
1460 }
1461
1462 return ret;
1463}
1464
1465static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev)
1466{
1467 struct hns_roce_v1_priv *priv = hr_dev->priv;
1468 struct hns_roce_free_mr *free_mr = &priv->free_mr;
1469
1470 destroy_workqueue(free_mr->free_mr_wq);
1471
1472 hns_roce_v1_release_lp_qp(hr_dev);
1473}
1474
1475/**
1476 * hns_roce_v1_reset - reset RoCE
1477 * @hr_dev: RoCE device struct pointer
1478 * @dereset: true -- drop reset, false -- reset
1479 * return 0 - success , negative --fail
1480 */
1481static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
1482{
1483 struct device_node *dsaf_node;
1484 struct device *dev = &hr_dev->pdev->dev;
1485 struct device_node *np = dev->of_node;
1486 struct fwnode_handle *fwnode;
1487 int ret;
1488
1489 /* check if this is DT/ACPI case */
1490 if (dev_of_node(dev)) {
1491 dsaf_node = of_parse_phandle(np, "dsaf-handle", 0);
1492 if (!dsaf_node) {
1493 dev_err(dev, "could not find dsaf-handle\n");
1494 return -EINVAL;
1495 }
1496 fwnode = &dsaf_node->fwnode;
1497 } else if (is_acpi_device_node(dev->fwnode)) {
1498 struct fwnode_reference_args args;
1499
1500 ret = acpi_node_get_property_reference(dev->fwnode,
1501 "dsaf-handle", 0, &args);
1502 if (ret) {
1503 dev_err(dev, "could not find dsaf-handle\n");
1504 return ret;
1505 }
1506 fwnode = args.fwnode;
1507 } else {
1508 dev_err(dev, "cannot read data from DT or ACPI\n");
1509 return -ENXIO;
1510 }
1511
1512 ret = hns_dsaf_roce_reset(fwnode, false);
1513 if (ret)
1514 return ret;
1515
1516 if (dereset) {
1517 msleep(SLEEP_TIME_INTERVAL);
1518 ret = hns_dsaf_roce_reset(fwnode, true);
1519 }
1520
1521 return ret;
1522}
1523
1524static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
1525{
1526 struct hns_roce_caps *caps = &hr_dev->caps;
1527 int i;
1528
1529 hr_dev->vendor_id = roce_read(hr_dev, ROCEE_VENDOR_ID_REG);
1530 hr_dev->vendor_part_id = roce_read(hr_dev, ROCEE_VENDOR_PART_ID_REG);
1531 hr_dev->sys_image_guid = roce_read(hr_dev, ROCEE_SYS_IMAGE_GUID_L_REG) |
1532 ((u64)roce_read(hr_dev,
1533 ROCEE_SYS_IMAGE_GUID_H_REG) << 32);
1534 hr_dev->hw_rev = HNS_ROCE_HW_VER1;
1535
1536 caps->num_qps = HNS_ROCE_V1_MAX_QP_NUM;
1537 caps->max_wqes = HNS_ROCE_V1_MAX_WQE_NUM;
1538 caps->min_wqes = HNS_ROCE_MIN_WQE_NUM;
1539 caps->num_cqs = HNS_ROCE_V1_MAX_CQ_NUM;
1540 caps->min_cqes = HNS_ROCE_MIN_CQE_NUM;
1541 caps->max_cqes = HNS_ROCE_V1_MAX_CQE_NUM;
1542 caps->max_sq_sg = HNS_ROCE_V1_SG_NUM;
1543 caps->max_rq_sg = HNS_ROCE_V1_SG_NUM;
1544 caps->max_sq_inline = HNS_ROCE_V1_INLINE_SIZE;
1545 caps->num_uars = HNS_ROCE_V1_UAR_NUM;
1546 caps->phy_num_uars = HNS_ROCE_V1_PHY_UAR_NUM;
1547 caps->num_aeq_vectors = HNS_ROCE_V1_AEQE_VEC_NUM;
1548 caps->num_comp_vectors = HNS_ROCE_V1_COMP_VEC_NUM;
1549 caps->num_other_vectors = HNS_ROCE_V1_ABNORMAL_VEC_NUM;
1550 caps->num_mtpts = HNS_ROCE_V1_MAX_MTPT_NUM;
1551 caps->num_mtt_segs = HNS_ROCE_V1_MAX_MTT_SEGS;
1552 caps->num_pds = HNS_ROCE_V1_MAX_PD_NUM;
1553 caps->max_qp_init_rdma = HNS_ROCE_V1_MAX_QP_INIT_RDMA;
1554 caps->max_qp_dest_rdma = HNS_ROCE_V1_MAX_QP_DEST_RDMA;
1555 caps->max_sq_desc_sz = HNS_ROCE_V1_MAX_SQ_DESC_SZ;
1556 caps->max_rq_desc_sz = HNS_ROCE_V1_MAX_RQ_DESC_SZ;
1557 caps->qpc_sz = HNS_ROCE_V1_QPC_SIZE;
1558 caps->irrl_entry_sz = HNS_ROCE_V1_IRRL_ENTRY_SIZE;
1559 caps->cqc_entry_sz = HNS_ROCE_V1_CQC_ENTRY_SIZE;
1560 caps->mtpt_entry_sz = HNS_ROCE_V1_MTPT_ENTRY_SIZE;
1561 caps->mtt_entry_sz = HNS_ROCE_V1_MTT_ENTRY_SIZE;
1562 caps->cqe_sz = HNS_ROCE_V1_CQE_SIZE;
1563 caps->page_size_cap = HNS_ROCE_V1_PAGE_SIZE_SUPPORT;
1564 caps->reserved_lkey = 0;
1565 caps->reserved_pds = 0;
1566 caps->reserved_mrws = 1;
1567 caps->reserved_uars = 0;
1568 caps->reserved_cqs = 0;
1569 caps->reserved_qps = 12; /* 2 SQP per port, six ports total 12 */
1570 caps->chunk_sz = HNS_ROCE_V1_TABLE_CHUNK_SIZE;
1571
1572 for (i = 0; i < caps->num_ports; i++)
1573 caps->pkey_table_len[i] = 1;
1574
1575 for (i = 0; i < caps->num_ports; i++) {
1576 /* Six ports shared 16 GID in v1 engine */
1577 if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports))
1578 caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
1579 caps->num_ports;
1580 else
1581 caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
1582 caps->num_ports + 1;
1583 }
1584
1585 caps->ceqe_depth = HNS_ROCE_V1_COMP_EQE_NUM;
1586 caps->aeqe_depth = HNS_ROCE_V1_ASYNC_EQE_NUM;
1587 caps->local_ca_ack_delay = roce_read(hr_dev, ROCEE_ACK_DELAY_REG);
1588 caps->max_mtu = IB_MTU_2048;
1589
1590 return 0;
1591}
1592
1593static int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
1594{
1595 int ret;
1596 u32 val;
1597 __le32 tmp;
1598 struct device *dev = &hr_dev->pdev->dev;
1599
1600 /* DMAE user config */
1601 val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG);
1602 tmp = cpu_to_le32(val);
1603 roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,
1604 ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);
1605 roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,
1606 ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,
1607 1 << PAGES_SHIFT_16);
1608 val = le32_to_cpu(tmp);
1609 roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val);
1610
1611 val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG);
1612 tmp = cpu_to_le32(val);
1613 roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,
1614 ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);
1615 roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,
1616 ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,
1617 1 << PAGES_SHIFT_16);
1618
1619 ret = hns_roce_db_init(hr_dev);
1620 if (ret) {
1621 dev_err(dev, "doorbell init failed!\n");
1622 return ret;
1623 }
1624
1625 ret = hns_roce_raq_init(hr_dev);
1626 if (ret) {
1627 dev_err(dev, "raq init failed!\n");
1628 goto error_failed_raq_init;
1629 }
1630
1631 ret = hns_roce_bt_init(hr_dev);
1632 if (ret) {
1633 dev_err(dev, "bt init failed!\n");
1634 goto error_failed_bt_init;
1635 }
1636
1637 ret = hns_roce_tptr_init(hr_dev);
1638 if (ret) {
1639 dev_err(dev, "tptr init failed!\n");
1640 goto error_failed_tptr_init;
1641 }
1642
1643 ret = hns_roce_free_mr_init(hr_dev);
1644 if (ret) {
1645 dev_err(dev, "free mr init failed!\n");
1646 goto error_failed_free_mr_init;
1647 }
1648
1649 hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
1650
1651 return 0;
1652
1653error_failed_free_mr_init:
1654 hns_roce_tptr_free(hr_dev);
1655
1656error_failed_tptr_init:
1657 hns_roce_bt_free(hr_dev);
1658
1659error_failed_bt_init:
1660 hns_roce_raq_free(hr_dev);
1661
1662error_failed_raq_init:
1663 hns_roce_db_free(hr_dev);
1664 return ret;
1665}
1666
1667static void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
1668{
1669 hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
1670 hns_roce_free_mr_free(hr_dev);
1671 hns_roce_tptr_free(hr_dev);
1672 hns_roce_bt_free(hr_dev);
1673 hns_roce_raq_free(hr_dev);
1674 hns_roce_db_free(hr_dev);
1675}
1676
1677static int hns_roce_v1_cmd_pending(struct hns_roce_dev *hr_dev)
1678{
1679 u32 status = readl(hr_dev->reg_base + ROCEE_MB6_REG);
1680
1681 return (!!(status & (1 << HCR_GO_BIT)));
1682}
1683
1684static int hns_roce_v1_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
1685 u64 out_param, u32 in_modifier, u8 op_modifier,
1686 u16 op, u16 token, int event)
1687{
1688 u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base + ROCEE_MB1_REG);
1689 unsigned long end;
1690 u32 val = 0;
1691 __le32 tmp;
1692
1693 end = msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS) + jiffies;
1694 while (hns_roce_v1_cmd_pending(hr_dev)) {
1695 if (time_after(jiffies, end)) {
1696 dev_err(hr_dev->dev, "jiffies=%d end=%d\n",
1697 (int)jiffies, (int)end);
1698 return -EAGAIN;
1699 }
1700 cond_resched();
1701 }
1702
1703 tmp = cpu_to_le32(val);
1704 roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_M, ROCEE_MB6_ROCEE_MB_CMD_S,
1705 op);
1706 roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_MDF_M,
1707 ROCEE_MB6_ROCEE_MB_CMD_MDF_S, op_modifier);
1708 roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_EVENT_S, event);
1709 roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_HW_RUN_S, 1);
1710 roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_TOKEN_M,
1711 ROCEE_MB6_ROCEE_MB_TOKEN_S, token);
1712
1713 val = le32_to_cpu(tmp);
1714 writeq(in_param, hcr + 0);
1715 writeq(out_param, hcr + 2);
1716 writel(in_modifier, hcr + 4);
1717 /* Memory barrier */
1718 wmb();
1719
1720 writel(val, hcr + 5);
1721
1722 return 0;
1723}
1724
1725static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev,
1726 unsigned int timeout)
1727{
1728 u8 __iomem *hcr = hr_dev->reg_base + ROCEE_MB1_REG;
1729 unsigned long end;
1730 u32 status = 0;
1731
1732 end = msecs_to_jiffies(timeout) + jiffies;
1733 while (hns_roce_v1_cmd_pending(hr_dev) && time_before(jiffies, end))
1734 cond_resched();
1735
1736 if (hns_roce_v1_cmd_pending(hr_dev)) {
1737 dev_err(hr_dev->dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
1738 return -ETIMEDOUT;
1739 }
1740
1741 status = le32_to_cpu((__force __le32)
1742 __raw_readl(hcr + HCR_STATUS_OFFSET));
1743 if ((status & STATUS_MASK) != 0x1) {
1744 dev_err(hr_dev->dev, "mailbox status 0x%x!\n", status);
1745 return -EBUSY;
1746 }
1747
1748 return 0;
1749}
1750
1751static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u32 port,
1752 int gid_index, const union ib_gid *gid,
1753 const struct ib_gid_attr *attr)
1754{
1755 unsigned long flags;
1756 u32 *p = NULL;
1757 u8 gid_idx;
1758
1759 gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
1760
1761 spin_lock_irqsave(&hr_dev->iboe.lock, flags);
1762
1763 p = (u32 *)&gid->raw[0];
1764 roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG +
1765 (HNS_ROCE_V1_GID_NUM * gid_idx));
1766
1767 p = (u32 *)&gid->raw[4];
1768 roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG +
1769 (HNS_ROCE_V1_GID_NUM * gid_idx));
1770
1771 p = (u32 *)&gid->raw[8];
1772 roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG +
1773 (HNS_ROCE_V1_GID_NUM * gid_idx));
1774
1775 p = (u32 *)&gid->raw[0xc];
1776 roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG +
1777 (HNS_ROCE_V1_GID_NUM * gid_idx));
1778
1779 spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
1780
1781 return 0;
1782}
1783
1784static int hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
1785 const u8 *addr)
1786{
1787 u32 reg_smac_l;
1788 u16 reg_smac_h;
1789 __le32 tmp;
1790 u16 *p_h;
1791 u32 *p;
1792 u32 val;
1793
1794 /*
1795 * When mac changed, loopback may fail
1796 * because of smac not equal to dmac.
1797 * We Need to release and create reserved qp again.
1798 */
1799 if (hr_dev->hw->dereg_mr) {
1800 int ret;
1801
1802 ret = hns_roce_v1_recreate_lp_qp(hr_dev);
1803 if (ret && ret != -ETIMEDOUT)
1804 return ret;
1805 }
1806
1807 p = (u32 *)(&addr[0]);
1808 reg_smac_l = *p;
1809 roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG +
1810 PHY_PORT_OFFSET * phy_port);
1811
1812 val = roce_read(hr_dev,
1813 ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1814 tmp = cpu_to_le32(val);
1815 p_h = (u16 *)(&addr[4]);
1816 reg_smac_h = *p_h;
1817 roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_SMAC_H_M,
1818 ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h);
1819 val = le32_to_cpu(tmp);
1820 roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1821 val);
1822
1823 return 0;
1824}
1825
1826static void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,
1827 enum ib_mtu mtu)
1828{
1829 __le32 tmp;
1830 u32 val;
1831
1832 val = roce_read(hr_dev,
1833 ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1834 tmp = cpu_to_le32(val);
1835 roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_PORT_MTU_M,
1836 ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu);
1837 val = le32_to_cpu(tmp);
1838 roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1839 val);
1840}
1841
1842static int hns_roce_v1_write_mtpt(struct hns_roce_dev *hr_dev, void *mb_buf,
1843 struct hns_roce_mr *mr,
1844 unsigned long mtpt_idx)
1845{
1846 u64 pages[HNS_ROCE_MAX_INNER_MTPT_NUM] = { 0 };
1847 struct ib_device *ibdev = &hr_dev->ib_dev;
1848 struct hns_roce_v1_mpt_entry *mpt_entry;
1849 dma_addr_t pbl_ba;
1850 int count;
1851 int i;
1852
1853 /* MPT filled into mailbox buf */
1854 mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf;
1855 memset(mpt_entry, 0, sizeof(*mpt_entry));
1856
1857 roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M,
1858 MPT_BYTE_4_KEY_STATE_S, KEY_VALID);
1859 roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M,
1860 MPT_BYTE_4_KEY_S, mr->key);
1861 roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M,
1862 MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K);
1863 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0);
1864 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S,
1865 (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1866 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0);
1867 roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M,
1868 MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type);
1869 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0);
1870 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S,
1871 (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1872 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S,
1873 (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1874 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S,
1875 (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1876 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S,
1877 0);
1878 roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0);
1879
1880 roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1881 MPT_BYTE_12_PBL_ADDR_H_S, 0);
1882 roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M,
1883 MPT_BYTE_12_MW_BIND_COUNTER_S, 0);
1884
1885 mpt_entry->virt_addr_l = cpu_to_le32((u32)mr->iova);
1886 mpt_entry->virt_addr_h = cpu_to_le32((u32)(mr->iova >> 32));
1887 mpt_entry->length = cpu_to_le32((u32)mr->size);
1888
1889 roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M,
1890 MPT_BYTE_28_PD_S, mr->pd);
1891 roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M,
1892 MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx);
1893 roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M,
1894 MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT);
1895
1896 /* DMA memory register */
1897 if (mr->type == MR_TYPE_DMA)
1898 return 0;
1899
1900 count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
1901 ARRAY_SIZE(pages), &pbl_ba);
1902 if (count < 1) {
1903 ibdev_err(ibdev, "failed to find PBL mtr, count = %d.", count);
1904 return -ENOBUFS;
1905 }
1906
1907 /* Register user mr */
1908 for (i = 0; i < count; i++) {
1909 switch (i) {
1910 case 0:
1911 mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i]));
1912 roce_set_field(mpt_entry->mpt_byte_36,
1913 MPT_BYTE_36_PA0_H_M,
1914 MPT_BYTE_36_PA0_H_S,
1915 (u32)(pages[i] >> PAGES_SHIFT_32));
1916 break;
1917 case 1:
1918 roce_set_field(mpt_entry->mpt_byte_36,
1919 MPT_BYTE_36_PA1_L_M,
1920 MPT_BYTE_36_PA1_L_S, (u32)(pages[i]));
1921 roce_set_field(mpt_entry->mpt_byte_40,
1922 MPT_BYTE_40_PA1_H_M,
1923 MPT_BYTE_40_PA1_H_S,
1924 (u32)(pages[i] >> PAGES_SHIFT_24));
1925 break;
1926 case 2:
1927 roce_set_field(mpt_entry->mpt_byte_40,
1928 MPT_BYTE_40_PA2_L_M,
1929 MPT_BYTE_40_PA2_L_S, (u32)(pages[i]));
1930 roce_set_field(mpt_entry->mpt_byte_44,
1931 MPT_BYTE_44_PA2_H_M,
1932 MPT_BYTE_44_PA2_H_S,
1933 (u32)(pages[i] >> PAGES_SHIFT_16));
1934 break;
1935 case 3:
1936 roce_set_field(mpt_entry->mpt_byte_44,
1937 MPT_BYTE_44_PA3_L_M,
1938 MPT_BYTE_44_PA3_L_S, (u32)(pages[i]));
1939 roce_set_field(mpt_entry->mpt_byte_48,
1940 MPT_BYTE_48_PA3_H_M,
1941 MPT_BYTE_48_PA3_H_S,
1942 (u32)(pages[i] >> PAGES_SHIFT_8));
1943 break;
1944 case 4:
1945 mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i]));
1946 roce_set_field(mpt_entry->mpt_byte_56,
1947 MPT_BYTE_56_PA4_H_M,
1948 MPT_BYTE_56_PA4_H_S,
1949 (u32)(pages[i] >> PAGES_SHIFT_32));
1950 break;
1951 case 5:
1952 roce_set_field(mpt_entry->mpt_byte_56,
1953 MPT_BYTE_56_PA5_L_M,
1954 MPT_BYTE_56_PA5_L_S, (u32)(pages[i]));
1955 roce_set_field(mpt_entry->mpt_byte_60,
1956 MPT_BYTE_60_PA5_H_M,
1957 MPT_BYTE_60_PA5_H_S,
1958 (u32)(pages[i] >> PAGES_SHIFT_24));
1959 break;
1960 case 6:
1961 roce_set_field(mpt_entry->mpt_byte_60,
1962 MPT_BYTE_60_PA6_L_M,
1963 MPT_BYTE_60_PA6_L_S, (u32)(pages[i]));
1964 roce_set_field(mpt_entry->mpt_byte_64,
1965 MPT_BYTE_64_PA6_H_M,
1966 MPT_BYTE_64_PA6_H_S,
1967 (u32)(pages[i] >> PAGES_SHIFT_16));
1968 break;
1969 default:
1970 break;
1971 }
1972 }
1973
1974 mpt_entry->pbl_addr_l = cpu_to_le32(pbl_ba);
1975 roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1976 MPT_BYTE_12_PBL_ADDR_H_S, upper_32_bits(pbl_ba));
1977
1978 return 0;
1979}
1980
1981static void *get_cqe(struct hns_roce_cq *hr_cq, int n)
1982{
1983 return hns_roce_buf_offset(hr_cq->mtr.kmem, n * HNS_ROCE_V1_CQE_SIZE);
1984}
1985
1986static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)
1987{
1988 struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe);
1989
1990 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1991 return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^
1992 !!(n & hr_cq->cq_depth)) ? hr_cqe : NULL;
1993}
1994
1995static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq)
1996{
1997 return get_sw_cqe(hr_cq, hr_cq->cons_index);
1998}
1999
2000static void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2001{
2002 __le32 doorbell[2];
2003
2004 doorbell[0] = cpu_to_le32(cons_index & ((hr_cq->cq_depth << 1) - 1));
2005 doorbell[1] = 0;
2006 roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
2007 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
2008 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
2009 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
2010 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0);
2011 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
2012 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn);
2013
2014 hns_roce_write64_k(doorbell, hr_cq->db_reg);
2015}
2016
2017static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2018 struct hns_roce_srq *srq)
2019{
2020 struct hns_roce_cqe *cqe, *dest;
2021 u32 prod_index;
2022 int nfreed = 0;
2023 u8 owner_bit;
2024
2025 for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index);
2026 ++prod_index) {
2027 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
2028 break;
2029 }
2030
2031 /*
2032 * Now backwards through the CQ, removing CQ entries
2033 * that match our QP by overwriting them with next entries.
2034 */
2035 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
2036 cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe);
2037 if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2038 CQE_BYTE_16_LOCAL_QPN_S) &
2039 HNS_ROCE_CQE_QPN_MASK) == qpn) {
2040 /* In v1 engine, not support SRQ */
2041 ++nfreed;
2042 } else if (nfreed) {
2043 dest = get_cqe(hr_cq, (prod_index + nfreed) &
2044 hr_cq->ib_cq.cqe);
2045 owner_bit = roce_get_bit(dest->cqe_byte_4,
2046 CQE_BYTE_4_OWNER_S);
2047 memcpy(dest, cqe, sizeof(*cqe));
2048 roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
2049 owner_bit);
2050 }
2051 }
2052
2053 if (nfreed) {
2054 hr_cq->cons_index += nfreed;
2055 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
2056 }
2057}
2058
2059static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2060 struct hns_roce_srq *srq)
2061{
2062 spin_lock_irq(&hr_cq->lock);
2063 __hns_roce_v1_cq_clean(hr_cq, qpn, srq);
2064 spin_unlock_irq(&hr_cq->lock);
2065}
2066
2067static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
2068 struct hns_roce_cq *hr_cq, void *mb_buf,
2069 u64 *mtts, dma_addr_t dma_handle)
2070{
2071 struct hns_roce_v1_priv *priv = hr_dev->priv;
2072 struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf;
2073 struct hns_roce_cq_context *cq_context = mb_buf;
2074 dma_addr_t tptr_dma_addr;
2075 int offset;
2076
2077 memset(cq_context, 0, sizeof(*cq_context));
2078
2079 /* Get the tptr for this CQ. */
2080 offset = hr_cq->cqn * HNS_ROCE_V1_TPTR_ENTRY_SIZE;
2081 tptr_dma_addr = tptr_buf->map + offset;
2082 hr_cq->tptr_addr = (u16 *)(tptr_buf->buf + offset);
2083
2084 /* Register cq_context members */
2085 roce_set_field(cq_context->cqc_byte_4,
2086 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M,
2087 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID);
2088 roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M,
2089 CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn);
2090
2091 cq_context->cq_bt_l = cpu_to_le32((u32)dma_handle);
2092
2093 roce_set_field(cq_context->cqc_byte_12,
2094 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M,
2095 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S,
2096 ((u64)dma_handle >> 32));
2097 roce_set_field(cq_context->cqc_byte_12,
2098 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M,
2099 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S,
2100 ilog2(hr_cq->cq_depth));
2101 roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M,
2102 CQ_CONTEXT_CQC_BYTE_12_CEQN_S, hr_cq->vector);
2103
2104 cq_context->cur_cqe_ba0_l = cpu_to_le32((u32)(mtts[0]));
2105
2106 roce_set_field(cq_context->cqc_byte_20,
2107 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M,
2108 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S, (mtts[0]) >> 32);
2109 /* Dedicated hardware, directly set 0 */
2110 roce_set_field(cq_context->cqc_byte_20,
2111 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M,
2112 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0);
2113 /**
2114 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
2115 * using 4K page, and shift more 32 because of
2116 * calculating the high 32 bit value evaluated to hardware.
2117 */
2118 roce_set_field(cq_context->cqc_byte_20,
2119 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,
2120 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,
2121 tptr_dma_addr >> 44);
2122
2123 cq_context->cqe_tptr_addr_l = cpu_to_le32((u32)(tptr_dma_addr >> 12));
2124
2125 roce_set_field(cq_context->cqc_byte_32,
2126 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,
2127 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0);
2128 roce_set_bit(cq_context->cqc_byte_32,
2129 CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0);
2130 roce_set_bit(cq_context->cqc_byte_32,
2131 CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0);
2132 roce_set_bit(cq_context->cqc_byte_32,
2133 CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0);
2134 roce_set_bit(cq_context->cqc_byte_32,
2135 CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,
2136 0);
2137 /* The initial value of cq's ci is 0 */
2138 roce_set_field(cq_context->cqc_byte_32,
2139 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,
2140 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
2141}
2142
2143static int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq,
2144 enum ib_cq_notify_flags flags)
2145{
2146 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2147 u32 notification_flag;
2148 __le32 doorbell[2] = {};
2149
2150 notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
2151 IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
2152 /*
2153 * flags = 0; Notification Flag = 1, next
2154 * flags = 1; Notification Flag = 0, solocited
2155 */
2156 doorbell[0] =
2157 cpu_to_le32(hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
2158 roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
2159 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
2160 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
2161 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
2162 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1);
2163 roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
2164 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S,
2165 hr_cq->cqn | notification_flag);
2166
2167 hns_roce_write64_k(doorbell, hr_cq->db_reg);
2168
2169 return 0;
2170}
2171
2172static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
2173 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2174{
2175 int qpn;
2176 int is_send;
2177 u16 wqe_ctr;
2178 u32 status;
2179 u32 opcode;
2180 struct hns_roce_cqe *cqe;
2181 struct hns_roce_qp *hr_qp;
2182 struct hns_roce_wq *wq;
2183 struct hns_roce_wqe_ctrl_seg *sq_wqe;
2184 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2185 struct device *dev = &hr_dev->pdev->dev;
2186
2187 /* Find cqe according consumer index */
2188 cqe = next_cqe_sw(hr_cq);
2189 if (!cqe)
2190 return -EAGAIN;
2191
2192 ++hr_cq->cons_index;
2193 /* Memory barrier */
2194 rmb();
2195 /* 0->SQ, 1->RQ */
2196 is_send = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S));
2197
2198 /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
2199 if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2200 CQE_BYTE_16_LOCAL_QPN_S) <= 1) {
2201 qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M,
2202 CQE_BYTE_20_PORT_NUM_S) +
2203 roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2204 CQE_BYTE_16_LOCAL_QPN_S) *
2205 HNS_ROCE_MAX_PORTS;
2206 } else {
2207 qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2208 CQE_BYTE_16_LOCAL_QPN_S);
2209 }
2210
2211 if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) {
2212 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2213 if (unlikely(!hr_qp)) {
2214 dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n",
2215 hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK));
2216 return -EINVAL;
2217 }
2218
2219 *cur_qp = hr_qp;
2220 }
2221
2222 wc->qp = &(*cur_qp)->ibqp;
2223 wc->vendor_err = 0;
2224
2225 status = roce_get_field(cqe->cqe_byte_4,
2226 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,
2227 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) &
2228 HNS_ROCE_CQE_STATUS_MASK;
2229 switch (status) {
2230 case HNS_ROCE_CQE_SUCCESS:
2231 wc->status = IB_WC_SUCCESS;
2232 break;
2233 case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR:
2234 wc->status = IB_WC_LOC_LEN_ERR;
2235 break;
2236 case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR:
2237 wc->status = IB_WC_LOC_QP_OP_ERR;
2238 break;
2239 case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR:
2240 wc->status = IB_WC_LOC_PROT_ERR;
2241 break;
2242 case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR:
2243 wc->status = IB_WC_WR_FLUSH_ERR;
2244 break;
2245 case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR:
2246 wc->status = IB_WC_MW_BIND_ERR;
2247 break;
2248 case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR:
2249 wc->status = IB_WC_BAD_RESP_ERR;
2250 break;
2251 case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR:
2252 wc->status = IB_WC_LOC_ACCESS_ERR;
2253 break;
2254 case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
2255 wc->status = IB_WC_REM_INV_REQ_ERR;
2256 break;
2257 case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR:
2258 wc->status = IB_WC_REM_ACCESS_ERR;
2259 break;
2260 case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR:
2261 wc->status = IB_WC_REM_OP_ERR;
2262 break;
2263 case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
2264 wc->status = IB_WC_RETRY_EXC_ERR;
2265 break;
2266 case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
2267 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2268 break;
2269 default:
2270 wc->status = IB_WC_GENERAL_ERR;
2271 break;
2272 }
2273
2274 /* CQE status error, directly return */
2275 if (wc->status != IB_WC_SUCCESS)
2276 return 0;
2277
2278 if (is_send) {
2279 /* SQ conrespond to CQE */
2280 sq_wqe = hns_roce_get_send_wqe(*cur_qp,
2281 roce_get_field(cqe->cqe_byte_4,
2282 CQE_BYTE_4_WQE_INDEX_M,
2283 CQE_BYTE_4_WQE_INDEX_S) &
2284 ((*cur_qp)->sq.wqe_cnt-1));
2285 switch (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_OPCODE_MASK) {
2286 case HNS_ROCE_WQE_OPCODE_SEND:
2287 wc->opcode = IB_WC_SEND;
2288 break;
2289 case HNS_ROCE_WQE_OPCODE_RDMA_READ:
2290 wc->opcode = IB_WC_RDMA_READ;
2291 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2292 break;
2293 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE:
2294 wc->opcode = IB_WC_RDMA_WRITE;
2295 break;
2296 case HNS_ROCE_WQE_OPCODE_LOCAL_INV:
2297 wc->opcode = IB_WC_LOCAL_INV;
2298 break;
2299 case HNS_ROCE_WQE_OPCODE_UD_SEND:
2300 wc->opcode = IB_WC_SEND;
2301 break;
2302 default:
2303 wc->status = IB_WC_GENERAL_ERR;
2304 break;
2305 }
2306 wc->wc_flags = (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_IMM ?
2307 IB_WC_WITH_IMM : 0);
2308
2309 wq = &(*cur_qp)->sq;
2310 if ((*cur_qp)->sq_signal_bits) {
2311 /*
2312 * If sg_signal_bit is 1,
2313 * firstly tail pointer updated to wqe
2314 * which current cqe correspond to
2315 */
2316 wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4,
2317 CQE_BYTE_4_WQE_INDEX_M,
2318 CQE_BYTE_4_WQE_INDEX_S);
2319 wq->tail += (wqe_ctr - (u16)wq->tail) &
2320 (wq->wqe_cnt - 1);
2321 }
2322 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2323 ++wq->tail;
2324 } else {
2325 /* RQ conrespond to CQE */
2326 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2327 opcode = roce_get_field(cqe->cqe_byte_4,
2328 CQE_BYTE_4_OPERATION_TYPE_M,
2329 CQE_BYTE_4_OPERATION_TYPE_S) &
2330 HNS_ROCE_CQE_OPCODE_MASK;
2331 switch (opcode) {
2332 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:
2333 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2334 wc->wc_flags = IB_WC_WITH_IMM;
2335 wc->ex.imm_data =
2336 cpu_to_be32(le32_to_cpu(cqe->immediate_data));
2337 break;
2338 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:
2339 if (roce_get_bit(cqe->cqe_byte_4,
2340 CQE_BYTE_4_IMM_INDICATOR_S)) {
2341 wc->opcode = IB_WC_RECV;
2342 wc->wc_flags = IB_WC_WITH_IMM;
2343 wc->ex.imm_data = cpu_to_be32(
2344 le32_to_cpu(cqe->immediate_data));
2345 } else {
2346 wc->opcode = IB_WC_RECV;
2347 wc->wc_flags = 0;
2348 }
2349 break;
2350 default:
2351 wc->status = IB_WC_GENERAL_ERR;
2352 break;
2353 }
2354
2355 /* Update tail pointer, record wr_id */
2356 wq = &(*cur_qp)->rq;
2357 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2358 ++wq->tail;
2359 wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M,
2360 CQE_BYTE_20_SL_S);
2361 wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20,
2362 CQE_BYTE_20_REMOTE_QPN_M,
2363 CQE_BYTE_20_REMOTE_QPN_S);
2364 wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20,
2365 CQE_BYTE_20_GRH_PRESENT_S) ?
2366 IB_WC_GRH : 0);
2367 wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28,
2368 CQE_BYTE_28_P_KEY_IDX_M,
2369 CQE_BYTE_28_P_KEY_IDX_S);
2370 }
2371
2372 return 0;
2373}
2374
2375int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2376{
2377 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2378 struct hns_roce_qp *cur_qp = NULL;
2379 unsigned long flags;
2380 int npolled;
2381 int ret;
2382
2383 spin_lock_irqsave(&hr_cq->lock, flags);
2384
2385 for (npolled = 0; npolled < num_entries; ++npolled) {
2386 ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled);
2387 if (ret)
2388 break;
2389 }
2390
2391 if (npolled) {
2392 *hr_cq->tptr_addr = hr_cq->cons_index &
2393 ((hr_cq->cq_depth << 1) - 1);
2394
2395 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
2396 }
2397
2398 spin_unlock_irqrestore(&hr_cq->lock, flags);
2399
2400 if (ret == 0 || ret == -EAGAIN)
2401 return npolled;
2402 else
2403 return ret;
2404}
2405
2406static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
2407 struct hns_roce_hem_table *table, int obj,
2408 int step_idx)
2409{
2410 struct hns_roce_v1_priv *priv = hr_dev->priv;
2411 struct device *dev = &hr_dev->pdev->dev;
2412 long end = HW_SYNC_TIMEOUT_MSECS;
2413 __le32 bt_cmd_val[2] = {0};
2414 unsigned long flags = 0;
2415 void __iomem *bt_cmd;
2416 u64 bt_ba = 0;
2417
2418 switch (table->type) {
2419 case HEM_TYPE_QPC:
2420 bt_ba = priv->bt_table.qpc_buf.map >> 12;
2421 break;
2422 case HEM_TYPE_MTPT:
2423 bt_ba = priv->bt_table.mtpt_buf.map >> 12;
2424 break;
2425 case HEM_TYPE_CQC:
2426 bt_ba = priv->bt_table.cqc_buf.map >> 12;
2427 break;
2428 case HEM_TYPE_SRQC:
2429 dev_dbg(dev, "HEM_TYPE_SRQC not support.\n");
2430 return -EINVAL;
2431 default:
2432 return 0;
2433 }
2434 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
2435 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type);
2436 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
2437 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
2438 roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
2439 roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
2440
2441 spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags);
2442
2443 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
2444
2445 while (1) {
2446 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
2447 if (!end) {
2448 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
2449 spin_unlock_irqrestore(&hr_dev->bt_cmd_lock,
2450 flags);
2451 return -EBUSY;
2452 }
2453 } else {
2454 break;
2455 }
2456 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
2457 end -= HW_SYNC_SLEEP_TIME_INTERVAL;
2458 }
2459
2460 bt_cmd_val[0] = cpu_to_le32(bt_ba);
2461 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
2462 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, bt_ba >> 32);
2463 hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
2464
2465 spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags);
2466
2467 return 0;
2468}
2469
2470static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
2471 enum hns_roce_qp_state cur_state,
2472 enum hns_roce_qp_state new_state,
2473 struct hns_roce_qp_context *context,
2474 struct hns_roce_qp *hr_qp)
2475{
2476 static const u16
2477 op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = {
2478 [HNS_ROCE_QP_STATE_RST] = {
2479 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2480 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2481 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
2482 },
2483 [HNS_ROCE_QP_STATE_INIT] = {
2484 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2485 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2486 /* Note: In v1 engine, HW doesn't support RST2INIT.
2487 * We use RST2INIT cmd instead of INIT2INIT.
2488 */
2489 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
2490 [HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP,
2491 },
2492 [HNS_ROCE_QP_STATE_RTR] = {
2493 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2494 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2495 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP,
2496 },
2497 [HNS_ROCE_QP_STATE_RTS] = {
2498 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2499 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2500 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP,
2501 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP,
2502 },
2503 [HNS_ROCE_QP_STATE_SQD] = {
2504 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2505 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2506 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP,
2507 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP,
2508 },
2509 [HNS_ROCE_QP_STATE_ERR] = {
2510 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2511 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2512 }
2513 };
2514
2515 struct hns_roce_cmd_mailbox *mailbox;
2516 struct device *dev = &hr_dev->pdev->dev;
2517 int ret;
2518
2519 if (cur_state >= HNS_ROCE_QP_NUM_STATE ||
2520 new_state >= HNS_ROCE_QP_NUM_STATE ||
2521 !op[cur_state][new_state]) {
2522 dev_err(dev, "[modify_qp]not support state %d to %d\n",
2523 cur_state, new_state);
2524 return -EINVAL;
2525 }
2526
2527 if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)
2528 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
2529 HNS_ROCE_CMD_2RST_QP,
2530 HNS_ROCE_CMD_TIMEOUT_MSECS);
2531
2532 if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)
2533 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
2534 HNS_ROCE_CMD_2ERR_QP,
2535 HNS_ROCE_CMD_TIMEOUT_MSECS);
2536
2537 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2538 if (IS_ERR(mailbox))
2539 return PTR_ERR(mailbox);
2540
2541 memcpy(mailbox->buf, context, sizeof(*context));
2542
2543 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
2544 op[cur_state][new_state],
2545 HNS_ROCE_CMD_TIMEOUT_MSECS);
2546
2547 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2548 return ret;
2549}
2550
2551static int find_wqe_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
2552 u64 *sq_ba, u64 *rq_ba, dma_addr_t *bt_ba)
2553{
2554 struct ib_device *ibdev = &hr_dev->ib_dev;
2555 int count;
2556
2557 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, sq_ba, 1, bt_ba);
2558 if (count < 1) {
2559 ibdev_err(ibdev, "Failed to find SQ ba\n");
2560 return -ENOBUFS;
2561 }
2562
2563 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, rq_ba,
2564 1, NULL);
2565 if (!count) {
2566 ibdev_err(ibdev, "Failed to find RQ ba\n");
2567 return -ENOBUFS;
2568 }
2569
2570 return 0;
2571}
2572
2573static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2574 int attr_mask, enum ib_qp_state cur_state,
2575 enum ib_qp_state new_state)
2576{
2577 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2578 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2579 struct hns_roce_sqp_context *context;
2580 dma_addr_t dma_handle = 0;
2581 u32 __iomem *addr;
2582 u64 sq_ba = 0;
2583 u64 rq_ba = 0;
2584 __le32 tmp;
2585 u32 reg_val;
2586
2587 context = kzalloc(sizeof(*context), GFP_KERNEL);
2588 if (!context)
2589 return -ENOMEM;
2590
2591 /* Search QP buf's MTTs */
2592 if (find_wqe_mtt(hr_dev, hr_qp, &sq_ba, &rq_ba, &dma_handle))
2593 goto out;
2594
2595 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2596 roce_set_field(context->qp1c_bytes_4,
2597 QP1C_BYTES_4_SQ_WQE_SHIFT_M,
2598 QP1C_BYTES_4_SQ_WQE_SHIFT_S,
2599 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2600 roce_set_field(context->qp1c_bytes_4,
2601 QP1C_BYTES_4_RQ_WQE_SHIFT_M,
2602 QP1C_BYTES_4_RQ_WQE_SHIFT_S,
2603 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2604 roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M,
2605 QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn);
2606
2607 context->sq_rq_bt_l = cpu_to_le32(dma_handle);
2608 roce_set_field(context->qp1c_bytes_12,
2609 QP1C_BYTES_12_SQ_RQ_BT_H_M,
2610 QP1C_BYTES_12_SQ_RQ_BT_H_S,
2611 upper_32_bits(dma_handle));
2612
2613 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,
2614 QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head);
2615 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M,
2616 QP1C_BYTES_16_PORT_NUM_S, hr_qp->phy_port);
2617 roce_set_bit(context->qp1c_bytes_16,
2618 QP1C_BYTES_16_SIGNALING_TYPE_S,
2619 hr_qp->sq_signal_bits);
2620 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S,
2621 1);
2622 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S,
2623 1);
2624 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S,
2625 0);
2626
2627 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M,
2628 QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head);
2629 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,
2630 QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index);
2631
2632 context->cur_rq_wqe_ba_l = cpu_to_le32(rq_ba);
2633
2634 roce_set_field(context->qp1c_bytes_28,
2635 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,
2636 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,
2637 upper_32_bits(rq_ba));
2638 roce_set_field(context->qp1c_bytes_28,
2639 QP1C_BYTES_28_RQ_CUR_IDX_M,
2640 QP1C_BYTES_28_RQ_CUR_IDX_S, 0);
2641
2642 roce_set_field(context->qp1c_bytes_32,
2643 QP1C_BYTES_32_RX_CQ_NUM_M,
2644 QP1C_BYTES_32_RX_CQ_NUM_S,
2645 to_hr_cq(ibqp->recv_cq)->cqn);
2646 roce_set_field(context->qp1c_bytes_32,
2647 QP1C_BYTES_32_TX_CQ_NUM_M,
2648 QP1C_BYTES_32_TX_CQ_NUM_S,
2649 to_hr_cq(ibqp->send_cq)->cqn);
2650
2651 context->cur_sq_wqe_ba_l = cpu_to_le32(sq_ba);
2652
2653 roce_set_field(context->qp1c_bytes_40,
2654 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,
2655 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,
2656 upper_32_bits(sq_ba));
2657 roce_set_field(context->qp1c_bytes_40,
2658 QP1C_BYTES_40_SQ_CUR_IDX_M,
2659 QP1C_BYTES_40_SQ_CUR_IDX_S, 0);
2660
2661 /* Copy context to QP1C register */
2662 addr = (u32 __iomem *)(hr_dev->reg_base +
2663 ROCEE_QP1C_CFG0_0_REG +
2664 hr_qp->phy_port * sizeof(*context));
2665
2666 writel(le32_to_cpu(context->qp1c_bytes_4), addr);
2667 writel(le32_to_cpu(context->sq_rq_bt_l), addr + 1);
2668 writel(le32_to_cpu(context->qp1c_bytes_12), addr + 2);
2669 writel(le32_to_cpu(context->qp1c_bytes_16), addr + 3);
2670 writel(le32_to_cpu(context->qp1c_bytes_20), addr + 4);
2671 writel(le32_to_cpu(context->cur_rq_wqe_ba_l), addr + 5);
2672 writel(le32_to_cpu(context->qp1c_bytes_28), addr + 6);
2673 writel(le32_to_cpu(context->qp1c_bytes_32), addr + 7);
2674 writel(le32_to_cpu(context->cur_sq_wqe_ba_l), addr + 8);
2675 writel(le32_to_cpu(context->qp1c_bytes_40), addr + 9);
2676 }
2677
2678 /* Modify QP1C status */
2679 reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG +
2680 hr_qp->phy_port * sizeof(*context));
2681 tmp = cpu_to_le32(reg_val);
2682 roce_set_field(tmp, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M,
2683 ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state);
2684 reg_val = le32_to_cpu(tmp);
2685 roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG +
2686 hr_qp->phy_port * sizeof(*context), reg_val);
2687
2688 hr_qp->state = new_state;
2689 if (new_state == IB_QPS_RESET) {
2690 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
2691 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
2692 if (ibqp->send_cq != ibqp->recv_cq)
2693 hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
2694 hr_qp->qpn, NULL);
2695
2696 hr_qp->rq.head = 0;
2697 hr_qp->rq.tail = 0;
2698 hr_qp->sq.head = 0;
2699 hr_qp->sq.tail = 0;
2700 }
2701
2702 kfree(context);
2703 return 0;
2704
2705out:
2706 kfree(context);
2707 return -EINVAL;
2708}
2709
2710static bool check_qp_state(enum ib_qp_state cur_state,
2711 enum ib_qp_state new_state)
2712{
2713 static const bool sm[][IB_QPS_ERR + 1] = {
2714 [IB_QPS_RESET] = { [IB_QPS_RESET] = true,
2715 [IB_QPS_INIT] = true },
2716 [IB_QPS_INIT] = { [IB_QPS_RESET] = true,
2717 [IB_QPS_INIT] = true,
2718 [IB_QPS_RTR] = true,
2719 [IB_QPS_ERR] = true },
2720 [IB_QPS_RTR] = { [IB_QPS_RESET] = true,
2721 [IB_QPS_RTS] = true,
2722 [IB_QPS_ERR] = true },
2723 [IB_QPS_RTS] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true },
2724 [IB_QPS_SQD] = {},
2725 [IB_QPS_SQE] = {},
2726 [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }
2727 };
2728
2729 return sm[cur_state][new_state];
2730}
2731
2732static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2733 int attr_mask, enum ib_qp_state cur_state,
2734 enum ib_qp_state new_state)
2735{
2736 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2737 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2738 struct device *dev = &hr_dev->pdev->dev;
2739 struct hns_roce_qp_context *context;
2740 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2741 dma_addr_t dma_handle_2 = 0;
2742 dma_addr_t dma_handle = 0;
2743 __le32 doorbell[2] = {0};
2744 u64 *mtts_2 = NULL;
2745 int ret = -EINVAL;
2746 const u8 *smac;
2747 u64 sq_ba = 0;
2748 u64 rq_ba = 0;
2749 u32 port;
2750 u32 port_num;
2751 u8 *dmac;
2752
2753 if (!check_qp_state(cur_state, new_state)) {
2754 ibdev_err(ibqp->device,
2755 "not support QP(%u) status from %d to %d\n",
2756 ibqp->qp_num, cur_state, new_state);
2757 return -EINVAL;
2758 }
2759
2760 context = kzalloc(sizeof(*context), GFP_KERNEL);
2761 if (!context)
2762 return -ENOMEM;
2763
2764 /* Search qp buf's mtts */
2765 if (find_wqe_mtt(hr_dev, hr_qp, &sq_ba, &rq_ba, &dma_handle))
2766 goto out;
2767
2768 /* Search IRRL's mtts */
2769 mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
2770 hr_qp->qpn, &dma_handle_2);
2771 if (mtts_2 == NULL) {
2772 dev_err(dev, "qp irrl_table find failed\n");
2773 goto out;
2774 }
2775
2776 /*
2777 * Reset to init
2778 * Mandatory param:
2779 * IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
2780 * Optional param: NA
2781 */
2782 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2783 roce_set_field(context->qpc_bytes_4,
2784 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2785 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2786 to_hr_qp_type(hr_qp->ibqp.qp_type));
2787
2788 roce_set_bit(context->qpc_bytes_4,
2789 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2790 roce_set_bit(context->qpc_bytes_4,
2791 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2792 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
2793 roce_set_bit(context->qpc_bytes_4,
2794 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2795 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
2796 );
2797 roce_set_bit(context->qpc_bytes_4,
2798 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S,
2799 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)
2800 );
2801 roce_set_bit(context->qpc_bytes_4,
2802 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2803 roce_set_field(context->qpc_bytes_4,
2804 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2805 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2806 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2807 roce_set_field(context->qpc_bytes_4,
2808 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2809 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2810 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2811 roce_set_field(context->qpc_bytes_4,
2812 QP_CONTEXT_QPC_BYTES_4_PD_M,
2813 QP_CONTEXT_QPC_BYTES_4_PD_S,
2814 to_hr_pd(ibqp->pd)->pdn);
2815 hr_qp->access_flags = attr->qp_access_flags;
2816 roce_set_field(context->qpc_bytes_8,
2817 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2818 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2819 to_hr_cq(ibqp->send_cq)->cqn);
2820 roce_set_field(context->qpc_bytes_8,
2821 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2822 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2823 to_hr_cq(ibqp->recv_cq)->cqn);
2824
2825 if (ibqp->srq)
2826 roce_set_field(context->qpc_bytes_12,
2827 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2828 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2829 to_hr_srq(ibqp->srq)->srqn);
2830
2831 roce_set_field(context->qpc_bytes_12,
2832 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2833 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2834 attr->pkey_index);
2835 hr_qp->pkey_index = attr->pkey_index;
2836 roce_set_field(context->qpc_bytes_16,
2837 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2838 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2839 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
2840 roce_set_field(context->qpc_bytes_4,
2841 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2842 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2843 to_hr_qp_type(hr_qp->ibqp.qp_type));
2844 roce_set_bit(context->qpc_bytes_4,
2845 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2846 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2847 roce_set_bit(context->qpc_bytes_4,
2848 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2849 !!(attr->qp_access_flags &
2850 IB_ACCESS_REMOTE_READ));
2851 roce_set_bit(context->qpc_bytes_4,
2852 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2853 !!(attr->qp_access_flags &
2854 IB_ACCESS_REMOTE_WRITE));
2855 } else {
2856 roce_set_bit(context->qpc_bytes_4,
2857 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2858 !!(hr_qp->access_flags &
2859 IB_ACCESS_REMOTE_READ));
2860 roce_set_bit(context->qpc_bytes_4,
2861 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2862 !!(hr_qp->access_flags &
2863 IB_ACCESS_REMOTE_WRITE));
2864 }
2865
2866 roce_set_bit(context->qpc_bytes_4,
2867 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2868 roce_set_field(context->qpc_bytes_4,
2869 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2870 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2871 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2872 roce_set_field(context->qpc_bytes_4,
2873 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2874 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2875 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2876 roce_set_field(context->qpc_bytes_4,
2877 QP_CONTEXT_QPC_BYTES_4_PD_M,
2878 QP_CONTEXT_QPC_BYTES_4_PD_S,
2879 to_hr_pd(ibqp->pd)->pdn);
2880
2881 roce_set_field(context->qpc_bytes_8,
2882 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2883 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2884 to_hr_cq(ibqp->send_cq)->cqn);
2885 roce_set_field(context->qpc_bytes_8,
2886 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2887 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2888 to_hr_cq(ibqp->recv_cq)->cqn);
2889
2890 if (ibqp->srq)
2891 roce_set_field(context->qpc_bytes_12,
2892 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2893 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2894 to_hr_srq(ibqp->srq)->srqn);
2895 if (attr_mask & IB_QP_PKEY_INDEX)
2896 roce_set_field(context->qpc_bytes_12,
2897 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2898 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2899 attr->pkey_index);
2900 else
2901 roce_set_field(context->qpc_bytes_12,
2902 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2903 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2904 hr_qp->pkey_index);
2905
2906 roce_set_field(context->qpc_bytes_16,
2907 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2908 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2909 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2910 if ((attr_mask & IB_QP_ALT_PATH) ||
2911 (attr_mask & IB_QP_ACCESS_FLAGS) ||
2912 (attr_mask & IB_QP_PKEY_INDEX) ||
2913 (attr_mask & IB_QP_QKEY)) {
2914 dev_err(dev, "INIT2RTR attr_mask error\n");
2915 goto out;
2916 }
2917
2918 dmac = (u8 *)attr->ah_attr.roce.dmac;
2919
2920 context->sq_rq_bt_l = cpu_to_le32(dma_handle);
2921 roce_set_field(context->qpc_bytes_24,
2922 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M,
2923 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S,
2924 upper_32_bits(dma_handle));
2925 roce_set_bit(context->qpc_bytes_24,
2926 QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S,
2927 1);
2928 roce_set_field(context->qpc_bytes_24,
2929 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
2930 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S,
2931 attr->min_rnr_timer);
2932 context->irrl_ba_l = cpu_to_le32((u32)(dma_handle_2));
2933 roce_set_field(context->qpc_bytes_32,
2934 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M,
2935 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S,
2936 ((u32)(dma_handle_2 >> 32)) &
2937 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M);
2938 roce_set_field(context->qpc_bytes_32,
2939 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M,
2940 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0);
2941 roce_set_bit(context->qpc_bytes_32,
2942 QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S,
2943 1);
2944 roce_set_bit(context->qpc_bytes_32,
2945 QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S,
2946 hr_qp->sq_signal_bits);
2947
2948 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) :
2949 hr_qp->port;
2950 smac = (const u8 *)hr_dev->dev_addr[port];
2951 /* when dmac equals smac or loop_idc is 1, it should loopback */
2952 if (ether_addr_equal_unaligned(dmac, smac) ||
2953 hr_dev->loop_idc == 0x1)
2954 roce_set_bit(context->qpc_bytes_32,
2955 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);
2956
2957 roce_set_bit(context->qpc_bytes_32,
2958 QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S,
2959 rdma_ah_get_ah_flags(&attr->ah_attr));
2960 roce_set_field(context->qpc_bytes_32,
2961 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
2962 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S,
2963 ilog2((unsigned int)attr->max_dest_rd_atomic));
2964
2965 if (attr_mask & IB_QP_DEST_QPN)
2966 roce_set_field(context->qpc_bytes_36,
2967 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
2968 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S,
2969 attr->dest_qp_num);
2970
2971 /* Configure GID index */
2972 port_num = rdma_ah_get_port_num(&attr->ah_attr);
2973 roce_set_field(context->qpc_bytes_36,
2974 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
2975 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S,
2976 hns_get_gid_index(hr_dev,
2977 port_num - 1,
2978 grh->sgid_index));
2979
2980 memcpy(&(context->dmac_l), dmac, 4);
2981
2982 roce_set_field(context->qpc_bytes_44,
2983 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2984 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S,
2985 *((u16 *)(&dmac[4])));
2986 roce_set_field(context->qpc_bytes_44,
2987 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M,
2988 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S,
2989 rdma_ah_get_static_rate(&attr->ah_attr));
2990 roce_set_field(context->qpc_bytes_44,
2991 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
2992 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S,
2993 grh->hop_limit);
2994
2995 roce_set_field(context->qpc_bytes_48,
2996 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
2997 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S,
2998 grh->flow_label);
2999 roce_set_field(context->qpc_bytes_48,
3000 QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
3001 QP_CONTEXT_QPC_BYTES_48_TCLASS_S,
3002 grh->traffic_class);
3003 roce_set_field(context->qpc_bytes_48,
3004 QP_CONTEXT_QPC_BYTES_48_MTU_M,
3005 QP_CONTEXT_QPC_BYTES_48_MTU_S, attr->path_mtu);
3006
3007 memcpy(context->dgid, grh->dgid.raw,
3008 sizeof(grh->dgid.raw));
3009
3010 dev_dbg(dev, "dmac:%x :%lx\n", context->dmac_l,
3011 roce_get_field(context->qpc_bytes_44,
3012 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
3013 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S));
3014
3015 roce_set_field(context->qpc_bytes_68,
3016 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M,
3017 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S,
3018 hr_qp->rq.head);
3019 roce_set_field(context->qpc_bytes_68,
3020 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M,
3021 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0);
3022
3023 context->cur_rq_wqe_ba_l = cpu_to_le32(rq_ba);
3024
3025 roce_set_field(context->qpc_bytes_76,
3026 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M,
3027 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S,
3028 upper_32_bits(rq_ba));
3029 roce_set_field(context->qpc_bytes_76,
3030 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M,
3031 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0);
3032
3033 context->rx_rnr_time = 0;
3034
3035 roce_set_field(context->qpc_bytes_84,
3036 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M,
3037 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S,
3038 attr->rq_psn - 1);
3039 roce_set_field(context->qpc_bytes_84,
3040 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M,
3041 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0);
3042
3043 roce_set_field(context->qpc_bytes_88,
3044 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
3045 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S,
3046 attr->rq_psn);
3047 roce_set_bit(context->qpc_bytes_88,
3048 QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0);
3049 roce_set_bit(context->qpc_bytes_88,
3050 QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0);
3051 roce_set_field(context->qpc_bytes_88,
3052 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M,
3053 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S,
3054 0);
3055 roce_set_field(context->qpc_bytes_88,
3056 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M,
3057 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S,
3058 0);
3059
3060 context->dma_length = 0;
3061 context->r_key = 0;
3062 context->va_l = 0;
3063 context->va_h = 0;
3064
3065 roce_set_field(context->qpc_bytes_108,
3066 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M,
3067 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0);
3068 roce_set_bit(context->qpc_bytes_108,
3069 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0);
3070 roce_set_bit(context->qpc_bytes_108,
3071 QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0);
3072
3073 roce_set_field(context->qpc_bytes_112,
3074 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M,
3075 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0);
3076 roce_set_field(context->qpc_bytes_112,
3077 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M,
3078 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0);
3079
3080 /* For chip resp ack */
3081 roce_set_field(context->qpc_bytes_156,
3082 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
3083 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
3084 hr_qp->phy_port);
3085 roce_set_field(context->qpc_bytes_156,
3086 QP_CONTEXT_QPC_BYTES_156_SL_M,
3087 QP_CONTEXT_QPC_BYTES_156_SL_S,
3088 rdma_ah_get_sl(&attr->ah_attr));
3089 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
3090 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
3091 /* If exist optional param, return error */
3092 if ((attr_mask & IB_QP_ALT_PATH) ||
3093 (attr_mask & IB_QP_ACCESS_FLAGS) ||
3094 (attr_mask & IB_QP_QKEY) ||
3095 (attr_mask & IB_QP_PATH_MIG_STATE) ||
3096 (attr_mask & IB_QP_CUR_STATE) ||
3097 (attr_mask & IB_QP_MIN_RNR_TIMER)) {
3098 dev_err(dev, "RTR2RTS attr_mask error\n");
3099 goto out;
3100 }
3101
3102 context->rx_cur_sq_wqe_ba_l = cpu_to_le32(sq_ba);
3103
3104 roce_set_field(context->qpc_bytes_120,
3105 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M,
3106 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S,
3107 upper_32_bits(sq_ba));
3108
3109 roce_set_field(context->qpc_bytes_124,
3110 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M,
3111 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0);
3112 roce_set_field(context->qpc_bytes_124,
3113 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M,
3114 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0);
3115
3116 roce_set_field(context->qpc_bytes_128,
3117 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M,
3118 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S,
3119 attr->sq_psn);
3120 roce_set_bit(context->qpc_bytes_128,
3121 QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0);
3122 roce_set_field(context->qpc_bytes_128,
3123 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M,
3124 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S,
3125 0);
3126 roce_set_bit(context->qpc_bytes_128,
3127 QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0);
3128
3129 roce_set_field(context->qpc_bytes_132,
3130 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M,
3131 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0);
3132 roce_set_field(context->qpc_bytes_132,
3133 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M,
3134 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0);
3135
3136 roce_set_field(context->qpc_bytes_136,
3137 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M,
3138 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S,
3139 attr->sq_psn);
3140 roce_set_field(context->qpc_bytes_136,
3141 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M,
3142 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S,
3143 attr->sq_psn);
3144
3145 roce_set_field(context->qpc_bytes_140,
3146 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M,
3147 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S,
3148 (attr->sq_psn >> SQ_PSN_SHIFT));
3149 roce_set_field(context->qpc_bytes_140,
3150 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M,
3151 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0);
3152 roce_set_bit(context->qpc_bytes_140,
3153 QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0);
3154
3155 roce_set_field(context->qpc_bytes_148,
3156 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M,
3157 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0);
3158 roce_set_field(context->qpc_bytes_148,
3159 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
3160 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S,
3161 attr->retry_cnt);
3162 roce_set_field(context->qpc_bytes_148,
3163 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M,
3164 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S,
3165 attr->rnr_retry);
3166 roce_set_field(context->qpc_bytes_148,
3167 QP_CONTEXT_QPC_BYTES_148_LSN_M,
3168 QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100);
3169
3170 context->rnr_retry = 0;
3171
3172 roce_set_field(context->qpc_bytes_156,
3173 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M,
3174 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S,
3175 attr->retry_cnt);
3176 if (attr->timeout < 0x12) {
3177 dev_info(dev, "ack timeout value(0x%x) must bigger than 0x12.\n",
3178 attr->timeout);
3179 roce_set_field(context->qpc_bytes_156,
3180 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
3181 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
3182 0x12);
3183 } else {
3184 roce_set_field(context->qpc_bytes_156,
3185 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
3186 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
3187 attr->timeout);
3188 }
3189 roce_set_field(context->qpc_bytes_156,
3190 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M,
3191 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S,
3192 attr->rnr_retry);
3193 roce_set_field(context->qpc_bytes_156,
3194 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
3195 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
3196 hr_qp->phy_port);
3197 roce_set_field(context->qpc_bytes_156,
3198 QP_CONTEXT_QPC_BYTES_156_SL_M,
3199 QP_CONTEXT_QPC_BYTES_156_SL_S,
3200 rdma_ah_get_sl(&attr->ah_attr));
3201 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
3202 roce_set_field(context->qpc_bytes_156,
3203 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
3204 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S,
3205 ilog2((unsigned int)attr->max_rd_atomic));
3206 roce_set_field(context->qpc_bytes_156,
3207 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M,
3208 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0);
3209 context->pkt_use_len = 0;
3210
3211 roce_set_field(context->qpc_bytes_164,
3212 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
3213 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr->sq_psn);
3214 roce_set_field(context->qpc_bytes_164,
3215 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M,
3216 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0);
3217
3218 roce_set_field(context->qpc_bytes_168,
3219 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M,
3220 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S,
3221 attr->sq_psn);
3222 roce_set_field(context->qpc_bytes_168,
3223 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M,
3224 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0);
3225 roce_set_field(context->qpc_bytes_168,
3226 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M,
3227 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0);
3228 roce_set_bit(context->qpc_bytes_168,
3229 QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0);
3230 roce_set_bit(context->qpc_bytes_168,
3231 QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0);
3232 roce_set_bit(context->qpc_bytes_168,
3233 QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0);
3234 context->sge_use_len = 0;
3235
3236 roce_set_field(context->qpc_bytes_176,
3237 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M,
3238 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0);
3239 roce_set_field(context->qpc_bytes_176,
3240 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M,
3241 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S,
3242 0);
3243 roce_set_field(context->qpc_bytes_180,
3244 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M,
3245 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0);
3246 roce_set_field(context->qpc_bytes_180,
3247 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M,
3248 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0);
3249
3250 context->tx_cur_sq_wqe_ba_l = cpu_to_le32(sq_ba);
3251
3252 roce_set_field(context->qpc_bytes_188,
3253 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M,
3254 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S,
3255 upper_32_bits(sq_ba));
3256 roce_set_bit(context->qpc_bytes_188,
3257 QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0);
3258 roce_set_field(context->qpc_bytes_188,
3259 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M,
3260 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S,
3261 0);
3262 }
3263
3264 /* Every status migrate must change state */
3265 roce_set_field(context->qpc_bytes_144,
3266 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
3267 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, new_state);
3268
3269 /* SW pass context to HW */
3270 ret = hns_roce_v1_qp_modify(hr_dev, to_hns_roce_state(cur_state),
3271 to_hns_roce_state(new_state), context,
3272 hr_qp);
3273 if (ret) {
3274 dev_err(dev, "hns_roce_qp_modify failed\n");
3275 goto out;
3276 }
3277
3278 /*
3279 * Use rst2init to instead of init2init with drv,
3280 * need to hw to flash RQ HEAD by DB again
3281 */
3282 if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
3283 roce_set_field(doorbell[0], RQ_DOORBELL_U32_4_RQ_HEAD_M,
3284 RQ_DOORBELL_U32_4_RQ_HEAD_S, hr_qp->rq.head);
3285 roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_QPN_M,
3286 RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
3287 roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_CMD_M,
3288 RQ_DOORBELL_U32_8_CMD_S, 1);
3289 roce_set_bit(doorbell[1], RQ_DOORBELL_U32_8_HW_SYNC_S, 1);
3290
3291 if (ibqp->uobject) {
3292 hr_qp->rq.db_reg = hr_dev->reg_base +
3293 hr_dev->odb_offset +
3294 DB_REG_OFFSET * hr_dev->priv_uar.index;
3295 }
3296
3297 hns_roce_write64_k(doorbell, hr_qp->rq.db_reg);
3298 }
3299
3300 hr_qp->state = new_state;
3301
3302 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
3303 hr_qp->resp_depth = attr->max_dest_rd_atomic;
3304 if (attr_mask & IB_QP_PORT) {
3305 hr_qp->port = attr->port_num - 1;
3306 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
3307 }
3308
3309 if (new_state == IB_QPS_RESET && !ibqp->uobject) {
3310 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
3311 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
3312 if (ibqp->send_cq != ibqp->recv_cq)
3313 hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
3314 hr_qp->qpn, NULL);
3315
3316 hr_qp->rq.head = 0;
3317 hr_qp->rq.tail = 0;
3318 hr_qp->sq.head = 0;
3319 hr_qp->sq.tail = 0;
3320 }
3321out:
3322 kfree(context);
3323 return ret;
3324}
3325
3326static int hns_roce_v1_modify_qp(struct ib_qp *ibqp,
3327 const struct ib_qp_attr *attr, int attr_mask,
3328 enum ib_qp_state cur_state,
3329 enum ib_qp_state new_state)
3330{
3331 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
3332 return -EOPNOTSUPP;
3333
3334 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
3335 return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state,
3336 new_state);
3337 else
3338 return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state,
3339 new_state);
3340}
3341
3342static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state)
3343{
3344 switch (state) {
3345 case HNS_ROCE_QP_STATE_RST:
3346 return IB_QPS_RESET;
3347 case HNS_ROCE_QP_STATE_INIT:
3348 return IB_QPS_INIT;
3349 case HNS_ROCE_QP_STATE_RTR:
3350 return IB_QPS_RTR;
3351 case HNS_ROCE_QP_STATE_RTS:
3352 return IB_QPS_RTS;
3353 case HNS_ROCE_QP_STATE_SQD:
3354 return IB_QPS_SQD;
3355 case HNS_ROCE_QP_STATE_ERR:
3356 return IB_QPS_ERR;
3357 default:
3358 return IB_QPS_ERR;
3359 }
3360}
3361
3362static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,
3363 struct hns_roce_qp *hr_qp,
3364 struct hns_roce_qp_context *hr_context)
3365{
3366 struct hns_roce_cmd_mailbox *mailbox;
3367 int ret;
3368
3369 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3370 if (IS_ERR(mailbox))
3371 return PTR_ERR(mailbox);
3372
3373 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
3374 HNS_ROCE_CMD_QUERY_QP,
3375 HNS_ROCE_CMD_TIMEOUT_MSECS);
3376 if (!ret)
3377 memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
3378 else
3379 dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n");
3380
3381 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3382
3383 return ret;
3384}
3385
3386static int hns_roce_v1_q_sqp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
3387 int qp_attr_mask,
3388 struct ib_qp_init_attr *qp_init_attr)
3389{
3390 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3391 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3392 struct hns_roce_sqp_context context;
3393 u32 addr;
3394
3395 mutex_lock(&hr_qp->mutex);
3396
3397 if (hr_qp->state == IB_QPS_RESET) {
3398 qp_attr->qp_state = IB_QPS_RESET;
3399 goto done;
3400 }
3401
3402 addr = ROCEE_QP1C_CFG0_0_REG +
3403 hr_qp->port * sizeof(struct hns_roce_sqp_context);
3404 context.qp1c_bytes_4 = cpu_to_le32(roce_read(hr_dev, addr));
3405 context.sq_rq_bt_l = cpu_to_le32(roce_read(hr_dev, addr + 1));
3406 context.qp1c_bytes_12 = cpu_to_le32(roce_read(hr_dev, addr + 2));
3407 context.qp1c_bytes_16 = cpu_to_le32(roce_read(hr_dev, addr + 3));
3408 context.qp1c_bytes_20 = cpu_to_le32(roce_read(hr_dev, addr + 4));
3409 context.cur_rq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 5));
3410 context.qp1c_bytes_28 = cpu_to_le32(roce_read(hr_dev, addr + 6));
3411 context.qp1c_bytes_32 = cpu_to_le32(roce_read(hr_dev, addr + 7));
3412 context.cur_sq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 8));
3413 context.qp1c_bytes_40 = cpu_to_le32(roce_read(hr_dev, addr + 9));
3414
3415 hr_qp->state = roce_get_field(context.qp1c_bytes_4,
3416 QP1C_BYTES_4_QP_STATE_M,
3417 QP1C_BYTES_4_QP_STATE_S);
3418 qp_attr->qp_state = hr_qp->state;
3419 qp_attr->path_mtu = IB_MTU_256;
3420 qp_attr->path_mig_state = IB_MIG_ARMED;
3421 qp_attr->qkey = QKEY_VAL;
3422 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
3423 qp_attr->rq_psn = 0;
3424 qp_attr->sq_psn = 0;
3425 qp_attr->dest_qp_num = 1;
3426 qp_attr->qp_access_flags = 6;
3427
3428 qp_attr->pkey_index = roce_get_field(context.qp1c_bytes_20,
3429 QP1C_BYTES_20_PKEY_IDX_M,
3430 QP1C_BYTES_20_PKEY_IDX_S);
3431 qp_attr->port_num = hr_qp->port + 1;
3432 qp_attr->sq_draining = 0;
3433 qp_attr->max_rd_atomic = 0;
3434 qp_attr->max_dest_rd_atomic = 0;
3435 qp_attr->min_rnr_timer = 0;
3436 qp_attr->timeout = 0;
3437 qp_attr->retry_cnt = 0;
3438 qp_attr->rnr_retry = 0;
3439 qp_attr->alt_timeout = 0;
3440
3441done:
3442 qp_attr->cur_qp_state = qp_attr->qp_state;
3443 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
3444 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
3445 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
3446 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
3447 qp_attr->cap.max_inline_data = 0;
3448 qp_init_attr->cap = qp_attr->cap;
3449 qp_init_attr->create_flags = 0;
3450
3451 mutex_unlock(&hr_qp->mutex);
3452
3453 return 0;
3454}
3455
3456static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
3457 int qp_attr_mask,
3458 struct ib_qp_init_attr *qp_init_attr)
3459{
3460 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3461 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3462 struct device *dev = &hr_dev->pdev->dev;
3463 struct hns_roce_qp_context *context;
3464 int tmp_qp_state;
3465 int ret = 0;
3466 int state;
3467
3468 context = kzalloc(sizeof(*context), GFP_KERNEL);
3469 if (!context)
3470 return -ENOMEM;
3471
3472 memset(qp_attr, 0, sizeof(*qp_attr));
3473 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
3474
3475 mutex_lock(&hr_qp->mutex);
3476
3477 if (hr_qp->state == IB_QPS_RESET) {
3478 qp_attr->qp_state = IB_QPS_RESET;
3479 goto done;
3480 }
3481
3482 ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context);
3483 if (ret) {
3484 dev_err(dev, "query qpc error\n");
3485 ret = -EINVAL;
3486 goto out;
3487 }
3488
3489 state = roce_get_field(context->qpc_bytes_144,
3490 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
3491 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S);
3492 tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state);
3493 if (tmp_qp_state == -1) {
3494 dev_err(dev, "to_ib_qp_state error\n");
3495 ret = -EINVAL;
3496 goto out;
3497 }
3498 hr_qp->state = (u8)tmp_qp_state;
3499 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
3500 qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->qpc_bytes_48,
3501 QP_CONTEXT_QPC_BYTES_48_MTU_M,
3502 QP_CONTEXT_QPC_BYTES_48_MTU_S);
3503 qp_attr->path_mig_state = IB_MIG_ARMED;
3504 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
3505 if (hr_qp->ibqp.qp_type == IB_QPT_UD)
3506 qp_attr->qkey = QKEY_VAL;
3507
3508 qp_attr->rq_psn = roce_get_field(context->qpc_bytes_88,
3509 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
3510 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S);
3511 qp_attr->sq_psn = (u32)roce_get_field(context->qpc_bytes_164,
3512 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
3513 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S);
3514 qp_attr->dest_qp_num = (u8)roce_get_field(context->qpc_bytes_36,
3515 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
3516 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S);
3517 qp_attr->qp_access_flags = ((roce_get_bit(context->qpc_bytes_4,
3518 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) << 2) |
3519 ((roce_get_bit(context->qpc_bytes_4,
3520 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) << 1) |
3521 ((roce_get_bit(context->qpc_bytes_4,
3522 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3);
3523
3524 if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
3525 struct ib_global_route *grh =
3526 rdma_ah_retrieve_grh(&qp_attr->ah_attr);
3527
3528 rdma_ah_set_sl(&qp_attr->ah_attr,
3529 roce_get_field(context->qpc_bytes_156,
3530 QP_CONTEXT_QPC_BYTES_156_SL_M,
3531 QP_CONTEXT_QPC_BYTES_156_SL_S));
3532 rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH);
3533 grh->flow_label =
3534 roce_get_field(context->qpc_bytes_48,
3535 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
3536 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S);
3537 grh->sgid_index =
3538 roce_get_field(context->qpc_bytes_36,
3539 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
3540 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S);
3541 grh->hop_limit =
3542 roce_get_field(context->qpc_bytes_44,
3543 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
3544 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S);
3545 grh->traffic_class =
3546 roce_get_field(context->qpc_bytes_48,
3547 QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
3548 QP_CONTEXT_QPC_BYTES_48_TCLASS_S);
3549
3550 memcpy(grh->dgid.raw, context->dgid,
3551 sizeof(grh->dgid.raw));
3552 }
3553
3554 qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12,
3555 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
3556 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S);
3557 qp_attr->port_num = hr_qp->port + 1;
3558 qp_attr->sq_draining = 0;
3559 qp_attr->max_rd_atomic = 1 << roce_get_field(context->qpc_bytes_156,
3560 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
3561 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S);
3562 qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->qpc_bytes_32,
3563 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
3564 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S);
3565 qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24,
3566 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
3567 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S));
3568 qp_attr->timeout = (u8)(roce_get_field(context->qpc_bytes_156,
3569 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
3570 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S));
3571 qp_attr->retry_cnt = roce_get_field(context->qpc_bytes_148,
3572 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
3573 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S);
3574 qp_attr->rnr_retry = (u8)le32_to_cpu(context->rnr_retry);
3575
3576done:
3577 qp_attr->cur_qp_state = qp_attr->qp_state;
3578 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
3579 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
3580
3581 if (!ibqp->uobject) {
3582 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
3583 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
3584 } else {
3585 qp_attr->cap.max_send_wr = 0;
3586 qp_attr->cap.max_send_sge = 0;
3587 }
3588
3589 qp_init_attr->cap = qp_attr->cap;
3590
3591out:
3592 mutex_unlock(&hr_qp->mutex);
3593 kfree(context);
3594 return ret;
3595}
3596
3597static int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
3598 int qp_attr_mask,
3599 struct ib_qp_init_attr *qp_init_attr)
3600{
3601 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3602
3603 return hr_qp->doorbell_qpn <= 1 ?
3604 hns_roce_v1_q_sqp(ibqp, qp_attr, qp_attr_mask, qp_init_attr) :
3605 hns_roce_v1_q_qp(ibqp, qp_attr, qp_attr_mask, qp_init_attr);
3606}
3607
3608int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
3609{
3610 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3611 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3612 struct hns_roce_cq *send_cq, *recv_cq;
3613 int ret;
3614
3615 ret = hns_roce_v1_modify_qp(ibqp, NULL, 0, hr_qp->state, IB_QPS_RESET);
3616 if (ret)
3617 return ret;
3618
3619 send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;
3620 recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;
3621
3622 hns_roce_lock_cqs(send_cq, recv_cq);
3623 if (!udata) {
3624 if (recv_cq)
3625 __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn,
3626 (hr_qp->ibqp.srq ?
3627 to_hr_srq(hr_qp->ibqp.srq) :
3628 NULL));
3629
3630 if (send_cq && send_cq != recv_cq)
3631 __hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL);
3632 }
3633 hns_roce_qp_remove(hr_dev, hr_qp);
3634 hns_roce_unlock_cqs(send_cq, recv_cq);
3635
3636 hns_roce_qp_destroy(hr_dev, hr_qp, udata);
3637
3638 return 0;
3639}
3640
3641static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
3642{
3643 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3644 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3645 struct device *dev = &hr_dev->pdev->dev;
3646 u32 cqe_cnt_ori;
3647 u32 cqe_cnt_cur;
3648 int wait_time = 0;
3649
3650 /*
3651 * Before freeing cq buffer, we need to ensure that the outstanding CQE
3652 * have been written by checking the CQE counter.
3653 */
3654 cqe_cnt_ori = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT);
3655 while (1) {
3656 if (roce_read(hr_dev, ROCEE_CAEP_CQE_WCMD_EMPTY) &
3657 HNS_ROCE_CQE_WCMD_EMPTY_BIT)
3658 break;
3659
3660 cqe_cnt_cur = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT);
3661 if ((cqe_cnt_cur - cqe_cnt_ori) >= HNS_ROCE_MIN_CQE_CNT)
3662 break;
3663
3664 msleep(HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS);
3665 if (wait_time > HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) {
3666 dev_warn(dev, "Destroy cq 0x%lx timeout!\n",
3667 hr_cq->cqn);
3668 break;
3669 }
3670 wait_time++;
3671 }
3672 return 0;
3673}
3674
3675static void set_eq_cons_index_v1(struct hns_roce_eq *eq, u32 req_not)
3676{
3677 roce_raw_write((eq->cons_index & HNS_ROCE_V1_CONS_IDX_M) |
3678 (req_not << eq->log_entries), eq->db_reg);
3679}
3680
3681static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
3682 struct hns_roce_aeqe *aeqe, int qpn)
3683{
3684 struct device *dev = &hr_dev->pdev->dev;
3685
3686 dev_warn(dev, "Local Work Queue Catastrophic Error.\n");
3687 switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
3688 HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
3689 case HNS_ROCE_LWQCE_QPC_ERROR:
3690 dev_warn(dev, "QP %d, QPC error.\n", qpn);
3691 break;
3692 case HNS_ROCE_LWQCE_MTU_ERROR:
3693 dev_warn(dev, "QP %d, MTU error.\n", qpn);
3694 break;
3695 case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
3696 dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
3697 break;
3698 case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
3699 dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
3700 break;
3701 case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
3702 dev_warn(dev, "QP %d, WQE shift error\n", qpn);
3703 break;
3704 case HNS_ROCE_LWQCE_SL_ERROR:
3705 dev_warn(dev, "QP %d, SL error.\n", qpn);
3706 break;
3707 case HNS_ROCE_LWQCE_PORT_ERROR:
3708 dev_warn(dev, "QP %d, port error.\n", qpn);
3709 break;
3710 default:
3711 break;
3712 }
3713}
3714
3715static void hns_roce_v1_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
3716 struct hns_roce_aeqe *aeqe,
3717 int qpn)
3718{
3719 struct device *dev = &hr_dev->pdev->dev;
3720
3721 dev_warn(dev, "Local Access Violation Work Queue Error.\n");
3722 switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
3723 HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
3724 case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
3725 dev_warn(dev, "QP %d, R_key violation.\n", qpn);
3726 break;
3727 case HNS_ROCE_LAVWQE_LENGTH_ERROR:
3728 dev_warn(dev, "QP %d, length error.\n", qpn);
3729 break;
3730 case HNS_ROCE_LAVWQE_VA_ERROR:
3731 dev_warn(dev, "QP %d, VA error.\n", qpn);
3732 break;
3733 case HNS_ROCE_LAVWQE_PD_ERROR:
3734 dev_err(dev, "QP %d, PD error.\n", qpn);
3735 break;
3736 case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
3737 dev_warn(dev, "QP %d, rw acc error.\n", qpn);
3738 break;
3739 case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
3740 dev_warn(dev, "QP %d, key state error.\n", qpn);
3741 break;
3742 case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
3743 dev_warn(dev, "QP %d, MR operation error.\n", qpn);
3744 break;
3745 default:
3746 break;
3747 }
3748}
3749
3750static void hns_roce_v1_qp_err_handle(struct hns_roce_dev *hr_dev,
3751 struct hns_roce_aeqe *aeqe,
3752 int event_type)
3753{
3754 struct device *dev = &hr_dev->pdev->dev;
3755 int phy_port;
3756 int qpn;
3757
3758 qpn = roce_get_field(aeqe->event.queue_event.num,
3759 HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
3760 HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);
3761 phy_port = roce_get_field(aeqe->event.queue_event.num,
3762 HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M,
3763 HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S);
3764 if (qpn <= 1)
3765 qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port;
3766
3767 switch (event_type) {
3768 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
3769 dev_warn(dev, "Invalid Req Local Work Queue Error.\n"
3770 "QP %d, phy_port %d.\n", qpn, phy_port);
3771 break;
3772 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
3773 hns_roce_v1_wq_catas_err_handle(hr_dev, aeqe, qpn);
3774 break;
3775 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
3776 hns_roce_v1_local_wq_access_err_handle(hr_dev, aeqe, qpn);
3777 break;
3778 default:
3779 break;
3780 }
3781
3782 hns_roce_qp_event(hr_dev, qpn, event_type);
3783}
3784
3785static void hns_roce_v1_cq_err_handle(struct hns_roce_dev *hr_dev,
3786 struct hns_roce_aeqe *aeqe,
3787 int event_type)
3788{
3789 struct device *dev = &hr_dev->pdev->dev;
3790 u32 cqn;
3791
3792 cqn = roce_get_field(aeqe->event.queue_event.num,
3793 HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
3794 HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S);
3795
3796 switch (event_type) {
3797 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
3798 dev_warn(dev, "CQ 0x%x access err.\n", cqn);
3799 break;
3800 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
3801 dev_warn(dev, "CQ 0x%x overflow\n", cqn);
3802 break;
3803 case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
3804 dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn);
3805 break;
3806 default:
3807 break;
3808 }
3809
3810 hns_roce_cq_event(hr_dev, cqn, event_type);
3811}
3812
3813static void hns_roce_v1_db_overflow_handle(struct hns_roce_dev *hr_dev,
3814 struct hns_roce_aeqe *aeqe)
3815{
3816 struct device *dev = &hr_dev->pdev->dev;
3817
3818 switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
3819 HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
3820 case HNS_ROCE_DB_SUBTYPE_SDB_OVF:
3821 dev_warn(dev, "SDB overflow.\n");
3822 break;
3823 case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF:
3824 dev_warn(dev, "SDB almost overflow.\n");
3825 break;
3826 case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP:
3827 dev_warn(dev, "SDB almost empty.\n");
3828 break;
3829 case HNS_ROCE_DB_SUBTYPE_ODB_OVF:
3830 dev_warn(dev, "ODB overflow.\n");
3831 break;
3832 case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF:
3833 dev_warn(dev, "ODB almost overflow.\n");
3834 break;
3835 case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP:
3836 dev_warn(dev, "SDB almost empty.\n");
3837 break;
3838 default:
3839 break;
3840 }
3841}
3842
3843static struct hns_roce_aeqe *get_aeqe_v1(struct hns_roce_eq *eq, u32 entry)
3844{
3845 unsigned long off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQE_SIZE;
3846
3847 return (struct hns_roce_aeqe *)((u8 *)
3848 (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
3849 off % HNS_ROCE_BA_SIZE);
3850}
3851
3852static struct hns_roce_aeqe *next_aeqe_sw_v1(struct hns_roce_eq *eq)
3853{
3854 struct hns_roce_aeqe *aeqe = get_aeqe_v1(eq, eq->cons_index);
3855
3856 return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^
3857 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
3858}
3859
3860static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev,
3861 struct hns_roce_eq *eq)
3862{
3863 struct device *dev = &hr_dev->pdev->dev;
3864 struct hns_roce_aeqe *aeqe;
3865 int aeqes_found = 0;
3866 int event_type;
3867
3868 while ((aeqe = next_aeqe_sw_v1(eq))) {
3869 /* Make sure we read the AEQ entry after we have checked the
3870 * ownership bit
3871 */
3872 dma_rmb();
3873
3874 dev_dbg(dev, "aeqe = %pK, aeqe->asyn.event_type = 0x%lx\n",
3875 aeqe,
3876 roce_get_field(aeqe->asyn,
3877 HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
3878 HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
3879 event_type = roce_get_field(aeqe->asyn,
3880 HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
3881 HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S);
3882 switch (event_type) {
3883 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
3884 dev_warn(dev, "PATH MIG not supported\n");
3885 break;
3886 case HNS_ROCE_EVENT_TYPE_COMM_EST:
3887 dev_warn(dev, "COMMUNICATION established\n");
3888 break;
3889 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
3890 dev_warn(dev, "SQ DRAINED not supported\n");
3891 break;
3892 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
3893 dev_warn(dev, "PATH MIG failed\n");
3894 break;
3895 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
3896 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
3897 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
3898 hns_roce_v1_qp_err_handle(hr_dev, aeqe, event_type);
3899 break;
3900 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
3901 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
3902 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
3903 dev_warn(dev, "SRQ not support!\n");
3904 break;
3905 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
3906 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
3907 case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
3908 hns_roce_v1_cq_err_handle(hr_dev, aeqe, event_type);
3909 break;
3910 case HNS_ROCE_EVENT_TYPE_PORT_CHANGE:
3911 dev_warn(dev, "port change.\n");
3912 break;
3913 case HNS_ROCE_EVENT_TYPE_MB:
3914 hns_roce_cmd_event(hr_dev,
3915 le16_to_cpu(aeqe->event.cmd.token),
3916 aeqe->event.cmd.status,
3917 le64_to_cpu(aeqe->event.cmd.out_param
3918 ));
3919 break;
3920 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
3921 hns_roce_v1_db_overflow_handle(hr_dev, aeqe);
3922 break;
3923 default:
3924 dev_warn(dev, "Unhandled event %d on EQ %d at idx %u.\n",
3925 event_type, eq->eqn, eq->cons_index);
3926 break;
3927 }
3928
3929 eq->cons_index++;
3930 aeqes_found = 1;
3931
3932 if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1)
3933 eq->cons_index = 0;
3934 }
3935
3936 set_eq_cons_index_v1(eq, 0);
3937
3938 return aeqes_found;
3939}
3940
3941static struct hns_roce_ceqe *get_ceqe_v1(struct hns_roce_eq *eq, u32 entry)
3942{
3943 unsigned long off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQE_SIZE;
3944
3945 return (struct hns_roce_ceqe *)((u8 *)
3946 (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
3947 off % HNS_ROCE_BA_SIZE);
3948}
3949
3950static struct hns_roce_ceqe *next_ceqe_sw_v1(struct hns_roce_eq *eq)
3951{
3952 struct hns_roce_ceqe *ceqe = get_ceqe_v1(eq, eq->cons_index);
3953
3954 return (!!(roce_get_bit(ceqe->comp,
3955 HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^
3956 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
3957}
3958
3959static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev,
3960 struct hns_roce_eq *eq)
3961{
3962 struct hns_roce_ceqe *ceqe;
3963 int ceqes_found = 0;
3964 u32 cqn;
3965
3966 while ((ceqe = next_ceqe_sw_v1(eq))) {
3967 /* Make sure we read CEQ entry after we have checked the
3968 * ownership bit
3969 */
3970 dma_rmb();
3971
3972 cqn = roce_get_field(ceqe->comp,
3973 HNS_ROCE_CEQE_CEQE_COMP_CQN_M,
3974 HNS_ROCE_CEQE_CEQE_COMP_CQN_S);
3975 hns_roce_cq_completion(hr_dev, cqn);
3976
3977 ++eq->cons_index;
3978 ceqes_found = 1;
3979
3980 if (eq->cons_index >
3981 EQ_DEPTH_COEFF * hr_dev->caps.ceqe_depth - 1)
3982 eq->cons_index = 0;
3983 }
3984
3985 set_eq_cons_index_v1(eq, 0);
3986
3987 return ceqes_found;
3988}
3989
3990static irqreturn_t hns_roce_v1_msix_interrupt_eq(int irq, void *eq_ptr)
3991{
3992 struct hns_roce_eq *eq = eq_ptr;
3993 struct hns_roce_dev *hr_dev = eq->hr_dev;
3994 int int_work;
3995
3996 if (eq->type_flag == HNS_ROCE_CEQ)
3997 /* CEQ irq routine, CEQ is pulse irq, not clear */
3998 int_work = hns_roce_v1_ceq_int(hr_dev, eq);
3999 else
4000 /* AEQ irq routine, AEQ is pulse irq, not clear */
4001 int_work = hns_roce_v1_aeq_int(hr_dev, eq);
4002
4003 return IRQ_RETVAL(int_work);
4004}
4005
4006static irqreturn_t hns_roce_v1_msix_interrupt_abn(int irq, void *dev_id)
4007{
4008 struct hns_roce_dev *hr_dev = dev_id;
4009 struct device *dev = &hr_dev->pdev->dev;
4010 int int_work = 0;
4011 u32 caepaemask_val;
4012 u32 cealmovf_val;
4013 u32 caepaest_val;
4014 u32 aeshift_val;
4015 u32 ceshift_val;
4016 u32 cemask_val;
4017 __le32 tmp;
4018 int i;
4019
4020 /*
4021 * Abnormal interrupt:
4022 * AEQ overflow, ECC multi-bit err, CEQ overflow must clear
4023 * interrupt, mask irq, clear irq, cancel mask operation
4024 */
4025 aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG);
4026 tmp = cpu_to_le32(aeshift_val);
4027
4028 /* AEQE overflow */
4029 if (roce_get_bit(tmp,
4030 ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) {
4031 dev_warn(dev, "AEQ overflow!\n");
4032
4033 /* Set mask */
4034 caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
4035 tmp = cpu_to_le32(caepaemask_val);
4036 roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
4037 HNS_ROCE_INT_MASK_ENABLE);
4038 caepaemask_val = le32_to_cpu(tmp);
4039 roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
4040
4041 /* Clear int state(INT_WC : write 1 clear) */
4042 caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG);
4043 tmp = cpu_to_le32(caepaest_val);
4044 roce_set_bit(tmp, ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1);
4045 caepaest_val = le32_to_cpu(tmp);
4046 roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val);
4047
4048 /* Clear mask */
4049 caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
4050 tmp = cpu_to_le32(caepaemask_val);
4051 roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
4052 HNS_ROCE_INT_MASK_DISABLE);
4053 caepaemask_val = le32_to_cpu(tmp);
4054 roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
4055 }
4056
4057 /* CEQ almost overflow */
4058 for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
4059 ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG +
4060 i * CEQ_REG_OFFSET);
4061 tmp = cpu_to_le32(ceshift_val);
4062
4063 if (roce_get_bit(tmp,
4064 ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) {
4065 dev_warn(dev, "CEQ[%d] almost overflow!\n", i);
4066 int_work++;
4067
4068 /* Set mask */
4069 cemask_val = roce_read(hr_dev,
4070 ROCEE_CAEP_CE_IRQ_MASK_0_REG +
4071 i * CEQ_REG_OFFSET);
4072 tmp = cpu_to_le32(cemask_val);
4073 roce_set_bit(tmp,
4074 ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
4075 HNS_ROCE_INT_MASK_ENABLE);
4076 cemask_val = le32_to_cpu(tmp);
4077 roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
4078 i * CEQ_REG_OFFSET, cemask_val);
4079
4080 /* Clear int state(INT_WC : write 1 clear) */
4081 cealmovf_val = roce_read(hr_dev,
4082 ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
4083 i * CEQ_REG_OFFSET);
4084 tmp = cpu_to_le32(cealmovf_val);
4085 roce_set_bit(tmp,
4086 ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S,
4087 1);
4088 cealmovf_val = le32_to_cpu(tmp);
4089 roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
4090 i * CEQ_REG_OFFSET, cealmovf_val);
4091
4092 /* Clear mask */
4093 cemask_val = roce_read(hr_dev,
4094 ROCEE_CAEP_CE_IRQ_MASK_0_REG +
4095 i * CEQ_REG_OFFSET);
4096 tmp = cpu_to_le32(cemask_val);
4097 roce_set_bit(tmp,
4098 ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
4099 HNS_ROCE_INT_MASK_DISABLE);
4100 cemask_val = le32_to_cpu(tmp);
4101 roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
4102 i * CEQ_REG_OFFSET, cemask_val);
4103 }
4104 }
4105
4106 /* ECC multi-bit error alarm */
4107 dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n",
4108 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG),
4109 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG),
4110 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG));
4111
4112 dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n",
4113 roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG),
4114 roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG),
4115 roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG));
4116
4117 return IRQ_RETVAL(int_work);
4118}
4119
4120static void hns_roce_v1_int_mask_enable(struct hns_roce_dev *hr_dev)
4121{
4122 u32 aemask_val;
4123 int masken = 0;
4124 __le32 tmp;
4125 int i;
4126
4127 /* AEQ INT */
4128 aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
4129 tmp = cpu_to_le32(aemask_val);
4130 roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
4131 masken);
4132 roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken);
4133 aemask_val = le32_to_cpu(tmp);
4134 roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val);
4135
4136 /* CEQ INT */
4137 for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
4138 /* IRQ mask */
4139 roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
4140 i * CEQ_REG_OFFSET, masken);
4141 }
4142}
4143
4144static void hns_roce_v1_free_eq(struct hns_roce_dev *hr_dev,
4145 struct hns_roce_eq *eq)
4146{
4147 int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) +
4148 HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;
4149 int i;
4150
4151 if (!eq->buf_list)
4152 return;
4153
4154 for (i = 0; i < npages; ++i)
4155 dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE,
4156 eq->buf_list[i].buf, eq->buf_list[i].map);
4157
4158 kfree(eq->buf_list);
4159}
4160
4161static void hns_roce_v1_enable_eq(struct hns_roce_dev *hr_dev, int eq_num,
4162 int enable_flag)
4163{
4164 void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num];
4165 __le32 tmp;
4166 u32 val;
4167
4168 val = readl(eqc);
4169 tmp = cpu_to_le32(val);
4170
4171 if (enable_flag)
4172 roce_set_field(tmp,
4173 ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
4174 ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
4175 HNS_ROCE_EQ_STAT_VALID);
4176 else
4177 roce_set_field(tmp,
4178 ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
4179 ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
4180 HNS_ROCE_EQ_STAT_INVALID);
4181
4182 val = le32_to_cpu(tmp);
4183 writel(val, eqc);
4184}
4185
4186static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev,
4187 struct hns_roce_eq *eq)
4188{
4189 void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn];
4190 struct device *dev = &hr_dev->pdev->dev;
4191 dma_addr_t tmp_dma_addr;
4192 u32 eqcuridx_val;
4193 u32 eqconsindx_val;
4194 u32 eqshift_val;
4195 __le32 tmp2 = 0;
4196 __le32 tmp1 = 0;
4197 __le32 tmp = 0;
4198 int num_bas;
4199 int ret;
4200 int i;
4201
4202 num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) +
4203 HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;
4204
4205 if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) {
4206 dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n",
4207 (eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE,
4208 num_bas);
4209 return -EINVAL;
4210 }
4211
4212 eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL);
4213 if (!eq->buf_list)
4214 return -ENOMEM;
4215
4216 for (i = 0; i < num_bas; ++i) {
4217 eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE,
4218 &tmp_dma_addr,
4219 GFP_KERNEL);
4220 if (!eq->buf_list[i].buf) {
4221 ret = -ENOMEM;
4222 goto err_out_free_pages;
4223 }
4224
4225 eq->buf_list[i].map = tmp_dma_addr;
4226 }
4227 eq->cons_index = 0;
4228 roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
4229 ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
4230 HNS_ROCE_EQ_STAT_INVALID);
4231 roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M,
4232 ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S,
4233 eq->log_entries);
4234 eqshift_val = le32_to_cpu(tmp);
4235 writel(eqshift_val, eqc);
4236
4237 /* Configure eq extended address 12~44bit */
4238 writel((u32)(eq->buf_list[0].map >> 12), eqc + 4);
4239
4240 /*
4241 * Configure eq extended address 45~49 bit.
4242 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
4243 * using 4K page, and shift more 32 because of
4244 * calculating the high 32 bit value evaluated to hardware.
4245 */
4246 roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M,
4247 ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S,
4248 eq->buf_list[0].map >> 44);
4249 roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M,
4250 ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0);
4251 eqcuridx_val = le32_to_cpu(tmp1);
4252 writel(eqcuridx_val, eqc + 8);
4253
4254 /* Configure eq consumer index */
4255 roce_set_field(tmp2, ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M,
4256 ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0);
4257 eqconsindx_val = le32_to_cpu(tmp2);
4258 writel(eqconsindx_val, eqc + 0xc);
4259
4260 return 0;
4261
4262err_out_free_pages:
4263 for (i -= 1; i >= 0; i--)
4264 dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf,
4265 eq->buf_list[i].map);
4266
4267 kfree(eq->buf_list);
4268 return ret;
4269}
4270
4271static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev)
4272{
4273 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
4274 struct device *dev = &hr_dev->pdev->dev;
4275 struct hns_roce_eq *eq;
4276 int irq_num;
4277 int eq_num;
4278 int ret;
4279 int i, j;
4280
4281 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
4282 irq_num = eq_num + hr_dev->caps.num_other_vectors;
4283
4284 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
4285 if (!eq_table->eq)
4286 return -ENOMEM;
4287
4288 eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base),
4289 GFP_KERNEL);
4290 if (!eq_table->eqc_base) {
4291 ret = -ENOMEM;
4292 goto err_eqc_base_alloc_fail;
4293 }
4294
4295 for (i = 0; i < eq_num; i++) {
4296 eq = &eq_table->eq[i];
4297 eq->hr_dev = hr_dev;
4298 eq->eqn = i;
4299 eq->irq = hr_dev->irq[i];
4300 eq->log_page_size = PAGE_SHIFT;
4301
4302 if (i < hr_dev->caps.num_comp_vectors) {
4303 /* CEQ */
4304 eq_table->eqc_base[i] = hr_dev->reg_base +
4305 ROCEE_CAEP_CEQC_SHIFT_0_REG +
4306 CEQ_REG_OFFSET * i;
4307 eq->type_flag = HNS_ROCE_CEQ;
4308 eq->db_reg = hr_dev->reg_base +
4309 ROCEE_CAEP_CEQC_CONS_IDX_0_REG +
4310 CEQ_REG_OFFSET * i;
4311 eq->entries = hr_dev->caps.ceqe_depth;
4312 eq->log_entries = ilog2(eq->entries);
4313 eq->eqe_size = HNS_ROCE_CEQE_SIZE;
4314 } else {
4315 /* AEQ */
4316 eq_table->eqc_base[i] = hr_dev->reg_base +
4317 ROCEE_CAEP_AEQC_AEQE_SHIFT_REG;
4318 eq->type_flag = HNS_ROCE_AEQ;
4319 eq->db_reg = hr_dev->reg_base +
4320 ROCEE_CAEP_AEQE_CONS_IDX_REG;
4321 eq->entries = hr_dev->caps.aeqe_depth;
4322 eq->log_entries = ilog2(eq->entries);
4323 eq->eqe_size = HNS_ROCE_AEQE_SIZE;
4324 }
4325 }
4326
4327 /* Disable irq */
4328 hns_roce_v1_int_mask_enable(hr_dev);
4329
4330 /* Configure ce int interval */
4331 roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG,
4332 HNS_ROCE_CEQ_DEFAULT_INTERVAL);
4333
4334 /* Configure ce int burst num */
4335 roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG,
4336 HNS_ROCE_CEQ_DEFAULT_BURST_NUM);
4337
4338 for (i = 0; i < eq_num; i++) {
4339 ret = hns_roce_v1_create_eq(hr_dev, &eq_table->eq[i]);
4340 if (ret) {
4341 dev_err(dev, "eq create failed\n");
4342 goto err_create_eq_fail;
4343 }
4344 }
4345
4346 for (j = 0; j < irq_num; j++) {
4347 if (j < eq_num)
4348 ret = request_irq(hr_dev->irq[j],
4349 hns_roce_v1_msix_interrupt_eq, 0,
4350 hr_dev->irq_names[j],
4351 &eq_table->eq[j]);
4352 else
4353 ret = request_irq(hr_dev->irq[j],
4354 hns_roce_v1_msix_interrupt_abn, 0,
4355 hr_dev->irq_names[j], hr_dev);
4356
4357 if (ret) {
4358 dev_err(dev, "request irq error!\n");
4359 goto err_request_irq_fail;
4360 }
4361 }
4362
4363 for (i = 0; i < eq_num; i++)
4364 hns_roce_v1_enable_eq(hr_dev, i, EQ_ENABLE);
4365
4366 return 0;
4367
4368err_request_irq_fail:
4369 for (j -= 1; j >= 0; j--)
4370 free_irq(hr_dev->irq[j], &eq_table->eq[j]);
4371
4372err_create_eq_fail:
4373 for (i -= 1; i >= 0; i--)
4374 hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]);
4375
4376 kfree(eq_table->eqc_base);
4377
4378err_eqc_base_alloc_fail:
4379 kfree(eq_table->eq);
4380
4381 return ret;
4382}
4383
4384static void hns_roce_v1_cleanup_eq_table(struct hns_roce_dev *hr_dev)
4385{
4386 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
4387 int irq_num;
4388 int eq_num;
4389 int i;
4390
4391 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
4392 irq_num = eq_num + hr_dev->caps.num_other_vectors;
4393 for (i = 0; i < eq_num; i++) {
4394 /* Disable EQ */
4395 hns_roce_v1_enable_eq(hr_dev, i, EQ_DISABLE);
4396
4397 free_irq(hr_dev->irq[i], &eq_table->eq[i]);
4398
4399 hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]);
4400 }
4401 for (i = eq_num; i < irq_num; i++)
4402 free_irq(hr_dev->irq[i], hr_dev);
4403
4404 kfree(eq_table->eqc_base);
4405 kfree(eq_table->eq);
4406}
4407
4408static const struct ib_device_ops hns_roce_v1_dev_ops = {
4409 .destroy_qp = hns_roce_v1_destroy_qp,
4410 .poll_cq = hns_roce_v1_poll_cq,
4411 .post_recv = hns_roce_v1_post_recv,
4412 .post_send = hns_roce_v1_post_send,
4413 .query_qp = hns_roce_v1_query_qp,
4414 .req_notify_cq = hns_roce_v1_req_notify_cq,
4415};
4416
4417static const struct hns_roce_hw hns_roce_hw_v1 = {
4418 .reset = hns_roce_v1_reset,
4419 .hw_profile = hns_roce_v1_profile,
4420 .hw_init = hns_roce_v1_init,
4421 .hw_exit = hns_roce_v1_exit,
4422 .post_mbox = hns_roce_v1_post_mbox,
4423 .poll_mbox_done = hns_roce_v1_chk_mbox,
4424 .set_gid = hns_roce_v1_set_gid,
4425 .set_mac = hns_roce_v1_set_mac,
4426 .set_mtu = hns_roce_v1_set_mtu,
4427 .write_mtpt = hns_roce_v1_write_mtpt,
4428 .write_cqc = hns_roce_v1_write_cqc,
4429 .set_hem = hns_roce_v1_set_hem,
4430 .clear_hem = hns_roce_v1_clear_hem,
4431 .modify_qp = hns_roce_v1_modify_qp,
4432 .dereg_mr = hns_roce_v1_dereg_mr,
4433 .destroy_cq = hns_roce_v1_destroy_cq,
4434 .init_eq = hns_roce_v1_init_eq_table,
4435 .cleanup_eq = hns_roce_v1_cleanup_eq_table,
4436 .hns_roce_dev_ops = &hns_roce_v1_dev_ops,
4437};
4438
4439static const struct of_device_id hns_roce_of_match[] = {
4440 { .compatible = "hisilicon,hns-roce-v1", .data = &hns_roce_hw_v1, },
4441 {},
4442};
4443MODULE_DEVICE_TABLE(of, hns_roce_of_match);
4444
4445static const struct acpi_device_id hns_roce_acpi_match[] = {
4446 { "HISI00D1", (kernel_ulong_t)&hns_roce_hw_v1 },
4447 {},
4448};
4449MODULE_DEVICE_TABLE(acpi, hns_roce_acpi_match);
4450
4451static struct
4452platform_device *hns_roce_find_pdev(struct fwnode_handle *fwnode)
4453{
4454 struct device *dev;
4455
4456 /* get the 'device' corresponding to the matching 'fwnode' */
4457 dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
4458 /* get the platform device */
4459 return dev ? to_platform_device(dev) : NULL;
4460}
4461
4462static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev)
4463{
4464 struct device *dev = &hr_dev->pdev->dev;
4465 struct platform_device *pdev = NULL;
4466 struct net_device *netdev = NULL;
4467 struct device_node *net_node;
4468 int port_cnt = 0;
4469 u8 phy_port;
4470 int ret;
4471 int i;
4472
4473 /* check if we are compatible with the underlying SoC */
4474 if (dev_of_node(dev)) {
4475 const struct of_device_id *of_id;
4476
4477 of_id = of_match_node(hns_roce_of_match, dev->of_node);
4478 if (!of_id) {
4479 dev_err(dev, "device is not compatible!\n");
4480 return -ENXIO;
4481 }
4482 hr_dev->hw = (const struct hns_roce_hw *)of_id->data;
4483 if (!hr_dev->hw) {
4484 dev_err(dev, "couldn't get H/W specific DT data!\n");
4485 return -ENXIO;
4486 }
4487 } else if (is_acpi_device_node(dev->fwnode)) {
4488 const struct acpi_device_id *acpi_id;
4489
4490 acpi_id = acpi_match_device(hns_roce_acpi_match, dev);
4491 if (!acpi_id) {
4492 dev_err(dev, "device is not compatible!\n");
4493 return -ENXIO;
4494 }
4495 hr_dev->hw = (const struct hns_roce_hw *) acpi_id->driver_data;
4496 if (!hr_dev->hw) {
4497 dev_err(dev, "couldn't get H/W specific ACPI data!\n");
4498 return -ENXIO;
4499 }
4500 } else {
4501 dev_err(dev, "can't read compatibility data from DT or ACPI\n");
4502 return -ENXIO;
4503 }
4504
4505 /* get the mapped register base address */
4506 hr_dev->reg_base = devm_platform_ioremap_resource(hr_dev->pdev, 0);
4507 if (IS_ERR(hr_dev->reg_base))
4508 return PTR_ERR(hr_dev->reg_base);
4509
4510 /* read the node_guid of IB device from the DT or ACPI */
4511 ret = device_property_read_u8_array(dev, "node-guid",
4512 (u8 *)&hr_dev->ib_dev.node_guid,
4513 GUID_LEN);
4514 if (ret) {
4515 dev_err(dev, "couldn't get node_guid from DT or ACPI!\n");
4516 return ret;
4517 }
4518
4519 /* get the RoCE associated ethernet ports or netdevices */
4520 for (i = 0; i < HNS_ROCE_MAX_PORTS; i++) {
4521 if (dev_of_node(dev)) {
4522 net_node = of_parse_phandle(dev->of_node, "eth-handle",
4523 i);
4524 if (!net_node)
4525 continue;
4526 pdev = of_find_device_by_node(net_node);
4527 } else if (is_acpi_device_node(dev->fwnode)) {
4528 struct fwnode_reference_args args;
4529
4530 ret = acpi_node_get_property_reference(dev->fwnode,
4531 "eth-handle",
4532 i, &args);
4533 if (ret)
4534 continue;
4535 pdev = hns_roce_find_pdev(args.fwnode);
4536 } else {
4537 dev_err(dev, "cannot read data from DT or ACPI\n");
4538 return -ENXIO;
4539 }
4540
4541 if (pdev) {
4542 netdev = platform_get_drvdata(pdev);
4543 phy_port = (u8)i;
4544 if (netdev) {
4545 hr_dev->iboe.netdevs[port_cnt] = netdev;
4546 hr_dev->iboe.phy_port[port_cnt] = phy_port;
4547 } else {
4548 dev_err(dev, "no netdev found with pdev %s\n",
4549 pdev->name);
4550 return -ENODEV;
4551 }
4552 port_cnt++;
4553 }
4554 }
4555
4556 if (port_cnt == 0) {
4557 dev_err(dev, "unable to get eth-handle for available ports!\n");
4558 return -EINVAL;
4559 }
4560
4561 hr_dev->caps.num_ports = port_cnt;
4562
4563 /* cmd issue mode: 0 is poll, 1 is event */
4564 hr_dev->cmd_mod = 1;
4565 hr_dev->loop_idc = 0;
4566 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
4567 hr_dev->odb_offset = ROCEE_DB_OTHERS_L_0_REG;
4568
4569 /* read the interrupt names from the DT or ACPI */
4570 ret = device_property_read_string_array(dev, "interrupt-names",
4571 hr_dev->irq_names,
4572 HNS_ROCE_V1_MAX_IRQ_NUM);
4573 if (ret < 0) {
4574 dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n");
4575 return ret;
4576 }
4577
4578 /* fetch the interrupt numbers */
4579 for (i = 0; i < HNS_ROCE_V1_MAX_IRQ_NUM; i++) {
4580 hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i);
4581 if (hr_dev->irq[i] <= 0)
4582 return -EINVAL;
4583 }
4584
4585 return 0;
4586}
4587
4588/**
4589 * hns_roce_probe - RoCE driver entrance
4590 * @pdev: pointer to platform device
4591 * Return : int
4592 *
4593 */
4594static int hns_roce_probe(struct platform_device *pdev)
4595{
4596 int ret;
4597 struct hns_roce_dev *hr_dev;
4598 struct device *dev = &pdev->dev;
4599
4600 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
4601 if (!hr_dev)
4602 return -ENOMEM;
4603
4604 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v1_priv), GFP_KERNEL);
4605 if (!hr_dev->priv) {
4606 ret = -ENOMEM;
4607 goto error_failed_kzalloc;
4608 }
4609
4610 hr_dev->pdev = pdev;
4611 hr_dev->dev = dev;
4612 platform_set_drvdata(pdev, hr_dev);
4613
4614 if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64ULL)) &&
4615 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32ULL))) {
4616 dev_err(dev, "Not usable DMA addressing mode\n");
4617 ret = -EIO;
4618 goto error_failed_get_cfg;
4619 }
4620
4621 ret = hns_roce_get_cfg(hr_dev);
4622 if (ret) {
4623 dev_err(dev, "Get Configuration failed!\n");
4624 goto error_failed_get_cfg;
4625 }
4626
4627 ret = hns_roce_init(hr_dev);
4628 if (ret) {
4629 dev_err(dev, "RoCE engine init failed!\n");
4630 goto error_failed_get_cfg;
4631 }
4632
4633 return 0;
4634
4635error_failed_get_cfg:
4636 kfree(hr_dev->priv);
4637
4638error_failed_kzalloc:
4639 ib_dealloc_device(&hr_dev->ib_dev);
4640
4641 return ret;
4642}
4643
4644/**
4645 * hns_roce_remove - remove RoCE device
4646 * @pdev: pointer to platform device
4647 */
4648static int hns_roce_remove(struct platform_device *pdev)
4649{
4650 struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev);
4651
4652 hns_roce_exit(hr_dev);
4653 kfree(hr_dev->priv);
4654 ib_dealloc_device(&hr_dev->ib_dev);
4655
4656 return 0;
4657}
4658
4659static struct platform_driver hns_roce_driver = {
4660 .probe = hns_roce_probe,
4661 .remove = hns_roce_remove,
4662 .driver = {
4663 .name = DRV_NAME,
4664 .of_match_table = hns_roce_of_match,
4665 .acpi_match_table = ACPI_PTR(hns_roce_acpi_match),
4666 },
4667};
4668
4669module_platform_driver(hns_roce_driver);
4670
4671MODULE_LICENSE("Dual BSD/GPL");
4672MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
4673MODULE_AUTHOR("Nenglong Zhao <zhaonenglong@hisilicon.com>");
4674MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
4675MODULE_DESCRIPTION("Hisilicon Hip06 Family RoCE Driver");
drivers/infiniband/hw/hns/hns_roce_hw_v1.h+0−1147
@@ -1,1147 +0,0 @@
1/*
2 * Copyright (c) 2016 Hisilicon Limited.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#ifndef _HNS_ROCE_HW_V1_H
34#define _HNS_ROCE_HW_V1_H
35
36#define CQ_STATE_VALID 2
37
38#define HNS_ROCE_V1_MAX_PD_NUM 0x8000
39#define HNS_ROCE_V1_MAX_CQ_NUM 0x10000
40#define HNS_ROCE_V1_MAX_CQE_NUM 0x8000
41
42#define HNS_ROCE_V1_MAX_QP_NUM 0x40000
43#define HNS_ROCE_V1_MAX_WQE_NUM 0x4000
44
45#define HNS_ROCE_V1_MAX_MTPT_NUM 0x80000
46
47#define HNS_ROCE_V1_MAX_MTT_SEGS 0x100000
48
49#define HNS_ROCE_V1_MAX_QP_INIT_RDMA 128
50#define HNS_ROCE_V1_MAX_QP_DEST_RDMA 128
51
52#define HNS_ROCE_V1_MAX_SQ_DESC_SZ 64
53#define HNS_ROCE_V1_MAX_RQ_DESC_SZ 64
54#define HNS_ROCE_V1_SG_NUM 2
55#define HNS_ROCE_V1_INLINE_SIZE 32
56
57#define HNS_ROCE_V1_UAR_NUM 256
58#define HNS_ROCE_V1_PHY_UAR_NUM 8
59
60#define HNS_ROCE_V1_GID_NUM 16
61#define HNS_ROCE_V1_RESV_QP 8
62
63#define HNS_ROCE_V1_MAX_IRQ_NUM 34
64#define HNS_ROCE_V1_COMP_VEC_NUM 32
65#define HNS_ROCE_V1_AEQE_VEC_NUM 1
66#define HNS_ROCE_V1_ABNORMAL_VEC_NUM 1
67
68#define HNS_ROCE_V1_COMP_EQE_NUM 0x8000
69#define HNS_ROCE_V1_ASYNC_EQE_NUM 0x400
70
71#define HNS_ROCE_V1_QPC_SIZE 256
72#define HNS_ROCE_V1_IRRL_ENTRY_SIZE 8
73#define HNS_ROCE_V1_CQC_ENTRY_SIZE 64
74#define HNS_ROCE_V1_MTPT_ENTRY_SIZE 64
75#define HNS_ROCE_V1_MTT_ENTRY_SIZE 64
76
77#define HNS_ROCE_V1_CQE_SIZE 32
78#define HNS_ROCE_V1_PAGE_SIZE_SUPPORT 0xFFFFF000
79
80#define HNS_ROCE_V1_TABLE_CHUNK_SIZE (1 << 17)
81
82#define HNS_ROCE_V1_EXT_RAQ_WF 8
83#define HNS_ROCE_V1_RAQ_ENTRY 64
84#define HNS_ROCE_V1_RAQ_DEPTH 32768
85#define HNS_ROCE_V1_RAQ_SIZE (HNS_ROCE_V1_RAQ_ENTRY * HNS_ROCE_V1_RAQ_DEPTH)
86
87#define HNS_ROCE_V1_SDB_DEPTH 0x400
88#define HNS_ROCE_V1_ODB_DEPTH 0x400
89
90#define HNS_ROCE_V1_DB_RSVD 0x80
91
92#define HNS_ROCE_V1_SDB_ALEPT HNS_ROCE_V1_DB_RSVD
93#define HNS_ROCE_V1_SDB_ALFUL (HNS_ROCE_V1_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD)
94#define HNS_ROCE_V1_ODB_ALEPT HNS_ROCE_V1_DB_RSVD
95#define HNS_ROCE_V1_ODB_ALFUL (HNS_ROCE_V1_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)
96
97#define HNS_ROCE_V1_EXT_SDB_DEPTH 0x4000
98#define HNS_ROCE_V1_EXT_ODB_DEPTH 0x4000
99#define HNS_ROCE_V1_EXT_SDB_ENTRY 16
100#define HNS_ROCE_V1_EXT_ODB_ENTRY 16
101#define HNS_ROCE_V1_EXT_SDB_SIZE \
102 (HNS_ROCE_V1_EXT_SDB_DEPTH * HNS_ROCE_V1_EXT_SDB_ENTRY)
103#define HNS_ROCE_V1_EXT_ODB_SIZE \
104 (HNS_ROCE_V1_EXT_ODB_DEPTH * HNS_ROCE_V1_EXT_ODB_ENTRY)
105
106#define HNS_ROCE_V1_EXT_SDB_ALEPT HNS_ROCE_V1_DB_RSVD
107#define HNS_ROCE_V1_EXT_SDB_ALFUL \
108 (HNS_ROCE_V1_EXT_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD)
109#define HNS_ROCE_V1_EXT_ODB_ALEPT HNS_ROCE_V1_DB_RSVD
110#define HNS_ROCE_V1_EXT_ODB_ALFUL \
111 (HNS_ROCE_V1_EXT_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)
112
113#define HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS 50000
114#define HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS 10000
115#define HNS_ROCE_V1_FREE_MR_WAIT_VALUE 5
116#define HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE 20
117
118#define HNS_ROCE_BT_RSV_BUF_SIZE (1 << 17)
119
120#define HNS_ROCE_V1_TPTR_ENTRY_SIZE 2
121#define HNS_ROCE_V1_TPTR_BUF_SIZE \
122 (HNS_ROCE_V1_TPTR_ENTRY_SIZE * HNS_ROCE_V1_MAX_CQ_NUM)
123
124#define HNS_ROCE_ODB_POLL_MODE 0
125
126#define HNS_ROCE_SDB_NORMAL_MODE 0
127#define HNS_ROCE_SDB_EXTEND_MODE 1
128
129#define HNS_ROCE_ODB_EXTEND_MODE 1
130
131#define KEY_VALID 0x02
132
133#define HNS_ROCE_CQE_QPN_MASK 0x3ffff
134#define HNS_ROCE_CQE_STATUS_MASK 0x1f
135#define HNS_ROCE_CQE_OPCODE_MASK 0xf
136
137#define HNS_ROCE_CQE_SUCCESS 0x00
138#define HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR 0x01
139#define HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR 0x02
140#define HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR 0x03
141#define HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR 0x04
142#define HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR 0x05
143#define HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR 0x06
144#define HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR 0x07
145#define HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR 0x08
146#define HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR 0x09
147#define HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR 0x0a
148#define HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR 0x0b
149#define HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR 0x0c
150
151#define QP1C_CFGN_OFFSET 0x28
152#define PHY_PORT_OFFSET 0x8
153#define MTPT_IDX_SHIFT 16
154#define ALL_PORT_VAL_OPEN 0x3f
155#define POL_TIME_INTERVAL_VAL 0x80
156#define SLEEP_TIME_INTERVAL 20
157#define SQ_PSN_SHIFT 8
158#define QKEY_VAL 0x80010000
159#define SDB_INV_CNT_OFFSET 8
160
161#define HNS_ROCE_CEQ_DEFAULT_INTERVAL 0x10
162#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x10
163
164#define HNS_ROCE_INT_MASK_DISABLE 0
165#define HNS_ROCE_INT_MASK_ENABLE 1
166
167#define CEQ_REG_OFFSET 0x18
168
169#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S 0
170
171#define HNS_ROCE_V1_CONS_IDX_M GENMASK(15, 0)
172
173#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16
174#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M GENMASK(31, 16)
175
176#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16
177#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M GENMASK(23, 16)
178
179#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24
180#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M GENMASK(30, 24)
181
182#define HNS_ROCE_AEQE_U32_4_OWNER_S 31
183
184#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0
185#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M GENMASK(23, 0)
186
187#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S 25
188#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M GENMASK(27, 25)
189
190#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0
191#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M GENMASK(15, 0)
192
193#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0
194#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M GENMASK(4, 0)
195
196/* Local Work Queue Catastrophic Error,SUBTYPE 0x5 */
197enum {
198 HNS_ROCE_LWQCE_QPC_ERROR = 1,
199 HNS_ROCE_LWQCE_MTU_ERROR,
200 HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR,
201 HNS_ROCE_LWQCE_WQE_ADDR_ERROR,
202 HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR,
203 HNS_ROCE_LWQCE_SL_ERROR,
204 HNS_ROCE_LWQCE_PORT_ERROR,
205};
206
207/* Local Access Violation Work Queue Error,SUBTYPE 0x7 */
208enum {
209 HNS_ROCE_LAVWQE_R_KEY_VIOLATION = 1,
210 HNS_ROCE_LAVWQE_LENGTH_ERROR,
211 HNS_ROCE_LAVWQE_VA_ERROR,
212 HNS_ROCE_LAVWQE_PD_ERROR,
213 HNS_ROCE_LAVWQE_RW_ACC_ERROR,
214 HNS_ROCE_LAVWQE_KEY_STATE_ERROR,
215 HNS_ROCE_LAVWQE_MR_OPERATION_ERROR,
216};
217
218/* DOORBELL overflow subtype */
219enum {
220 HNS_ROCE_DB_SUBTYPE_SDB_OVF = 1,
221 HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF,
222 HNS_ROCE_DB_SUBTYPE_ODB_OVF,
223 HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF,
224 HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP,
225 HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP,
226};
227
228enum {
229 /* RQ&SRQ related operations */
230 HNS_ROCE_OPCODE_SEND_DATA_RECEIVE = 0x06,
231 HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE,
232};
233
234enum {
235 HNS_ROCE_PORT_DOWN = 0,
236 HNS_ROCE_PORT_UP,
237};
238
239struct hns_roce_cq_context {
240 __le32 cqc_byte_4;
241 __le32 cq_bt_l;
242 __le32 cqc_byte_12;
243 __le32 cur_cqe_ba0_l;
244 __le32 cqc_byte_20;
245 __le32 cqe_tptr_addr_l;
246 __le32 cur_cqe_ba1_l;
247 __le32 cqc_byte_32;
248};
249
250#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S 0
251#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M \
252 (((1UL << 2) - 1) << CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S)
253
254#define CQ_CONTEXT_CQC_BYTE_4_CQN_S 16
255#define CQ_CONTEXT_CQC_BYTE_4_CQN_M \
256 (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_4_CQN_S)
257
258#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S 0
259#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M \
260 (((1UL << 17) - 1) << CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S)
261
262#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S 20
263#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M \
264 (((1UL << 4) - 1) << CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S)
265
266#define CQ_CONTEXT_CQC_BYTE_12_CEQN_S 24
267#define CQ_CONTEXT_CQC_BYTE_12_CEQN_M \
268 (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_12_CEQN_S)
269
270#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S 0
271#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M \
272 (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S)
273
274#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S 16
275#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M \
276 (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S)
277
278#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S 8
279#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M \
280 (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S)
281
282#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S 0
283#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M \
284 (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S)
285
286#define CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S 9
287
288#define CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S 8
289#define CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S 14
290#define CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S 15
291
292#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S 16
293#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M \
294 (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S)
295
296struct hns_roce_cqe {
297 __le32 cqe_byte_4;
298 union {
299 __le32 r_key;
300 __le32 immediate_data;
301 };
302 __le32 byte_cnt;
303 __le32 cqe_byte_16;
304 __le32 cqe_byte_20;
305 __le32 s_mac_l;
306 __le32 cqe_byte_28;
307 __le32 reserved;
308};
309
310#define CQE_BYTE_4_OWNER_S 7
311#define CQE_BYTE_4_SQ_RQ_FLAG_S 14
312
313#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_S 8
314#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_M \
315 (((1UL << 5) - 1) << CQE_BYTE_4_STATUS_OF_THE_OPERATION_S)
316
317#define CQE_BYTE_4_WQE_INDEX_S 16
318#define CQE_BYTE_4_WQE_INDEX_M (((1UL << 14) - 1) << CQE_BYTE_4_WQE_INDEX_S)
319
320#define CQE_BYTE_4_OPERATION_TYPE_S 0
321#define CQE_BYTE_4_OPERATION_TYPE_M \
322 (((1UL << 4) - 1) << CQE_BYTE_4_OPERATION_TYPE_S)
323
324#define CQE_BYTE_4_IMM_INDICATOR_S 15
325
326#define CQE_BYTE_16_LOCAL_QPN_S 0
327#define CQE_BYTE_16_LOCAL_QPN_M (((1UL << 24) - 1) << CQE_BYTE_16_LOCAL_QPN_S)
328
329#define CQE_BYTE_20_PORT_NUM_S 26
330#define CQE_BYTE_20_PORT_NUM_M (((1UL << 3) - 1) << CQE_BYTE_20_PORT_NUM_S)
331
332#define CQE_BYTE_20_SL_S 24
333#define CQE_BYTE_20_SL_M (((1UL << 2) - 1) << CQE_BYTE_20_SL_S)
334
335#define CQE_BYTE_20_REMOTE_QPN_S 0
336#define CQE_BYTE_20_REMOTE_QPN_M \
337 (((1UL << 24) - 1) << CQE_BYTE_20_REMOTE_QPN_S)
338
339#define CQE_BYTE_20_GRH_PRESENT_S 29
340
341#define CQE_BYTE_28_P_KEY_IDX_S 16
342#define CQE_BYTE_28_P_KEY_IDX_M (((1UL << 16) - 1) << CQE_BYTE_28_P_KEY_IDX_S)
343
344#define CQ_DB_REQ_NOT_SOL 0
345#define CQ_DB_REQ_NOT (1 << 16)
346
347struct hns_roce_v1_mpt_entry {
348 __le32 mpt_byte_4;
349 __le32 pbl_addr_l;
350 __le32 mpt_byte_12;
351 __le32 virt_addr_l;
352 __le32 virt_addr_h;
353 __le32 length;
354 __le32 mpt_byte_28;
355 __le32 pa0_l;
356 __le32 mpt_byte_36;
357 __le32 mpt_byte_40;
358 __le32 mpt_byte_44;
359 __le32 mpt_byte_48;
360 __le32 pa4_l;
361 __le32 mpt_byte_56;
362 __le32 mpt_byte_60;
363 __le32 mpt_byte_64;
364};
365
366#define MPT_BYTE_4_KEY_STATE_S 0
367#define MPT_BYTE_4_KEY_STATE_M (((1UL << 2) - 1) << MPT_BYTE_4_KEY_STATE_S)
368
369#define MPT_BYTE_4_KEY_S 8
370#define MPT_BYTE_4_KEY_M (((1UL << 8) - 1) << MPT_BYTE_4_KEY_S)
371
372#define MPT_BYTE_4_PAGE_SIZE_S 16
373#define MPT_BYTE_4_PAGE_SIZE_M (((1UL << 2) - 1) << MPT_BYTE_4_PAGE_SIZE_S)
374
375#define MPT_BYTE_4_MW_TYPE_S 20
376
377#define MPT_BYTE_4_MW_BIND_ENABLE_S 21
378
379#define MPT_BYTE_4_OWN_S 22
380
381#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_S 24
382#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_M \
383 (((1UL << 2) - 1) << MPT_BYTE_4_MEMORY_LOCATION_TYPE_S)
384
385#define MPT_BYTE_4_REMOTE_ATOMIC_S 26
386#define MPT_BYTE_4_LOCAL_WRITE_S 27
387#define MPT_BYTE_4_REMOTE_WRITE_S 28
388#define MPT_BYTE_4_REMOTE_READ_S 29
389#define MPT_BYTE_4_REMOTE_INVAL_ENABLE_S 30
390#define MPT_BYTE_4_ADDRESS_TYPE_S 31
391
392#define MPT_BYTE_12_PBL_ADDR_H_S 0
393#define MPT_BYTE_12_PBL_ADDR_H_M \
394 (((1UL << 17) - 1) << MPT_BYTE_12_PBL_ADDR_H_S)
395
396#define MPT_BYTE_12_MW_BIND_COUNTER_S 17
397#define MPT_BYTE_12_MW_BIND_COUNTER_M \
398 (((1UL << 15) - 1) << MPT_BYTE_12_MW_BIND_COUNTER_S)
399
400#define MPT_BYTE_28_PD_S 0
401#define MPT_BYTE_28_PD_M (((1UL << 16) - 1) << MPT_BYTE_28_PD_S)
402
403#define MPT_BYTE_28_L_KEY_IDX_L_S 16
404#define MPT_BYTE_28_L_KEY_IDX_L_M \
405 (((1UL << 16) - 1) << MPT_BYTE_28_L_KEY_IDX_L_S)
406
407#define MPT_BYTE_36_PA0_H_S 0
408#define MPT_BYTE_36_PA0_H_M (((1UL << 5) - 1) << MPT_BYTE_36_PA0_H_S)
409
410#define MPT_BYTE_36_PA1_L_S 8
411#define MPT_BYTE_36_PA1_L_M (((1UL << 24) - 1) << MPT_BYTE_36_PA1_L_S)
412
413#define MPT_BYTE_40_PA1_H_S 0
414#define MPT_BYTE_40_PA1_H_M (((1UL << 13) - 1) << MPT_BYTE_40_PA1_H_S)
415
416#define MPT_BYTE_40_PA2_L_S 16
417#define MPT_BYTE_40_PA2_L_M (((1UL << 16) - 1) << MPT_BYTE_40_PA2_L_S)
418
419#define MPT_BYTE_44_PA2_H_S 0
420#define MPT_BYTE_44_PA2_H_M (((1UL << 21) - 1) << MPT_BYTE_44_PA2_H_S)
421
422#define MPT_BYTE_44_PA3_L_S 24
423#define MPT_BYTE_44_PA3_L_M (((1UL << 8) - 1) << MPT_BYTE_44_PA3_L_S)
424
425#define MPT_BYTE_48_PA3_H_S 0
426#define MPT_BYTE_48_PA3_H_M (((1UL << 29) - 1) << MPT_BYTE_48_PA3_H_S)
427
428#define MPT_BYTE_56_PA4_H_S 0
429#define MPT_BYTE_56_PA4_H_M (((1UL << 5) - 1) << MPT_BYTE_56_PA4_H_S)
430
431#define MPT_BYTE_56_PA5_L_S 8
432#define MPT_BYTE_56_PA5_L_M (((1UL << 24) - 1) << MPT_BYTE_56_PA5_L_S)
433
434#define MPT_BYTE_60_PA5_H_S 0
435#define MPT_BYTE_60_PA5_H_M (((1UL << 13) - 1) << MPT_BYTE_60_PA5_H_S)
436
437#define MPT_BYTE_60_PA6_L_S 16
438#define MPT_BYTE_60_PA6_L_M (((1UL << 16) - 1) << MPT_BYTE_60_PA6_L_S)
439
440#define MPT_BYTE_64_PA6_H_S 0
441#define MPT_BYTE_64_PA6_H_M (((1UL << 21) - 1) << MPT_BYTE_64_PA6_H_S)
442
443#define MPT_BYTE_64_L_KEY_IDX_H_S 24
444#define MPT_BYTE_64_L_KEY_IDX_H_M \
445 (((1UL << 8) - 1) << MPT_BYTE_64_L_KEY_IDX_H_S)
446
447struct hns_roce_wqe_ctrl_seg {
448 __le32 sgl_pa_h;
449 __le32 flag;
450 union {
451 __be32 imm_data;
452 __le32 inv_key;
453 };
454 __le32 msg_length;
455};
456
457struct hns_roce_wqe_data_seg {
458 __le64 addr;
459 __le32 lkey;
460 __le32 len;
461};
462
463struct hns_roce_wqe_raddr_seg {
464 __le32 rkey;
465 __le32 len; /* reserved */
466 __le64 raddr;
467};
468
469struct hns_roce_rq_wqe_ctrl {
470 __le32 rwqe_byte_4;
471 __le32 rocee_sgl_ba_l;
472 __le32 rwqe_byte_12;
473 __le32 reserved[5];
474};
475
476#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S 16
477#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M \
478 (((1UL << 6) - 1) << RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S)
479
480#define HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS 10000
481
482#define GID_LEN 16
483
484struct hns_roce_ud_send_wqe {
485 __le32 dmac_h;
486 __le32 u32_8;
487 __le32 immediate_data;
488
489 __le32 u32_16;
490 union {
491 unsigned char dgid[GID_LEN];
492 struct {
493 __le32 u32_20;
494 __le32 u32_24;
495 __le32 u32_28;
496 __le32 u32_32;
497 };
498 };
499
500 __le32 u32_36;
501 __le32 u32_40;
502
503 __le32 va0_l;
504 __le32 va0_h;
505 __le32 l_key0;
506
507 __le32 va1_l;
508 __le32 va1_h;
509 __le32 l_key1;
510};
511
512#define UD_SEND_WQE_U32_4_DMAC_0_S 0
513#define UD_SEND_WQE_U32_4_DMAC_0_M \
514 (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_0_S)
515
516#define UD_SEND_WQE_U32_4_DMAC_1_S 8
517#define UD_SEND_WQE_U32_4_DMAC_1_M \
518 (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_1_S)
519
520#define UD_SEND_WQE_U32_4_DMAC_2_S 16
521#define UD_SEND_WQE_U32_4_DMAC_2_M \
522 (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_2_S)
523
524#define UD_SEND_WQE_U32_4_DMAC_3_S 24
525#define UD_SEND_WQE_U32_4_DMAC_3_M \
526 (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_3_S)
527
528#define UD_SEND_WQE_U32_8_DMAC_4_S 0
529#define UD_SEND_WQE_U32_8_DMAC_4_M \
530 (((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_4_S)
531
532#define UD_SEND_WQE_U32_8_DMAC_5_S 8
533#define UD_SEND_WQE_U32_8_DMAC_5_M \
534 (((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_5_S)
535
536#define UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S 22
537
538#define UD_SEND_WQE_U32_8_OPERATION_TYPE_S 16
539#define UD_SEND_WQE_U32_8_OPERATION_TYPE_M \
540 (((1UL << 4) - 1) << UD_SEND_WQE_U32_8_OPERATION_TYPE_S)
541
542#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S 24
543#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M \
544 (((1UL << 6) - 1) << UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S)
545
546#define UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S 31
547
548#define UD_SEND_WQE_U32_16_DEST_QP_S 0
549#define UD_SEND_WQE_U32_16_DEST_QP_M \
550 (((1UL << 24) - 1) << UD_SEND_WQE_U32_16_DEST_QP_S)
551
552#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S 24
553#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M \
554 (((1UL << 8) - 1) << UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S)
555
556#define UD_SEND_WQE_U32_36_FLOW_LABEL_S 0
557#define UD_SEND_WQE_U32_36_FLOW_LABEL_M \
558 (((1UL << 20) - 1) << UD_SEND_WQE_U32_36_FLOW_LABEL_S)
559
560#define UD_SEND_WQE_U32_36_PRIORITY_S 20
561#define UD_SEND_WQE_U32_36_PRIORITY_M \
562 (((1UL << 4) - 1) << UD_SEND_WQE_U32_36_PRIORITY_S)
563
564#define UD_SEND_WQE_U32_36_SGID_INDEX_S 24
565#define UD_SEND_WQE_U32_36_SGID_INDEX_M \
566 (((1UL << 8) - 1) << UD_SEND_WQE_U32_36_SGID_INDEX_S)
567
568#define UD_SEND_WQE_U32_40_HOP_LIMIT_S 0
569#define UD_SEND_WQE_U32_40_HOP_LIMIT_M \
570 (((1UL << 8) - 1) << UD_SEND_WQE_U32_40_HOP_LIMIT_S)
571
572#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S 8
573#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M \
574 (((1UL << 8) - 1) << UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S)
575
576struct hns_roce_sqp_context {
577 __le32 qp1c_bytes_4;
578 __le32 sq_rq_bt_l;
579 __le32 qp1c_bytes_12;
580 __le32 qp1c_bytes_16;
581 __le32 qp1c_bytes_20;
582 __le32 cur_rq_wqe_ba_l;
583 __le32 qp1c_bytes_28;
584 __le32 qp1c_bytes_32;
585 __le32 cur_sq_wqe_ba_l;
586 __le32 qp1c_bytes_40;
587};
588
589#define QP1C_BYTES_4_QP_STATE_S 0
590#define QP1C_BYTES_4_QP_STATE_M \
591 (((1UL << 3) - 1) << QP1C_BYTES_4_QP_STATE_S)
592
593#define QP1C_BYTES_4_SQ_WQE_SHIFT_S 8
594#define QP1C_BYTES_4_SQ_WQE_SHIFT_M \
595 (((1UL << 4) - 1) << QP1C_BYTES_4_SQ_WQE_SHIFT_S)
596
597#define QP1C_BYTES_4_RQ_WQE_SHIFT_S 12
598#define QP1C_BYTES_4_RQ_WQE_SHIFT_M \
599 (((1UL << 4) - 1) << QP1C_BYTES_4_RQ_WQE_SHIFT_S)
600
601#define QP1C_BYTES_4_PD_S 16
602#define QP1C_BYTES_4_PD_M (((1UL << 16) - 1) << QP1C_BYTES_4_PD_S)
603
604#define QP1C_BYTES_12_SQ_RQ_BT_H_S 0
605#define QP1C_BYTES_12_SQ_RQ_BT_H_M \
606 (((1UL << 17) - 1) << QP1C_BYTES_12_SQ_RQ_BT_H_S)
607
608#define QP1C_BYTES_16_RQ_HEAD_S 0
609#define QP1C_BYTES_16_RQ_HEAD_M (((1UL << 15) - 1) << QP1C_BYTES_16_RQ_HEAD_S)
610
611#define QP1C_BYTES_16_PORT_NUM_S 16
612#define QP1C_BYTES_16_PORT_NUM_M \
613 (((1UL << 3) - 1) << QP1C_BYTES_16_PORT_NUM_S)
614
615#define QP1C_BYTES_16_SIGNALING_TYPE_S 27
616#define QP1C_BYTES_16_LOCAL_ENABLE_E2E_CREDIT_S 28
617#define QP1C_BYTES_16_RQ_BA_FLG_S 29
618#define QP1C_BYTES_16_SQ_BA_FLG_S 30
619#define QP1C_BYTES_16_QP1_ERR_S 31
620
621#define QP1C_BYTES_20_SQ_HEAD_S 0
622#define QP1C_BYTES_20_SQ_HEAD_M (((1UL << 15) - 1) << QP1C_BYTES_20_SQ_HEAD_S)
623
624#define QP1C_BYTES_20_PKEY_IDX_S 16
625#define QP1C_BYTES_20_PKEY_IDX_M \
626 (((1UL << 16) - 1) << QP1C_BYTES_20_PKEY_IDX_S)
627
628#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S 0
629#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M \
630 (((1UL << 5) - 1) << QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S)
631
632#define QP1C_BYTES_28_RQ_CUR_IDX_S 16
633#define QP1C_BYTES_28_RQ_CUR_IDX_M \
634 (((1UL << 15) - 1) << QP1C_BYTES_28_RQ_CUR_IDX_S)
635
636#define QP1C_BYTES_32_TX_CQ_NUM_S 0
637#define QP1C_BYTES_32_TX_CQ_NUM_M \
638 (((1UL << 16) - 1) << QP1C_BYTES_32_TX_CQ_NUM_S)
639
640#define QP1C_BYTES_32_RX_CQ_NUM_S 16
641#define QP1C_BYTES_32_RX_CQ_NUM_M \
642 (((1UL << 16) - 1) << QP1C_BYTES_32_RX_CQ_NUM_S)
643
644#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S 0
645#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M \
646 (((1UL << 5) - 1) << QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S)
647
648#define QP1C_BYTES_40_SQ_CUR_IDX_S 16
649#define QP1C_BYTES_40_SQ_CUR_IDX_M \
650 (((1UL << 15) - 1) << QP1C_BYTES_40_SQ_CUR_IDX_S)
651
652#define HNS_ROCE_WQE_INLINE (1UL<<31)
653#define HNS_ROCE_WQE_SE (1UL<<30)
654
655#define HNS_ROCE_WQE_SGE_NUM_BIT 24
656#define HNS_ROCE_WQE_IMM (1UL<<23)
657#define HNS_ROCE_WQE_FENCE (1UL<<21)
658#define HNS_ROCE_WQE_CQ_NOTIFY (1UL<<20)
659
660#define HNS_ROCE_WQE_OPCODE_SEND (0<<16)
661#define HNS_ROCE_WQE_OPCODE_RDMA_READ (1<<16)
662#define HNS_ROCE_WQE_OPCODE_RDMA_WRITE (2<<16)
663#define HNS_ROCE_WQE_OPCODE_LOCAL_INV (4<<16)
664#define HNS_ROCE_WQE_OPCODE_UD_SEND (7<<16)
665#define HNS_ROCE_WQE_OPCODE_MASK (15<<16)
666
667struct hns_roce_qp_context {
668 __le32 qpc_bytes_4;
669 __le32 qpc_bytes_8;
670 __le32 qpc_bytes_12;
671 __le32 qpc_bytes_16;
672 __le32 sq_rq_bt_l;
673 __le32 qpc_bytes_24;
674 __le32 irrl_ba_l;
675 __le32 qpc_bytes_32;
676 __le32 qpc_bytes_36;
677 __le32 dmac_l;
678 __le32 qpc_bytes_44;
679 __le32 qpc_bytes_48;
680 u8 dgid[16];
681 __le32 qpc_bytes_68;
682 __le32 cur_rq_wqe_ba_l;
683 __le32 qpc_bytes_76;
684 __le32 rx_rnr_time;
685 __le32 qpc_bytes_84;
686 __le32 qpc_bytes_88;
687 union {
688 __le32 rx_sge_len;
689 __le32 dma_length;
690 };
691 union {
692 __le32 rx_sge_num;
693 __le32 rx_send_pktn;
694 __le32 r_key;
695 };
696 __le32 va_l;
697 __le32 va_h;
698 __le32 qpc_bytes_108;
699 __le32 qpc_bytes_112;
700 __le32 rx_cur_sq_wqe_ba_l;
701 __le32 qpc_bytes_120;
702 __le32 qpc_bytes_124;
703 __le32 qpc_bytes_128;
704 __le32 qpc_bytes_132;
705 __le32 qpc_bytes_136;
706 __le32 qpc_bytes_140;
707 __le32 qpc_bytes_144;
708 __le32 qpc_bytes_148;
709 union {
710 __le32 rnr_retry;
711 __le32 ack_time;
712 };
713 __le32 qpc_bytes_156;
714 __le32 pkt_use_len;
715 __le32 qpc_bytes_164;
716 __le32 qpc_bytes_168;
717 union {
718 __le32 sge_use_len;
719 __le32 pa_use_len;
720 };
721 __le32 qpc_bytes_176;
722 __le32 qpc_bytes_180;
723 __le32 tx_cur_sq_wqe_ba_l;
724 __le32 qpc_bytes_188;
725 __le32 rvd21;
726};
727
728#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S 0
729#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M \
730 (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S)
731
732#define QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S 3
733#define QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S 4
734#define QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S 5
735#define QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S 6
736#define QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S 7
737
738#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S 8
739#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M \
740 (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S)
741
742#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S 12
743#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M \
744 (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S)
745
746#define QP_CONTEXT_QPC_BYTES_4_PD_S 16
747#define QP_CONTEXT_QPC_BYTES_4_PD_M \
748 (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_4_PD_S)
749
750#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S 0
751#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M \
752 (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S)
753
754#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S 16
755#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M \
756 (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S)
757
758#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S 0
759#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M \
760 (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S)
761
762#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S 16
763#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M \
764 (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S)
765
766#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_S 0
767#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_M \
768 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_16_QP_NUM_S)
769
770#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S 0
771#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M \
772 (((1UL << 17) - 1) << QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S)
773
774#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S 18
775#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M \
776 (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S)
777
778#define QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S 23
779
780#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S 0
781#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M \
782 (((1UL << 17) - 1) << QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S)
783
784#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S 18
785#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M \
786 (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S)
787
788#define QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S 20
789#define QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S 21
790#define QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S 22
791#define QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S 23
792
793#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S 24
794#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M \
795 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S)
796
797#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_S 0
798#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_M \
799 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_36_DEST_QP_S)
800
801#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S 24
802#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M \
803 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S)
804
805#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_S 0
806#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_M \
807 (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_44_DMAC_H_S)
808
809#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S 16
810#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M \
811 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S)
812
813#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_S 24
814#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_M \
815 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_44_HOPLMT_S)
816
817#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S 0
818#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M \
819 (((1UL << 20) - 1) << QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S)
820
821#define QP_CONTEXT_QPC_BYTES_48_TCLASS_S 20
822#define QP_CONTEXT_QPC_BYTES_48_TCLASS_M \
823 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_48_TCLASS_S)
824
825#define QP_CONTEXT_QPC_BYTES_48_MTU_S 28
826#define QP_CONTEXT_QPC_BYTES_48_MTU_M \
827 (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_48_MTU_S)
828
829#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S 0
830#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M \
831 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S)
832
833#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S 16
834#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M \
835 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S)
836
837#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S 0
838#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M \
839 (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S)
840
841#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S 8
842#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M \
843 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S)
844
845#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S 0
846#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M \
847 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S)
848
849#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S 24
850#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M \
851 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S)
852
853#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S 0
854#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M \
855 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S)
856
857#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S 24
858#define QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S 25
859
860#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S 26
861#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M \
862 (((1UL << 2) - 1) << \
863 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S)
864
865#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S 29
866#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M \
867 (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S)
868
869#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S 0
870#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M \
871 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S)
872
873#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S 24
874#define QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S 25
875
876#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S 0
877#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M \
878 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S)
879
880#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S 24
881#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M \
882 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S)
883
884#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S 0
885#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M \
886 (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S)
887
888#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S 0
889#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M \
890 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S)
891
892#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S 16
893#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M \
894 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S)
895
896#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S 0
897#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M \
898 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S)
899
900#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S 24
901
902#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S 25
903#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M \
904 (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S)
905
906#define QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S 27
907
908#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S 0
909#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M \
910 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S)
911
912#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S 24
913#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M \
914 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S)
915
916#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S 0
917#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M \
918 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S)
919
920#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S 24
921#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M \
922 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S)
923
924#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S 0
925#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M \
926 (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S)
927
928#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S 16
929#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M \
930 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S)
931
932#define QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S 31
933
934#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_S 0
935#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_M \
936 (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_144_QP_STATE_S)
937
938#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S 0
939#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M \
940 (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S)
941
942#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S 2
943#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M \
944 (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S)
945
946#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S 5
947#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M \
948 (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S)
949
950#define QP_CONTEXT_QPC_BYTES_148_LSN_S 8
951#define QP_CONTEXT_QPC_BYTES_148_LSN_M \
952 (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_148_LSN_S)
953
954#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S 0
955#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M \
956 (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S)
957
958#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S 3
959#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M \
960 (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S)
961
962#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S 8
963#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M \
964 (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S)
965
966#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S 11
967#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M \
968 (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S)
969
970#define QP_CONTEXT_QPC_BYTES_156_SL_S 14
971#define QP_CONTEXT_QPC_BYTES_156_SL_M \
972 (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_156_SL_S)
973
974#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S 16
975#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M \
976 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S)
977
978#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S 24
979#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M \
980 (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S)
981
982#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S 0
983#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M \
984 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S)
985
986#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S 24
987#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M \
988 (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S)
989
990#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S 0
991#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M \
992 (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S)
993
994#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S 24
995#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M \
996 (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S)
997
998#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S 26
999#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M \
1000 (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S)
1001
1002#define QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S 28
1003#define QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S 29
1004#define QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S 30
1005
1006#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S 0
1007#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M \
1008 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S)
1009
1010#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S 16
1011#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M \
1012 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S)
1013
1014#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S 0
1015#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M \
1016 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S)
1017
1018#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S 16
1019#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M \
1020 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S)
1021
1022#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S 0
1023#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M \
1024 (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S)
1025
1026#define QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S 8
1027
1028#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S 16
1029#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M \
1030 (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S)
1031
1032#define STATUS_MASK 0xff
1033#define GO_BIT_TIMEOUT_MSECS 10000
1034#define HCR_STATUS_OFFSET 0x18
1035#define HCR_GO_BIT 15
1036
1037struct hns_roce_rq_db {
1038 __le32 u32_4;
1039 __le32 u32_8;
1040};
1041
1042#define RQ_DOORBELL_U32_4_RQ_HEAD_S 0
1043#define RQ_DOORBELL_U32_4_RQ_HEAD_M \
1044 (((1UL << 15) - 1) << RQ_DOORBELL_U32_4_RQ_HEAD_S)
1045
1046#define RQ_DOORBELL_U32_8_QPN_S 0
1047#define RQ_DOORBELL_U32_8_QPN_M (((1UL << 24) - 1) << RQ_DOORBELL_U32_8_QPN_S)
1048
1049#define RQ_DOORBELL_U32_8_CMD_S 28
1050#define RQ_DOORBELL_U32_8_CMD_M (((1UL << 3) - 1) << RQ_DOORBELL_U32_8_CMD_S)
1051
1052#define RQ_DOORBELL_U32_8_HW_SYNC_S 31
1053
1054struct hns_roce_sq_db {
1055 __le32 u32_4;
1056 __le32 u32_8;
1057};
1058
1059#define SQ_DOORBELL_U32_4_SQ_HEAD_S 0
1060#define SQ_DOORBELL_U32_4_SQ_HEAD_M \
1061 (((1UL << 15) - 1) << SQ_DOORBELL_U32_4_SQ_HEAD_S)
1062
1063#define SQ_DOORBELL_U32_4_SL_S 16
1064#define SQ_DOORBELL_U32_4_SL_M \
1065 (((1UL << 2) - 1) << SQ_DOORBELL_U32_4_SL_S)
1066
1067#define SQ_DOORBELL_U32_4_PORT_S 18
1068#define SQ_DOORBELL_U32_4_PORT_M (((1UL << 3) - 1) << SQ_DOORBELL_U32_4_PORT_S)
1069
1070#define SQ_DOORBELL_U32_8_QPN_S 0
1071#define SQ_DOORBELL_U32_8_QPN_M (((1UL << 24) - 1) << SQ_DOORBELL_U32_8_QPN_S)
1072
1073#define SQ_DOORBELL_HW_SYNC_S 31
1074
1075struct hns_roce_ext_db {
1076 int esdb_dep;
1077 int eodb_dep;
1078 struct hns_roce_buf_list *sdb_buf_list;
1079 struct hns_roce_buf_list *odb_buf_list;
1080};
1081
1082struct hns_roce_db_table {
1083 int sdb_ext_mod;
1084 int odb_ext_mod;
1085 struct hns_roce_ext_db *ext_db;
1086};
1087
1088#define HW_SYNC_SLEEP_TIME_INTERVAL 20
1089#define HW_SYNC_TIMEOUT_MSECS (25 * HW_SYNC_SLEEP_TIME_INTERVAL)
1090#define BT_CMD_SYNC_SHIFT 31
1091#define HNS_ROCE_BA_SIZE (32 * 4096)
1092
1093struct hns_roce_bt_table {
1094 struct hns_roce_buf_list qpc_buf;
1095 struct hns_roce_buf_list mtpt_buf;
1096 struct hns_roce_buf_list cqc_buf;
1097};
1098
1099struct hns_roce_tptr_table {
1100 struct hns_roce_buf_list tptr_buf;
1101};
1102
1103struct hns_roce_qp_work {
1104 struct work_struct work;
1105 struct ib_device *ib_dev;
1106 struct hns_roce_qp *qp;
1107 u32 db_wait_stage;
1108 u32 sdb_issue_ptr;
1109 u32 sdb_inv_cnt;
1110 u32 sche_cnt;
1111};
1112
1113struct hns_roce_mr_free_work {
1114 struct work_struct work;
1115 struct ib_device *ib_dev;
1116 struct completion *comp;
1117 int comp_flag;
1118 void *mr;
1119};
1120
1121struct hns_roce_recreate_lp_qp_work {
1122 struct work_struct work;
1123 struct ib_device *ib_dev;
1124 struct completion *comp;
1125 int comp_flag;
1126};
1127
1128struct hns_roce_free_mr {
1129 struct workqueue_struct *free_mr_wq;
1130 struct hns_roce_qp *mr_free_qp[HNS_ROCE_V1_RESV_QP];
1131 struct hns_roce_cq *mr_free_cq;
1132 struct hns_roce_pd *mr_free_pd;
1133};
1134
1135struct hns_roce_v1_priv {
1136 struct hns_roce_db_table db_table;
1137 struct hns_roce_raq_table raq_table;
1138 struct hns_roce_bt_table bt_table;
1139 struct hns_roce_tptr_table tptr_table;
1140 struct hns_roce_free_mr free_mr;
1141};
1142
1143int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool dereset);
1144int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
1145int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
1146
1147#endif
drivers/infiniband/hw/hns/hns_roce_hw_v2.c+23−26
@@ -678,6 +678,7 @@ static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val, · 677 unmodified lines
678static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,678static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
679 void *wqe)679 void *wqe)
680{680{
681#define HNS_ROCE_SL_SHIFT 2
681 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;682 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
682 683
683 /* All kinds of DirectWQE have the same header field layout */684 /* All kinds of DirectWQE have the same header field layout */
@@ -685,7 +686,8 @@ static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, · 1 unmodified line
685 roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_L_M,686 roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_L_M,
686 V2_RC_SEND_WQE_BYTE_4_DB_SL_L_S, qp->sl);687 V2_RC_SEND_WQE_BYTE_4_DB_SL_L_S, qp->sl);
687 roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_H_M,688 roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_H_M,
688 V2_RC_SEND_WQE_BYTE_4_DB_SL_H_S, qp->sl >> 2);689 V2_RC_SEND_WQE_BYTE_4_DB_SL_H_S,
690 qp->sl >> HNS_ROCE_SL_SHIFT);
689 roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_M,691 roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_M,
690 V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_S, qp->sq.head);692 V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_S, qp->sq.head);
691 693
@@ -1305,14 +1307,14 @@ static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, · 613 unmodified lines
1305 continue;1307 continue;
1306 1308
1307 dev_err_ratelimited(hr_dev->dev,1309 dev_err_ratelimited(hr_dev->dev,
1308 "Cmdq IO error, opcode = %x, return = %x\n",1310 "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n",
1309 desc->opcode, desc_ret);1311 desc->opcode, desc_ret);
1310 ret = -EIO;1312 ret = -EIO;
1311 }1313 }
1312 } else {1314 } else {
1313 /* FW/HW reset or incorrect number of desc */1315 /* FW/HW reset or incorrect number of desc */
1314 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);1316 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);
1315 dev_warn(hr_dev->dev, "CMDQ move tail from %d to %d\n",1317 dev_warn(hr_dev->dev, "CMDQ move tail from %u to %u.\n",
1316 csq->head, tail);1318 csq->head, tail);
1317 csq->head = tail;1319 csq->head = tail;
1318 1320
@@ -1571,7 +1573,7 @@ static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev) · 252 unmodified lines
1571 struct hns_roce_cmq_desc desc;1573 struct hns_roce_cmq_desc desc;
1572 int ret;1574 int ret;
1573 1575
1574 if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) {1576 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
1575 hr_dev->func_num = 1;1577 hr_dev->func_num = 1;
1576 return 0;1578 return 0;
1577 }1579 }
@@ -2003,7 +2005,8 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) · 425 unmodified lines
2003 caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;2005 caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
2004 2006
2005 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {2007 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2006 caps->flags |= HNS_ROCE_CAP_FLAG_STASH;2008 caps->flags |= HNS_ROCE_CAP_FLAG_STASH |
2009 HNS_ROCE_CAP_FLAG_DIRECT_WQE;
2007 caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE;2010 caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE;
2008 } else {2011 } else {
2009 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;2012 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
@@ -2144,7 +2147,6 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) · 134 unmodified lines
2144 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;2147 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
2145 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;2148 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
2146 2149
2147 caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
2148 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;2150 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
2149 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;2151 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2150 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;2152 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
@@ -2161,6 +2163,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) · 10 unmodified lines
2161 (u32)priv->handle->rinfo.num_vectors - 2);2163 (u32)priv->handle->rinfo.num_vectors - 2);
2162 2164
2163 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {2165 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2166 caps->eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM;
2164 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;2167 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
2165 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;2168 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
2166 2169
@@ -2181,6 +2184,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) · 14 unmodified lines
2181 } else {2184 } else {
2182 u32 func_num = max_t(u32, 1, hr_dev->func_num);2185 u32 func_num = max_t(u32, 1, hr_dev->func_num);
2183 2186
2187 caps->eqe_hop_num = HNS_ROCE_V2_EQE_HOP_NUM;
2184 caps->ceqe_size = HNS_ROCE_CEQE_SIZE;2188 caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
2185 caps->aeqe_size = HNS_ROCE_AEQE_SIZE;2189 caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
2186 caps->gid_table_len[0] /= func_num;2190 caps->gid_table_len[0] /= func_num;
@@ -2393,7 +2397,7 @@ static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev) · 206 unmodified lines
2393 struct hns_roce_caps *caps = &hr_dev->caps;2397 struct hns_roce_caps *caps = &hr_dev->caps;
2394 int ret;2398 int ret;
2395 2399
2396 if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)2400 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
2397 return 0;2401 return 0;
2398 2402
2399 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE,2403 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE,
@@ -2967,8 +2971,8 @@ static int config_gmv_table(struct hns_roce_dev *hr_dev, · 567 unmodified lines
2967 return hns_roce_cmq_send(hr_dev, desc, 2);2971 return hns_roce_cmq_send(hr_dev, desc, 2);
2968}2972}
2969 2973
2970static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u32 port,2974static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index,
2971 int gid_index, const union ib_gid *gid,2975 const union ib_gid *gid,
2972 const struct ib_gid_attr *attr)2976 const struct ib_gid_attr *attr)
2973{2977{
2974 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;2978 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
@@ -3063,8 +3067,7 @@ static int set_mtpt_pbl(struct hns_roce_dev *hr_dev, · 88 unmodified lines
3063}3067}
3064 3068
3065static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,3069static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
3066 void *mb_buf, struct hns_roce_mr *mr,3070 void *mb_buf, struct hns_roce_mr *mr)
3067 unsigned long mtpt_idx)
3068{3071{
3069 struct hns_roce_v2_mpt_entry *mpt_entry;3072 struct hns_roce_v2_mpt_entry *mpt_entry;
3070 int ret;3073 int ret;
@@ -4488,14 +4491,6 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, · 1417 unmodified lines
4488 return 0;4491 return 0;
4489}4492}
4490 4493
4491static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
4492{
4493 if (!fl)
4494 fl = rdma_calc_flow_label(lqpn, rqpn);
4495
4496 return rdma_flow_label_to_udp_sport(fl);
4497}
4498
4499static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr,4494static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
4500 u32 *dip_idx)4495 u32 *dip_idx)
4501{4496{
@@ -4712,8 +4707,9 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, · 210 unmodified lines
4712 }4707 }
4713 4708
4714 hr_reg_write(context, QPC_UDPSPN,4709 hr_reg_write(context, QPC_UDPSPN,
4715 is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num,4710 is_udp ? rdma_get_udp_sport(grh->flow_label, ibqp->qp_num,
4716 attr->dest_qp_num) : 0);4711 attr->dest_qp_num) :
4712 0);
4717 4713
4718 hr_reg_clear(qpc_mask, QPC_UDPSPN);4714 hr_reg_clear(qpc_mask, QPC_UDPSPN);
4719 4715
@@ -4739,7 +4735,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, · 19 unmodified lines
4739 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);4735 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4740 if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) {4736 if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) {
4741 ibdev_err(ibdev,4737 ibdev_err(ibdev,
4742 "failed to fill QPC, sl (%d) shouldn't be larger than %d.\n",4738 "failed to fill QPC, sl (%u) shouldn't be larger than %d.\n",
4743 hr_qp->sl, MAX_SERVICE_LEVEL);4739 hr_qp->sl, MAX_SERVICE_LEVEL);
4744 return -EINVAL;4740 return -EINVAL;
4745 }4741 }
@@ -4768,7 +4764,8 @@ static bool check_qp_state(enum ib_qp_state cur_state, · 22 unmodified lines
4768 [IB_QPS_ERR] = true },4764 [IB_QPS_ERR] = true },
4769 [IB_QPS_SQD] = {},4765 [IB_QPS_SQD] = {},
4770 [IB_QPS_SQE] = {},4766 [IB_QPS_SQE] = {},
4771 [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }4767 [IB_QPS_ERR] = { [IB_QPS_RESET] = true,
4768 [IB_QPS_ERR] = true }
4772 };4769 };
4773 4770
4774 return sm[cur_state][new_state];4771 return sm[cur_state][new_state];
@@ -5868,7 +5865,7 @@ static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, · 1093 unmodified lines
5868 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag);5865 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag);
5869}5866}
5870 5867
5871static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)5868static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, u32 eqn)
5872{5869{
5873 struct device *dev = hr_dev->dev;5870 struct device *dev = hr_dev->dev;
5874 int ret;5871 int ret;
@@ -5882,7 +5879,7 @@ static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) · 7 unmodified lines
5882 0, HNS_ROCE_CMD_DESTROY_AEQC,5879 0, HNS_ROCE_CMD_DESTROY_AEQC,
5883 HNS_ROCE_CMD_TIMEOUT_MSECS);5880 HNS_ROCE_CMD_TIMEOUT_MSECS);
5884 if (ret)5881 if (ret)
5885 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);5882 dev_err(dev, "[mailbox cmd] destroy eqc(%u) failed.\n", eqn);
5886}5883}
5887 5884
5888static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)5885static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
@@ -6394,7 +6391,7 @@ static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) · 505 unmodified lines
6394 if (!id)6391 if (!id)
6395 return 0;6392 return 0;
6396 6393
6397 if (id->driver_data && handle->pdev->revision < PCI_REVISION_ID_HIP09)6394 if (id->driver_data && handle->pdev->revision == PCI_REVISION_ID_HIP08)
6398 return 0;6395 return 0;
6399 6396
6400 ret = __hns_roce_hw_v2_init_instance(handle);6397 ret = __hns_roce_hw_v2_init_instance(handle);
drivers/infiniband/hw/hns/hns_roce_hw_v2.h+4−18
@@ -35,26 +35,15 @@ · 34 unmodified lines
35 35
36#include <linux/bitops.h>36#include <linux/bitops.h>
37 37
38#define HNS_ROCE_VF_QPC_BT_NUM 256
39#define HNS_ROCE_VF_SCCC_BT_NUM 64
40#define HNS_ROCE_VF_SRQC_BT_NUM 64
41#define HNS_ROCE_VF_CQC_BT_NUM 64
42#define HNS_ROCE_VF_MPT_BT_NUM 64
43#define HNS_ROCE_VF_SMAC_NUM 32
44#define HNS_ROCE_VF_SL_NUM 8
45#define HNS_ROCE_VF_GMV_BT_NUM 256
46
47#define HNS_ROCE_V2_MAX_QP_NUM 0x100038#define HNS_ROCE_V2_MAX_QP_NUM 0x1000
48#define HNS_ROCE_V2_MAX_QPC_TIMER_NUM 0x20039#define HNS_ROCE_V2_MAX_QPC_TIMER_NUM 0x200
49#define HNS_ROCE_V2_MAX_WQE_NUM 0x800040#define HNS_ROCE_V2_MAX_WQE_NUM 0x8000
50#define HNS_ROCE_V2_MAX_SRQ 0x100000
51#define HNS_ROCE_V2_MAX_SRQ_WR 0x800041#define HNS_ROCE_V2_MAX_SRQ_WR 0x8000
52#define HNS_ROCE_V2_MAX_SRQ_SGE 6442#define HNS_ROCE_V2_MAX_SRQ_SGE 64
53#define HNS_ROCE_V2_MAX_CQ_NUM 0x10000043#define HNS_ROCE_V2_MAX_CQ_NUM 0x100000
54#define HNS_ROCE_V2_MAX_CQC_TIMER_NUM 0x10044#define HNS_ROCE_V2_MAX_CQC_TIMER_NUM 0x100
55#define HNS_ROCE_V2_MAX_SRQ_NUM 0x10000045#define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000
56#define HNS_ROCE_V2_MAX_CQE_NUM 0x40000046#define HNS_ROCE_V2_MAX_CQE_NUM 0x400000
57#define HNS_ROCE_V2_MAX_SRQWQE_NUM 0x8000
58#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 6447#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 64
59#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 6448#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64
60#define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x20000049#define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000
@@ -63,13 +52,10 @@ · 2 unmodified lines
63#define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 3252#define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32
64#define HNS_ROCE_V2_UAR_NUM 25653#define HNS_ROCE_V2_UAR_NUM 256
65#define HNS_ROCE_V2_PHY_UAR_NUM 154#define HNS_ROCE_V2_PHY_UAR_NUM 1
66#define HNS_ROCE_V2_MAX_IRQ_NUM 65
67#define HNS_ROCE_V2_COMP_VEC_NUM 63
68#define HNS_ROCE_V2_AEQE_VEC_NUM 155#define HNS_ROCE_V2_AEQE_VEC_NUM 1
69#define HNS_ROCE_V2_ABNORMAL_VEC_NUM 156#define HNS_ROCE_V2_ABNORMAL_VEC_NUM 1
70#define HNS_ROCE_V2_MAX_MTPT_NUM 0x10000057#define HNS_ROCE_V2_MAX_MTPT_NUM 0x100000
71#define HNS_ROCE_V2_MAX_MTT_SEGS 0x100000058#define HNS_ROCE_V2_MAX_MTT_SEGS 0x1000000
72#define HNS_ROCE_V2_MAX_CQE_SEGS 0x1000000
73#define HNS_ROCE_V2_MAX_SRQWQE_SEGS 0x100000059#define HNS_ROCE_V2_MAX_SRQWQE_SEGS 0x1000000
74#define HNS_ROCE_V2_MAX_IDX_SEGS 0x100000060#define HNS_ROCE_V2_MAX_IDX_SEGS 0x1000000
75#define HNS_ROCE_V2_MAX_PD_NUM 0x100000061#define HNS_ROCE_V2_MAX_PD_NUM 0x1000000
@@ -81,7 +67,6 @@ · 5 unmodified lines
81#define HNS_ROCE_V2_MAX_RQ_DESC_SZ 1667#define HNS_ROCE_V2_MAX_RQ_DESC_SZ 16
82#define HNS_ROCE_V2_MAX_SRQ_DESC_SZ 6468#define HNS_ROCE_V2_MAX_SRQ_DESC_SZ 64
83#define HNS_ROCE_V2_IRRL_ENTRY_SZ 6469#define HNS_ROCE_V2_IRRL_ENTRY_SZ 64
84#define HNS_ROCE_V2_TRRL_ENTRY_SZ 48
85#define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 10070#define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 100
86#define HNS_ROCE_V2_CQC_ENTRY_SZ 6471#define HNS_ROCE_V2_CQC_ENTRY_SZ 64
87#define HNS_ROCE_V2_SRQC_ENTRY_SZ 6472#define HNS_ROCE_V2_SRQC_ENTRY_SZ 64
@@ -103,7 +88,6 @@ · 15 unmodified lines
103#define HNS_ROCE_INVALID_LKEY 0x088#define HNS_ROCE_INVALID_LKEY 0x0
104#define HNS_ROCE_INVALID_SGE_LENGTH 0x8000000089#define HNS_ROCE_INVALID_SGE_LENGTH 0x80000000
105#define HNS_ROCE_CMQ_TX_TIMEOUT 3000090#define HNS_ROCE_CMQ_TX_TIMEOUT 30000
106#define HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE 2
107#define HNS_ROCE_V2_RSV_QPS 891#define HNS_ROCE_V2_RSV_QPS 8
108 92
109#define HNS_ROCE_V2_HW_RST_TIMEOUT 100093#define HNS_ROCE_V2_HW_RST_TIMEOUT 1000
@@ -117,12 +101,14 @@ · 7 unmodified lines
117#define HNS_ROCE_CQE_HOP_NUM 1101#define HNS_ROCE_CQE_HOP_NUM 1
118#define HNS_ROCE_SRQWQE_HOP_NUM 1102#define HNS_ROCE_SRQWQE_HOP_NUM 1
119#define HNS_ROCE_PBL_HOP_NUM 2103#define HNS_ROCE_PBL_HOP_NUM 2
120#define HNS_ROCE_EQE_HOP_NUM 2
121#define HNS_ROCE_IDX_HOP_NUM 1104#define HNS_ROCE_IDX_HOP_NUM 1
122#define HNS_ROCE_SQWQE_HOP_NUM 2105#define HNS_ROCE_SQWQE_HOP_NUM 2
123#define HNS_ROCE_EXT_SGE_HOP_NUM 1106#define HNS_ROCE_EXT_SGE_HOP_NUM 1
124#define HNS_ROCE_RQWQE_HOP_NUM 2107#define HNS_ROCE_RQWQE_HOP_NUM 2
125 108
109#define HNS_ROCE_V2_EQE_HOP_NUM 2
110#define HNS_ROCE_V3_EQE_HOP_NUM 1
111
126#define HNS_ROCE_BA_PG_SZ_SUPPORTED_256K 6112#define HNS_ROCE_BA_PG_SZ_SUPPORTED_256K 6
127#define HNS_ROCE_BA_PG_SZ_SUPPORTED_16K 2113#define HNS_ROCE_BA_PG_SZ_SUPPORTED_16K 2
128#define HNS_ROCE_V2_GID_INDEX_NUM 16114#define HNS_ROCE_V2_GID_INDEX_NUM 16
@@ -1441,7 +1427,7 @@ struct hns_roce_v2_priv { · 1312 unmodified lines
1441struct hns_roce_dip {1427struct hns_roce_dip {
1442 u8 dgid[GID_LEN_V2];1428 u8 dgid[GID_LEN_V2];
1443 u32 dip_idx;1429 u32 dip_idx;
1444 struct list_head node; /* all dips are on a list */1430 struct list_head node; /* all dips are on a list */
1445};1431};
1446 1432
1447/* only for RNR timeout issue of HIP08 */1433/* only for RNR timeout issue of HIP08 */
drivers/infiniband/hw/hns/hns_roce_main.c+30−55
@@ -31,7 +31,6 @@ · 30 unmodified lines
31 * SOFTWARE.31 * SOFTWARE.
32 */32 */
33#include <linux/acpi.h>33#include <linux/acpi.h>
34#include <linux/of_platform.h>
35#include <linux/module.h>34#include <linux/module.h>
36#include <linux/pci.h>35#include <linux/pci.h>
37#include <rdma/ib_addr.h>36#include <rdma/ib_addr.h>
@@ -70,7 +69,7 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) · 32 unmodified lines
70 if (port >= hr_dev->caps.num_ports)69 if (port >= hr_dev->caps.num_ports)
71 return -EINVAL;70 return -EINVAL;
72 71
73 ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr);72 ret = hr_dev->hw->set_gid(hr_dev, attr->index, &attr->gid, attr);
74 73
75 return ret;74 return ret;
76}75}
@@ -84,7 +83,7 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) · 7 unmodified lines
84 if (port >= hr_dev->caps.num_ports)83 if (port >= hr_dev->caps.num_ports)
85 return -EINVAL;84 return -EINVAL;
86 85
87 ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, NULL, NULL);86 ret = hr_dev->hw->set_gid(hr_dev, attr->index, NULL, NULL);
88 87
89 return ret;88 return ret;
90}89}
@@ -152,9 +151,6 @@ static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev) · 61 unmodified lines
152 u8 i;151 u8 i;
153 152
154 for (i = 0; i < hr_dev->caps.num_ports; i++) {153 for (i = 0; i < hr_dev->caps.num_ports; i++) {
155 if (hr_dev->hw->set_mtu)
156 hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i],
157 hr_dev->caps.max_mtu);
158 ret = hns_roce_set_mac(hr_dev, i,154 ret = hns_roce_set_mac(hr_dev, i,
159 hr_dev->iboe.netdevs[i]->dev_addr);155 hr_dev->iboe.netdevs[i]->dev_addr);
160 if (ret)156 if (ret)
@@ -270,6 +266,9 @@ static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device, · 109 unmodified lines
270static int hns_roce_query_pkey(struct ib_device *ib_dev, u32 port, u16 index,266static int hns_roce_query_pkey(struct ib_device *ib_dev, u32 port, u16 index,
271 u16 *pkey)267 u16 *pkey)
272{268{
269 if (index > 0)
270 return -EINVAL;
271
273 *pkey = PKEY_ID;272 *pkey = PKEY_ID;
274 273
275 return 0;274 return 0;
@@ -307,9 +306,22 @@ hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, · 31 unmodified lines
307 entry->address = address;306 entry->address = address;
308 entry->mmap_type = mmap_type;307 entry->mmap_type = mmap_type;
309 308
310 ret = rdma_user_mmap_entry_insert_exact(309 switch (mmap_type) {
311 ucontext, &entry->rdma_entry, length,310 /* pgoff 0 must be used by DB for compatibility */
312 mmap_type == HNS_ROCE_MMAP_TYPE_DB ? 0 : 1);311 case HNS_ROCE_MMAP_TYPE_DB:
312 ret = rdma_user_mmap_entry_insert_exact(
313 ucontext, &entry->rdma_entry, length, 0);
314 break;
315 case HNS_ROCE_MMAP_TYPE_DWQE:
316 ret = rdma_user_mmap_entry_insert_range(
317 ucontext, &entry->rdma_entry, length, 1,
318 U32_MAX);
319 break;
320 default:
321 ret = -EINVAL;
322 break;
323 }
324
313 if (ret) {325 if (ret) {
314 kfree(entry);326 kfree(entry);
315 return NULL;327 return NULL;
@@ -323,18 +335,12 @@ static void hns_roce_dealloc_uar_entry(struct hns_roce_ucontext *context) · 7 unmodified lines
323 if (context->db_mmap_entry)335 if (context->db_mmap_entry)
324 rdma_user_mmap_entry_remove(336 rdma_user_mmap_entry_remove(
325 &context->db_mmap_entry->rdma_entry);337 &context->db_mmap_entry->rdma_entry);
326
327 if (context->tptr_mmap_entry)
328 rdma_user_mmap_entry_remove(
329 &context->tptr_mmap_entry->rdma_entry);
330}338}
331 339
332static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx)340static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx)
333{341{
334 struct hns_roce_ucontext *context = to_hr_ucontext(uctx);342 struct hns_roce_ucontext *context = to_hr_ucontext(uctx);
335 struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device);
336 u64 address;343 u64 address;
337 int ret;
338 344
339 address = context->uar.pfn << PAGE_SHIFT;345 address = context->uar.pfn << PAGE_SHIFT;
340 context->db_mmap_entry = hns_roce_user_mmap_entry_insert(346 context->db_mmap_entry = hns_roce_user_mmap_entry_insert(
@@ -342,27 +348,7 @@ static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx) · 1 unmodified line
342 if (!context->db_mmap_entry)348 if (!context->db_mmap_entry)
343 return -ENOMEM;349 return -ENOMEM;
344 350
345 if (!hr_dev->tptr_dma_addr || !hr_dev->tptr_size)
346 return 0;
347
348 /*
349 * FIXME: using io_remap_pfn_range on the dma address returned
350 * by dma_alloc_coherent is totally wrong.
351 */
352 context->tptr_mmap_entry =
353 hns_roce_user_mmap_entry_insert(uctx, hr_dev->tptr_dma_addr,
354 hr_dev->tptr_size,
355 HNS_ROCE_MMAP_TYPE_TPTR);
356 if (!context->tptr_mmap_entry) {
357 ret = -ENOMEM;
358 goto err;
359 }
360
361 return 0;351 return 0;
362
363err:
364 hns_roce_dealloc_uar_entry(context);
365 return ret;
366}352}
367 353
368static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,354static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
@@ -436,10 +422,15 @@ static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) · 67 unmodified lines
436 422
437 entry = to_hns_mmap(rdma_entry);423 entry = to_hns_mmap(rdma_entry);
438 pfn = entry->address >> PAGE_SHIFT;424 pfn = entry->address >> PAGE_SHIFT;
439 prot = vma->vm_page_prot;
440 425
441 if (entry->mmap_type != HNS_ROCE_MMAP_TYPE_TPTR)426 switch (entry->mmap_type) {
442 prot = pgprot_noncached(prot);427 case HNS_ROCE_MMAP_TYPE_DB:
428 case HNS_ROCE_MMAP_TYPE_DWQE:
429 prot = pgprot_device(vma->vm_page_prot);
430 break;
431 default:
432 return -EINVAL;
433 }
443 434
444 ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE,435 ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE,
445 prot, rdma_entry);436 prot, rdma_entry);
@@ -816,7 +807,6 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) · 370 unmodified lines
816 int ret;807 int ret;
817 808
818 spin_lock_init(&hr_dev->sm_lock);809 spin_lock_init(&hr_dev->sm_lock);
819 spin_lock_init(&hr_dev->bt_cmd_lock);
820 810
821 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||811 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||
822 hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) {812 hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) {
@@ -907,20 +897,13 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) · 84 unmodified lines
907 struct device *dev = hr_dev->dev;897 struct device *dev = hr_dev->dev;
908 int ret;898 int ret;
909 899
910 if (hr_dev->hw->reset) {
911 ret = hr_dev->hw->reset(hr_dev, true);
912 if (ret) {
913 dev_err(dev, "Reset RoCE engine failed!\n");
914 return ret;
915 }
916 }
917 hr_dev->is_reset = false;900 hr_dev->is_reset = false;
918 901
919 if (hr_dev->hw->cmq_init) {902 if (hr_dev->hw->cmq_init) {
920 ret = hr_dev->hw->cmq_init(hr_dev);903 ret = hr_dev->hw->cmq_init(hr_dev);
921 if (ret) {904 if (ret) {
922 dev_err(dev, "Init RoCE Command Queue failed!\n");905 dev_err(dev, "Init RoCE Command Queue failed!\n");
923 goto error_failed_cmq_init;906 return ret;
924 }907 }
925 }908 }
926 909
@@ -1003,12 +986,6 @@ error_failed_cmd_init: · 76 unmodified lines
1003 if (hr_dev->hw->cmq_exit)986 if (hr_dev->hw->cmq_exit)
1004 hr_dev->hw->cmq_exit(hr_dev);987 hr_dev->hw->cmq_exit(hr_dev);
1005 988
1006error_failed_cmq_init:
1007 if (hr_dev->hw->reset) {
1008 if (hr_dev->hw->reset(hr_dev, false))
1009 dev_err(dev, "Dereset RoCE engine failed!\n");
1010 }
1011
1012 return ret;989 return ret;
1013}990}
1014 991
@@ -1028,8 +1005,6 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev) · 13 unmodified lines
1028 hns_roce_cmd_cleanup(hr_dev);1005 hns_roce_cmd_cleanup(hr_dev);
1029 if (hr_dev->hw->cmq_exit)1006 if (hr_dev->hw->cmq_exit)
1030 hr_dev->hw->cmq_exit(hr_dev);1007 hr_dev->hw->cmq_exit(hr_dev);
1031 if (hr_dev->hw->reset)
1032 hr_dev->hw->reset(hr_dev, false);
1033}1008}
1034 1009
1035MODULE_LICENSE("Dual BSD/GPL");1010MODULE_LICENSE("Dual BSD/GPL");
drivers/infiniband/hw/hns/hns_roce_mr.c+10−22
@@ -31,7 +31,6 @@ · 30 unmodified lines
31 * SOFTWARE.31 * SOFTWARE.
32 */32 */
33 33
34#include <linux/platform_device.h>
35#include <linux/vmalloc.h>34#include <linux/vmalloc.h>
36#include <rdma/ib_umem.h>35#include <rdma/ib_umem.h>
37#include "hns_roce_device.h"36#include "hns_roce_device.h"
@@ -81,7 +80,7 @@ static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) · 43 unmodified lines
81 return -ENOMEM;80 return -ENOMEM;
82 }81 }
83 82
84 mr->key = hw_index_to_key(id); /* MR key */83 mr->key = hw_index_to_key(id); /* MR key */
85 84
86 err = hns_roce_table_get(hr_dev, &hr_dev->mr_table.mtpt_table,85 err = hns_roce_table_get(hr_dev, &hr_dev->mr_table.mtpt_table,
87 (unsigned long)id);86 (unsigned long)id);
@@ -173,8 +172,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, · 85 unmodified lines
173 }172 }
174 173
175 if (mr->type != MR_TYPE_FRMR)174 if (mr->type != MR_TYPE_FRMR)
176 ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr,175 ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr);
177 mtpt_idx);
178 else176 else
179 ret = hr_dev->hw->frmr_write_mtpt(hr_dev, mailbox->buf, mr);177 ret = hr_dev->hw->frmr_write_mtpt(hr_dev, mailbox->buf, mr);
180 if (ret) {178 if (ret) {
@@ -363,12 +361,8 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) · 182 unmodified lines
363 struct hns_roce_mr *mr = to_hr_mr(ibmr);361 struct hns_roce_mr *mr = to_hr_mr(ibmr);
364 int ret = 0;362 int ret = 0;
365 363
366 if (hr_dev->hw->dereg_mr) {364 hns_roce_mr_free(hr_dev, mr);
367 ret = hr_dev->hw->dereg_mr(hr_dev, mr, udata);365 kfree(mr);
368 } else {
369 hns_roce_mr_free(hr_dev, mr);
370 kfree(mr);
371 }
372 366
373 return ret;367 return ret;
374}368}
@@ -614,10 +608,7 @@ static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 239 unmodified lines
614 return -ENOBUFS;608 return -ENOBUFS;
615 609
616 for (i = 0; i < count && npage < max_count; i++) {610 for (i = 0; i < count && npage < max_count; i++) {
617 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)611 addr = pages[npage];
618 addr = to_hr_hw_page_addr(pages[npage]);
619 else
620 addr = pages[npage];
621 612
622 mtts[i] = cpu_to_le64(addr);613 mtts[i] = cpu_to_le64(addr);
623 npage++;614 npage++;
@@ -824,11 +815,11 @@ int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 200 unmodified lines
824}815}
825 816
826int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,817int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
827 int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr)818 u32 offset, u64 *mtt_buf, int mtt_max, u64 *base_addr)
828{819{
829 struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;820 struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;
830 int mtt_count, left;821 int mtt_count, left;
831 int start_index;822 u32 start_index;
832 int total = 0;823 int total = 0;
833 __le64 *mtts;824 __le64 *mtts;
834 u32 npage;825 u32 npage;
@@ -847,10 +838,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, · 12 unmodified lines
847 continue;838 continue;
848 839
849 addr = cfg->root_ba + (npage << HNS_HW_PAGE_SHIFT);840 addr = cfg->root_ba + (npage << HNS_HW_PAGE_SHIFT);
850 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)841 mtt_buf[total] = addr;
851 mtt_buf[total] = to_hr_hw_page_addr(addr);
852 else
853 mtt_buf[total] = addr;
854 842
855 total++;843 total++;
856 }844 }
@@ -884,10 +872,10 @@ done: · 27 unmodified lines
884static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev,872static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev,
885 struct hns_roce_buf_attr *attr,873 struct hns_roce_buf_attr *attr,
886 struct hns_roce_hem_cfg *cfg,874 struct hns_roce_hem_cfg *cfg,
887 unsigned int *buf_page_shift, int unalinged_size)875 unsigned int *buf_page_shift, u64 unalinged_size)
888{876{
889 struct hns_roce_buf_region *r;877 struct hns_roce_buf_region *r;
890 int first_region_padding;878 u64 first_region_padding;
891 int page_cnt, region_cnt;879 int page_cnt, region_cnt;
892 unsigned int page_shift;880 unsigned int page_shift;
893 size_t buf_size;881 size_t buf_size;
drivers/infiniband/hw/hns/hns_roce_pd.c+3−14
@@ -30,7 +30,6 @@ · 29 unmodified lines
30 * SOFTWARE.30 * SOFTWARE.
31 */31 */
32 32
33#include <linux/platform_device.h>
34#include <linux/pci.h>33#include <linux/pci.h>
35#include "hns_roce_device.h"34#include "hns_roce_device.h"
36 35
@@ -86,7 +85,6 @@ int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) · 49 unmodified lines
86int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)85int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
87{86{
88 struct hns_roce_ida *uar_ida = &hr_dev->uar_ida;87 struct hns_roce_ida *uar_ida = &hr_dev->uar_ida;
89 struct resource *res;
90 int id;88 int id;
91 89
92 /* Using bitmap to manager UAR index */90 /* Using bitmap to manager UAR index */
@@ -104,18 +102,9 @@ int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) · 11 unmodified lines
104 else102 else
105 uar->index = 0;103 uar->index = 0;
106 104
107 if (!dev_is_pci(hr_dev->dev)) {105 uar->pfn = ((pci_resource_start(hr_dev->pci_dev, 2)) >> PAGE_SHIFT);
108 res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0);106 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE)
109 if (!res) {107 hr_dev->dwqe_page = pci_resource_start(hr_dev->pci_dev, 4);
110 ida_free(&uar_ida->ida, id);
111 dev_err(&hr_dev->pdev->dev, "memory resource not found!\n");
112 return -EINVAL;
113 }
114 uar->pfn = ((res->start) >> PAGE_SHIFT) + uar->index;
115 } else {
116 uar->pfn = ((pci_resource_start(hr_dev->pci_dev, 2))
117 >> PAGE_SHIFT);
118 }
119 108
120 return 0;109 return 0;
121}110}
drivers/infiniband/hw/hns/hns_roce_qp.c+61−32
@@ -32,7 +32,6 @@ · 31 unmodified lines
32 */32 */
33 33
34#include <linux/pci.h>34#include <linux/pci.h>
35#include <linux/platform_device.h>
36#include <rdma/ib_addr.h>35#include <rdma/ib_addr.h>
37#include <rdma/ib_umem.h>36#include <rdma/ib_umem.h>
38#include <rdma/uverbs_ioctl.h>37#include <rdma/uverbs_ioctl.h>
@@ -110,12 +109,11 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) · 71 unmodified lines
110 return;109 return;
111 }110 }
112 111
113 if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 &&112 if (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR ||
114 (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR ||113 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR ||
115 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR ||114 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR ||
116 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR ||115 event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION ||
117 event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION ||116 event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH) {
118 event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH)) {
119 qp->state = IB_QPS_ERR;117 qp->state = IB_QPS_ERR;
120 118
121 flush_cqe(hr_dev, qp);119 flush_cqe(hr_dev, qp);
@@ -219,13 +217,7 @@ static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) · 97 unmodified lines
219 int ret;217 int ret;
220 218
221 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {219 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
222 /* when hw version is v1, the sqpn is allocated */220 num = 1;
223 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
224 num = HNS_ROCE_MAX_PORTS +
225 hr_dev->iboe.phy_port[hr_qp->port];
226 else
227 num = 1;
228
229 hr_qp->doorbell_qpn = 1;221 hr_qp->doorbell_qpn = 1;
230 } else {222 } else {
231 mutex_lock(&qp_table->bank_mutex);223 mutex_lock(&qp_table->bank_mutex);
@@ -324,11 +316,6 @@ static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) · 92 unmodified lines
324 if (!hr_qp->qpn)316 if (!hr_qp->qpn)
325 return -EINVAL;317 return -EINVAL;
326 318
327 /* In v1 engine, GSI QP context is saved in the RoCE hw's register */
328 if (hr_qp->ibqp.qp_type == IB_QPT_GSI &&
329 hr_dev->hw_rev == HNS_ROCE_HW_VER1)
330 return 0;
331
332 /* Alloc memory for QPC */319 /* Alloc memory for QPC */
333 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);320 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
334 if (ret) {321 if (ret) {
@@ -379,6 +366,11 @@ err_out: · 44 unmodified lines
379 return ret;366 return ret;
380}367}
381 368
369static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp)
370{
371 rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry);
372}
373
382void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)374void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
383{375{
384 struct xarray *xa = &hr_dev->qp_table_xa;376 struct xarray *xa = &hr_dev->qp_table_xa;
@@ -402,11 +394,6 @@ static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) · 17 unmodified lines
402{394{
403 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;395 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
404 396
405 /* In v1 engine, GSI QP context is saved in the RoCE hw's register */
406 if (hr_qp->ibqp.qp_type == IB_QPT_GSI &&
407 hr_dev->hw_rev == HNS_ROCE_HW_VER1)
408 return;
409
410 if (hr_dev->caps.trrl_entry_sz)397 if (hr_dev->caps.trrl_entry_sz)
411 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);398 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
412 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);399 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
@@ -535,11 +522,6 @@ static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, · 122 unmodified lines
535 522
536 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;523 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
537 524
538 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
539 hr_qp->sq.max_gs = HNS_ROCE_SGE_IN_WQE;
540 return;
541 }
542
543 hr_qp->sq.max_gs = max(1U, cap->max_send_sge);525 hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
544 526
545 wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp);527 wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp);
@@ -780,7 +762,11 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 234 unmodified lines
780 goto err_inline;762 goto err_inline;
781 }763 }
782 764
765 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE)
766 hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE;
767
783 return 0;768 return 0;
769
784err_inline:770err_inline:
785 free_rq_inline_buf(hr_qp);771 free_rq_inline_buf(hr_qp);
786 772
@@ -822,6 +808,35 @@ static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, · 35 unmodified lines
822 hns_roce_qp_has_rq(init_attr));808 hns_roce_qp_has_rq(init_attr));
823}809}
824 810
811static int qp_mmap_entry(struct hns_roce_qp *hr_qp,
812 struct hns_roce_dev *hr_dev,
813 struct ib_udata *udata,
814 struct hns_roce_ib_create_qp_resp *resp)
815{
816 struct hns_roce_ucontext *uctx =
817 rdma_udata_to_drv_context(udata,
818 struct hns_roce_ucontext, ibucontext);
819 struct rdma_user_mmap_entry *rdma_entry;
820 u64 address;
821
822 address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE;
823
824 hr_qp->dwqe_mmap_entry =
825 hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address,
826 HNS_ROCE_DWQE_SIZE,
827 HNS_ROCE_MMAP_TYPE_DWQE);
828
829 if (!hr_qp->dwqe_mmap_entry) {
830 ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n");
831 return -ENOMEM;
832 }
833
834 rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry;
835 resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry);
836
837 return 0;
838}
839
825static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,840static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,
826 struct hns_roce_qp *hr_qp,841 struct hns_roce_qp *hr_qp,
827 struct ib_qp_init_attr *init_attr,842 struct ib_qp_init_attr *init_attr,
@@ -909,10 +924,16 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 81 unmodified lines
909 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;924 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;
910 925
911 if (udata) {926 if (udata) {
927 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) {
928 ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp);
929 if (ret)
930 return ret;
931 }
932
912 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd,933 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd,
913 resp);934 resp);
914 if (ret)935 if (ret)
915 return ret;936 goto err_remove_qp;
916 } else {937 } else {
917 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr);938 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr);
918 if (ret)939 if (ret)
@@ -920,6 +941,12 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 1 unmodified line
920 }941 }
921 942
922 return 0;943 return 0;
944
945err_remove_qp:
946 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
947 qp_user_mmap_entry_remove(hr_qp);
948
949 return ret;
923}950}
924 951
925static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,952static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
@@ -933,6 +960,8 @@ static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, · 7 unmodified lines
933 hns_roce_db_unmap_user(uctx, &hr_qp->rdb);960 hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
934 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)961 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
935 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);962 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
963 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
964 qp_user_mmap_entry_remove(hr_qp);
936 } else {965 } else {
937 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)966 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
938 hns_roce_free_db(hr_dev, &hr_qp->rdb);967 hns_roce_free_db(hr_dev, &hr_qp->rdb);
@@ -1158,7 +1187,7 @@ static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, · 219 unmodified lines
1158 goto out;1187 goto out;
1159 break;1188 break;
1160 case IB_QPT_UD:1189 case IB_QPT_UD:
1161 if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 &&1190 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 &&
1162 is_user)1191 is_user)
1163 goto out;1192 goto out;
1164 break;1193 break;
@@ -1391,7 +1420,7 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, · 226 unmodified lines
1391 }1420 }
1392}1421}
1393 1422
1394static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset)1423static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset)
1395{1424{
1396 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);1425 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);
1397}1426}
drivers/infiniband/hw/irdma/i40iw_if.c+1−1
@@ -198,7 +198,7 @@ static void i40iw_remove(struct auxiliary_device *aux_dev) · 197 unmodified lines
198 aux_dev);198 aux_dev);
199 struct i40e_info *cdev_info = i40e_adev->ldev;199 struct i40e_info *cdev_info = i40e_adev->ldev;
200 200
201 return i40e_client_device_unregister(cdev_info);201 i40e_client_device_unregister(cdev_info);
202}202}
203 203
204static const struct auxiliary_device_id i40iw_auxiliary_id_table[] = {204static const struct auxiliary_device_id i40iw_auxiliary_id_table[] = {
drivers/infiniband/hw/irdma/pble.h+1−1
@@ -69,7 +69,7 @@ struct irdma_add_page_info { · 68 unmodified lines
69struct irdma_chunk {69struct irdma_chunk {
70 struct list_head list;70 struct list_head list;
71 struct irdma_dma_info dmainfo;71 struct irdma_dma_info dmainfo;
72 void *bitmapbuf;72 unsigned long *bitmapbuf;
73 73
74 u32 sizeofbitmap;74 u32 sizeofbitmap;
75 u64 size;75 u64 size;
drivers/infiniband/hw/irdma/verbs.c+10−21
@@ -21,7 +21,8 @@ static int irdma_query_device(struct ib_device *ibdev, · 20 unmodified lines
21 return -EINVAL;21 return -EINVAL;
22 22
23 memset(props, 0, sizeof(*props));23 memset(props, 0, sizeof(*props));
24 ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr);24 addrconf_addr_eui48((u8 *)&props->sys_image_guid,
25 iwdev->netdev->dev_addr);
25 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 |26 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 |
26 irdma_fw_minor_ver(&rf->sc_dev);27 irdma_fw_minor_ver(&rf->sc_dev);
27 props->device_cap_flags = iwdev->device_cap_flags;28 props->device_cap_flags = iwdev->device_cap_flags;
@@ -1170,6 +1171,10 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 1142 unmodified lines
1170 udp_info->ttl = attr->ah_attr.grh.hop_limit;1171 udp_info->ttl = attr->ah_attr.grh.hop_limit;
1171 udp_info->flow_label = attr->ah_attr.grh.flow_label;1172 udp_info->flow_label = attr->ah_attr.grh.flow_label;
1172 udp_info->tos = attr->ah_attr.grh.traffic_class;1173 udp_info->tos = attr->ah_attr.grh.traffic_class;
1174 udp_info->src_port =
1175 rdma_get_udp_sport(udp_info->flow_label,
1176 ibqp->qp_num,
1177 roce_info->dest_qp);
1173 irdma_qp_rem_qos(&iwqp->sc_qp);1178 irdma_qp_rem_qos(&iwqp->sc_qp);
1174 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri);1179 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri);
1175 ctx_info->user_pri = rt_tos2priority(udp_info->tos);1180 ctx_info->user_pri = rt_tos2priority(udp_info->tos);
@@ -4321,24 +4326,6 @@ static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, · 3145 unmodified lines
4321 return IB_LINK_LAYER_ETHERNET;4326 return IB_LINK_LAYER_ETHERNET;
4322}4327}
4323 4328
4324static __be64 irdma_mac_to_guid(struct net_device *ndev)
4325{
4326 const unsigned char *mac = ndev->dev_addr;
4327 __be64 guid;
4328 unsigned char *dst = (unsigned char *)&guid;
4329
4330 dst[0] = mac[0] ^ 2;
4331 dst[1] = mac[1];
4332 dst[2] = mac[2];
4333 dst[3] = 0xff;
4334 dst[4] = 0xfe;
4335 dst[5] = mac[3];
4336 dst[6] = mac[4];
4337 dst[7] = mac[5];
4338
4339 return guid;
4340}
4341
4342static const struct ib_device_ops irdma_roce_dev_ops = {4329static const struct ib_device_ops irdma_roce_dev_ops = {
4343 .attach_mcast = irdma_attach_mcast,4330 .attach_mcast = irdma_attach_mcast,
4344 .create_ah = irdma_create_ah,4331 .create_ah = irdma_create_ah,
@@ -4408,7 +4395,8 @@ static const struct ib_device_ops irdma_dev_ops = { · 63 unmodified lines
4408static void irdma_init_roce_device(struct irdma_device *iwdev)4395static void irdma_init_roce_device(struct irdma_device *iwdev)
4409{4396{
4410 iwdev->ibdev.node_type = RDMA_NODE_IB_CA;4397 iwdev->ibdev.node_type = RDMA_NODE_IB_CA;
4411 iwdev->ibdev.node_guid = irdma_mac_to_guid(iwdev->netdev);4398 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid,
4399 iwdev->netdev->dev_addr);
4412 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops);4400 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops);
4413}4401}
4414 4402
@@ -4421,7 +4409,8 @@ static int irdma_init_iw_device(struct irdma_device *iwdev) · 6 unmodified lines
4421 struct net_device *netdev = iwdev->netdev;4409 struct net_device *netdev = iwdev->netdev;
4422 4410
4423 iwdev->ibdev.node_type = RDMA_NODE_RNIC;4411 iwdev->ibdev.node_type = RDMA_NODE_RNIC;
4424 ether_addr_copy((u8 *)&iwdev->ibdev.node_guid, netdev->dev_addr);4412 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid,
4413 netdev->dev_addr);
4425 iwdev->ibdev.ops.iw_add_ref = irdma_qp_add_ref;4414 iwdev->ibdev.ops.iw_add_ref = irdma_qp_add_ref;
4426 iwdev->ibdev.ops.iw_rem_ref = irdma_qp_rem_ref;4415 iwdev->ibdev.ops.iw_rem_ref = irdma_qp_rem_ref;
4427 iwdev->ibdev.ops.iw_get_qp = irdma_get_qp;4416 iwdev->ibdev.ops.iw_get_qp = irdma_get_qp;
drivers/infiniband/hw/mlx4/main.c+12−22
@@ -85,14 +85,6 @@ static enum rdma_link_layer mlx4_ib_port_link_layer(struct ib_device *device, · 84 unmodified lines
85 85
86static struct workqueue_struct *wq;86static struct workqueue_struct *wq;
87 87
88static void init_query_mad(struct ib_smp *mad)
89{
90 mad->base_version = 1;
91 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
92 mad->class_version = 1;
93 mad->method = IB_MGMT_METHOD_GET;
94}
95
96static int check_flow_steering_support(struct mlx4_dev *dev)88static int check_flow_steering_support(struct mlx4_dev *dev)
97{89{
98 int eth_num_ports = 0;90 int eth_num_ports = 0;
@@ -471,7 +463,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, · 372 unmodified lines
471 if (!in_mad || !out_mad)463 if (!in_mad || !out_mad)
472 goto out;464 goto out;
473 465
474 init_query_mad(in_mad);466 ib_init_query_mad(in_mad);
475 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;467 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
476 468
477 err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS,469 err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS,
@@ -669,7 +661,7 @@ static int ib_link_query_port(struct ib_device *ibdev, u32 port, · 191 unmodified lines
669 if (!in_mad || !out_mad)661 if (!in_mad || !out_mad)
670 goto out;662 goto out;
671 663
672 init_query_mad(in_mad);664 ib_init_query_mad(in_mad);
673 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;665 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
674 in_mad->attr_mod = cpu_to_be32(port);666 in_mad->attr_mod = cpu_to_be32(port);
675 667
@@ -721,7 +713,7 @@ static int ib_link_query_port(struct ib_device *ibdev, u32 port, · 45 unmodified lines
721 713
722 /* If reported active speed is QDR, check if is FDR-10 */714 /* If reported active speed is QDR, check if is FDR-10 */
723 if (props->active_speed == IB_SPEED_QDR) {715 if (props->active_speed == IB_SPEED_QDR) {
724 init_query_mad(in_mad);716 ib_init_query_mad(in_mad);
725 in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO;717 in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO;
726 in_mad->attr_mod = cpu_to_be32(port);718 in_mad->attr_mod = cpu_to_be32(port);
727 719
@@ -848,7 +840,7 @@ int __mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index, · 120 unmodified lines
848 if (!in_mad || !out_mad)840 if (!in_mad || !out_mad)
849 goto out;841 goto out;
850 842
851 init_query_mad(in_mad);843 ib_init_query_mad(in_mad);
852 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;844 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
853 in_mad->attr_mod = cpu_to_be32(port);845 in_mad->attr_mod = cpu_to_be32(port);
854 846
@@ -870,7 +862,7 @@ int __mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index, · 15 unmodified lines
870 }862 }
871 }863 }
872 864
873 init_query_mad(in_mad);865 ib_init_query_mad(in_mad);
874 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;866 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;
875 in_mad->attr_mod = cpu_to_be32(index / 8);867 in_mad->attr_mod = cpu_to_be32(index / 8);
876 868
@@ -917,7 +909,7 @@ static int mlx4_ib_query_sl2vl(struct ib_device *ibdev, u32 port, · 40 unmodified lines
917 if (!in_mad || !out_mad)909 if (!in_mad || !out_mad)
918 goto out;910 goto out;
919 911
920 init_query_mad(in_mad);912 ib_init_query_mad(in_mad);
921 in_mad->attr_id = IB_SMP_ATTR_SL_TO_VL_TABLE;913 in_mad->attr_id = IB_SMP_ATTR_SL_TO_VL_TABLE;
922 in_mad->attr_mod = 0;914 in_mad->attr_mod = 0;
923 915
@@ -971,7 +963,7 @@ int __mlx4_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, · 47 unmodified lines
971 if (!in_mad || !out_mad)963 if (!in_mad || !out_mad)
972 goto out;964 goto out;
973 965
974 init_query_mad(in_mad);966 ib_init_query_mad(in_mad);
975 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;967 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;
976 in_mad->attr_mod = cpu_to_be32(index / 32);968 in_mad->attr_mod = cpu_to_be32(index / 32);
977 969
@@ -1990,7 +1982,7 @@ static int init_node_data(struct mlx4_ib_dev *dev) · 1012 unmodified lines
1990 if (!in_mad || !out_mad)1982 if (!in_mad || !out_mad)
1991 goto out;1983 goto out;
1992 1984
1993 init_query_mad(in_mad);1985 ib_init_query_mad(in_mad);
1994 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;1986 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
1995 if (mlx4_is_master(dev->dev))1987 if (mlx4_is_master(dev->dev))
1996 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;1988 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
@@ -2784,10 +2776,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) · 787 unmodified lines
2784 if (err)2776 if (err)
2785 goto err_counter;2777 goto err_counter;
2786 2778
2787 ibdev->ib_uc_qpns_bitmap =2779 ibdev->ib_uc_qpns_bitmap = bitmap_alloc(ibdev->steer_qpn_count,
2788 kmalloc_array(BITS_TO_LONGS(ibdev->steer_qpn_count),2780 GFP_KERNEL);
2789 sizeof(long),
2790 GFP_KERNEL);
2791 if (!ibdev->ib_uc_qpns_bitmap)2781 if (!ibdev->ib_uc_qpns_bitmap)
2792 goto err_steer_qp_release;2782 goto err_steer_qp_release;
2793 2783
@@ -2875,7 +2865,7 @@ err_diag_counters: · 81 unmodified lines
2875 mlx4_ib_diag_cleanup(ibdev);2865 mlx4_ib_diag_cleanup(ibdev);
2876 2866
2877err_steer_free_bitmap:2867err_steer_free_bitmap:
2878 kfree(ibdev->ib_uc_qpns_bitmap);2868 bitmap_free(ibdev->ib_uc_qpns_bitmap);
2879 2869
2880err_steer_qp_release:2870err_steer_qp_release:
2881 mlx4_qp_release_range(dev, ibdev->steer_qpn_base,2871 mlx4_qp_release_range(dev, ibdev->steer_qpn_base,
@@ -2988,7 +2978,7 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) · 106 unmodified lines
2988 2978
2989 mlx4_qp_release_range(dev, ibdev->steer_qpn_base,2979 mlx4_qp_release_range(dev, ibdev->steer_qpn_base,
2990 ibdev->steer_qpn_count);2980 ibdev->steer_qpn_count);
2991 kfree(ibdev->ib_uc_qpns_bitmap);2981 bitmap_free(ibdev->ib_uc_qpns_bitmap);
2992 2982
2993 iounmap(ibdev->uar_map);2983 iounmap(ibdev->uar_map);
2994 for (p = 0; p < ibdev->num_ports; ++p)2984 for (p = 0; p < ibdev->num_ports; ++p)
drivers/infiniband/hw/mlx5/cq.c+4−1
@@ -328,8 +328,11 @@ static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev, · 327 unmodified lines
328 }328 }
329 329
330 wc->vendor_err = cqe->vendor_err_synd;330 wc->vendor_err = cqe->vendor_err_synd;
331 if (dump)331 if (dump) {
332 mlx5_ib_warn(dev, "WC error: %d, Message: %s\n", wc->status,
333 ib_wc_status_msg(wc->status));
332 dump_cqe(dev, cqe);334 dump_cqe(dev, cqe);
335 }
333}336}
334 337
335static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,338static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,
drivers/infiniband/hw/mlx5/mad.c+14−9
@@ -291,7 +291,7 @@ int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, unsigned int port) · 290 unmodified lines
291 if (!in_mad || !out_mad)291 if (!in_mad || !out_mad)
292 goto out;292 goto out;
293 293
294 init_query_mad(in_mad);294 ib_init_query_mad(in_mad);
295 in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;295 in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;
296 in_mad->attr_mod = cpu_to_be32(port);296 in_mad->attr_mod = cpu_to_be32(port);
297 297
@@ -318,7 +318,7 @@ static int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev, · 20 unmodified lines
318 if (!in_mad)318 if (!in_mad)
319 return -ENOMEM;319 return -ENOMEM;
320 320
321 init_query_mad(in_mad);321 ib_init_query_mad(in_mad);
322 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;322 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
323 323
324 err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, 1, NULL, NULL, in_mad,324 err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, 1, NULL, NULL, in_mad,
@@ -405,7 +405,7 @@ int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc) · 80 unmodified lines
405 if (!in_mad || !out_mad)405 if (!in_mad || !out_mad)
406 goto out;406 goto out;
407 407
408 init_query_mad(in_mad);408 ib_init_query_mad(in_mad);
409 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;409 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
410 410
411 err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);411 err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);
@@ -430,7 +430,7 @@ int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid) · 18 unmodified lines
430 if (!in_mad || !out_mad)430 if (!in_mad || !out_mad)
431 goto out;431 goto out;
432 432
433 init_query_mad(in_mad);433 ib_init_query_mad(in_mad);
434 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;434 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
435 435
436 err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);436 err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);
@@ -456,7 +456,7 @@ int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u32 port, u16 index, · 19 unmodified lines
456 if (!in_mad || !out_mad)456 if (!in_mad || !out_mad)
457 goto out;457 goto out;
458 458
459 init_query_mad(in_mad);459 ib_init_query_mad(in_mad);
460 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;460 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;
461 in_mad->attr_mod = cpu_to_be32(index / 32);461 in_mad->attr_mod = cpu_to_be32(index / 32);
462 462
@@ -485,7 +485,7 @@ int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u32 port, int index, · 22 unmodified lines
485 if (!in_mad || !out_mad)485 if (!in_mad || !out_mad)
486 goto out;486 goto out;
487 487
488 init_query_mad(in_mad);488 ib_init_query_mad(in_mad);
489 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;489 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
490 in_mad->attr_mod = cpu_to_be32(port);490 in_mad->attr_mod = cpu_to_be32(port);
491 491
@@ -496,7 +496,7 @@ int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u32 port, int index, · 4 unmodified lines
496 496
497 memcpy(gid->raw, out_mad->data + 8, 8);497 memcpy(gid->raw, out_mad->data + 8, 8);
498 498
499 init_query_mad(in_mad);499 ib_init_query_mad(in_mad);
500 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;500 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;
501 in_mad->attr_mod = cpu_to_be32(index / 8);501 in_mad->attr_mod = cpu_to_be32(index / 8);
502 502
@@ -530,7 +530,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, · 27 unmodified lines
530 530
531 /* props being zeroed by the caller, avoid zeroing it here */531 /* props being zeroed by the caller, avoid zeroing it here */
532 532
533 init_query_mad(in_mad);533 ib_init_query_mad(in_mad);
534 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;534 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
535 in_mad->attr_mod = cpu_to_be32(port);535 in_mad->attr_mod = cpu_to_be32(port);
536 536
@@ -584,6 +584,11 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, · 47 unmodified lines
584 props->port_cap_flags2 & IB_PORT_LINK_SPEED_HDR_SUP)584 props->port_cap_flags2 & IB_PORT_LINK_SPEED_HDR_SUP)
585 props->active_speed = IB_SPEED_HDR;585 props->active_speed = IB_SPEED_HDR;
586 break;586 break;
587 case 8:
588 if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP &&
589 props->port_cap_flags2 & IB_PORT_LINK_SPEED_NDR_SUP)
590 props->active_speed = IB_SPEED_NDR;
591 break;
587 }592 }
588 }593 }
589 594
@@ -591,7 +596,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, · 1 unmodified line
591 if (props->active_speed == 4) {596 if (props->active_speed == 4) {
592 if (dev->port_caps[port - 1].ext_port_cap &597 if (dev->port_caps[port - 1].ext_port_cap &
593 MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) {598 MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) {
594 init_query_mad(in_mad);599 ib_init_query_mad(in_mad);
595 in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;600 in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;
596 in_mad->attr_mod = cpu_to_be32(port);601 in_mad->attr_mod = cpu_to_be32(port);
597 602
drivers/infiniband/hw/mlx5/mlx5_ib.h+2−10
@@ -665,9 +665,9 @@ struct mlx5_ib_mr { · 664 unmodified lines
665 665
666 /* User MR data */666 /* User MR data */
667 struct mlx5_cache_ent *cache_ent;667 struct mlx5_cache_ent *cache_ent;
668 /* Everything after cache_ent is zero'd when MR allocated */
668 struct ib_umem *umem;669 struct ib_umem *umem;
669 670
670 /* This is zero'd when the MR is allocated */
671 union {671 union {
672 /* Used only while the MR is in the cache */672 /* Used only while the MR is in the cache */
673 struct {673 struct {
@@ -719,7 +719,7 @@ struct mlx5_ib_mr { · 45 unmodified lines
719/* Zero the fields in the mr that are variant depending on usage */719/* Zero the fields in the mr that are variant depending on usage */
720static inline void mlx5_clear_mr(struct mlx5_ib_mr *mr)720static inline void mlx5_clear_mr(struct mlx5_ib_mr *mr)
721{721{
722 memset(mr->out, 0, sizeof(*mr) - offsetof(struct mlx5_ib_mr, out));722 memset_after(mr, 0, cache_ent);
723}723}
724 724
725static inline bool is_odp_mr(struct mlx5_ib_mr *mr)725static inline bool is_odp_mr(struct mlx5_ib_mr *mr)
@@ -1466,14 +1466,6 @@ extern const struct uapi_definition mlx5_ib_flow_defs[]; · 740 unmodified lines
1466extern const struct uapi_definition mlx5_ib_qos_defs[];1466extern const struct uapi_definition mlx5_ib_qos_defs[];
1467extern const struct uapi_definition mlx5_ib_std_types_defs[];1467extern const struct uapi_definition mlx5_ib_std_types_defs[];
1468 1468
1469static inline void init_query_mad(struct ib_smp *mad)
1470{
1471 mad->base_version = 1;
1472 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
1473 mad->class_version = 1;
1474 mad->method = IB_MGMT_METHOD_GET;
1475}
1476
1477static inline int is_qp1(enum ib_qp_type qp_type)1469static inline int is_qp1(enum ib_qp_type qp_type)
1478{1470{
1479 return qp_type == MLX5_IB_QPT_HW_GSI || qp_type == IB_QPT_GSI;1471 return qp_type == MLX5_IB_QPT_HW_GSI || qp_type == IB_QPT_GSI;
drivers/infiniband/hw/mthca/mthca_allocator.c+5−10
@@ -51,7 +51,7 @@ u32 mthca_alloc(struct mthca_alloc *alloc) · 50 unmodified lines
51 }51 }
52 52
53 if (obj < alloc->max) {53 if (obj < alloc->max) {
54 set_bit(obj, alloc->table);54 __set_bit(obj, alloc->table);
55 obj |= alloc->top;55 obj |= alloc->top;
56 } else56 } else
57 obj = -1;57 obj = -1;
@@ -69,7 +69,7 @@ void mthca_free(struct mthca_alloc *alloc, u32 obj) · 11 unmodified lines
69 69
70 spin_lock_irqsave(&alloc->lock, flags);70 spin_lock_irqsave(&alloc->lock, flags);
71 71
72 clear_bit(obj, alloc->table);72 __clear_bit(obj, alloc->table);
73 alloc->last = min(alloc->last, obj);73 alloc->last = min(alloc->last, obj);
74 alloc->top = (alloc->top + alloc->max) & alloc->mask;74 alloc->top = (alloc->top + alloc->max) & alloc->mask;
75 75
@@ -79,8 +79,6 @@ void mthca_free(struct mthca_alloc *alloc, u32 obj) · 3 unmodified lines
79int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask,79int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask,
80 u32 reserved)80 u32 reserved)
81{81{
82 int i;
83
84 /* num must be a power of 2 */82 /* num must be a power of 2 */
85 if (num != 1 << (ffs(num) - 1))83 if (num != 1 << (ffs(num) - 1))
86 return -EINVAL;84 return -EINVAL;
@@ -90,21 +88,18 @@ int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, · 3 unmodified lines
90 alloc->max = num;88 alloc->max = num;
91 alloc->mask = mask;89 alloc->mask = mask;
92 spin_lock_init(&alloc->lock);90 spin_lock_init(&alloc->lock);
93 alloc->table = kmalloc_array(BITS_TO_LONGS(num), sizeof(long),91 alloc->table = bitmap_zalloc(num, GFP_KERNEL);
94 GFP_KERNEL);
95 if (!alloc->table)92 if (!alloc->table)
96 return -ENOMEM;93 return -ENOMEM;
97 94
98 bitmap_zero(alloc->table, num);95 bitmap_set(alloc->table, 0, reserved);
99 for (i = 0; i < reserved; ++i)
100 set_bit(i, alloc->table);
101 96
102 return 0;97 return 0;
103}98}
104 99
105void mthca_alloc_cleanup(struct mthca_alloc *alloc)100void mthca_alloc_cleanup(struct mthca_alloc *alloc)
106{101{
107 kfree(alloc->table);102 bitmap_free(alloc->table);
108}103}
109 104
110/*105/*
drivers/infiniband/hw/mthca/mthca_mr.c+11−14
@@ -101,13 +101,13 @@ static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order) · 100 unmodified lines
101 return -1;101 return -1;
102 102
103 found:103 found:
104 clear_bit(seg, buddy->bits[o]);104 __clear_bit(seg, buddy->bits[o]);
105 --buddy->num_free[o];105 --buddy->num_free[o];
106 106
107 while (o > order) {107 while (o > order) {
108 --o;108 --o;
109 seg <<= 1;109 seg <<= 1;
110 set_bit(seg ^ 1, buddy->bits[o]);110 __set_bit(seg ^ 1, buddy->bits[o]);
111 ++buddy->num_free[o];111 ++buddy->num_free[o];
112 }112 }
113 113
@@ -125,13 +125,13 @@ static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) · 11 unmodified lines
125 spin_lock(&buddy->lock);125 spin_lock(&buddy->lock);
126 126
127 while (test_bit(seg ^ 1, buddy->bits[order])) {127 while (test_bit(seg ^ 1, buddy->bits[order])) {
128 clear_bit(seg ^ 1, buddy->bits[order]);128 __clear_bit(seg ^ 1, buddy->bits[order]);
129 --buddy->num_free[order];129 --buddy->num_free[order];
130 seg >>= 1;130 seg >>= 1;
131 ++order;131 ++order;
132 }132 }
133 133
134 set_bit(seg, buddy->bits[order]);134 __set_bit(seg, buddy->bits[order]);
135 ++buddy->num_free[order];135 ++buddy->num_free[order];
136 136
137 spin_unlock(&buddy->lock);137 spin_unlock(&buddy->lock);
@@ -139,7 +139,7 @@ static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) · 1 unmodified line
139 139
140static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order)140static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order)
141{141{
142 int i, s;142 int i;
143 143
144 buddy->max_order = max_order;144 buddy->max_order = max_order;
145 spin_lock_init(&buddy->lock);145 spin_lock_init(&buddy->lock);
@@ -152,22 +152,20 @@ static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order) · 6 unmodified lines
152 goto err_out;152 goto err_out;
153 153
154 for (i = 0; i <= buddy->max_order; ++i) {154 for (i = 0; i <= buddy->max_order; ++i) {
155 s = BITS_TO_LONGS(1 << (buddy->max_order - i));155 buddy->bits[i] = bitmap_zalloc(1 << (buddy->max_order - i),
156 buddy->bits[i] = kmalloc_array(s, sizeof(long), GFP_KERNEL);156 GFP_KERNEL);
157 if (!buddy->bits[i])157 if (!buddy->bits[i])
158 goto err_out_free;158 goto err_out_free;
159 bitmap_zero(buddy->bits[i],
160 1 << (buddy->max_order - i));
161 }159 }
162 160
163 set_bit(0, buddy->bits[buddy->max_order]);161 __set_bit(0, buddy->bits[buddy->max_order]);
164 buddy->num_free[buddy->max_order] = 1;162 buddy->num_free[buddy->max_order] = 1;
165 163
166 return 0;164 return 0;
167 165
168err_out_free:166err_out_free:
169 for (i = 0; i <= buddy->max_order; ++i)167 for (i = 0; i <= buddy->max_order; ++i)
170 kfree(buddy->bits[i]);168 bitmap_free(buddy->bits[i]);
171 169
172err_out:170err_out:
173 kfree(buddy->bits);171 kfree(buddy->bits);
@@ -181,7 +179,7 @@ static void mthca_buddy_cleanup(struct mthca_buddy *buddy) · 7 unmodified lines
181 int i;179 int i;
182 180
183 for (i = 0; i <= buddy->max_order; ++i)181 for (i = 0; i <= buddy->max_order; ++i)
184 kfree(buddy->bits[i]);182 bitmap_free(buddy->bits[i]);
185 183
186 kfree(buddy->bits);184 kfree(buddy->bits);
187 kfree(buddy->num_free);185 kfree(buddy->num_free);
@@ -469,8 +467,7 @@ int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, · 281 unmodified lines
469 mpt_entry->start = cpu_to_be64(iova);467 mpt_entry->start = cpu_to_be64(iova);
470 mpt_entry->length = cpu_to_be64(total_size);468 mpt_entry->length = cpu_to_be64(total_size);
471 469
472 memset(&mpt_entry->lkey, 0,470 memset_startat(mpt_entry, 0, lkey);
473 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
474 471
475 if (mr->mtt)472 if (mr->mtt)
476 mpt_entry->mtt_seg =473 mpt_entry->mtt_seg =
drivers/infiniband/hw/mthca/mthca_provider.c+6−14
@@ -50,14 +50,6 @@ · 49 unmodified lines
50#include <rdma/mthca-abi.h>50#include <rdma/mthca-abi.h>
51#include "mthca_memfree.h"51#include "mthca_memfree.h"
52 52
53static void init_query_mad(struct ib_smp *mad)
54{
55 mad->base_version = 1;
56 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
57 mad->class_version = 1;
58 mad->method = IB_MGMT_METHOD_GET;
59}
60
61static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,53static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
62 struct ib_udata *uhw)54 struct ib_udata *uhw)
63{55{
@@ -78,7 +70,7 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr · 14 unmodified lines
78 70
79 props->fw_ver = mdev->fw_ver;71 props->fw_ver = mdev->fw_ver;
80 72
81 init_query_mad(in_mad);73 ib_init_query_mad(in_mad);
82 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;74 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
83 75
84 err = mthca_MAD_IFC(mdev, 1, 1,76 err = mthca_MAD_IFC(mdev, 1, 1,
@@ -140,7 +132,7 @@ static int mthca_query_port(struct ib_device *ibdev, · 55 unmodified lines
140 132
141 /* props being zeroed by the caller, avoid zeroing it here */133 /* props being zeroed by the caller, avoid zeroing it here */
142 134
143 init_query_mad(in_mad);135 ib_init_query_mad(in_mad);
144 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;136 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
145 in_mad->attr_mod = cpu_to_be32(port);137 in_mad->attr_mod = cpu_to_be32(port);
146 138
@@ -234,7 +226,7 @@ static int mthca_query_pkey(struct ib_device *ibdev, · 87 unmodified lines
234 if (!in_mad || !out_mad)226 if (!in_mad || !out_mad)
235 goto out;227 goto out;
236 228
237 init_query_mad(in_mad);229 ib_init_query_mad(in_mad);
238 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;230 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;
239 in_mad->attr_mod = cpu_to_be32(index / 32);231 in_mad->attr_mod = cpu_to_be32(index / 32);
240 232
@@ -263,7 +255,7 @@ static int mthca_query_gid(struct ib_device *ibdev, u32 port, · 22 unmodified lines
263 if (!in_mad || !out_mad)255 if (!in_mad || !out_mad)
264 goto out;256 goto out;
265 257
266 init_query_mad(in_mad);258 ib_init_query_mad(in_mad);
267 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;259 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
268 in_mad->attr_mod = cpu_to_be32(port);260 in_mad->attr_mod = cpu_to_be32(port);
269 261
@@ -274,7 +266,7 @@ static int mthca_query_gid(struct ib_device *ibdev, u32 port, · 4 unmodified lines
274 266
275 memcpy(gid->raw, out_mad->data + 8, 8);267 memcpy(gid->raw, out_mad->data + 8, 8);
276 268
277 init_query_mad(in_mad);269 ib_init_query_mad(in_mad);
278 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;270 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;
279 in_mad->attr_mod = cpu_to_be32(index / 8);271 in_mad->attr_mod = cpu_to_be32(index / 8);
280 272
@@ -1006,7 +998,7 @@ static int mthca_init_node_data(struct mthca_dev *dev) · 725 unmodified lines
1006 if (!in_mad || !out_mad)998 if (!in_mad || !out_mad)
1007 goto out;999 goto out;
1008 1000
1009 init_query_mad(in_mad);1001 ib_init_query_mad(in_mad);
1010 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;1002 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
1011 1003
1012 err = mthca_MAD_IFC(dev, 1, 1,1004 err = mthca_MAD_IFC(dev, 1, 1,
drivers/infiniband/hw/ocrdma/ocrdma_hw.c+6−10
@@ -1506,7 +1506,6 @@ int ocrdma_mbx_dealloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd) · 1505 unmodified lines
1506static int ocrdma_mbx_alloc_pd_range(struct ocrdma_dev *dev)1506static int ocrdma_mbx_alloc_pd_range(struct ocrdma_dev *dev)
1507{1507{
1508 int status = -ENOMEM;1508 int status = -ENOMEM;
1509 size_t pd_bitmap_size;
1510 struct ocrdma_alloc_pd_range *cmd;1509 struct ocrdma_alloc_pd_range *cmd;
1511 struct ocrdma_alloc_pd_range_rsp *rsp;1510 struct ocrdma_alloc_pd_range_rsp *rsp;
1512 1511
@@ -1528,10 +1527,8 @@ static int ocrdma_mbx_alloc_pd_range(struct ocrdma_dev *dev) · 15 unmodified lines
1528 dev->pd_mgr->pd_dpp_start = rsp->dpp_page_pdid &1527 dev->pd_mgr->pd_dpp_start = rsp->dpp_page_pdid &
1529 OCRDMA_ALLOC_PD_RNG_RSP_START_PDID_MASK;1528 OCRDMA_ALLOC_PD_RNG_RSP_START_PDID_MASK;
1530 dev->pd_mgr->max_dpp_pd = rsp->pd_count;1529 dev->pd_mgr->max_dpp_pd = rsp->pd_count;
1531 pd_bitmap_size =1530 dev->pd_mgr->pd_dpp_bitmap = bitmap_zalloc(rsp->pd_count,
1532 BITS_TO_LONGS(rsp->pd_count) * sizeof(long);1531 GFP_KERNEL);
1533 dev->pd_mgr->pd_dpp_bitmap = kzalloc(pd_bitmap_size,
1534 GFP_KERNEL);
1535 }1532 }
1536 kfree(cmd);1533 kfree(cmd);
1537 }1534 }
@@ -1547,9 +1544,8 @@ static int ocrdma_mbx_alloc_pd_range(struct ocrdma_dev *dev) · 9 unmodified lines
1547 dev->pd_mgr->pd_norm_start = rsp->dpp_page_pdid &1544 dev->pd_mgr->pd_norm_start = rsp->dpp_page_pdid &
1548 OCRDMA_ALLOC_PD_RNG_RSP_START_PDID_MASK;1545 OCRDMA_ALLOC_PD_RNG_RSP_START_PDID_MASK;
1549 dev->pd_mgr->max_normal_pd = rsp->pd_count;1546 dev->pd_mgr->max_normal_pd = rsp->pd_count;
1550 pd_bitmap_size = BITS_TO_LONGS(rsp->pd_count) * sizeof(long);1547 dev->pd_mgr->pd_norm_bitmap = bitmap_zalloc(rsp->pd_count,
1551 dev->pd_mgr->pd_norm_bitmap = kzalloc(pd_bitmap_size,1548 GFP_KERNEL);
1552 GFP_KERNEL);
1553 }1549 }
1554 kfree(cmd);1550 kfree(cmd);
1555 1551
@@ -1611,8 +1607,8 @@ void ocrdma_alloc_pd_pool(struct ocrdma_dev *dev) · 55 unmodified lines
1611static void ocrdma_free_pd_pool(struct ocrdma_dev *dev)1607static void ocrdma_free_pd_pool(struct ocrdma_dev *dev)
1612{1608{
1613 ocrdma_mbx_dealloc_pd_range(dev);1609 ocrdma_mbx_dealloc_pd_range(dev);
1614 kfree(dev->pd_mgr->pd_norm_bitmap);1610 bitmap_free(dev->pd_mgr->pd_norm_bitmap);
1615 kfree(dev->pd_mgr->pd_dpp_bitmap);1611 bitmap_free(dev->pd_mgr->pd_dpp_bitmap);
1616 kfree(dev->pd_mgr);1612 kfree(dev->pd_mgr);
1617}1613}
1618 1614
drivers/infiniband/hw/ocrdma/ocrdma_main.c+2−15
@@ -62,20 +62,6 @@ MODULE_DESCRIPTION(OCRDMA_ROCE_DRV_DESC " " OCRDMA_ROCE_DRV_VERSION); · 61 unmodified lines
62MODULE_AUTHOR("Emulex Corporation");62MODULE_AUTHOR("Emulex Corporation");
63MODULE_LICENSE("Dual BSD/GPL");63MODULE_LICENSE("Dual BSD/GPL");
64 64
65void ocrdma_get_guid(struct ocrdma_dev *dev, u8 *guid)
66{
67 u8 mac_addr[6];
68
69 memcpy(&mac_addr[0], &dev->nic_info.mac_addr[0], ETH_ALEN);
70 guid[0] = mac_addr[0] ^ 2;
71 guid[1] = mac_addr[1];
72 guid[2] = mac_addr[2];
73 guid[3] = 0xff;
74 guid[4] = 0xfe;
75 guid[5] = mac_addr[3];
76 guid[6] = mac_addr[4];
77 guid[7] = mac_addr[5];
78}
79static enum rdma_link_layer ocrdma_link_layer(struct ib_device *device,65static enum rdma_link_layer ocrdma_link_layer(struct ib_device *device,
80 u32 port_num)66 u32 port_num)
81{67{
@@ -203,7 +189,8 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) · 121 unmodified lines
203{189{
204 int ret;190 int ret;
205 191
206 ocrdma_get_guid(dev, (u8 *)&dev->ibdev.node_guid);192 addrconf_addr_eui48((u8 *)&dev->ibdev.node_guid,
193 dev->nic_info.mac_addr);
207 BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);194 BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
208 memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC,195 memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC,
209 sizeof(OCRDMA_NODE_DESC));196 sizeof(OCRDMA_NODE_DESC));
drivers/infiniband/hw/ocrdma/ocrdma_verbs.c+8−10
@@ -41,6 +41,7 @@ · 40 unmodified lines
41 */41 */
42 42
43#include <linux/dma-mapping.h>43#include <linux/dma-mapping.h>
44#include <net/addrconf.h>
44#include <rdma/ib_verbs.h>45#include <rdma/ib_verbs.h>
45#include <rdma/ib_user_verbs.h>46#include <rdma/ib_user_verbs.h>
46#include <rdma/iw_cm.h>47#include <rdma/iw_cm.h>
@@ -74,7 +75,8 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, · 27 unmodified lines
74 memset(attr, 0, sizeof *attr);75 memset(attr, 0, sizeof *attr);
75 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],76 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
76 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));77 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
77 ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid);78 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
79 dev->nic_info.mac_addr);
78 attr->max_mr_size = dev->attr.max_mr_size;80 attr->max_mr_size = dev->attr.max_mr_size;
79 attr->page_size_cap = 0xffff000;81 attr->page_size_cap = 0xffff000;
80 attr->vendor_id = dev->nic_info.pdev->vendor;82 attr->vendor_id = dev->nic_info.pdev->vendor;
@@ -245,13 +247,13 @@ static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, · 164 unmodified lines
245static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool)247static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool)
246{248{
247 u16 pd_bitmap_idx = 0;249 u16 pd_bitmap_idx = 0;
248 const unsigned long *pd_bitmap;250 unsigned long *pd_bitmap;
249 251
250 if (dpp_pool) {252 if (dpp_pool) {
251 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap;253 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap;
252 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,254 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,
253 dev->pd_mgr->max_dpp_pd);255 dev->pd_mgr->max_dpp_pd);
254 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_dpp_bitmap);256 __set_bit(pd_bitmap_idx, pd_bitmap);
255 dev->pd_mgr->pd_dpp_count++;257 dev->pd_mgr->pd_dpp_count++;
256 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh)258 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh)
257 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count;259 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count;
@@ -259,7 +261,7 @@ static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) · 1 unmodified line
259 pd_bitmap = dev->pd_mgr->pd_norm_bitmap;261 pd_bitmap = dev->pd_mgr->pd_norm_bitmap;
260 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,262 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,
261 dev->pd_mgr->max_normal_pd);263 dev->pd_mgr->max_normal_pd);
262 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_norm_bitmap);264 __set_bit(pd_bitmap_idx, pd_bitmap);
263 dev->pd_mgr->pd_norm_count++;265 dev->pd_mgr->pd_norm_count++;
264 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh)266 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh)
265 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count;267 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count;
@@ -1844,12 +1846,10 @@ int ocrdma_modify_srq(struct ib_srq *ibsrq, · 1578 unmodified lines
1844 1846
1845int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)1847int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
1846{1848{
1847 int status;
1848 struct ocrdma_srq *srq;1849 struct ocrdma_srq *srq;
1849 1850
1850 srq = get_ocrdma_srq(ibsrq);1851 srq = get_ocrdma_srq(ibsrq);
1851 status = ocrdma_mbx_query_srq(srq, srq_attr);1852 return ocrdma_mbx_query_srq(srq, srq_attr);
1852 return status;
1853}1853}
1854 1854
1855int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)1855int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
@@ -1960,7 +1960,6 @@ static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, · 104 unmodified lines
1960static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,1960static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1961 const struct ib_send_wr *wr)1961 const struct ib_send_wr *wr)
1962{1962{
1963 int status;
1964 struct ocrdma_sge *sge;1963 struct ocrdma_sge *sge;
1965 u32 wqe_size = sizeof(*hdr);1964 u32 wqe_size = sizeof(*hdr);
1966 1965
@@ -1972,8 +1971,7 @@ static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, · 5 unmodified lines
1972 sge = (struct ocrdma_sge *)(hdr + 1);1971 sge = (struct ocrdma_sge *)(hdr + 1);
1973 }1972 }
1974 1973
1975 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);1974 return ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1976 return status;
1977}1975}
1978 1976
1979static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,1977static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
drivers/infiniband/hw/ocrdma/ocrdma_verbs.h+0−1
@@ -59,7 +59,6 @@ int ocrdma_query_port(struct ib_device *ibdev, u32 port, · 58 unmodified lines
59enum rdma_protocol_type59enum rdma_protocol_type
60ocrdma_query_protocol(struct ib_device *device, u32 port_num);60ocrdma_query_protocol(struct ib_device *device, u32 port_num);
61 61
62void ocrdma_get_guid(struct ocrdma_dev *, u8 *guid);
63int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);62int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);
64 63
65int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata);64int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata);
drivers/infiniband/hw/qedr/verbs.c+2−0
@@ -1931,6 +1931,7 @@ static int qedr_create_user_qp(struct qedr_dev *dev, · 1930 unmodified lines
1931 /* db offset was calculated in copy_qp_uresp, now set in the user q */1931 /* db offset was calculated in copy_qp_uresp, now set in the user q */
1932 if (qedr_qp_has_sq(qp)) {1932 if (qedr_qp_has_sq(qp)) {
1933 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;1933 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1934 qp->sq.max_wr = attrs->cap.max_send_wr;
1934 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,1935 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1935 &qp->usq.db_rec_data->db_data,1936 &qp->usq.db_rec_data->db_data,
1936 DB_REC_WIDTH_32B,1937 DB_REC_WIDTH_32B,
@@ -1941,6 +1942,7 @@ static int qedr_create_user_qp(struct qedr_dev *dev, · 4 unmodified lines
1941 1942
1942 if (qedr_qp_has_rq(qp)) {1943 if (qedr_qp_has_rq(qp)) {
1943 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;1944 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1945 qp->rq.max_wr = attrs->cap.max_recv_wr;
1944 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,1946 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1945 &qp->urq.db_rec_data->db_data,1947 &qp->urq.db_rec_data->db_data,
1946 DB_REC_WIDTH_32B,1948 DB_REC_WIDTH_32B,
drivers/infiniband/hw/qib/qib_iba6120.c+1−1
@@ -3030,7 +3030,7 @@ static int qib_6120_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs) · 3029 unmodified lines
3030 3030
3031/* Does read/modify/write to appropriate registers to3031/* Does read/modify/write to appropriate registers to
3032 * set output and direction bits selected by mask.3032 * set output and direction bits selected by mask.
3033 * these are in their canonical postions (e.g. lsb of3033 * these are in their canonical positions (e.g. lsb of
3034 * dir will end up in D48 of extctrl on existing chips).3034 * dir will end up in D48 of extctrl on existing chips).
3035 * returns contents of GP Inputs.3035 * returns contents of GP Inputs.
3036 */3036 */
drivers/infiniband/hw/qib/qib_iba7220.c+1−1
@@ -3742,7 +3742,7 @@ static int qib_7220_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs) · 3741 unmodified lines
3742/*3742/*
3743 * Does read/modify/write to appropriate registers to3743 * Does read/modify/write to appropriate registers to
3744 * set output and direction bits selected by mask.3744 * set output and direction bits selected by mask.
3745 * these are in their canonical postions (e.g. lsb of3745 * these are in their canonical positions (e.g. lsb of
3746 * dir will end up in D48 of extctrl on existing chips).3746 * dir will end up in D48 of extctrl on existing chips).
3747 * returns contents of GP Inputs.3747 * returns contents of GP Inputs.
3748 */3748 */
drivers/infiniband/hw/qib/qib_iba7322.c+1−1
@@ -5665,7 +5665,7 @@ static int qib_7322_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs) · 5664 unmodified lines
5665/*5665/*
5666 * Does read/modify/write to appropriate registers to5666 * Does read/modify/write to appropriate registers to
5667 * set output and direction bits selected by mask.5667 * set output and direction bits selected by mask.
5668 * these are in their canonical postions (e.g. lsb of5668 * these are in their canonical positions (e.g. lsb of
5669 * dir will end up in D48 of extctrl on existing chips).5669 * dir will end up in D48 of extctrl on existing chips).
5670 * returns contents of GP Inputs.5670 * returns contents of GP Inputs.
5671 */5671 */
drivers/infiniband/hw/usnic/usnic_ib_sysfs.c+2−1
@@ -243,10 +243,11 @@ static struct attribute *usnic_ib_qpn_default_attrs[] = { · 242 unmodified lines
243 &qpn_attr_summary.attr,243 &qpn_attr_summary.attr,
244 NULL244 NULL
245};245};
246ATTRIBUTE_GROUPS(usnic_ib_qpn_default);
246 247
247static struct kobj_type usnic_ib_qpn_type = {248static struct kobj_type usnic_ib_qpn_type = {
248 .sysfs_ops = &usnic_ib_qpn_sysfs_ops,249 .sysfs_ops = &usnic_ib_qpn_sysfs_ops,
249 .default_attrs = usnic_ib_qpn_default_attrs250 .default_groups = usnic_ib_qpn_default_groups,
250};251};
251 252
252int usnic_ib_sysfs_register_usdev(struct usnic_ib_dev *us_ibdev)253int usnic_ib_sysfs_register_usdev(struct usnic_ib_dev *us_ibdev)
drivers/infiniband/hw/usnic/usnic_ib_verbs.c+3−5
@@ -442,12 +442,10 @@ int usnic_ib_query_gid(struct ib_device *ibdev, u32 port, int index, · 441 unmodified lines
442int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)442int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
443{443{
444 struct usnic_ib_pd *pd = to_upd(ibpd);444 struct usnic_ib_pd *pd = to_upd(ibpd);
445 void *umem_pd;
446 445
447 umem_pd = pd->umem_pd = usnic_uiom_alloc_pd();446 pd->umem_pd = usnic_uiom_alloc_pd();
448 if (IS_ERR_OR_NULL(umem_pd)) {447 if (IS_ERR(pd->umem_pd))
449 return umem_pd ? PTR_ERR(umem_pd) : -ENOMEM;448 return PTR_ERR(pd->umem_pd);
450 }
451 449
452 return 0;450 return 0;
453}451}
drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c+5−5
@@ -63,12 +63,12 @@ int pvrdma_uar_table_init(struct pvrdma_dev *dev) · 62 unmodified lines
63 tbl->max = num;63 tbl->max = num;
64 tbl->mask = mask;64 tbl->mask = mask;
65 spin_lock_init(&tbl->lock);65 spin_lock_init(&tbl->lock);
66 tbl->table = kcalloc(BITS_TO_LONGS(num), sizeof(long), GFP_KERNEL);66 tbl->table = bitmap_zalloc(num, GFP_KERNEL);
67 if (!tbl->table)67 if (!tbl->table)
68 return -ENOMEM;68 return -ENOMEM;
69 69
70 /* 0th UAR is taken by the device. */70 /* 0th UAR is taken by the device. */
71 set_bit(0, tbl->table);71 __set_bit(0, tbl->table);
72 72
73 return 0;73 return 0;
74}74}
@@ -77,7 +77,7 @@ void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev) · 2 unmodified lines
77{77{
78 struct pvrdma_id_table *tbl = &dev->uar_table.tbl;78 struct pvrdma_id_table *tbl = &dev->uar_table.tbl;
79 79
80 kfree(tbl->table);80 bitmap_free(tbl->table);
81}81}
82 82
83int pvrdma_uar_alloc(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar)83int pvrdma_uar_alloc(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar)
@@ -100,7 +100,7 @@ int pvrdma_uar_alloc(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar) · 16 unmodified lines
100 return -ENOMEM;100 return -ENOMEM;
101 }101 }
102 102
103 set_bit(obj, tbl->table);103 __set_bit(obj, tbl->table);
104 obj |= tbl->top;104 obj |= tbl->top;
105 105
106 spin_unlock_irqrestore(&tbl->lock, flags);106 spin_unlock_irqrestore(&tbl->lock, flags);
@@ -120,7 +120,7 @@ void pvrdma_uar_free(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar) · 13 unmodified lines
120 120
121 obj = uar->index & (tbl->max - 1);121 obj = uar->index & (tbl->max - 1);
122 spin_lock_irqsave(&tbl->lock, flags);122 spin_lock_irqsave(&tbl->lock, flags);
123 clear_bit(obj, tbl->table);123 __clear_bit(obj, tbl->table);
124 tbl->last = min(tbl->last, obj);124 tbl->last = min(tbl->last, obj);
125 tbl->top = (tbl->top + tbl->max) & tbl->mask;125 tbl->top = (tbl->top + tbl->max) & tbl->mask;
126 spin_unlock_irqrestore(&tbl->lock, flags);126 spin_unlock_irqrestore(&tbl->lock, flags);
drivers/infiniband/sw/rxe/Makefile+0−1
@@ -22,5 +22,4 @@ rdma_rxe-y := \ · 21 unmodified lines
22 rxe_mcast.o \22 rxe_mcast.o \
23 rxe_task.o \23 rxe_task.o \
24 rxe_net.o \24 rxe_net.o \
25 rxe_sysfs.o \
26 rxe_hw_counters.o25 rxe_hw_counters.o
drivers/infiniband/sw/rxe/rxe.c+0−4
@@ -13,8 +13,6 @@ MODULE_AUTHOR("Bob Pearson, Frank Zago, John Groves, Kamal Heib"); · 12 unmodified lines
13MODULE_DESCRIPTION("Soft RDMA transport");13MODULE_DESCRIPTION("Soft RDMA transport");
14MODULE_LICENSE("Dual BSD/GPL");14MODULE_LICENSE("Dual BSD/GPL");
15 15
16bool rxe_initialized;
17
18/* free resources for a rxe device all objects created for this device must16/* free resources for a rxe device all objects created for this device must
19 * have been destroyed17 * have been destroyed
20 */18 */
@@ -290,7 +288,6 @@ static int __init rxe_module_init(void) · 269 unmodified lines
290 return err;288 return err;
291 289
292 rdma_link_register(&rxe_link_ops);290 rdma_link_register(&rxe_link_ops);
293 rxe_initialized = true;
294 pr_info("loaded\n");291 pr_info("loaded\n");
295 return 0;292 return 0;
296}293}
@@ -301,7 +298,6 @@ static void __exit rxe_module_exit(void) · 4 unmodified lines
301 ib_unregister_driver(RDMA_DRIVER_RXE);298 ib_unregister_driver(RDMA_DRIVER_RXE);
302 rxe_net_exit();299 rxe_net_exit();
303 300
304 rxe_initialized = false;
305 pr_info("unloaded\n");301 pr_info("unloaded\n");
306}302}
307 303
drivers/infiniband/sw/rxe/rxe.h+0−2
@@ -39,8 +39,6 @@ · 38 unmodified lines
39 39
40#define RXE_ROCE_V2_SPORT (0xc000)40#define RXE_ROCE_V2_SPORT (0xc000)
41 41
42extern bool rxe_initialized;
43
44void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);42void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
45 43
46int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name);44int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name);
drivers/infiniband/sw/rxe/rxe_comp.c+3−5
@@ -458,8 +458,6 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, · 457 unmodified lines
458 struct rxe_pkt_info *pkt,458 struct rxe_pkt_info *pkt,
459 struct rxe_send_wqe *wqe)459 struct rxe_send_wqe *wqe)
460{460{
461 unsigned long flags;
462
463 if (wqe->has_rd_atomic) {461 if (wqe->has_rd_atomic) {
464 wqe->has_rd_atomic = 0;462 wqe->has_rd_atomic = 0;
465 atomic_inc(&qp->req.rd_atomic);463 atomic_inc(&qp->req.rd_atomic);
@@ -472,11 +470,11 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, · 6 unmodified lines
472 470
473 if (unlikely(qp->req.state == QP_STATE_DRAIN)) {471 if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
474 /* state_lock used by requester & completer */472 /* state_lock used by requester & completer */
475 spin_lock_irqsave(&qp->state_lock, flags);473 spin_lock_bh(&qp->state_lock);
476 if ((qp->req.state == QP_STATE_DRAIN) &&474 if ((qp->req.state == QP_STATE_DRAIN) &&
477 (qp->comp.psn == qp->req.psn)) {475 (qp->comp.psn == qp->req.psn)) {
478 qp->req.state = QP_STATE_DRAINED;476 qp->req.state = QP_STATE_DRAINED;
479 spin_unlock_irqrestore(&qp->state_lock, flags);477 spin_unlock_bh(&qp->state_lock);
480 478
481 if (qp->ibqp.event_handler) {479 if (qp->ibqp.event_handler) {
482 struct ib_event ev;480 struct ib_event ev;
@@ -488,7 +486,7 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, · 5 unmodified lines
488 qp->ibqp.qp_context);486 qp->ibqp.qp_context);
489 }487 }
490 } else {488 } else {
491 spin_unlock_irqrestore(&qp->state_lock, flags);489 spin_unlock_bh(&qp->state_lock);
492 }490 }
493 }491 }
494 492
drivers/infiniband/sw/rxe/rxe_cq.c+10−14
@@ -42,14 +42,13 @@ err1: · 41 unmodified lines
42static void rxe_send_complete(struct tasklet_struct *t)42static void rxe_send_complete(struct tasklet_struct *t)
43{43{
44 struct rxe_cq *cq = from_tasklet(cq, t, comp_task);44 struct rxe_cq *cq = from_tasklet(cq, t, comp_task);
45 unsigned long flags;
46 45
47 spin_lock_irqsave(&cq->cq_lock, flags);46 spin_lock_bh(&cq->cq_lock);
48 if (cq->is_dying) {47 if (cq->is_dying) {
49 spin_unlock_irqrestore(&cq->cq_lock, flags);48 spin_unlock_bh(&cq->cq_lock);
50 return;49 return;
51 }50 }
52 spin_unlock_irqrestore(&cq->cq_lock, flags);51 spin_unlock_bh(&cq->cq_lock);
53 52
54 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);53 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
55}54}
@@ -106,15 +105,14 @@ int rxe_cq_resize_queue(struct rxe_cq *cq, int cqe, · 50 unmodified lines
106int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)105int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)
107{106{
108 struct ib_event ev;107 struct ib_event ev;
109 unsigned long flags;
110 int full;108 int full;
111 void *addr;109 void *addr;
112 110
113 spin_lock_irqsave(&cq->cq_lock, flags);111 spin_lock_bh(&cq->cq_lock);
114 112
115 full = queue_full(cq->queue, QUEUE_TYPE_TO_CLIENT);113 full = queue_full(cq->queue, QUEUE_TYPE_TO_CLIENT);
116 if (unlikely(full)) {114 if (unlikely(full)) {
117 spin_unlock_irqrestore(&cq->cq_lock, flags);115 spin_unlock_bh(&cq->cq_lock);
118 if (cq->ibcq.event_handler) {116 if (cq->ibcq.event_handler) {
119 ev.device = cq->ibcq.device;117 ev.device = cq->ibcq.device;
120 ev.element.cq = &cq->ibcq;118 ev.element.cq = &cq->ibcq;
@@ -130,7 +128,7 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited) · 9 unmodified lines
130 128
131 queue_advance_producer(cq->queue, QUEUE_TYPE_TO_CLIENT);129 queue_advance_producer(cq->queue, QUEUE_TYPE_TO_CLIENT);
132 130
133 spin_unlock_irqrestore(&cq->cq_lock, flags);131 spin_unlock_bh(&cq->cq_lock);
134 132
135 if ((cq->notify == IB_CQ_NEXT_COMP) ||133 if ((cq->notify == IB_CQ_NEXT_COMP) ||
136 (cq->notify == IB_CQ_SOLICITED && solicited)) {134 (cq->notify == IB_CQ_SOLICITED && solicited)) {
@@ -143,16 +141,14 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited) · 6 unmodified lines
143 141
144void rxe_cq_disable(struct rxe_cq *cq)142void rxe_cq_disable(struct rxe_cq *cq)
145{143{
146 unsigned long flags;144 spin_lock_bh(&cq->cq_lock);
147
148 spin_lock_irqsave(&cq->cq_lock, flags);
149 cq->is_dying = true;145 cq->is_dying = true;
150 spin_unlock_irqrestore(&cq->cq_lock, flags);146 spin_unlock_bh(&cq->cq_lock);
151}147}
152 148
153void rxe_cq_cleanup(struct rxe_pool_entry *arg)149void rxe_cq_cleanup(struct rxe_pool_elem *elem)
154{150{
155 struct rxe_cq *cq = container_of(arg, typeof(*cq), pelem);151 struct rxe_cq *cq = container_of(elem, typeof(*cq), elem);
156 152
157 if (cq->queue)153 if (cq->queue)
158 rxe_queue_cleanup(cq->queue);154 rxe_queue_cleanup(cq->queue);
drivers/infiniband/sw/rxe/rxe_loc.h+5−5
@@ -37,7 +37,7 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited); · 36 unmodified lines
37 37
38void rxe_cq_disable(struct rxe_cq *cq);38void rxe_cq_disable(struct rxe_cq *cq);
39 39
40void rxe_cq_cleanup(struct rxe_pool_entry *arg);40void rxe_cq_cleanup(struct rxe_pool_elem *arg);
41 41
42/* rxe_mcast.c */42/* rxe_mcast.c */
43int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid,43int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid,
@@ -51,7 +51,7 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, · 7 unmodified lines
51 51
52void rxe_drop_all_mcast_groups(struct rxe_qp *qp);52void rxe_drop_all_mcast_groups(struct rxe_qp *qp);
53 53
54void rxe_mc_cleanup(struct rxe_pool_entry *arg);54void rxe_mc_cleanup(struct rxe_pool_elem *arg);
55 55
56/* rxe_mmap.c */56/* rxe_mmap.c */
57struct rxe_mmap_info {57struct rxe_mmap_info {
@@ -89,7 +89,7 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey); · 31 unmodified lines
89int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);89int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
90int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr);90int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr);
91int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);91int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
92void rxe_mr_cleanup(struct rxe_pool_entry *arg);92void rxe_mr_cleanup(struct rxe_pool_elem *arg);
93 93
94/* rxe_mw.c */94/* rxe_mw.c */
95int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);95int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
@@ -97,7 +97,7 @@ int rxe_dealloc_mw(struct ib_mw *ibmw); · 1 unmodified line
97int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe);97int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
98int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey);98int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey);
99struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey);99struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey);
100void rxe_mw_cleanup(struct rxe_pool_entry *arg);100void rxe_mw_cleanup(struct rxe_pool_elem *arg);
101 101
102/* rxe_net.c */102/* rxe_net.c */
103struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,103struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
@@ -131,7 +131,7 @@ void rxe_qp_error(struct rxe_qp *qp); · 27 unmodified lines
131 131
132void rxe_qp_destroy(struct rxe_qp *qp);132void rxe_qp_destroy(struct rxe_qp *qp);
133 133
134void rxe_qp_cleanup(struct rxe_pool_entry *arg);134void rxe_qp_cleanup(struct rxe_pool_elem *elem);
135 135
136static inline int qp_num(struct rxe_qp *qp)136static inline int qp_num(struct rxe_qp *qp)
137{137{
drivers/infiniband/sw/rxe/rxe_mcast.c+5−6
@@ -40,12 +40,11 @@ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid, · 39 unmodified lines
40 int err;40 int err;
41 struct rxe_mc_grp *grp;41 struct rxe_mc_grp *grp;
42 struct rxe_pool *pool = &rxe->mc_grp_pool;42 struct rxe_pool *pool = &rxe->mc_grp_pool;
43 unsigned long flags;
44 43
45 if (rxe->attr.max_mcast_qp_attach == 0)44 if (rxe->attr.max_mcast_qp_attach == 0)
46 return -EINVAL;45 return -EINVAL;
47 46
48 write_lock_irqsave(&pool->pool_lock, flags);47 write_lock_bh(&pool->pool_lock);
49 48
50 grp = rxe_pool_get_key_locked(pool, mgid);49 grp = rxe_pool_get_key_locked(pool, mgid);
51 if (grp)50 if (grp)
@@ -53,13 +52,13 @@ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid, · 1 unmodified line
53 52
54 grp = create_grp(rxe, pool, mgid);53 grp = create_grp(rxe, pool, mgid);
55 if (IS_ERR(grp)) {54 if (IS_ERR(grp)) {
56 write_unlock_irqrestore(&pool->pool_lock, flags);55 write_unlock_bh(&pool->pool_lock);
57 err = PTR_ERR(grp);56 err = PTR_ERR(grp);
58 return err;57 return err;
59 }58 }
60 59
61done:60done:
62 write_unlock_irqrestore(&pool->pool_lock, flags);61 write_unlock_bh(&pool->pool_lock);
63 *grp_p = grp;62 *grp_p = grp;
64 return 0;63 return 0;
65}64}
@@ -169,9 +168,9 @@ void rxe_drop_all_mcast_groups(struct rxe_qp *qp) · 103 unmodified lines
169 }168 }
170}169}
171 170
172void rxe_mc_cleanup(struct rxe_pool_entry *arg)171void rxe_mc_cleanup(struct rxe_pool_elem *elem)
173{172{
174 struct rxe_mc_grp *grp = container_of(arg, typeof(*grp), pelem);173 struct rxe_mc_grp *grp = container_of(elem, typeof(*grp), elem);
175 struct rxe_dev *rxe = grp->rxe;174 struct rxe_dev *rxe = grp->rxe;
176 175
177 rxe_drop_key(grp);176 rxe_drop_key(grp);
drivers/infiniband/sw/rxe/rxe_mr.c+3−3
@@ -50,7 +50,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) · 49 unmodified lines
50 50
51static void rxe_mr_init(int access, struct rxe_mr *mr)51static void rxe_mr_init(int access, struct rxe_mr *mr)
52{52{
53 u32 lkey = mr->pelem.index << 8 | rxe_get_next_key(-1);53 u32 lkey = mr->elem.index << 8 | rxe_get_next_key(-1);
54 u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;54 u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;
55 55
56 /* set ibmr->l/rkey and also copy into private l/rkey56 /* set ibmr->l/rkey and also copy into private l/rkey
@@ -697,9 +697,9 @@ int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) · 640 unmodified lines
697 return 0;697 return 0;
698}698}
699 699
700void rxe_mr_cleanup(struct rxe_pool_entry *arg)700void rxe_mr_cleanup(struct rxe_pool_elem *elem)
701{701{
702 struct rxe_mr *mr = container_of(arg, typeof(*mr), pelem);702 struct rxe_mr *mr = container_of(elem, typeof(*mr), elem);
703 703
704 ib_umem_release(mr->umem);704 ib_umem_release(mr->umem);
705 705
drivers/infiniband/sw/rxe/rxe_mw.c+9−12
@@ -21,7 +21,7 @@ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) · 20 unmodified lines
21 }21 }
22 22
23 rxe_add_index(mw);23 rxe_add_index(mw);
24 mw->rkey = ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);24 mw->rkey = ibmw->rkey = (mw->elem.index << 8) | rxe_get_next_key(-1);
25 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?25 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?
26 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;26 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;
27 spin_lock_init(&mw->lock);27 spin_lock_init(&mw->lock);
@@ -56,11 +56,10 @@ int rxe_dealloc_mw(struct ib_mw *ibmw) · 28 unmodified lines
56{56{
57 struct rxe_mw *mw = to_rmw(ibmw);57 struct rxe_mw *mw = to_rmw(ibmw);
58 struct rxe_pd *pd = to_rpd(ibmw->pd);58 struct rxe_pd *pd = to_rpd(ibmw->pd);
59 unsigned long flags;
60 59
61 spin_lock_irqsave(&mw->lock, flags);60 spin_lock_bh(&mw->lock);
62 rxe_do_dealloc_mw(mw);61 rxe_do_dealloc_mw(mw);
63 spin_unlock_irqrestore(&mw->lock, flags);62 spin_unlock_bh(&mw->lock);
64 63
65 rxe_drop_ref(mw);64 rxe_drop_ref(mw);
66 rxe_drop_ref(pd);65 rxe_drop_ref(pd);
@@ -197,7 +196,6 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) · 130 unmodified lines
197 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);196 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
198 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;197 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;
199 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;198 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;
200 unsigned long flags;
201 199
202 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);200 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);
203 if (unlikely(!mw)) {201 if (unlikely(!mw)) {
@@ -225,7 +223,7 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) · 21 unmodified lines
225 mr = NULL;223 mr = NULL;
226 }224 }
227 225
228 spin_lock_irqsave(&mw->lock, flags);226 spin_lock_bh(&mw->lock);
229 227
230 ret = rxe_check_bind_mw(qp, wqe, mw, mr);228 ret = rxe_check_bind_mw(qp, wqe, mw, mr);
231 if (ret)229 if (ret)
@@ -233,7 +231,7 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) · 1 unmodified line
233 231
234 rxe_do_bind_mw(qp, wqe, mw, mr);232 rxe_do_bind_mw(qp, wqe, mw, mr);
235err_unlock:233err_unlock:
236 spin_unlock_irqrestore(&mw->lock, flags);234 spin_unlock_bh(&mw->lock);
237err_drop_mr:235err_drop_mr:
238 if (mr)236 if (mr)
239 rxe_drop_ref(mr);237 rxe_drop_ref(mr);
@@ -280,7 +278,6 @@ static void rxe_do_invalidate_mw(struct rxe_mw *mw) · 40 unmodified lines
280int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)278int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)
281{279{
282 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);280 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
283 unsigned long flags;
284 struct rxe_mw *mw;281 struct rxe_mw *mw;
285 int ret;282 int ret;
286 283
@@ -295,7 +292,7 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) · 8 unmodified lines
295 goto err_drop_ref;292 goto err_drop_ref;
296 }293 }
297 294
298 spin_lock_irqsave(&mw->lock, flags);295 spin_lock_bh(&mw->lock);
299 296
300 ret = rxe_check_invalidate_mw(qp, mw);297 ret = rxe_check_invalidate_mw(qp, mw);
301 if (ret)298 if (ret)
@@ -303,7 +300,7 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) · 1 unmodified line
303 300
304 rxe_do_invalidate_mw(mw);301 rxe_do_invalidate_mw(mw);
305err_unlock:302err_unlock:
306 spin_unlock_irqrestore(&mw->lock, flags);303 spin_unlock_bh(&mw->lock);
307err_drop_ref:304err_drop_ref:
308 rxe_drop_ref(mw);305 rxe_drop_ref(mw);
309err:306err:
@@ -333,9 +330,9 @@ struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey) · 23 unmodified lines
333 return mw;330 return mw;
334}331}
335 332
336void rxe_mw_cleanup(struct rxe_pool_entry *elem)333void rxe_mw_cleanup(struct rxe_pool_elem *elem)
337{334{
338 struct rxe_mw *mw = container_of(elem, typeof(*mw), pelem);335 struct rxe_mw *mw = container_of(elem, typeof(*mw), elem);
339 336
340 rxe_drop_index(mw);337 rxe_drop_index(mw);
341}338}
drivers/infiniband/sw/rxe/rxe_net.c+2−7
@@ -22,24 +22,20 @@ static struct rxe_recv_sockets recv_sockets; · 21 unmodified lines
22 22
23int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)23int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)
24{24{
25 int err;
26 unsigned char ll_addr[ETH_ALEN];25 unsigned char ll_addr[ETH_ALEN];
27 26
28 ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr);27 ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr);
29 err = dev_mc_add(rxe->ndev, ll_addr);
30 28
31 return err;29 return dev_mc_add(rxe->ndev, ll_addr);
32}30}
33 31
34int rxe_mcast_delete(struct rxe_dev *rxe, union ib_gid *mgid)32int rxe_mcast_delete(struct rxe_dev *rxe, union ib_gid *mgid)
35{33{
36 int err;
37 unsigned char ll_addr[ETH_ALEN];34 unsigned char ll_addr[ETH_ALEN];
38 35
39 ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr);36 ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr);
40 err = dev_mc_del(rxe->ndev, ll_addr);
41 37
42 return err;38 return dev_mc_del(rxe->ndev, ll_addr);
43}39}
44 40
45static struct dst_entry *rxe_find_route4(struct net_device *ndev,41static struct dst_entry *rxe_find_route4(struct net_device *ndev,
@@ -444,7 +440,6 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, · 398 unmodified lines
444 else440 else
445 err = rxe_send(skb, pkt);441 err = rxe_send(skb, pkt);
446 if (err) {442 if (err) {
447 rxe->xmit_errors++;
448 rxe_counter_inc(rxe, RXE_CNT_SEND_ERR);443 rxe_counter_inc(rxe, RXE_CNT_SEND_ERR);
449 return err;444 return err;
450 }445 }
drivers/infiniband/sw/rxe/rxe_opcode.c+370−369
@@ -108,8 +108,8 @@ struct rxe_wr_opcode_info rxe_wr_opcode_info[] = { · 107 unmodified lines
108struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = {108struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = {
109 [IB_OPCODE_RC_SEND_FIRST] = {109 [IB_OPCODE_RC_SEND_FIRST] = {
110 .name = "IB_OPCODE_RC_SEND_FIRST",110 .name = "IB_OPCODE_RC_SEND_FIRST",
111 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_RWR_MASK111 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_RWR_MASK |
112 | RXE_SEND_MASK | RXE_START_MASK,112 RXE_SEND_MASK | RXE_START_MASK,
113 .length = RXE_BTH_BYTES,113 .length = RXE_BTH_BYTES,
114 .offset = {114 .offset = {
115 [RXE_BTH] = 0,115 [RXE_BTH] = 0,
@@ -117,9 +117,9 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 1 unmodified line
117 }117 }
118 },118 },
119 [IB_OPCODE_RC_SEND_MIDDLE] = {119 [IB_OPCODE_RC_SEND_MIDDLE] = {
120 .name = "IB_OPCODE_RC_SEND_MIDDLE]",120 .name = "IB_OPCODE_RC_SEND_MIDDLE",
121 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK121 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK |
122 | RXE_MIDDLE_MASK,122 RXE_MIDDLE_MASK,
123 .length = RXE_BTH_BYTES,123 .length = RXE_BTH_BYTES,
124 .offset = {124 .offset = {
125 [RXE_BTH] = 0,125 [RXE_BTH] = 0,
@@ -128,8 +128,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
128 },128 },
129 [IB_OPCODE_RC_SEND_LAST] = {129 [IB_OPCODE_RC_SEND_LAST] = {
130 .name = "IB_OPCODE_RC_SEND_LAST",130 .name = "IB_OPCODE_RC_SEND_LAST",
131 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK131 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK |
132 | RXE_SEND_MASK | RXE_END_MASK,132 RXE_SEND_MASK | RXE_END_MASK,
133 .length = RXE_BTH_BYTES,133 .length = RXE_BTH_BYTES,
134 .offset = {134 .offset = {
135 [RXE_BTH] = 0,135 [RXE_BTH] = 0,
@@ -138,21 +138,21 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
138 },138 },
139 [IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE] = {139 [IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE] = {
140 .name = "IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE",140 .name = "IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE",
141 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK141 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
142 | RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK,142 RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK,
143 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,143 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,
144 .offset = {144 .offset = {
145 [RXE_BTH] = 0,145 [RXE_BTH] = 0,
146 [RXE_IMMDT] = RXE_BTH_BYTES,146 [RXE_IMMDT] = RXE_BTH_BYTES,
147 [RXE_PAYLOAD] = RXE_BTH_BYTES147 [RXE_PAYLOAD] = RXE_BTH_BYTES +
148 + RXE_IMMDT_BYTES,148 RXE_IMMDT_BYTES,
149 }149 }
150 },150 },
151 [IB_OPCODE_RC_SEND_ONLY] = {151 [IB_OPCODE_RC_SEND_ONLY] = {
152 .name = "IB_OPCODE_RC_SEND_ONLY",152 .name = "IB_OPCODE_RC_SEND_ONLY",
153 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK153 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK |
154 | RXE_RWR_MASK | RXE_SEND_MASK154 RXE_RWR_MASK | RXE_SEND_MASK |
155 | RXE_START_MASK | RXE_END_MASK,155 RXE_START_MASK | RXE_END_MASK,
156 .length = RXE_BTH_BYTES,156 .length = RXE_BTH_BYTES,
157 .offset = {157 .offset = {
158 [RXE_BTH] = 0,158 [RXE_BTH] = 0,
@@ -161,33 +161,33 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
161 },161 },
162 [IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE] = {162 [IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE] = {
163 .name = "IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE",163 .name = "IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE",
164 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK164 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
165 | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK165 RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK |
166 | RXE_START_MASK | RXE_END_MASK,166 RXE_START_MASK | RXE_END_MASK,
167 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,167 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,
168 .offset = {168 .offset = {
169 [RXE_BTH] = 0,169 [RXE_BTH] = 0,
170 [RXE_IMMDT] = RXE_BTH_BYTES,170 [RXE_IMMDT] = RXE_BTH_BYTES,
171 [RXE_PAYLOAD] = RXE_BTH_BYTES171 [RXE_PAYLOAD] = RXE_BTH_BYTES +
172 + RXE_IMMDT_BYTES,172 RXE_IMMDT_BYTES,
173 }173 }
174 },174 },
175 [IB_OPCODE_RC_RDMA_WRITE_FIRST] = {175 [IB_OPCODE_RC_RDMA_WRITE_FIRST] = {
176 .name = "IB_OPCODE_RC_RDMA_WRITE_FIRST",176 .name = "IB_OPCODE_RC_RDMA_WRITE_FIRST",
177 .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK177 .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
178 | RXE_WRITE_MASK | RXE_START_MASK,178 RXE_WRITE_MASK | RXE_START_MASK,
179 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,179 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,
180 .offset = {180 .offset = {
181 [RXE_BTH] = 0,181 [RXE_BTH] = 0,
182 [RXE_RETH] = RXE_BTH_BYTES,182 [RXE_RETH] = RXE_BTH_BYTES,
183 [RXE_PAYLOAD] = RXE_BTH_BYTES183 [RXE_PAYLOAD] = RXE_BTH_BYTES +
184 + RXE_RETH_BYTES,184 RXE_RETH_BYTES,
185 }185 }
186 },186 },
187 [IB_OPCODE_RC_RDMA_WRITE_MIDDLE] = {187 [IB_OPCODE_RC_RDMA_WRITE_MIDDLE] = {
188 .name = "IB_OPCODE_RC_RDMA_WRITE_MIDDLE",188 .name = "IB_OPCODE_RC_RDMA_WRITE_MIDDLE",
189 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK189 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK |
190 | RXE_MIDDLE_MASK,190 RXE_MIDDLE_MASK,
191 .length = RXE_BTH_BYTES,191 .length = RXE_BTH_BYTES,
192 .offset = {192 .offset = {
193 [RXE_BTH] = 0,193 [RXE_BTH] = 0,
@@ -196,8 +196,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
196 },196 },
197 [IB_OPCODE_RC_RDMA_WRITE_LAST] = {197 [IB_OPCODE_RC_RDMA_WRITE_LAST] = {
198 .name = "IB_OPCODE_RC_RDMA_WRITE_LAST",198 .name = "IB_OPCODE_RC_RDMA_WRITE_LAST",
199 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK199 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK |
200 | RXE_END_MASK,200 RXE_END_MASK,
201 .length = RXE_BTH_BYTES,201 .length = RXE_BTH_BYTES,
202 .offset = {202 .offset = {
203 [RXE_BTH] = 0,203 [RXE_BTH] = 0,
@@ -206,69 +206,69 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
206 },206 },
207 [IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {207 [IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {
208 .name = "IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE",208 .name = "IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE",
209 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK209 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
210 | RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK210 RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK |
211 | RXE_END_MASK,211 RXE_END_MASK,
212 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,212 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,
213 .offset = {213 .offset = {
214 [RXE_BTH] = 0,214 [RXE_BTH] = 0,
215 [RXE_IMMDT] = RXE_BTH_BYTES,215 [RXE_IMMDT] = RXE_BTH_BYTES,
216 [RXE_PAYLOAD] = RXE_BTH_BYTES216 [RXE_PAYLOAD] = RXE_BTH_BYTES +
217 + RXE_IMMDT_BYTES,217 RXE_IMMDT_BYTES,
218 }218 }
219 },219 },
220 [IB_OPCODE_RC_RDMA_WRITE_ONLY] = {220 [IB_OPCODE_RC_RDMA_WRITE_ONLY] = {
221 .name = "IB_OPCODE_RC_RDMA_WRITE_ONLY",221 .name = "IB_OPCODE_RC_RDMA_WRITE_ONLY",
222 .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK222 .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
223 | RXE_WRITE_MASK | RXE_START_MASK223 RXE_WRITE_MASK | RXE_START_MASK |
224 | RXE_END_MASK,224 RXE_END_MASK,
225 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,225 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,
226 .offset = {226 .offset = {
227 [RXE_BTH] = 0,227 [RXE_BTH] = 0,
228 [RXE_RETH] = RXE_BTH_BYTES,228 [RXE_RETH] = RXE_BTH_BYTES,
229 [RXE_PAYLOAD] = RXE_BTH_BYTES229 [RXE_PAYLOAD] = RXE_BTH_BYTES +
230 + RXE_RETH_BYTES,230 RXE_RETH_BYTES,
231 }231 }
232 },232 },
233 [IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {233 [IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {
234 .name = "IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE",234 .name = "IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE",
235 .mask = RXE_RETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK235 .mask = RXE_RETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK |
236 | RXE_REQ_MASK | RXE_WRITE_MASK236 RXE_REQ_MASK | RXE_WRITE_MASK |
237 | RXE_COMP_MASK | RXE_RWR_MASK237 RXE_COMP_MASK | RXE_RWR_MASK |
238 | RXE_START_MASK | RXE_END_MASK,238 RXE_START_MASK | RXE_END_MASK,
239 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES,239 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES,
240 .offset = {240 .offset = {
241 [RXE_BTH] = 0,241 [RXE_BTH] = 0,
242 [RXE_RETH] = RXE_BTH_BYTES,242 [RXE_RETH] = RXE_BTH_BYTES,
243 [RXE_IMMDT] = RXE_BTH_BYTES243 [RXE_IMMDT] = RXE_BTH_BYTES +
244 + RXE_RETH_BYTES,244 RXE_RETH_BYTES,
245 [RXE_PAYLOAD] = RXE_BTH_BYTES245 [RXE_PAYLOAD] = RXE_BTH_BYTES +
246 + RXE_RETH_BYTES246 RXE_RETH_BYTES +
247 + RXE_IMMDT_BYTES,247 RXE_IMMDT_BYTES,
248 }248 }
249 },249 },
250 [IB_OPCODE_RC_RDMA_READ_REQUEST] = {250 [IB_OPCODE_RC_RDMA_READ_REQUEST] = {
251 .name = "IB_OPCODE_RC_RDMA_READ_REQUEST",251 .name = "IB_OPCODE_RC_RDMA_READ_REQUEST",
252 .mask = RXE_RETH_MASK | RXE_REQ_MASK | RXE_READ_MASK252 .mask = RXE_RETH_MASK | RXE_REQ_MASK | RXE_READ_MASK |
253 | RXE_START_MASK | RXE_END_MASK,253 RXE_START_MASK | RXE_END_MASK,
254 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,254 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,
255 .offset = {255 .offset = {
256 [RXE_BTH] = 0,256 [RXE_BTH] = 0,
257 [RXE_RETH] = RXE_BTH_BYTES,257 [RXE_RETH] = RXE_BTH_BYTES,
258 [RXE_PAYLOAD] = RXE_BTH_BYTES258 [RXE_PAYLOAD] = RXE_BTH_BYTES +
259 + RXE_RETH_BYTES,259 RXE_RETH_BYTES,
260 }260 }
261 },261 },
262 [IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST] = {262 [IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST] = {
263 .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST",263 .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST",
264 .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK264 .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK |
265 | RXE_START_MASK,265 RXE_START_MASK,
266 .length = RXE_BTH_BYTES + RXE_AETH_BYTES,266 .length = RXE_BTH_BYTES + RXE_AETH_BYTES,
267 .offset = {267 .offset = {
268 [RXE_BTH] = 0,268 [RXE_BTH] = 0,
269 [RXE_AETH] = RXE_BTH_BYTES,269 [RXE_AETH] = RXE_BTH_BYTES,
270 [RXE_PAYLOAD] = RXE_BTH_BYTES270 [RXE_PAYLOAD] = RXE_BTH_BYTES +
271 + RXE_AETH_BYTES,271 RXE_AETH_BYTES,
272 }272 }
273 },273 },
274 [IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE] = {274 [IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE] = {
@@ -282,109 +282,110 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 7 unmodified lines
282 },282 },
283 [IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST] = {283 [IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST] = {
284 .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST",284 .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST",
285 .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK285 .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK |
286 | RXE_END_MASK,286 RXE_END_MASK,
287 .length = RXE_BTH_BYTES + RXE_AETH_BYTES,287 .length = RXE_BTH_BYTES + RXE_AETH_BYTES,
288 .offset = {288 .offset = {
289 [RXE_BTH] = 0,289 [RXE_BTH] = 0,
290 [RXE_AETH] = RXE_BTH_BYTES,290 [RXE_AETH] = RXE_BTH_BYTES,
291 [RXE_PAYLOAD] = RXE_BTH_BYTES291 [RXE_PAYLOAD] = RXE_BTH_BYTES +
292 + RXE_AETH_BYTES,292 RXE_AETH_BYTES,
293 }293 }
294 },294 },
295 [IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY] = {295 [IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY] = {
296 .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY",296 .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY",
297 .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK297 .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK |
298 | RXE_START_MASK | RXE_END_MASK,298 RXE_START_MASK | RXE_END_MASK,
299 .length = RXE_BTH_BYTES + RXE_AETH_BYTES,299 .length = RXE_BTH_BYTES + RXE_AETH_BYTES,
300 .offset = {300 .offset = {
301 [RXE_BTH] = 0,301 [RXE_BTH] = 0,
302 [RXE_AETH] = RXE_BTH_BYTES,302 [RXE_AETH] = RXE_BTH_BYTES,
303 [RXE_PAYLOAD] = RXE_BTH_BYTES303 [RXE_PAYLOAD] = RXE_BTH_BYTES +
304 + RXE_AETH_BYTES,304 RXE_AETH_BYTES,
305 }305 }
306 },306 },
307 [IB_OPCODE_RC_ACKNOWLEDGE] = {307 [IB_OPCODE_RC_ACKNOWLEDGE] = {
308 .name = "IB_OPCODE_RC_ACKNOWLEDGE",308 .name = "IB_OPCODE_RC_ACKNOWLEDGE",
309 .mask = RXE_AETH_MASK | RXE_ACK_MASK | RXE_START_MASK309 .mask = RXE_AETH_MASK | RXE_ACK_MASK | RXE_START_MASK |
310 | RXE_END_MASK,310 RXE_END_MASK,
311 .length = RXE_BTH_BYTES + RXE_AETH_BYTES,311 .length = RXE_BTH_BYTES + RXE_AETH_BYTES,
312 .offset = {312 .offset = {
313 [RXE_BTH] = 0,313 [RXE_BTH] = 0,
314 [RXE_AETH] = RXE_BTH_BYTES,314 [RXE_AETH] = RXE_BTH_BYTES,
315 [RXE_PAYLOAD] = RXE_BTH_BYTES315 [RXE_PAYLOAD] = RXE_BTH_BYTES +
316 + RXE_AETH_BYTES,316 RXE_AETH_BYTES,
317 }317 }
318 },318 },
319 [IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = {319 [IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = {
320 .name = "IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE",320 .name = "IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE",
321 .mask = RXE_AETH_MASK | RXE_ATMACK_MASK | RXE_ACK_MASK321 .mask = RXE_AETH_MASK | RXE_ATMACK_MASK | RXE_ACK_MASK |
322 | RXE_START_MASK | RXE_END_MASK,322 RXE_START_MASK | RXE_END_MASK,
323 .length = RXE_BTH_BYTES + RXE_ATMACK_BYTES + RXE_AETH_BYTES,323 .length = RXE_BTH_BYTES + RXE_ATMACK_BYTES + RXE_AETH_BYTES,
324 .offset = {324 .offset = {
325 [RXE_BTH] = 0,325 [RXE_BTH] = 0,
326 [RXE_AETH] = RXE_BTH_BYTES,326 [RXE_AETH] = RXE_BTH_BYTES,
327 [RXE_ATMACK] = RXE_BTH_BYTES327 [RXE_ATMACK] = RXE_BTH_BYTES +
328 + RXE_AETH_BYTES,328 RXE_AETH_BYTES,
329 [RXE_PAYLOAD] = RXE_BTH_BYTES329 [RXE_PAYLOAD] = RXE_BTH_BYTES +
330 + RXE_ATMACK_BYTES + RXE_AETH_BYTES,330 RXE_ATMACK_BYTES +
331 RXE_AETH_BYTES,
331 }332 }
332 },333 },
333 [IB_OPCODE_RC_COMPARE_SWAP] = {334 [IB_OPCODE_RC_COMPARE_SWAP] = {
334 .name = "IB_OPCODE_RC_COMPARE_SWAP",335 .name = "IB_OPCODE_RC_COMPARE_SWAP",
335 .mask = RXE_ATMETH_MASK | RXE_REQ_MASK | RXE_ATOMIC_MASK336 .mask = RXE_ATMETH_MASK | RXE_REQ_MASK | RXE_ATOMIC_MASK |
336 | RXE_START_MASK | RXE_END_MASK,337 RXE_START_MASK | RXE_END_MASK,
337 .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES,338 .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES,
338 .offset = {339 .offset = {
339 [RXE_BTH] = 0,340 [RXE_BTH] = 0,
340 [RXE_ATMETH] = RXE_BTH_BYTES,341 [RXE_ATMETH] = RXE_BTH_BYTES,
341 [RXE_PAYLOAD] = RXE_BTH_BYTES342 [RXE_PAYLOAD] = RXE_BTH_BYTES +
342 + RXE_ATMETH_BYTES,343 RXE_ATMETH_BYTES,
343 }344 }
344 },345 },
345 [IB_OPCODE_RC_FETCH_ADD] = {346 [IB_OPCODE_RC_FETCH_ADD] = {
346 .name = "IB_OPCODE_RC_FETCH_ADD",347 .name = "IB_OPCODE_RC_FETCH_ADD",
347 .mask = RXE_ATMETH_MASK | RXE_REQ_MASK | RXE_ATOMIC_MASK348 .mask = RXE_ATMETH_MASK | RXE_REQ_MASK | RXE_ATOMIC_MASK |
348 | RXE_START_MASK | RXE_END_MASK,349 RXE_START_MASK | RXE_END_MASK,
349 .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES,350 .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES,
350 .offset = {351 .offset = {
351 [RXE_BTH] = 0,352 [RXE_BTH] = 0,
352 [RXE_ATMETH] = RXE_BTH_BYTES,353 [RXE_ATMETH] = RXE_BTH_BYTES,
353 [RXE_PAYLOAD] = RXE_BTH_BYTES354 [RXE_PAYLOAD] = RXE_BTH_BYTES +
354 + RXE_ATMETH_BYTES,355 RXE_ATMETH_BYTES,
355 }356 }
356 },357 },
357 [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = {358 [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = {
358 .name = "IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE",359 .name = "IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE",
359 .mask = RXE_IETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK360 .mask = RXE_IETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
360 | RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK,361 RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK,
361 .length = RXE_BTH_BYTES + RXE_IETH_BYTES,362 .length = RXE_BTH_BYTES + RXE_IETH_BYTES,
362 .offset = {363 .offset = {
363 [RXE_BTH] = 0,364 [RXE_BTH] = 0,
364 [RXE_IETH] = RXE_BTH_BYTES,365 [RXE_IETH] = RXE_BTH_BYTES,
365 [RXE_PAYLOAD] = RXE_BTH_BYTES366 [RXE_PAYLOAD] = RXE_BTH_BYTES +
366 + RXE_IETH_BYTES,367 RXE_IETH_BYTES,
367 }368 }
368 },369 },
369 [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = {370 [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = {
370 .name = "IB_OPCODE_RC_SEND_ONLY_INV",371 .name = "IB_OPCODE_RC_SEND_ONLY_INV",
371 .mask = RXE_IETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK372 .mask = RXE_IETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
372 | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK373 RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK |
373 | RXE_END_MASK | RXE_START_MASK,374 RXE_END_MASK | RXE_START_MASK,
374 .length = RXE_BTH_BYTES + RXE_IETH_BYTES,375 .length = RXE_BTH_BYTES + RXE_IETH_BYTES,
375 .offset = {376 .offset = {
376 [RXE_BTH] = 0,377 [RXE_BTH] = 0,
377 [RXE_IETH] = RXE_BTH_BYTES,378 [RXE_IETH] = RXE_BTH_BYTES,
378 [RXE_PAYLOAD] = RXE_BTH_BYTES379 [RXE_PAYLOAD] = RXE_BTH_BYTES +
379 + RXE_IETH_BYTES,380 RXE_IETH_BYTES,
380 }381 }
381 },382 },
382 383
383 /* UC */384 /* UC */
384 [IB_OPCODE_UC_SEND_FIRST] = {385 [IB_OPCODE_UC_SEND_FIRST] = {
385 .name = "IB_OPCODE_UC_SEND_FIRST",386 .name = "IB_OPCODE_UC_SEND_FIRST",
386 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_RWR_MASK387 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_RWR_MASK |
387 | RXE_SEND_MASK | RXE_START_MASK,388 RXE_SEND_MASK | RXE_START_MASK,
388 .length = RXE_BTH_BYTES,389 .length = RXE_BTH_BYTES,
389 .offset = {390 .offset = {
390 [RXE_BTH] = 0,391 [RXE_BTH] = 0,
@@ -393,8 +394,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
393 },394 },
394 [IB_OPCODE_UC_SEND_MIDDLE] = {395 [IB_OPCODE_UC_SEND_MIDDLE] = {
395 .name = "IB_OPCODE_UC_SEND_MIDDLE",396 .name = "IB_OPCODE_UC_SEND_MIDDLE",
396 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK397 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK |
397 | RXE_MIDDLE_MASK,398 RXE_MIDDLE_MASK,
398 .length = RXE_BTH_BYTES,399 .length = RXE_BTH_BYTES,
399 .offset = {400 .offset = {
400 [RXE_BTH] = 0,401 [RXE_BTH] = 0,
@@ -403,8 +404,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
403 },404 },
404 [IB_OPCODE_UC_SEND_LAST] = {405 [IB_OPCODE_UC_SEND_LAST] = {
405 .name = "IB_OPCODE_UC_SEND_LAST",406 .name = "IB_OPCODE_UC_SEND_LAST",
406 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK407 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK |
407 | RXE_SEND_MASK | RXE_END_MASK,408 RXE_SEND_MASK | RXE_END_MASK,
408 .length = RXE_BTH_BYTES,409 .length = RXE_BTH_BYTES,
409 .offset = {410 .offset = {
410 [RXE_BTH] = 0,411 [RXE_BTH] = 0,
@@ -413,21 +414,21 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
413 },414 },
414 [IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE] = {415 [IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE] = {
415 .name = "IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE",416 .name = "IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE",
416 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK417 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
417 | RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK,418 RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK,
418 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,419 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,
419 .offset = {420 .offset = {
420 [RXE_BTH] = 0,421 [RXE_BTH] = 0,
421 [RXE_IMMDT] = RXE_BTH_BYTES,422 [RXE_IMMDT] = RXE_BTH_BYTES,
422 [RXE_PAYLOAD] = RXE_BTH_BYTES423 [RXE_PAYLOAD] = RXE_BTH_BYTES +
423 + RXE_IMMDT_BYTES,424 RXE_IMMDT_BYTES,
424 }425 }
425 },426 },
426 [IB_OPCODE_UC_SEND_ONLY] = {427 [IB_OPCODE_UC_SEND_ONLY] = {
427 .name = "IB_OPCODE_UC_SEND_ONLY",428 .name = "IB_OPCODE_UC_SEND_ONLY",
428 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK429 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK |
429 | RXE_RWR_MASK | RXE_SEND_MASK430 RXE_RWR_MASK | RXE_SEND_MASK |
430 | RXE_START_MASK | RXE_END_MASK,431 RXE_START_MASK | RXE_END_MASK,
431 .length = RXE_BTH_BYTES,432 .length = RXE_BTH_BYTES,
432 .offset = {433 .offset = {
433 [RXE_BTH] = 0,434 [RXE_BTH] = 0,
@@ -436,33 +437,33 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
436 },437 },
437 [IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE] = {438 [IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE] = {
438 .name = "IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE",439 .name = "IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE",
439 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK440 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
440 | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK441 RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK |
441 | RXE_START_MASK | RXE_END_MASK,442 RXE_START_MASK | RXE_END_MASK,
442 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,443 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,
443 .offset = {444 .offset = {
444 [RXE_BTH] = 0,445 [RXE_BTH] = 0,
445 [RXE_IMMDT] = RXE_BTH_BYTES,446 [RXE_IMMDT] = RXE_BTH_BYTES,
446 [RXE_PAYLOAD] = RXE_BTH_BYTES447 [RXE_PAYLOAD] = RXE_BTH_BYTES +
447 + RXE_IMMDT_BYTES,448 RXE_IMMDT_BYTES,
448 }449 }
449 },450 },
450 [IB_OPCODE_UC_RDMA_WRITE_FIRST] = {451 [IB_OPCODE_UC_RDMA_WRITE_FIRST] = {
451 .name = "IB_OPCODE_UC_RDMA_WRITE_FIRST",452 .name = "IB_OPCODE_UC_RDMA_WRITE_FIRST",
452 .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK453 .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
453 | RXE_WRITE_MASK | RXE_START_MASK,454 RXE_WRITE_MASK | RXE_START_MASK,
454 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,455 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,
455 .offset = {456 .offset = {
456 [RXE_BTH] = 0,457 [RXE_BTH] = 0,
457 [RXE_RETH] = RXE_BTH_BYTES,458 [RXE_RETH] = RXE_BTH_BYTES,
458 [RXE_PAYLOAD] = RXE_BTH_BYTES459 [RXE_PAYLOAD] = RXE_BTH_BYTES +
459 + RXE_RETH_BYTES,460 RXE_RETH_BYTES,
460 }461 }
461 },462 },
462 [IB_OPCODE_UC_RDMA_WRITE_MIDDLE] = {463 [IB_OPCODE_UC_RDMA_WRITE_MIDDLE] = {
463 .name = "IB_OPCODE_UC_RDMA_WRITE_MIDDLE",464 .name = "IB_OPCODE_UC_RDMA_WRITE_MIDDLE",
464 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK465 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK |
465 | RXE_MIDDLE_MASK,466 RXE_MIDDLE_MASK,
466 .length = RXE_BTH_BYTES,467 .length = RXE_BTH_BYTES,
467 .offset = {468 .offset = {
468 [RXE_BTH] = 0,469 [RXE_BTH] = 0,
@@ -471,8 +472,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
471 },472 },
472 [IB_OPCODE_UC_RDMA_WRITE_LAST] = {473 [IB_OPCODE_UC_RDMA_WRITE_LAST] = {
473 .name = "IB_OPCODE_UC_RDMA_WRITE_LAST",474 .name = "IB_OPCODE_UC_RDMA_WRITE_LAST",
474 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK475 .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK |
475 | RXE_END_MASK,476 RXE_END_MASK,
476 .length = RXE_BTH_BYTES,477 .length = RXE_BTH_BYTES,
477 .offset = {478 .offset = {
478 [RXE_BTH] = 0,479 [RXE_BTH] = 0,
@@ -481,460 +482,460 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { · 2 unmodified lines
481 },482 },
482 [IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {483 [IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {
483 .name = "IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE",484 .name = "IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE",
484 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK485 .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
485 | RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK486 RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK |
486 | RXE_END_MASK,487 RXE_END_MASK,
487 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,488 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES,
488 .offset = {489 .offset = {
489 [RXE_BTH] = 0,490 [RXE_BTH] = 0,
490 [RXE_IMMDT] = RXE_BTH_BYTES,491 [RXE_IMMDT] = RXE_BTH_BYTES,
491 [RXE_PAYLOAD] = RXE_BTH_BYTES492 [RXE_PAYLOAD] = RXE_BTH_BYTES +
492 + RXE_IMMDT_BYTES,493 RXE_IMMDT_BYTES,
493 }494 }
494 },495 },
495 [IB_OPCODE_UC_RDMA_WRITE_ONLY] = {496 [IB_OPCODE_UC_RDMA_WRITE_ONLY] = {
496 .name = "IB_OPCODE_UC_RDMA_WRITE_ONLY",497 .name = "IB_OPCODE_UC_RDMA_WRITE_ONLY",
497 .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK498 .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
498 | RXE_WRITE_MASK | RXE_START_MASK499 RXE_WRITE_MASK | RXE_START_MASK |
499 | RXE_END_MASK,500 RXE_END_MASK,
500 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,501 .length = RXE_BTH_BYTES + RXE_RETH_BYTES,
501 .offset = {502 .offset = {
502 [RXE_BTH] = 0,503 [RXE_BTH] = 0,
503 [RXE_RETH] = RXE_BTH_BYTES,504 [RXE_RETH] = RXE_BTH_BYTES,
504 [RXE_PAYLOAD] = RXE_BTH_BYTES505 [RXE_PAYLOAD] = RXE_BTH_BYTES +
505 + RXE_RETH_BYTES,506 RXE_RETH_BYTES,
506 }507 }
507 },508 },
508 [IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {509 [IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {
509 .name = "IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE",510 .name = "IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE",
510 .mask = RXE_RETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK511 .mask = RXE_RETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK |
511 | RXE_REQ_MASK | RXE_WRITE_MASK512 RXE_REQ_MASK | RXE_WRITE_MASK |
512 | RXE_COMP_MASK | RXE_RWR_MASK513 RXE_COMP_MASK | RXE_RWR_MASK |
513 | RXE_START_MASK | RXE_END_MASK,514 RXE_START_MASK | RXE_END_MASK,
514 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES,515 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES,
515 .offset = {516 .offset = {
516 [RXE_BTH] = 0,517 [RXE_BTH] = 0,
517 [RXE_RETH] = RXE_BTH_BYTES,518 [RXE_RETH] = RXE_BTH_BYTES,
518 [RXE_IMMDT] = RXE_BTH_BYTES519 [RXE_IMMDT] = RXE_BTH_BYTES +
519 + RXE_RETH_BYTES,520 RXE_RETH_BYTES,
520 [RXE_PAYLOAD] = RXE_BTH_BYTES521 [RXE_PAYLOAD] = RXE_BTH_BYTES +
521 + RXE_RETH_BYTES522 RXE_RETH_BYTES +
522 + RXE_IMMDT_BYTES,523 RXE_IMMDT_BYTES,
523 }524 }
524 },525 },
525 526
526 /* RD */527 /* RD */
527 [IB_OPCODE_RD_SEND_FIRST] = {528 [IB_OPCODE_RD_SEND_FIRST] = {
528 .name = "IB_OPCODE_RD_SEND_FIRST",529 .name = "IB_OPCODE_RD_SEND_FIRST",
529 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK530 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK |
530 | RXE_REQ_MASK | RXE_RWR_MASK | RXE_SEND_MASK531 RXE_REQ_MASK | RXE_RWR_MASK | RXE_SEND_MASK |
531 | RXE_START_MASK,532 RXE_START_MASK,
532 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,533 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,
533 .offset = {534 .offset = {
534 [RXE_BTH] = 0,535 [RXE_BTH] = 0,
535 [RXE_RDETH] = RXE_BTH_BYTES,536 [RXE_RDETH] = RXE_BTH_BYTES,
536 [RXE_DETH] = RXE_BTH_BYTES537 [RXE_DETH] = RXE_BTH_BYTES +
537 + RXE_RDETH_BYTES,538 RXE_RDETH_BYTES,
538 [RXE_PAYLOAD] = RXE_BTH_BYTES539 [RXE_PAYLOAD] = RXE_BTH_BYTES +
539 + RXE_RDETH_BYTES540 RXE_RDETH_BYTES +
540 + RXE_DETH_BYTES,541 RXE_DETH_BYTES,
541 }542 }
542 },543 },
543 [IB_OPCODE_RD_SEND_MIDDLE] = {544 [IB_OPCODE_RD_SEND_MIDDLE] = {
544 .name = "IB_OPCODE_RD_SEND_MIDDLE",545 .name = "IB_OPCODE_RD_SEND_MIDDLE",
545 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK546 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK |
546 | RXE_REQ_MASK | RXE_SEND_MASK547 RXE_REQ_MASK | RXE_SEND_MASK |
547 | RXE_MIDDLE_MASK,548 RXE_MIDDLE_MASK,
548 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,549 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,
549 .offset = {550 .offset = {
550 [RXE_BTH] = 0,551 [RXE_BTH] = 0,
551 [RXE_RDETH] = RXE_BTH_BYTES,552 [RXE_RDETH] = RXE_BTH_BYTES,
552 [RXE_DETH] = RXE_BTH_BYTES553 [RXE_DETH] = RXE_BTH_BYTES +
553 + RXE_RDETH_BYTES,554 RXE_RDETH_BYTES,
554 [RXE_PAYLOAD] = RXE_BTH_BYTES555 [RXE_PAYLOAD] = RXE_BTH_BYTES +
555 + RXE_RDETH_BYTES556 RXE_RDETH_BYTES +
556 + RXE_DETH_BYTES,557 RXE_DETH_BYTES,
557 }558 }
558 },559 },
559 [IB_OPCODE_RD_SEND_LAST] = {560 [IB_OPCODE_RD_SEND_LAST] = {
560 .name = "IB_OPCODE_RD_SEND_LAST",561 .name = "IB_OPCODE_RD_SEND_LAST",
561 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK562 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK |
562 | RXE_REQ_MASK | RXE_COMP_MASK | RXE_SEND_MASK563 RXE_REQ_MASK | RXE_COMP_MASK | RXE_SEND_MASK |
563 | RXE_END_MASK,564 RXE_END_MASK,
564 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,565 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,
565 .offset = {566 .offset = {
566 [RXE_BTH] = 0,567 [RXE_BTH] = 0,
567 [RXE_RDETH] = RXE_BTH_BYTES,568 [RXE_RDETH] = RXE_BTH_BYTES,
568 [RXE_DETH] = RXE_BTH_BYTES569 [RXE_DETH] = RXE_BTH_BYTES +
569 + RXE_RDETH_BYTES,570 RXE_RDETH_BYTES,
570 [RXE_PAYLOAD] = RXE_BTH_BYTES571 [RXE_PAYLOAD] = RXE_BTH_BYTES +
571 + RXE_RDETH_BYTES572 RXE_RDETH_BYTES +
572 + RXE_DETH_BYTES,573 RXE_DETH_BYTES,
573 }574 }
574 },575 },
575 [IB_OPCODE_RD_SEND_LAST_WITH_IMMEDIATE] = {576 [IB_OPCODE_RD_SEND_LAST_WITH_IMMEDIATE] = {
576 .name = "IB_OPCODE_RD_SEND_LAST_WITH_IMMEDIATE",577 .name = "IB_OPCODE_RD_SEND_LAST_WITH_IMMEDIATE",
577 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK578 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK |
578 | RXE_PAYLOAD_MASK | RXE_REQ_MASK579 RXE_PAYLOAD_MASK | RXE_REQ_MASK |
579 | RXE_COMP_MASK | RXE_SEND_MASK580 RXE_COMP_MASK | RXE_SEND_MASK |
580 | RXE_END_MASK,581 RXE_END_MASK,
581 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES582 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES +
582 + RXE_RDETH_BYTES,583 RXE_RDETH_BYTES,
583 .offset = {584 .offset = {
584 [RXE_BTH] = 0,585 [RXE_BTH] = 0,
585 [RXE_RDETH] = RXE_BTH_BYTES,586 [RXE_RDETH] = RXE_BTH_BYTES,
586 [RXE_DETH] = RXE_BTH_BYTES587 [RXE_DETH] = RXE_BTH_BYTES +
587 + RXE_RDETH_BYTES,588 RXE_RDETH_BYTES,
588 [RXE_IMMDT] = RXE_BTH_BYTES589 [RXE_IMMDT] = RXE_BTH_BYTES +
589 + RXE_RDETH_BYTES590 RXE_RDETH_BYTES +
590 + RXE_DETH_BYTES,591 RXE_DETH_BYTES,
591 [RXE_PAYLOAD] = RXE_BTH_BYTES592 [RXE_PAYLOAD] = RXE_BTH_BYTES +
592 + RXE_RDETH_BYTES593 RXE_RDETH_BYTES +
593 + RXE_DETH_BYTES594 RXE_DETH_BYTES +
594 + RXE_IMMDT_BYTES,595 RXE_IMMDT_BYTES,
595 }596 }
596 },597 },
597 [IB_OPCODE_RD_SEND_ONLY] = {598 [IB_OPCODE_RD_SEND_ONLY] = {
598 .name = "IB_OPCODE_RD_SEND_ONLY",599 .name = "IB_OPCODE_RD_SEND_ONLY",
599 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK600 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK |
600 | RXE_REQ_MASK | RXE_COMP_MASK | RXE_RWR_MASK601 RXE_REQ_MASK | RXE_COMP_MASK | RXE_RWR_MASK |
601 | RXE_SEND_MASK | RXE_START_MASK | RXE_END_MASK,602 RXE_SEND_MASK | RXE_START_MASK | RXE_END_MASK,
602 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,603 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,
603 .offset = {604 .offset = {
604 [RXE_BTH] = 0,605 [RXE_BTH] = 0,
605 [RXE_RDETH] = RXE_BTH_BYTES,606 [RXE_RDETH] = RXE_BTH_BYTES,
606 [RXE_DETH] = RXE_BTH_BYTES607 [RXE_DETH] = RXE_BTH_BYTES +
607 + RXE_RDETH_BYTES,608 RXE_RDETH_BYTES,
608 [RXE_PAYLOAD] = RXE_BTH_BYTES609 [RXE_PAYLOAD] = RXE_BTH_BYTES +
609 + RXE_RDETH_BYTES610 RXE_RDETH_BYTES +
610 + RXE_DETH_BYTES,611 RXE_DETH_BYTES,
611 }612 }
612 },613 },
613 [IB_OPCODE_RD_SEND_ONLY_WITH_IMMEDIATE] = {614 [IB_OPCODE_RD_SEND_ONLY_WITH_IMMEDIATE] = {
614 .name = "IB_OPCODE_RD_SEND_ONLY_WITH_IMMEDIATE",615 .name = "IB_OPCODE_RD_SEND_ONLY_WITH_IMMEDIATE",
615 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK616 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK |
616 | RXE_PAYLOAD_MASK | RXE_REQ_MASK617 RXE_PAYLOAD_MASK | RXE_REQ_MASK |
617 | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK618 RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK |
618 | RXE_START_MASK | RXE_END_MASK,619 RXE_START_MASK | RXE_END_MASK,
619 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES620 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES +
620 + RXE_RDETH_BYTES,621 RXE_RDETH_BYTES,
621 .offset = {622 .offset = {
622 [RXE_BTH] = 0,623 [RXE_BTH] = 0,
623 [RXE_RDETH] = RXE_BTH_BYTES,624 [RXE_RDETH] = RXE_BTH_BYTES,
624 [RXE_DETH] = RXE_BTH_BYTES625 [RXE_DETH] = RXE_BTH_BYTES +
625 + RXE_RDETH_BYTES,626 RXE_RDETH_BYTES,
626 [RXE_IMMDT] = RXE_BTH_BYTES627 [RXE_IMMDT] = RXE_BTH_BYTES +
627 + RXE_RDETH_BYTES628 RXE_RDETH_BYTES +
628 + RXE_DETH_BYTES,629 RXE_DETH_BYTES,
629 [RXE_PAYLOAD] = RXE_BTH_BYTES630 [RXE_PAYLOAD] = RXE_BTH_BYTES +
630 + RXE_RDETH_BYTES631 RXE_RDETH_BYTES +
631 + RXE_DETH_BYTES632 RXE_DETH_BYTES +
632 + RXE_IMMDT_BYTES,633 RXE_IMMDT_BYTES,
633 }634 }
634 },635 },
635 [IB_OPCODE_RD_RDMA_WRITE_FIRST] = {636 [IB_OPCODE_RD_RDMA_WRITE_FIRST] = {
636 .name = "IB_OPCODE_RD_RDMA_WRITE_FIRST",637 .name = "IB_OPCODE_RD_RDMA_WRITE_FIRST",
637 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK638 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK |
638 | RXE_PAYLOAD_MASK | RXE_REQ_MASK639 RXE_PAYLOAD_MASK | RXE_REQ_MASK |
639 | RXE_WRITE_MASK | RXE_START_MASK,640 RXE_WRITE_MASK | RXE_START_MASK,
640 .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES641 .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES +
641 + RXE_RDETH_BYTES,642 RXE_RDETH_BYTES,
642 .offset = {643 .offset = {
643 [RXE_BTH] = 0,644 [RXE_BTH] = 0,
644 [RXE_RDETH] = RXE_BTH_BYTES,645 [RXE_RDETH] = RXE_BTH_BYTES,
645 [RXE_DETH] = RXE_BTH_BYTES646 [RXE_DETH] = RXE_BTH_BYTES +
646 + RXE_RDETH_BYTES,647 RXE_RDETH_BYTES,
647 [RXE_RETH] = RXE_BTH_BYTES648 [RXE_RETH] = RXE_BTH_BYTES +
648 + RXE_RDETH_BYTES649 RXE_RDETH_BYTES +
649 + RXE_DETH_BYTES,650 RXE_DETH_BYTES,
650 [RXE_PAYLOAD] = RXE_BTH_BYTES651 [RXE_PAYLOAD] = RXE_BTH_BYTES +
651 + RXE_RDETH_BYTES652 RXE_RDETH_BYTES +
652 + RXE_DETH_BYTES653 RXE_DETH_BYTES +
653 + RXE_RETH_BYTES,654 RXE_RETH_BYTES,
654 }655 }
655 },656 },
656 [IB_OPCODE_RD_RDMA_WRITE_MIDDLE] = {657 [IB_OPCODE_RD_RDMA_WRITE_MIDDLE] = {
657 .name = "IB_OPCODE_RD_RDMA_WRITE_MIDDLE",658 .name = "IB_OPCODE_RD_RDMA_WRITE_MIDDLE",
658 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK659 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK |
659 | RXE_REQ_MASK | RXE_WRITE_MASK660 RXE_REQ_MASK | RXE_WRITE_MASK |
660 | RXE_MIDDLE_MASK,661 RXE_MIDDLE_MASK,
661 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,662 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,
662 .offset = {663 .offset = {
663 [RXE_BTH] = 0,664 [RXE_BTH] = 0,
664 [RXE_RDETH] = RXE_BTH_BYTES,665 [RXE_RDETH] = RXE_BTH_BYTES,
665 [RXE_DETH] = RXE_BTH_BYTES666 [RXE_DETH] = RXE_BTH_BYTES +
666 + RXE_RDETH_BYTES,667 RXE_RDETH_BYTES,
667 [RXE_PAYLOAD] = RXE_BTH_BYTES668 [RXE_PAYLOAD] = RXE_BTH_BYTES +
668 + RXE_RDETH_BYTES669 RXE_RDETH_BYTES +
669 + RXE_DETH_BYTES,670 RXE_DETH_BYTES,
670 }671 }
671 },672 },
672 [IB_OPCODE_RD_RDMA_WRITE_LAST] = {673 [IB_OPCODE_RD_RDMA_WRITE_LAST] = {
673 .name = "IB_OPCODE_RD_RDMA_WRITE_LAST",674 .name = "IB_OPCODE_RD_RDMA_WRITE_LAST",
674 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK675 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK |
675 | RXE_REQ_MASK | RXE_WRITE_MASK676 RXE_REQ_MASK | RXE_WRITE_MASK |
676 | RXE_END_MASK,677 RXE_END_MASK,
677 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,678 .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES,
678 .offset = {679 .offset = {
679 [RXE_BTH] = 0,680 [RXE_BTH] = 0,
680 [RXE_RDETH] = RXE_BTH_BYTES,681 [RXE_RDETH] = RXE_BTH_BYTES,
681 [RXE_DETH] = RXE_BTH_BYTES682 [RXE_DETH] = RXE_BTH_BYTES +
682 + RXE_RDETH_BYTES,683 RXE_RDETH_BYTES,
683 [RXE_PAYLOAD] = RXE_BTH_BYTES684 [RXE_PAYLOAD] = RXE_BTH_BYTES +
684 + RXE_RDETH_BYTES685 RXE_RDETH_BYTES +
685 + RXE_DETH_BYTES,686 RXE_DETH_BYTES,
686 }687 }
687 },688 },
688 [IB_OPCODE_RD_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {689 [IB_OPCODE_RD_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {
689 .name = "IB_OPCODE_RD_RDMA_WRITE_LAST_WITH_IMMEDIATE",690 .name = "IB_OPCODE_RD_RDMA_WRITE_LAST_WITH_IMMEDIATE",
690 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK691 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK |
691 | RXE_PAYLOAD_MASK | RXE_REQ_MASK692 RXE_PAYLOAD_MASK | RXE_REQ_MASK |
692 | RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK693 RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK |
693 | RXE_END_MASK,694 RXE_END_MASK,
694 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES695 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES +
695 + RXE_RDETH_BYTES,696 RXE_RDETH_BYTES,
696 .offset = {697 .offset = {
697 [RXE_BTH] = 0,698 [RXE_BTH] = 0,
698 [RXE_RDETH] = RXE_BTH_BYTES,699 [RXE_RDETH] = RXE_BTH_BYTES,
699 [RXE_DETH] = RXE_BTH_BYTES700 [RXE_DETH] = RXE_BTH_BYTES +
700 + RXE_RDETH_BYTES,701 RXE_RDETH_BYTES,
701 [RXE_IMMDT] = RXE_BTH_BYTES702 [RXE_IMMDT] = RXE_BTH_BYTES +
702 + RXE_RDETH_BYTES703 RXE_RDETH_BYTES +
703 + RXE_DETH_BYTES,704 RXE_DETH_BYTES,
704 [RXE_PAYLOAD] = RXE_BTH_BYTES705 [RXE_PAYLOAD] = RXE_BTH_BYTES +
705 + RXE_RDETH_BYTES706 RXE_RDETH_BYTES +
706 + RXE_DETH_BYTES707 RXE_DETH_BYTES +
707 + RXE_IMMDT_BYTES,708 RXE_IMMDT_BYTES,
708 }709 }
709 },710 },
710 [IB_OPCODE_RD_RDMA_WRITE_ONLY] = {711 [IB_OPCODE_RD_RDMA_WRITE_ONLY] = {
711 .name = "IB_OPCODE_RD_RDMA_WRITE_ONLY",712 .name = "IB_OPCODE_RD_RDMA_WRITE_ONLY",
712 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK713 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK |
713 | RXE_PAYLOAD_MASK | RXE_REQ_MASK714 RXE_PAYLOAD_MASK | RXE_REQ_MASK |
714 | RXE_WRITE_MASK | RXE_START_MASK715 RXE_WRITE_MASK | RXE_START_MASK |
715 | RXE_END_MASK,716 RXE_END_MASK,
716 .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES717 .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES +
717 + RXE_RDETH_BYTES,718 RXE_RDETH_BYTES,
718 .offset = {719 .offset = {
719 [RXE_BTH] = 0,720 [RXE_BTH] = 0,
720 [RXE_RDETH] = RXE_BTH_BYTES,721 [RXE_RDETH] = RXE_BTH_BYTES,
721 [RXE_DETH] = RXE_BTH_BYTES722 [RXE_DETH] = RXE_BTH_BYTES +
722 + RXE_RDETH_BYTES,723 RXE_RDETH_BYTES,
723 [RXE_RETH] = RXE_BTH_BYTES724 [RXE_RETH] = RXE_BTH_BYTES +
724 + RXE_RDETH_BYTES725 RXE_RDETH_BYTES +
725 + RXE_DETH_BYTES,726 RXE_DETH_BYTES,
726 [RXE_PAYLOAD] = RXE_BTH_BYTES727 [RXE_PAYLOAD] = RXE_BTH_BYTES +
727 + RXE_RDETH_BYTES728 RXE_RDETH_BYTES +
728 + RXE_DETH_BYTES729 RXE_DETH_BYTES +
729 + RXE_RETH_BYTES,730 RXE_RETH_BYTES,
730 }731 }
731 },732 },
732 [IB_OPCODE_RD_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {733 [IB_OPCODE_RD_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {
733 .name = "IB_OPCODE_RD_RDMA_WRITE_ONLY_WITH_IMMEDIATE",734 .name = "IB_OPCODE_RD_RDMA_WRITE_ONLY_WITH_IMMEDIATE",
734 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK735 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK |
735 | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK736 RXE_IMMDT_MASK | RXE_PAYLOAD_MASK |
736 | RXE_REQ_MASK | RXE_WRITE_MASK737 RXE_REQ_MASK | RXE_WRITE_MASK |
737 | RXE_COMP_MASK | RXE_RWR_MASK738 RXE_COMP_MASK | RXE_RWR_MASK |
738 | RXE_START_MASK | RXE_END_MASK,739 RXE_START_MASK | RXE_END_MASK,
739 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES740 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES +
740 + RXE_DETH_BYTES + RXE_RDETH_BYTES,741 RXE_DETH_BYTES + RXE_RDETH_BYTES,
741 .offset = {742 .offset = {
742 [RXE_BTH] = 0,743 [RXE_BTH] = 0,
743 [RXE_RDETH] = RXE_BTH_BYTES,744 [RXE_RDETH] = RXE_BTH_BYTES,
744 [RXE_DETH] = RXE_BTH_BYTES745 [RXE_DETH] = RXE_BTH_BYTES +
745 + RXE_RDETH_BYTES,746 RXE_RDETH_BYTES,
746 [RXE_RETH] = RXE_BTH_BYTES747 [RXE_RETH] = RXE_BTH_BYTES +
747 + RXE_RDETH_BYTES748 RXE_RDETH_BYTES +
748 + RXE_DETH_BYTES,749 RXE_DETH_BYTES,
749 [RXE_IMMDT] = RXE_BTH_BYTES750 [RXE_IMMDT] = RXE_BTH_BYTES +
750 + RXE_RDETH_BYTES751 RXE_RDETH_BYTES +
751 + RXE_DETH_BYTES752 RXE_DETH_BYTES +
752 + RXE_RETH_BYTES,753 RXE_RETH_BYTES,
753 [RXE_PAYLOAD] = RXE_BTH_BYTES754 [RXE_PAYLOAD] = RXE_BTH_BYTES +
754 + RXE_RDETH_BYTES755 RXE_RDETH_BYTES +
755 + RXE_DETH_BYTES756 RXE_DETH_BYTES +
756 + RXE_RETH_BYTES757 RXE_RETH_BYTES +
757 + RXE_IMMDT_BYTES,758 RXE_IMMDT_BYTES,
758 }759 }
759 },760 },
760 [IB_OPCODE_RD_RDMA_READ_REQUEST] = {761 [IB_OPCODE_RD_RDMA_READ_REQUEST] = {
761 .name = "IB_OPCODE_RD_RDMA_READ_REQUEST",762 .name = "IB_OPCODE_RD_RDMA_READ_REQUEST",
762 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK763 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK |
763 | RXE_REQ_MASK | RXE_READ_MASK764 RXE_REQ_MASK | RXE_READ_MASK |
764 | RXE_START_MASK | RXE_END_MASK,765 RXE_START_MASK | RXE_END_MASK,
765 .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES766 .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES +
766 + RXE_RDETH_BYTES,767 RXE_RDETH_BYTES,
767 .offset = {768 .offset = {
768 [RXE_BTH] = 0,769 [RXE_BTH] = 0,
769 [RXE_RDETH] = RXE_BTH_BYTES,770 [RXE_RDETH] = RXE_BTH_BYTES,
770 [RXE_DETH] = RXE_BTH_BYTES771 [RXE_DETH] = RXE_BTH_BYTES +
771 + RXE_RDETH_BYTES,772 RXE_RDETH_BYTES,
772 [RXE_RETH] = RXE_BTH_BYTES773 [RXE_RETH] = RXE_BTH_BYTES +
773 + RXE_RDETH_BYTES774 RXE_RDETH_BYTES +
774 + RXE_DETH_BYTES,775 RXE_DETH_BYTES,
775 [RXE_PAYLOAD] = RXE_BTH_BYTES776 [RXE_PAYLOAD] = RXE_BTH_BYTES +
776 + RXE_RETH_BYTES777 RXE_RETH_BYTES +
777 + RXE_DETH_BYTES778 RXE_DETH_BYTES +
778 + RXE_RDETH_BYTES,779 RXE_RDETH_BYTES,
779 }780 }
780 },781 },
781 [IB_OPCODE_RD_RDMA_READ_RESPONSE_FIRST] = {782 [IB_OPCODE_RD_RDMA_READ_RESPONSE_FIRST] = {
782 .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_FIRST",783 .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_FIRST",
783 .mask = RXE_RDETH_MASK | RXE_AETH_MASK784 .mask = RXE_RDETH_MASK | RXE_AETH_MASK |
784 | RXE_PAYLOAD_MASK | RXE_ACK_MASK785 RXE_PAYLOAD_MASK | RXE_ACK_MASK |
785 | RXE_START_MASK,786 RXE_START_MASK,
786 .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES,787 .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES,
787 .offset = {788 .offset = {
788 [RXE_BTH] = 0,789 [RXE_BTH] = 0,
789 [RXE_RDETH] = RXE_BTH_BYTES,790 [RXE_RDETH] = RXE_BTH_BYTES,
790 [RXE_AETH] = RXE_BTH_BYTES791 [RXE_AETH] = RXE_BTH_BYTES +
791 + RXE_RDETH_BYTES,792 RXE_RDETH_BYTES,
792 [RXE_PAYLOAD] = RXE_BTH_BYTES793 [RXE_PAYLOAD] = RXE_BTH_BYTES +
793 + RXE_RDETH_BYTES794 RXE_RDETH_BYTES +
794 + RXE_AETH_BYTES,795 RXE_AETH_BYTES,
795 }796 }
796 },797 },
797 [IB_OPCODE_RD_RDMA_READ_RESPONSE_MIDDLE] = {798 [IB_OPCODE_RD_RDMA_READ_RESPONSE_MIDDLE] = {
798 .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_MIDDLE",799 .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_MIDDLE",
799 .mask = RXE_RDETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK800 .mask = RXE_RDETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK |
800 | RXE_MIDDLE_MASK,801 RXE_MIDDLE_MASK,
801 .length = RXE_BTH_BYTES + RXE_RDETH_BYTES,802 .length = RXE_BTH_BYTES + RXE_RDETH_BYTES,
802 .offset = {803 .offset = {
803 [RXE_BTH] = 0,804 [RXE_BTH] = 0,
804 [RXE_RDETH] = RXE_BTH_BYTES,805 [RXE_RDETH] = RXE_BTH_BYTES,
805 [RXE_PAYLOAD] = RXE_BTH_BYTES806 [RXE_PAYLOAD] = RXE_BTH_BYTES +
806 + RXE_RDETH_BYTES,807 RXE_RDETH_BYTES,
807 }808 }
808 },809 },
809 [IB_OPCODE_RD_RDMA_READ_RESPONSE_LAST] = {810 [IB_OPCODE_RD_RDMA_READ_RESPONSE_LAST] = {
810 .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_LAST",811 .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_LAST",
811 .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_PAYLOAD_MASK812 .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_PAYLOAD_MASK |
812 | RXE_ACK_MASK | RXE_END_MASK,813 RXE_ACK_MASK | RXE_END_MASK,
813 .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES,814 .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES,
814 .offset = {815 .offset = {
815 [RXE_BTH] = 0,816 [RXE_BTH] = 0,
816 [RXE_RDETH] = RXE_BTH_BYTES,817 [RXE_RDETH] = RXE_BTH_BYTES,
817 [RXE_AETH] = RXE_BTH_BYTES818 [RXE_AETH] = RXE_BTH_BYTES +
818 + RXE_RDETH_BYTES,819 RXE_RDETH_BYTES,
819 [RXE_PAYLOAD] = RXE_BTH_BYTES820 [RXE_PAYLOAD] = RXE_BTH_BYTES +
820 + RXE_RDETH_BYTES821 RXE_RDETH_BYTES +
821 + RXE_AETH_BYTES,822 RXE_AETH_BYTES,
822 }823 }
823 },824 },
824 [IB_OPCODE_RD_RDMA_READ_RESPONSE_ONLY] = {825 [IB_OPCODE_RD_RDMA_READ_RESPONSE_ONLY] = {
825 .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_ONLY",826 .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_ONLY",
826 .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_PAYLOAD_MASK827 .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_PAYLOAD_MASK |
827 | RXE_ACK_MASK | RXE_START_MASK | RXE_END_MASK,828 RXE_ACK_MASK | RXE_START_MASK | RXE_END_MASK,
828 .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES,829 .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES,
829 .offset = {830 .offset = {
830 [RXE_BTH] = 0,831 [RXE_BTH] = 0,
831 [RXE_RDETH] = RXE_BTH_BYTES,832 [RXE_RDETH] = RXE_BTH_BYTES,
832 [RXE_AETH] = RXE_BTH_BYTES833 [RXE_AETH] = RXE_BTH_BYTES +
833 + RXE_RDETH_BYTES,834 RXE_RDETH_BYTES,
834 [RXE_PAYLOAD] = RXE_BTH_BYTES835 [RXE_PAYLOAD] = RXE_BTH_BYTES +
835 + RXE_RDETH_BYTES836 RXE_RDETH_BYTES +
836 + RXE_AETH_BYTES,837 RXE_AETH_BYTES,
837 }838 }
838 },839 },
839 [IB_OPCODE_RD_ACKNOWLEDGE] = {840 [IB_OPCODE_RD_ACKNOWLEDGE] = {
840 .name = "IB_OPCODE_RD_ACKNOWLEDGE",841 .name = "IB_OPCODE_RD_ACKNOWLEDGE",
841 .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_ACK_MASK842 .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_ACK_MASK |
842 | RXE_START_MASK | RXE_END_MASK,843 RXE_START_MASK | RXE_END_MASK,
843 .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES,844 .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES,
844 .offset = {845 .offset = {
845 [RXE_BTH] = 0,846 [RXE_BTH] = 0,
846 [RXE_RDETH] = RXE_BTH_BYTES,847 [RXE_RDETH] = RXE_BTH_BYTES,
847 [RXE_AETH] = RXE_BTH_BYTES848 [RXE_AETH] = RXE_BTH_BYTES +
848 + RXE_RDETH_BYTES,849 RXE_RDETH_BYTES,
849 }850 }
850 },851 },
851 [IB_OPCODE_RD_ATOMIC_ACKNOWLEDGE] = {852 [IB_OPCODE_RD_ATOMIC_ACKNOWLEDGE] = {
852 .name = "IB_OPCODE_RD_ATOMIC_ACKNOWLEDGE",853 .name = "IB_OPCODE_RD_ATOMIC_ACKNOWLEDGE",
853 .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_ATMACK_MASK854 .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_ATMACK_MASK |
854 | RXE_ACK_MASK | RXE_START_MASK | RXE_END_MASK,855 RXE_ACK_MASK | RXE_START_MASK | RXE_END_MASK,
855 .length = RXE_BTH_BYTES + RXE_ATMACK_BYTES + RXE_AETH_BYTES856 .length = RXE_BTH_BYTES + RXE_ATMACK_BYTES + RXE_AETH_BYTES +
856 + RXE_RDETH_BYTES,857 RXE_RDETH_BYTES,
857 .offset = {858 .offset = {
858 [RXE_BTH] = 0,859 [RXE_BTH] = 0,
859 [RXE_RDETH] = RXE_BTH_BYTES,860 [RXE_RDETH] = RXE_BTH_BYTES,
860 [RXE_AETH] = RXE_BTH_BYTES861 [RXE_AETH] = RXE_BTH_BYTES +
861 + RXE_RDETH_BYTES,862 RXE_RDETH_BYTES,
862 [RXE_ATMACK] = RXE_BTH_BYTES863 [RXE_ATMACK] = RXE_BTH_BYTES +
863 + RXE_RDETH_BYTES864 RXE_RDETH_BYTES +
864 + RXE_AETH_BYTES,865 RXE_AETH_BYTES,
865 }866 }
866 },867 },
867 [IB_OPCODE_RD_COMPARE_SWAP] = {868 [IB_OPCODE_RD_COMPARE_SWAP] = {
868 .name = "RD_COMPARE_SWAP",869 .name = "RD_COMPARE_SWAP",
869 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_ATMETH_MASK870 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_ATMETH_MASK |
870 | RXE_REQ_MASK | RXE_ATOMIC_MASK871 RXE_REQ_MASK | RXE_ATOMIC_MASK |
871 | RXE_START_MASK | RXE_END_MASK,872 RXE_START_MASK | RXE_END_MASK,
872 .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES + RXE_DETH_BYTES873 .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES + RXE_DETH_BYTES +
873 + RXE_RDETH_BYTES,874 RXE_RDETH_BYTES,
874 .offset = {875 .offset = {
875 [RXE_BTH] = 0,876 [RXE_BTH] = 0,
876 [RXE_RDETH] = RXE_BTH_BYTES,877 [RXE_RDETH] = RXE_BTH_BYTES,
877 [RXE_DETH] = RXE_BTH_BYTES878 [RXE_DETH] = RXE_BTH_BYTES +
878 + RXE_RDETH_BYTES,879 RXE_RDETH_BYTES,
879 [RXE_ATMETH] = RXE_BTH_BYTES880 [RXE_ATMETH] = RXE_BTH_BYTES +
880 + RXE_RDETH_BYTES881 RXE_RDETH_BYTES +
881 + RXE_DETH_BYTES,882 RXE_DETH_BYTES,
882 [RXE_PAYLOAD] = RXE_BTH_BYTES +883 [RXE_PAYLOAD] = RXE_BTH_BYTES +
883 + RXE_ATMETH_BYTES884 RXE_ATMETH_BYTES +
884 + RXE_DETH_BYTES +885 RXE_DETH_BYTES +
885 + RXE_RDETH_BYTES,886 RXE_RDETH_BYTES,
886 }887 }
887 },888 },
888 [IB_OPCODE_RD_FETCH_ADD] = {889 [IB_OPCODE_RD_FETCH_ADD] = {
889 .name = "IB_OPCODE_RD_FETCH_ADD",890 .name = "IB_OPCODE_RD_FETCH_ADD",
890 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_ATMETH_MASK891 .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_ATMETH_MASK |
891 | RXE_REQ_MASK | RXE_ATOMIC_MASK892 RXE_REQ_MASK | RXE_ATOMIC_MASK |
892 | RXE_START_MASK | RXE_END_MASK,893 RXE_START_MASK | RXE_END_MASK,
893 .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES + RXE_DETH_BYTES894 .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES + RXE_DETH_BYTES +
894 + RXE_RDETH_BYTES,895 RXE_RDETH_BYTES,
895 .offset = {896 .offset = {
896 [RXE_BTH] = 0,897 [RXE_BTH] = 0,
897 [RXE_RDETH] = RXE_BTH_BYTES,898 [RXE_RDETH] = RXE_BTH_BYTES,
898 [RXE_DETH] = RXE_BTH_BYTES899 [RXE_DETH] = RXE_BTH_BYTES +
899 + RXE_RDETH_BYTES,900 RXE_RDETH_BYTES,
900 [RXE_ATMETH] = RXE_BTH_BYTES901 [RXE_ATMETH] = RXE_BTH_BYTES +
901 + RXE_RDETH_BYTES902 RXE_RDETH_BYTES +
902 + RXE_DETH_BYTES,903 RXE_DETH_BYTES,
903 [RXE_PAYLOAD] = RXE_BTH_BYTES +904 [RXE_PAYLOAD] = RXE_BTH_BYTES +
904 + RXE_ATMETH_BYTES905 RXE_ATMETH_BYTES +
905 + RXE_DETH_BYTES +906 RXE_DETH_BYTES +
906 + RXE_RDETH_BYTES,907 RXE_RDETH_BYTES,
907 }908 }
908 },909 },
909 910
910 /* UD */911 /* UD */
911 [IB_OPCODE_UD_SEND_ONLY] = {912 [IB_OPCODE_UD_SEND_ONLY] = {
912 .name = "IB_OPCODE_UD_SEND_ONLY",913 .name = "IB_OPCODE_UD_SEND_ONLY",
913 .mask = RXE_DETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK914 .mask = RXE_DETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK |
914 | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK915 RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK |
915 | RXE_START_MASK | RXE_END_MASK,916 RXE_START_MASK | RXE_END_MASK,
916 .length = RXE_BTH_BYTES + RXE_DETH_BYTES,917 .length = RXE_BTH_BYTES + RXE_DETH_BYTES,
917 .offset = {918 .offset = {
918 [RXE_BTH] = 0,919 [RXE_BTH] = 0,
919 [RXE_DETH] = RXE_BTH_BYTES,920 [RXE_DETH] = RXE_BTH_BYTES,
920 [RXE_PAYLOAD] = RXE_BTH_BYTES921 [RXE_PAYLOAD] = RXE_BTH_BYTES +
921 + RXE_DETH_BYTES,922 RXE_DETH_BYTES,
922 }923 }
923 },924 },
924 [IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE] = {925 [IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE] = {
925 .name = "IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE",926 .name = "IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE",
926 .mask = RXE_DETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK927 .mask = RXE_DETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK |
927 | RXE_REQ_MASK | RXE_COMP_MASK | RXE_RWR_MASK928 RXE_REQ_MASK | RXE_COMP_MASK | RXE_RWR_MASK |
928 | RXE_SEND_MASK | RXE_START_MASK | RXE_END_MASK,929 RXE_SEND_MASK | RXE_START_MASK | RXE_END_MASK,
929 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES,930 .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES,
930 .offset = {931 .offset = {
931 [RXE_BTH] = 0,932 [RXE_BTH] = 0,
932 [RXE_DETH] = RXE_BTH_BYTES,933 [RXE_DETH] = RXE_BTH_BYTES,
933 [RXE_IMMDT] = RXE_BTH_BYTES934 [RXE_IMMDT] = RXE_BTH_BYTES +
934 + RXE_DETH_BYTES,935 RXE_DETH_BYTES,
935 [RXE_PAYLOAD] = RXE_BTH_BYTES936 [RXE_PAYLOAD] = RXE_BTH_BYTES +
936 + RXE_DETH_BYTES937 RXE_DETH_BYTES +
937 + RXE_IMMDT_BYTES,938 RXE_IMMDT_BYTES,
938 }939 }
939 },940 },
940 941
drivers/infiniband/sw/rxe/rxe_pool.c+84−93
@@ -5,13 +5,14 @@ · 4 unmodified lines
5 */5 */
6 6
7#include "rxe.h"7#include "rxe.h"
8#include "rxe_loc.h"8
9#define RXE_POOL_ALIGN (16)
9 10
10static const struct rxe_type_info {11static const struct rxe_type_info {
11 const char *name;12 const char *name;
12 size_t size;13 size_t size;
13 size_t elem_offset;14 size_t elem_offset;
14 void (*cleanup)(struct rxe_pool_entry *obj);15 void (*cleanup)(struct rxe_pool_elem *obj);
15 enum rxe_pool_flags flags;16 enum rxe_pool_flags flags;
16 u32 min_index;17 u32 min_index;
17 u32 max_index;18 u32 max_index;
@@ -21,19 +22,19 @@ static const struct rxe_type_info { · 3 unmodified lines
21 [RXE_TYPE_UC] = {22 [RXE_TYPE_UC] = {
22 .name = "rxe-uc",23 .name = "rxe-uc",
23 .size = sizeof(struct rxe_ucontext),24 .size = sizeof(struct rxe_ucontext),
24 .elem_offset = offsetof(struct rxe_ucontext, pelem),25 .elem_offset = offsetof(struct rxe_ucontext, elem),
25 .flags = RXE_POOL_NO_ALLOC,26 .flags = RXE_POOL_NO_ALLOC,
26 },27 },
27 [RXE_TYPE_PD] = {28 [RXE_TYPE_PD] = {
28 .name = "rxe-pd",29 .name = "rxe-pd",
29 .size = sizeof(struct rxe_pd),30 .size = sizeof(struct rxe_pd),
30 .elem_offset = offsetof(struct rxe_pd, pelem),31 .elem_offset = offsetof(struct rxe_pd, elem),
31 .flags = RXE_POOL_NO_ALLOC,32 .flags = RXE_POOL_NO_ALLOC,
32 },33 },
33 [RXE_TYPE_AH] = {34 [RXE_TYPE_AH] = {
34 .name = "rxe-ah",35 .name = "rxe-ah",
35 .size = sizeof(struct rxe_ah),36 .size = sizeof(struct rxe_ah),
36 .elem_offset = offsetof(struct rxe_ah, pelem),37 .elem_offset = offsetof(struct rxe_ah, elem),
37 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,38 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
38 .min_index = RXE_MIN_AH_INDEX,39 .min_index = RXE_MIN_AH_INDEX,
39 .max_index = RXE_MAX_AH_INDEX,40 .max_index = RXE_MAX_AH_INDEX,
@@ -41,7 +42,7 @@ static const struct rxe_type_info { · 1 unmodified line
41 [RXE_TYPE_SRQ] = {42 [RXE_TYPE_SRQ] = {
42 .name = "rxe-srq",43 .name = "rxe-srq",
43 .size = sizeof(struct rxe_srq),44 .size = sizeof(struct rxe_srq),
44 .elem_offset = offsetof(struct rxe_srq, pelem),45 .elem_offset = offsetof(struct rxe_srq, elem),
45 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,46 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
46 .min_index = RXE_MIN_SRQ_INDEX,47 .min_index = RXE_MIN_SRQ_INDEX,
47 .max_index = RXE_MAX_SRQ_INDEX,48 .max_index = RXE_MAX_SRQ_INDEX,
@@ -49,7 +50,7 @@ static const struct rxe_type_info { · 1 unmodified line
49 [RXE_TYPE_QP] = {50 [RXE_TYPE_QP] = {
50 .name = "rxe-qp",51 .name = "rxe-qp",
51 .size = sizeof(struct rxe_qp),52 .size = sizeof(struct rxe_qp),
52 .elem_offset = offsetof(struct rxe_qp, pelem),53 .elem_offset = offsetof(struct rxe_qp, elem),
53 .cleanup = rxe_qp_cleanup,54 .cleanup = rxe_qp_cleanup,
54 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,55 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
55 .min_index = RXE_MIN_QP_INDEX,56 .min_index = RXE_MIN_QP_INDEX,
@@ -58,14 +59,14 @@ static const struct rxe_type_info { · 2 unmodified lines
58 [RXE_TYPE_CQ] = {59 [RXE_TYPE_CQ] = {
59 .name = "rxe-cq",60 .name = "rxe-cq",
60 .size = sizeof(struct rxe_cq),61 .size = sizeof(struct rxe_cq),
61 .elem_offset = offsetof(struct rxe_cq, pelem),62 .elem_offset = offsetof(struct rxe_cq, elem),
62 .flags = RXE_POOL_NO_ALLOC,63 .flags = RXE_POOL_NO_ALLOC,
63 .cleanup = rxe_cq_cleanup,64 .cleanup = rxe_cq_cleanup,
64 },65 },
65 [RXE_TYPE_MR] = {66 [RXE_TYPE_MR] = {
66 .name = "rxe-mr",67 .name = "rxe-mr",
67 .size = sizeof(struct rxe_mr),68 .size = sizeof(struct rxe_mr),
68 .elem_offset = offsetof(struct rxe_mr, pelem),69 .elem_offset = offsetof(struct rxe_mr, elem),
69 .cleanup = rxe_mr_cleanup,70 .cleanup = rxe_mr_cleanup,
70 .flags = RXE_POOL_INDEX,71 .flags = RXE_POOL_INDEX,
71 .min_index = RXE_MIN_MR_INDEX,72 .min_index = RXE_MIN_MR_INDEX,
@@ -74,7 +75,7 @@ static const struct rxe_type_info { · 2 unmodified lines
74 [RXE_TYPE_MW] = {75 [RXE_TYPE_MW] = {
75 .name = "rxe-mw",76 .name = "rxe-mw",
76 .size = sizeof(struct rxe_mw),77 .size = sizeof(struct rxe_mw),
77 .elem_offset = offsetof(struct rxe_mw, pelem),78 .elem_offset = offsetof(struct rxe_mw, elem),
78 .cleanup = rxe_mw_cleanup,79 .cleanup = rxe_mw_cleanup,
79 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,80 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
80 .min_index = RXE_MIN_MW_INDEX,81 .min_index = RXE_MIN_MW_INDEX,
@@ -83,7 +84,7 @@ static const struct rxe_type_info { · 2 unmodified lines
83 [RXE_TYPE_MC_GRP] = {84 [RXE_TYPE_MC_GRP] = {
84 .name = "rxe-mc_grp",85 .name = "rxe-mc_grp",
85 .size = sizeof(struct rxe_mc_grp),86 .size = sizeof(struct rxe_mc_grp),
86 .elem_offset = offsetof(struct rxe_mc_grp, pelem),87 .elem_offset = offsetof(struct rxe_mc_grp, elem),
87 .cleanup = rxe_mc_cleanup,88 .cleanup = rxe_mc_cleanup,
88 .flags = RXE_POOL_KEY,89 .flags = RXE_POOL_KEY,
89 .key_offset = offsetof(struct rxe_mc_grp, mgid),90 .key_offset = offsetof(struct rxe_mc_grp, mgid),
@@ -92,15 +93,10 @@ static const struct rxe_type_info { · 2 unmodified lines
92 [RXE_TYPE_MC_ELEM] = {93 [RXE_TYPE_MC_ELEM] = {
93 .name = "rxe-mc_elem",94 .name = "rxe-mc_elem",
94 .size = sizeof(struct rxe_mc_elem),95 .size = sizeof(struct rxe_mc_elem),
95 .elem_offset = offsetof(struct rxe_mc_elem, pelem),96 .elem_offset = offsetof(struct rxe_mc_elem, elem),
96 },97 },
97};98};
98 99
99static inline const char *pool_name(struct rxe_pool *pool)
100{
101 return rxe_type_info[pool->type].name;
102}
103
104static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)100static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
105{101{
106 int err = 0;102 int err = 0;
@@ -130,35 +126,36 @@ int rxe_pool_init( · 23 unmodified lines
130 enum rxe_elem_type type,126 enum rxe_elem_type type,
131 unsigned int max_elem)127 unsigned int max_elem)
132{128{
129 const struct rxe_type_info *info = &rxe_type_info[type];
133 int err = 0;130 int err = 0;
134 size_t size = rxe_type_info[type].size;
135 131
136 memset(pool, 0, sizeof(*pool));132 memset(pool, 0, sizeof(*pool));
137 133
138 pool->rxe = rxe;134 pool->rxe = rxe;
135 pool->name = info->name;
139 pool->type = type;136 pool->type = type;
140 pool->max_elem = max_elem;137 pool->max_elem = max_elem;
141 pool->elem_size = ALIGN(size, RXE_POOL_ALIGN);138 pool->elem_size = ALIGN(info->size, RXE_POOL_ALIGN);
142 pool->flags = rxe_type_info[type].flags;139 pool->elem_offset = info->elem_offset;
143 pool->index.tree = RB_ROOT;140 pool->flags = info->flags;
144 pool->key.tree = RB_ROOT;141 pool->cleanup = info->cleanup;
145 pool->cleanup = rxe_type_info[type].cleanup;
146 142
147 atomic_set(&pool->num_elem, 0);143 atomic_set(&pool->num_elem, 0);
148 144
149 rwlock_init(&pool->pool_lock);145 rwlock_init(&pool->pool_lock);
150 146
151 if (rxe_type_info[type].flags & RXE_POOL_INDEX) {147 if (pool->flags & RXE_POOL_INDEX) {
152 err = rxe_pool_init_index(pool,148 pool->index.tree = RB_ROOT;
153 rxe_type_info[type].max_index,149 err = rxe_pool_init_index(pool, info->max_index,
154 rxe_type_info[type].min_index);150 info->min_index);
155 if (err)151 if (err)
156 goto out;152 goto out;
157 }153 }
158 154
159 if (rxe_type_info[type].flags & RXE_POOL_KEY) {155 if (pool->flags & RXE_POOL_KEY) {
160 pool->key.key_offset = rxe_type_info[type].key_offset;156 pool->key.tree = RB_ROOT;
161 pool->key.key_size = rxe_type_info[type].key_size;157 pool->key.key_offset = info->key_offset;
158 pool->key.key_size = info->key_size;
162 }159 }
163 160
164out:161out:
@@ -169,9 +166,10 @@ void rxe_pool_cleanup(struct rxe_pool *pool) · 4 unmodified lines
169{166{
170 if (atomic_read(&pool->num_elem) > 0)167 if (atomic_read(&pool->num_elem) > 0)
171 pr_warn("%s pool destroyed with unfree'd elem\n",168 pr_warn("%s pool destroyed with unfree'd elem\n",
172 pool_name(pool));169 pool->name);
173 170
174 bitmap_free(pool->index.table);171 if (pool->flags & RXE_POOL_INDEX)
172 bitmap_free(pool->index.table);
175}173}
176 174
177static u32 alloc_index(struct rxe_pool *pool)175static u32 alloc_index(struct rxe_pool *pool)
@@ -189,15 +187,15 @@ static u32 alloc_index(struct rxe_pool *pool) · 11 unmodified lines
189 return index + pool->index.min_index;187 return index + pool->index.min_index;
190}188}
191 189
192static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)190static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_elem *new)
193{191{
194 struct rb_node **link = &pool->index.tree.rb_node;192 struct rb_node **link = &pool->index.tree.rb_node;
195 struct rb_node *parent = NULL;193 struct rb_node *parent = NULL;
196 struct rxe_pool_entry *elem;194 struct rxe_pool_elem *elem;
197 195
198 while (*link) {196 while (*link) {
199 parent = *link;197 parent = *link;
200 elem = rb_entry(parent, struct rxe_pool_entry, index_node);198 elem = rb_entry(parent, struct rxe_pool_elem, index_node);
201 199
202 if (elem->index == new->index) {200 if (elem->index == new->index) {
203 pr_warn("element already exists!\n");201 pr_warn("element already exists!\n");
@@ -216,19 +214,20 @@ static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new) · 12 unmodified lines
216 return 0;214 return 0;
217}215}
218 216
219static int rxe_insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)217static int rxe_insert_key(struct rxe_pool *pool, struct rxe_pool_elem *new)
220{218{
221 struct rb_node **link = &pool->key.tree.rb_node;219 struct rb_node **link = &pool->key.tree.rb_node;
222 struct rb_node *parent = NULL;220 struct rb_node *parent = NULL;
223 struct rxe_pool_entry *elem;221 struct rxe_pool_elem *elem;
224 int cmp;222 int cmp;
225 223
226 while (*link) {224 while (*link) {
227 parent = *link;225 parent = *link;
228 elem = rb_entry(parent, struct rxe_pool_entry, key_node);226 elem = rb_entry(parent, struct rxe_pool_elem, key_node);
229 227
230 cmp = memcmp((u8 *)elem + pool->key.key_offset,228 cmp = memcmp((u8 *)elem + pool->key.key_offset,
231 (u8 *)new + pool->key.key_offset, pool->key.key_size);229 (u8 *)new + pool->key.key_offset,
230 pool->key.key_size);
232 231
233 if (cmp == 0) {232 if (cmp == 0) {
234 pr_warn("key already exists!\n");233 pr_warn("key already exists!\n");
@@ -247,7 +246,7 @@ static int rxe_insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new) · 12 unmodified lines
247 return 0;246 return 0;
248}247}
249 248
250int __rxe_add_key_locked(struct rxe_pool_entry *elem, void *key)249int __rxe_add_key_locked(struct rxe_pool_elem *elem, void *key)
251{250{
252 struct rxe_pool *pool = elem->pool;251 struct rxe_pool *pool = elem->pool;
253 int err;252 int err;
@@ -258,37 +257,35 @@ int __rxe_add_key_locked(struct rxe_pool_entry *elem, void *key) · 4 unmodified lines
258 return err;257 return err;
259}258}
260 259
261int __rxe_add_key(struct rxe_pool_entry *elem, void *key)260int __rxe_add_key(struct rxe_pool_elem *elem, void *key)
262{261{
263 struct rxe_pool *pool = elem->pool;262 struct rxe_pool *pool = elem->pool;
264 unsigned long flags;
265 int err;263 int err;
266 264
267 write_lock_irqsave(&pool->pool_lock, flags);265 write_lock_bh(&pool->pool_lock);
268 err = __rxe_add_key_locked(elem, key);266 err = __rxe_add_key_locked(elem, key);
269 write_unlock_irqrestore(&pool->pool_lock, flags);267 write_unlock_bh(&pool->pool_lock);
270 268
271 return err;269 return err;
272}270}
273 271
274void __rxe_drop_key_locked(struct rxe_pool_entry *elem)272void __rxe_drop_key_locked(struct rxe_pool_elem *elem)
275{273{
276 struct rxe_pool *pool = elem->pool;274 struct rxe_pool *pool = elem->pool;
277 275
278 rb_erase(&elem->key_node, &pool->key.tree);276 rb_erase(&elem->key_node, &pool->key.tree);
279}277}
280 278
281void __rxe_drop_key(struct rxe_pool_entry *elem)279void __rxe_drop_key(struct rxe_pool_elem *elem)
282{280{
283 struct rxe_pool *pool = elem->pool;281 struct rxe_pool *pool = elem->pool;
284 unsigned long flags;
285 282
286 write_lock_irqsave(&pool->pool_lock, flags);283 write_lock_bh(&pool->pool_lock);
287 __rxe_drop_key_locked(elem);284 __rxe_drop_key_locked(elem);
288 write_unlock_irqrestore(&pool->pool_lock, flags);285 write_unlock_bh(&pool->pool_lock);
289}286}
290 287
291int __rxe_add_index_locked(struct rxe_pool_entry *elem)288int __rxe_add_index_locked(struct rxe_pool_elem *elem)
292{289{
293 struct rxe_pool *pool = elem->pool;290 struct rxe_pool *pool = elem->pool;
294 int err;291 int err;
@@ -299,20 +296,19 @@ int __rxe_add_index_locked(struct rxe_pool_entry *elem) · 4 unmodified lines
299 return err;296 return err;
300}297}
301 298
302int __rxe_add_index(struct rxe_pool_entry *elem)299int __rxe_add_index(struct rxe_pool_elem *elem)
303{300{
304 struct rxe_pool *pool = elem->pool;301 struct rxe_pool *pool = elem->pool;
305 unsigned long flags;
306 int err;302 int err;
307 303
308 write_lock_irqsave(&pool->pool_lock, flags);304 write_lock_bh(&pool->pool_lock);
309 err = __rxe_add_index_locked(elem);305 err = __rxe_add_index_locked(elem);
310 write_unlock_irqrestore(&pool->pool_lock, flags);306 write_unlock_bh(&pool->pool_lock);
311 307
312 return err;308 return err;
313}309}
314 310
315void __rxe_drop_index_locked(struct rxe_pool_entry *elem)311void __rxe_drop_index_locked(struct rxe_pool_elem *elem)
316{312{
317 struct rxe_pool *pool = elem->pool;313 struct rxe_pool *pool = elem->pool;
318 314
@@ -320,32 +316,31 @@ void __rxe_drop_index_locked(struct rxe_pool_entry *elem) · 1 unmodified line
320 rb_erase(&elem->index_node, &pool->index.tree);316 rb_erase(&elem->index_node, &pool->index.tree);
321}317}
322 318
323void __rxe_drop_index(struct rxe_pool_entry *elem)319void __rxe_drop_index(struct rxe_pool_elem *elem)
324{320{
325 struct rxe_pool *pool = elem->pool;321 struct rxe_pool *pool = elem->pool;
326 unsigned long flags;
327 322
328 write_lock_irqsave(&pool->pool_lock, flags);323 write_lock_bh(&pool->pool_lock);
329 __rxe_drop_index_locked(elem);324 __rxe_drop_index_locked(elem);
330 write_unlock_irqrestore(&pool->pool_lock, flags);325 write_unlock_bh(&pool->pool_lock);
331}326}
332 327
333void *rxe_alloc_locked(struct rxe_pool *pool)328void *rxe_alloc_locked(struct rxe_pool *pool)
334{329{
335 const struct rxe_type_info *info = &rxe_type_info[pool->type];330 struct rxe_pool_elem *elem;
336 struct rxe_pool_entry *elem;331 void *obj;
337 u8 *obj;
338 332
339 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)333 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
340 goto out_cnt;334 goto out_cnt;
341 335
342 obj = kzalloc(info->size, GFP_ATOMIC);336 obj = kzalloc(pool->elem_size, GFP_ATOMIC);
343 if (!obj)337 if (!obj)
344 goto out_cnt;338 goto out_cnt;
345 339
346 elem = (struct rxe_pool_entry *)(obj + info->elem_offset);340 elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset);
347 341
348 elem->pool = pool;342 elem->pool = pool;
343 elem->obj = obj;
349 kref_init(&elem->ref_cnt);344 kref_init(&elem->ref_cnt);
350 345
351 return obj;346 return obj;
@@ -357,20 +352,20 @@ out_cnt: · 5 unmodified lines
357 352
358void *rxe_alloc(struct rxe_pool *pool)353void *rxe_alloc(struct rxe_pool *pool)
359{354{
360 const struct rxe_type_info *info = &rxe_type_info[pool->type];355 struct rxe_pool_elem *elem;
361 struct rxe_pool_entry *elem;356 void *obj;
362 u8 *obj;
363 357
364 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)358 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
365 goto out_cnt;359 goto out_cnt;
366 360
367 obj = kzalloc(info->size, GFP_KERNEL);361 obj = kzalloc(pool->elem_size, GFP_KERNEL);
368 if (!obj)362 if (!obj)
369 goto out_cnt;363 goto out_cnt;
370 364
371 elem = (struct rxe_pool_entry *)(obj + info->elem_offset);365 elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset);
372 366
373 elem->pool = pool;367 elem->pool = pool;
368 elem->obj = obj;
374 kref_init(&elem->ref_cnt);369 kref_init(&elem->ref_cnt);
375 370
376 return obj;371 return obj;
@@ -380,12 +375,13 @@ out_cnt: · 3 unmodified lines
380 return NULL;375 return NULL;
381}376}
382 377
383int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)378int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
384{379{
385 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)380 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
386 goto out_cnt;381 goto out_cnt;
387 382
388 elem->pool = pool;383 elem->pool = pool;
384 elem->obj = (u8 *)elem - pool->elem_offset;
389 kref_init(&elem->ref_cnt);385 kref_init(&elem->ref_cnt);
390 386
391 return 0;387 return 0;
@@ -397,17 +393,16 @@ out_cnt: · 5 unmodified lines
397 393
398void rxe_elem_release(struct kref *kref)394void rxe_elem_release(struct kref *kref)
399{395{
400 struct rxe_pool_entry *elem =396 struct rxe_pool_elem *elem =
401 container_of(kref, struct rxe_pool_entry, ref_cnt);397 container_of(kref, struct rxe_pool_elem, ref_cnt);
402 struct rxe_pool *pool = elem->pool;398 struct rxe_pool *pool = elem->pool;
403 const struct rxe_type_info *info = &rxe_type_info[pool->type];399 void *obj;
404 u8 *obj;
405 400
406 if (pool->cleanup)401 if (pool->cleanup)
407 pool->cleanup(elem);402 pool->cleanup(elem);
408 403
409 if (!(pool->flags & RXE_POOL_NO_ALLOC)) {404 if (!(pool->flags & RXE_POOL_NO_ALLOC)) {
410 obj = (u8 *)elem - info->elem_offset;405 obj = elem->obj;
411 kfree(obj);406 kfree(obj);
412 }407 }
413 408
@@ -416,15 +411,14 @@ void rxe_elem_release(struct kref *kref) · 2 unmodified lines
416 411
417void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)412void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)
418{413{
419 const struct rxe_type_info *info = &rxe_type_info[pool->type];
420 struct rb_node *node;414 struct rb_node *node;
421 struct rxe_pool_entry *elem;415 struct rxe_pool_elem *elem;
422 u8 *obj;416 void *obj;
423 417
424 node = pool->index.tree.rb_node;418 node = pool->index.tree.rb_node;
425 419
426 while (node) {420 while (node) {
427 elem = rb_entry(node, struct rxe_pool_entry, index_node);421 elem = rb_entry(node, struct rxe_pool_elem, index_node);
428 422
429 if (elem->index > index)423 if (elem->index > index)
430 node = node->rb_left;424 node = node->rb_left;
@@ -436,7 +430,7 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) · 5 unmodified lines
436 430
437 if (node) {431 if (node) {
438 kref_get(&elem->ref_cnt);432 kref_get(&elem->ref_cnt);
439 obj = (u8 *)elem - info->elem_offset;433 obj = elem->obj;
440 } else {434 } else {
441 obj = NULL;435 obj = NULL;
442 }436 }
@@ -446,28 +440,26 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) · 3 unmodified lines
446 440
447void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)441void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
448{442{
449 u8 *obj;443 void *obj;
450 unsigned long flags;
451 444
452 read_lock_irqsave(&pool->pool_lock, flags);445 read_lock_bh(&pool->pool_lock);
453 obj = rxe_pool_get_index_locked(pool, index);446 obj = rxe_pool_get_index_locked(pool, index);
454 read_unlock_irqrestore(&pool->pool_lock, flags);447 read_unlock_bh(&pool->pool_lock);
455 448
456 return obj;449 return obj;
457}450}
458 451
459void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key)452void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key)
460{453{
461 const struct rxe_type_info *info = &rxe_type_info[pool->type];
462 struct rb_node *node;454 struct rb_node *node;
463 struct rxe_pool_entry *elem;455 struct rxe_pool_elem *elem;
464 u8 *obj;456 void *obj;
465 int cmp;457 int cmp;
466 458
467 node = pool->key.tree.rb_node;459 node = pool->key.tree.rb_node;
468 460
469 while (node) {461 while (node) {
470 elem = rb_entry(node, struct rxe_pool_entry, key_node);462 elem = rb_entry(node, struct rxe_pool_elem, key_node);
471 463
472 cmp = memcmp((u8 *)elem + pool->key.key_offset,464 cmp = memcmp((u8 *)elem + pool->key.key_offset,
473 key, pool->key.key_size);465 key, pool->key.key_size);
@@ -482,7 +474,7 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key) · 8 unmodified lines
482 474
483 if (node) {475 if (node) {
484 kref_get(&elem->ref_cnt);476 kref_get(&elem->ref_cnt);
485 obj = (u8 *)elem - info->elem_offset;477 obj = elem->obj;
486 } else {478 } else {
487 obj = NULL;479 obj = NULL;
488 }480 }
@@ -492,12 +484,11 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key) · 3 unmodified lines
492 484
493void *rxe_pool_get_key(struct rxe_pool *pool, void *key)485void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
494{486{
495 u8 *obj;487 void *obj;
496 unsigned long flags;
497 488
498 read_lock_irqsave(&pool->pool_lock, flags);489 read_lock_bh(&pool->pool_lock);
499 obj = rxe_pool_get_key_locked(pool, key);490 obj = rxe_pool_get_key_locked(pool, key);
500 read_unlock_irqrestore(&pool->pool_lock, flags);491 read_unlock_bh(&pool->pool_lock);
501 492
502 return obj;493 return obj;
503}494}
drivers/infiniband/sw/rxe/rxe_pool.h+26−28
@@ -7,9 +7,6 @@ · 6 unmodified lines
7#ifndef RXE_POOL_H7#ifndef RXE_POOL_H
8#define RXE_POOL_H8#define RXE_POOL_H
9 9
10#define RXE_POOL_ALIGN (16)
11#define RXE_POOL_CACHE_FLAGS (0)
12
13enum rxe_pool_flags {10enum rxe_pool_flags {
14 RXE_POOL_INDEX = BIT(1),11 RXE_POOL_INDEX = BIT(1),
15 RXE_POOL_KEY = BIT(2),12 RXE_POOL_KEY = BIT(2),
@@ -30,10 +27,9 @@ enum rxe_elem_type { · 14 unmodified lines
30 RXE_NUM_TYPES, /* keep me last */27 RXE_NUM_TYPES, /* keep me last */
31};28};
32 29
33struct rxe_pool_entry;30struct rxe_pool_elem {
34
35struct rxe_pool_entry {
36 struct rxe_pool *pool;31 struct rxe_pool *pool;
32 void *obj;
37 struct kref ref_cnt;33 struct kref ref_cnt;
38 struct list_head list;34 struct list_head list;
39 35
@@ -47,14 +43,16 @@ struct rxe_pool_entry { · 7 unmodified lines
47 43
48struct rxe_pool {44struct rxe_pool {
49 struct rxe_dev *rxe;45 struct rxe_dev *rxe;
46 const char *name;
50 rwlock_t pool_lock; /* protects pool add/del/search */47 rwlock_t pool_lock; /* protects pool add/del/search */
51 size_t elem_size;48 void (*cleanup)(struct rxe_pool_elem *obj);
52 void (*cleanup)(struct rxe_pool_entry *obj);
53 enum rxe_pool_flags flags;49 enum rxe_pool_flags flags;
54 enum rxe_elem_type type;50 enum rxe_elem_type type;
55 51
56 unsigned int max_elem;52 unsigned int max_elem;
57 atomic_t num_elem;53 atomic_t num_elem;
54 size_t elem_size;
55 size_t elem_offset;
58 56
59 /* only used if indexed */57 /* only used if indexed */
60 struct {58 struct {
@@ -89,51 +87,51 @@ void *rxe_alloc_locked(struct rxe_pool *pool); · 28 unmodified lines
89void *rxe_alloc(struct rxe_pool *pool);87void *rxe_alloc(struct rxe_pool *pool);
90 88
91/* connect already allocated object to pool */89/* connect already allocated object to pool */
92int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem);90int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem);
93 91
94#define rxe_add_to_pool(pool, obj) __rxe_add_to_pool(pool, &(obj)->pelem)92#define rxe_add_to_pool(pool, obj) __rxe_add_to_pool(pool, &(obj)->elem)
95 93
96/* assign an index to an indexed object and insert object into94/* assign an index to an indexed object and insert object into
97 * pool's rb tree holding and not holding the pool_lock95 * pool's rb tree holding and not holding the pool_lock
98 */96 */
99int __rxe_add_index_locked(struct rxe_pool_entry *elem);97int __rxe_add_index_locked(struct rxe_pool_elem *elem);
100 98
101#define rxe_add_index_locked(obj) __rxe_add_index_locked(&(obj)->pelem)99#define rxe_add_index_locked(obj) __rxe_add_index_locked(&(obj)->elem)
102 100
103int __rxe_add_index(struct rxe_pool_entry *elem);101int __rxe_add_index(struct rxe_pool_elem *elem);
104 102
105#define rxe_add_index(obj) __rxe_add_index(&(obj)->pelem)103#define rxe_add_index(obj) __rxe_add_index(&(obj)->elem)
106 104
107/* drop an index and remove object from rb tree105/* drop an index and remove object from rb tree
108 * holding and not holding the pool_lock106 * holding and not holding the pool_lock
109 */107 */
110void __rxe_drop_index_locked(struct rxe_pool_entry *elem);108void __rxe_drop_index_locked(struct rxe_pool_elem *elem);
111 109
112#define rxe_drop_index_locked(obj) __rxe_drop_index_locked(&(obj)->pelem)110#define rxe_drop_index_locked(obj) __rxe_drop_index_locked(&(obj)->elem)
113 111
114void __rxe_drop_index(struct rxe_pool_entry *elem);112void __rxe_drop_index(struct rxe_pool_elem *elem);
115 113
116#define rxe_drop_index(obj) __rxe_drop_index(&(obj)->pelem)114#define rxe_drop_index(obj) __rxe_drop_index(&(obj)->elem)
117 115
118/* assign a key to a keyed object and insert object into116/* assign a key to a keyed object and insert object into
119 * pool's rb tree holding and not holding pool_lock117 * pool's rb tree holding and not holding pool_lock
120 */118 */
121int __rxe_add_key_locked(struct rxe_pool_entry *elem, void *key);119int __rxe_add_key_locked(struct rxe_pool_elem *elem, void *key);
122 120
123#define rxe_add_key_locked(obj, key) __rxe_add_key_locked(&(obj)->pelem, key)121#define rxe_add_key_locked(obj, key) __rxe_add_key_locked(&(obj)->elem, key)
124 122
125int __rxe_add_key(struct rxe_pool_entry *elem, void *key);123int __rxe_add_key(struct rxe_pool_elem *elem, void *key);
126 124
127#define rxe_add_key(obj, key) __rxe_add_key(&(obj)->pelem, key)125#define rxe_add_key(obj, key) __rxe_add_key(&(obj)->elem, key)
128 126
129/* remove elem from rb tree holding and not holding the pool_lock */127/* remove elem from rb tree holding and not holding the pool_lock */
130void __rxe_drop_key_locked(struct rxe_pool_entry *elem);128void __rxe_drop_key_locked(struct rxe_pool_elem *elem);
131 129
132#define rxe_drop_key_locked(obj) __rxe_drop_key_locked(&(obj)->pelem)130#define rxe_drop_key_locked(obj) __rxe_drop_key_locked(&(obj)->elem)
133 131
134void __rxe_drop_key(struct rxe_pool_entry *elem);132void __rxe_drop_key(struct rxe_pool_elem *elem);
135 133
136#define rxe_drop_key(obj) __rxe_drop_key(&(obj)->pelem)134#define rxe_drop_key(obj) __rxe_drop_key(&(obj)->elem)
137 135
138/* lookup an indexed object from index holding and not holding the pool_lock.136/* lookup an indexed object from index holding and not holding the pool_lock.
139 * takes a reference on object137 * takes a reference on object
@@ -153,9 +151,9 @@ void *rxe_pool_get_key(struct rxe_pool *pool, void *key); · 13 unmodified lines
153void rxe_elem_release(struct kref *kref);151void rxe_elem_release(struct kref *kref);
154 152
155/* take a reference on an object */153/* take a reference on an object */
156#define rxe_add_ref(elem) kref_get(&(elem)->pelem.ref_cnt)154#define rxe_add_ref(obj) kref_get(&(obj)->elem.ref_cnt)
157 155
158/* drop a reference on an object */156/* drop a reference on an object */
159#define rxe_drop_ref(elem) kref_put(&(elem)->pelem.ref_cnt, rxe_elem_release)157#define rxe_drop_ref(obj) kref_put(&(obj)->elem.ref_cnt, rxe_elem_release)
160 158
161#endif /* RXE_POOL_H */159#endif /* RXE_POOL_H */
drivers/infiniband/sw/rxe/rxe_qp.c+3−3
@@ -167,7 +167,7 @@ static void rxe_qp_init_misc(struct rxe_dev *rxe, struct rxe_qp *qp, · 166 unmodified lines
167 qp->attr.path_mtu = 1;167 qp->attr.path_mtu = 1;
168 qp->mtu = ib_mtu_enum_to_int(qp->attr.path_mtu);168 qp->mtu = ib_mtu_enum_to_int(qp->attr.path_mtu);
169 169
170 qpn = qp->pelem.index;170 qpn = qp->elem.index;
171 port = &rxe->port;171 port = &rxe->port;
172 172
173 switch (init->qp_type) {173 switch (init->qp_type) {
@@ -832,9 +832,9 @@ static void rxe_qp_do_cleanup(struct work_struct *work) · 658 unmodified lines
832}832}
833 833
834/* called when the last reference to the qp is dropped */834/* called when the last reference to the qp is dropped */
835void rxe_qp_cleanup(struct rxe_pool_entry *arg)835void rxe_qp_cleanup(struct rxe_pool_elem *elem)
836{836{
837 struct rxe_qp *qp = container_of(arg, typeof(*qp), pelem);837 struct rxe_qp *qp = container_of(elem, typeof(*qp), elem);
838 838
839 execute_in_process_context(rxe_qp_do_cleanup, &qp->cleanup_work);839 execute_in_process_context(rxe_qp_do_cleanup, &qp->cleanup_work);
840}840}
drivers/infiniband/sw/rxe/rxe_queue.c+4−5
@@ -151,7 +151,6 @@ int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, · 150 unmodified lines
151 struct rxe_queue *new_q;151 struct rxe_queue *new_q;
152 unsigned int num_elem = *num_elem_p;152 unsigned int num_elem = *num_elem_p;
153 int err;153 int err;
154 unsigned long flags = 0, flags1;
155 154
156 new_q = rxe_queue_init(q->rxe, &num_elem, elem_size, q->type);155 new_q = rxe_queue_init(q->rxe, &num_elem, elem_size, q->type);
157 if (!new_q)156 if (!new_q)
@@ -165,17 +164,17 @@ int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, · 7 unmodified lines
165 goto err1;164 goto err1;
166 }165 }
167 166
168 spin_lock_irqsave(consumer_lock, flags1);167 spin_lock_bh(consumer_lock);
169 168
170 if (producer_lock) {169 if (producer_lock) {
171 spin_lock_irqsave(producer_lock, flags);170 spin_lock_bh(producer_lock);
172 err = resize_finish(q, new_q, num_elem);171 err = resize_finish(q, new_q, num_elem);
173 spin_unlock_irqrestore(producer_lock, flags);172 spin_unlock_bh(producer_lock);
174 } else {173 } else {
175 err = resize_finish(q, new_q, num_elem);174 err = resize_finish(q, new_q, num_elem);
176 }175 }
177 176
178 spin_unlock_irqrestore(consumer_lock, flags1);177 spin_unlock_bh(consumer_lock);
179 178
180 rxe_queue_cleanup(new_q); /* new/old dep on err */179 rxe_queue_cleanup(new_q); /* new/old dep on err */
181 if (err)180 if (err)
drivers/infiniband/sw/rxe/rxe_req.c+5−11
@@ -110,7 +110,6 @@ void rnr_nak_timer(struct timer_list *t) · 109 unmodified lines
110static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)110static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
111{111{
112 struct rxe_send_wqe *wqe;112 struct rxe_send_wqe *wqe;
113 unsigned long flags;
114 struct rxe_queue *q = qp->sq.queue;113 struct rxe_queue *q = qp->sq.queue;
115 unsigned int index = qp->req.wqe_index;114 unsigned int index = qp->req.wqe_index;
116 unsigned int cons;115 unsigned int cons;
@@ -124,25 +123,23 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) · 7 unmodified lines
124 /* check to see if we are drained;123 /* check to see if we are drained;
125 * state_lock used by requester and completer124 * state_lock used by requester and completer
126 */125 */
127 spin_lock_irqsave(&qp->state_lock, flags);126 spin_lock_bh(&qp->state_lock);
128 do {127 do {
129 if (qp->req.state != QP_STATE_DRAIN) {128 if (qp->req.state != QP_STATE_DRAIN) {
130 /* comp just finished */129 /* comp just finished */
131 spin_unlock_irqrestore(&qp->state_lock,130 spin_unlock_bh(&qp->state_lock);
132 flags);
133 break;131 break;
134 }132 }
135 133
136 if (wqe && ((index != cons) ||134 if (wqe && ((index != cons) ||
137 (wqe->state != wqe_state_posted))) {135 (wqe->state != wqe_state_posted))) {
138 /* comp not done yet */136 /* comp not done yet */
139 spin_unlock_irqrestore(&qp->state_lock,137 spin_unlock_bh(&qp->state_lock);
140 flags);
141 break;138 break;
142 }139 }
143 140
144 qp->req.state = QP_STATE_DRAINED;141 qp->req.state = QP_STATE_DRAINED;
145 spin_unlock_irqrestore(&qp->state_lock, flags);142 spin_unlock_bh(&qp->state_lock);
146 143
147 if (qp->ibqp.event_handler) {144 if (qp->ibqp.event_handler) {
148 struct ib_event ev;145 struct ib_event ev;
@@ -372,7 +369,6 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, · 223 unmodified lines
372 int pad = (-payload) & 0x3;369 int pad = (-payload) & 0x3;
373 int paylen;370 int paylen;
374 int solicited;371 int solicited;
375 u16 pkey;
376 u32 qp_num;372 u32 qp_num;
377 int ack_req;373 int ack_req;
378 374
@@ -404,8 +400,6 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, · 25 unmodified lines
404 (pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) ==400 (pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) ==
405 (RXE_WRITE_MASK | RXE_IMMDT_MASK));401 (RXE_WRITE_MASK | RXE_IMMDT_MASK));
406 402
407 pkey = IB_DEFAULT_PKEY_FULL;
408
409 qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn :403 qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn :
410 qp->attr.dest_qp_num;404 qp->attr.dest_qp_num;
411 405
@@ -414,7 +408,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, · 2 unmodified lines
414 if (ack_req)408 if (ack_req)
415 qp->req.noack_pkts = 0;409 qp->req.noack_pkts = 0;
416 410
417 bth_init(pkt, pkt->opcode, solicited, 0, pad, pkey, qp_num,411 bth_init(pkt, pkt->opcode, solicited, 0, pad, IB_DEFAULT_PKEY_FULL, qp_num,
418 ack_req, pkt->psn);412 ack_req, pkt->psn);
419 413
420 /* init optional headers */414 /* init optional headers */
drivers/infiniband/sw/rxe/rxe_srq.c+1−1
@@ -83,7 +83,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, · 82 unmodified lines
83 srq->ibsrq.event_handler = init->event_handler;83 srq->ibsrq.event_handler = init->event_handler;
84 srq->ibsrq.srq_context = init->srq_context;84 srq->ibsrq.srq_context = init->srq_context;
85 srq->limit = init->attr.srq_limit;85 srq->limit = init->attr.srq_limit;
86 srq->srq_num = srq->pelem.index;86 srq->srq_num = srq->elem.index;
87 srq->rq.max_wr = init->attr.max_wr;87 srq->rq.max_wr = init->attr.max_wr;
88 srq->rq.max_sge = init->attr.max_sge;88 srq->rq.max_sge = init->attr.max_sge;
89 89
drivers/infiniband/sw/rxe/rxe_sysfs.c+0−119
@@ -1,119 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7#include "rxe.h"
8#include "rxe_net.h"
9
10/* Copy argument and remove trailing CR. Return the new length. */
11static int sanitize_arg(const char *val, char *intf, int intf_len)
12{
13 int len;
14
15 if (!val)
16 return 0;
17
18 /* Remove newline. */
19 for (len = 0; len < intf_len - 1 && val[len] && val[len] != '\n'; len++)
20 intf[len] = val[len];
21 intf[len] = 0;
22
23 if (len == 0 || (val[len] != 0 && val[len] != '\n'))
24 return 0;
25
26 return len;
27}
28
29static int rxe_param_set_add(const char *val, const struct kernel_param *kp)
30{
31 int len;
32 int err = 0;
33 char intf[32];
34 struct net_device *ndev;
35 struct rxe_dev *exists;
36
37 if (!rxe_initialized) {
38 pr_err("Module parameters are not supported, use rdma link add or rxe_cfg\n");
39 return -EAGAIN;
40 }
41
42 len = sanitize_arg(val, intf, sizeof(intf));
43 if (!len) {
44 pr_err("add: invalid interface name\n");
45 return -EINVAL;
46 }
47
48 ndev = dev_get_by_name(&init_net, intf);
49 if (!ndev) {
50 pr_err("interface %s not found\n", intf);
51 return -EINVAL;
52 }
53
54 if (is_vlan_dev(ndev)) {
55 pr_err("rxe creation allowed on top of a real device only\n");
56 err = -EPERM;
57 goto err;
58 }
59
60 exists = rxe_get_dev_from_net(ndev);
61 if (exists) {
62 ib_device_put(&exists->ib_dev);
63 pr_err("already configured on %s\n", intf);
64 err = -EINVAL;
65 goto err;
66 }
67
68 err = rxe_net_add("rxe%d", ndev);
69 if (err) {
70 pr_err("failed to add %s\n", intf);
71 goto err;
72 }
73
74err:
75 dev_put(ndev);
76 return err;
77}
78
79static int rxe_param_set_remove(const char *val, const struct kernel_param *kp)
80{
81 int len;
82 char intf[32];
83 struct ib_device *ib_dev;
84
85 len = sanitize_arg(val, intf, sizeof(intf));
86 if (!len) {
87 pr_err("add: invalid interface name\n");
88 return -EINVAL;
89 }
90
91 if (strncmp("all", intf, len) == 0) {
92 pr_info("rxe_sys: remove all");
93 ib_unregister_driver(RDMA_DRIVER_RXE);
94 return 0;
95 }
96
97 ib_dev = ib_device_get_by_name(intf, RDMA_DRIVER_RXE);
98 if (!ib_dev) {
99 pr_err("not configured on %s\n", intf);
100 return -EINVAL;
101 }
102
103 ib_unregister_device_and_put(ib_dev);
104
105 return 0;
106}
107
108static const struct kernel_param_ops rxe_add_ops = {
109 .set = rxe_param_set_add,
110};
111
112static const struct kernel_param_ops rxe_remove_ops = {
113 .set = rxe_param_set_remove,
114};
115
116module_param_cb(add, &rxe_add_ops, NULL, 0200);
117MODULE_PARM_DESC(add, "DEPRECATED. Create RXE device over network interface");
118module_param_cb(remove, &rxe_remove_ops, NULL, 0200);
119MODULE_PARM_DESC(remove, "DEPRECATED. Remove RXE device over network interface");
drivers/infiniband/sw/rxe/rxe_task.c+8−10
@@ -32,25 +32,24 @@ void rxe_do_task(struct tasklet_struct *t) · 31 unmodified lines
32{32{
33 int cont;33 int cont;
34 int ret;34 int ret;
35 unsigned long flags;
36 struct rxe_task *task = from_tasklet(task, t, tasklet);35 struct rxe_task *task = from_tasklet(task, t, tasklet);
37 36
38 spin_lock_irqsave(&task->state_lock, flags);37 spin_lock_bh(&task->state_lock);
39 switch (task->state) {38 switch (task->state) {
40 case TASK_STATE_START:39 case TASK_STATE_START:
41 task->state = TASK_STATE_BUSY;40 task->state = TASK_STATE_BUSY;
42 spin_unlock_irqrestore(&task->state_lock, flags);41 spin_unlock_bh(&task->state_lock);
43 break;42 break;
44 43
45 case TASK_STATE_BUSY:44 case TASK_STATE_BUSY:
46 task->state = TASK_STATE_ARMED;45 task->state = TASK_STATE_ARMED;
47 fallthrough;46 fallthrough;
48 case TASK_STATE_ARMED:47 case TASK_STATE_ARMED:
49 spin_unlock_irqrestore(&task->state_lock, flags);48 spin_unlock_bh(&task->state_lock);
50 return;49 return;
51 50
52 default:51 default:
53 spin_unlock_irqrestore(&task->state_lock, flags);52 spin_unlock_bh(&task->state_lock);
54 pr_warn("%s failed with bad state %d\n", __func__, task->state);53 pr_warn("%s failed with bad state %d\n", __func__, task->state);
55 return;54 return;
56 }55 }
@@ -59,7 +58,7 @@ void rxe_do_task(struct tasklet_struct *t) · 2 unmodified lines
59 cont = 0;58 cont = 0;
60 ret = task->func(task->arg);59 ret = task->func(task->arg);
61 60
62 spin_lock_irqsave(&task->state_lock, flags);61 spin_lock_bh(&task->state_lock);
63 switch (task->state) {62 switch (task->state) {
64 case TASK_STATE_BUSY:63 case TASK_STATE_BUSY:
65 if (ret)64 if (ret)
@@ -81,7 +80,7 @@ void rxe_do_task(struct tasklet_struct *t) · 15 unmodified lines
81 pr_warn("%s failed with bad state %d\n", __func__,80 pr_warn("%s failed with bad state %d\n", __func__,
82 task->state);81 task->state);
83 }82 }
84 spin_unlock_irqrestore(&task->state_lock, flags);83 spin_unlock_bh(&task->state_lock);
85 } while (cont);84 } while (cont);
86 85
87 task->ret = ret;86 task->ret = ret;
@@ -106,7 +105,6 @@ int rxe_init_task(void *obj, struct rxe_task *task, · 18 unmodified lines
106 105
107void rxe_cleanup_task(struct rxe_task *task)106void rxe_cleanup_task(struct rxe_task *task)
108{107{
109 unsigned long flags;
110 bool idle;108 bool idle;
111 109
112 /*110 /*
@@ -116,9 +114,9 @@ void rxe_cleanup_task(struct rxe_task *task) · 3 unmodified lines
116 task->destroyed = true;114 task->destroyed = true;
117 115
118 do {116 do {
119 spin_lock_irqsave(&task->state_lock, flags);117 spin_lock_bh(&task->state_lock);
120 idle = (task->state == TASK_STATE_START);118 idle = (task->state == TASK_STATE_START);
121 spin_unlock_irqrestore(&task->state_lock, flags);119 spin_unlock_bh(&task->state_lock);
122 } while (!idle);120 } while (!idle);
123 121
124 tasklet_kill(&task->tasklet);122 tasklet_kill(&task->tasklet);
drivers/infiniband/sw/rxe/rxe_verbs.c+17−17
@@ -182,7 +182,7 @@ static int rxe_create_ah(struct ib_ah *ibah, · 181 unmodified lines
182 182
183 /* create index > 0 */183 /* create index > 0 */
184 rxe_add_index(ah);184 rxe_add_index(ah);
185 ah->ah_num = ah->pelem.index;185 ah->ah_num = ah->elem.index;
186 186
187 if (uresp) {187 if (uresp) {
188 /* only if new user provider */188 /* only if new user provider */
@@ -383,10 +383,9 @@ static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, · 194 unmodified lines
383 const struct ib_recv_wr **bad_wr)383 const struct ib_recv_wr **bad_wr)
384{384{
385 int err = 0;385 int err = 0;
386 unsigned long flags;
387 struct rxe_srq *srq = to_rsrq(ibsrq);386 struct rxe_srq *srq = to_rsrq(ibsrq);
388 387
389 spin_lock_irqsave(&srq->rq.producer_lock, flags);388 spin_lock_bh(&srq->rq.producer_lock);
390 389
391 while (wr) {390 while (wr) {
392 err = post_one_recv(&srq->rq, wr);391 err = post_one_recv(&srq->rq, wr);
@@ -395,7 +394,7 @@ static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, · 2 unmodified lines
395 wr = wr->next;394 wr = wr->next;
396 }395 }
397 396
398 spin_unlock_irqrestore(&srq->rq.producer_lock, flags);397 spin_unlock_bh(&srq->rq.producer_lock);
399 398
400 if (err)399 if (err)
401 *bad_wr = wr;400 *bad_wr = wr;
@@ -469,6 +468,11 @@ static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 67 unmodified lines
469 if (err)468 if (err)
470 goto err1;469 goto err1;
471 470
471 if ((mask & IB_QP_AV) && (attr->ah_attr.ah_flags & IB_AH_GRH))
472 qp->src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
473 qp->ibqp.qp_num,
474 qp->attr.dest_qp_num);
475
472 return 0;476 return 0;
473 477
474err1:478err1:
@@ -634,19 +638,18 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr, · 159 unmodified lines
634 int err;638 int err;
635 struct rxe_sq *sq = &qp->sq;639 struct rxe_sq *sq = &qp->sq;
636 struct rxe_send_wqe *send_wqe;640 struct rxe_send_wqe *send_wqe;
637 unsigned long flags;
638 int full;641 int full;
639 642
640 err = validate_send_wr(qp, ibwr, mask, length);643 err = validate_send_wr(qp, ibwr, mask, length);
641 if (err)644 if (err)
642 return err;645 return err;
643 646
644 spin_lock_irqsave(&qp->sq.sq_lock, flags);647 spin_lock_bh(&qp->sq.sq_lock);
645 648
646 full = queue_full(sq->queue, QUEUE_TYPE_TO_DRIVER);649 full = queue_full(sq->queue, QUEUE_TYPE_TO_DRIVER);
647 650
648 if (unlikely(full)) {651 if (unlikely(full)) {
649 spin_unlock_irqrestore(&qp->sq.sq_lock, flags);652 spin_unlock_bh(&qp->sq.sq_lock);
650 return -ENOMEM;653 return -ENOMEM;
651 }654 }
652 655
@@ -655,7 +658,7 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr, · 2 unmodified lines
655 658
656 queue_advance_producer(sq->queue, QUEUE_TYPE_TO_DRIVER);659 queue_advance_producer(sq->queue, QUEUE_TYPE_TO_DRIVER);
657 660
658 spin_unlock_irqrestore(&qp->sq.sq_lock, flags);661 spin_unlock_bh(&qp->sq.sq_lock);
659 662
660 return 0;663 return 0;
661}664}
@@ -735,7 +738,6 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, · 73 unmodified lines
735 int err = 0;738 int err = 0;
736 struct rxe_qp *qp = to_rqp(ibqp);739 struct rxe_qp *qp = to_rqp(ibqp);
737 struct rxe_rq *rq = &qp->rq;740 struct rxe_rq *rq = &qp->rq;
738 unsigned long flags;
739 741
740 if (unlikely((qp_state(qp) < IB_QPS_INIT) || !qp->valid)) {742 if (unlikely((qp_state(qp) < IB_QPS_INIT) || !qp->valid)) {
741 *bad_wr = wr;743 *bad_wr = wr;
@@ -749,7 +751,7 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, · 7 unmodified lines
749 goto err1;751 goto err1;
750 }752 }
751 753
752 spin_lock_irqsave(&rq->producer_lock, flags);754 spin_lock_bh(&rq->producer_lock);
753 755
754 while (wr) {756 while (wr) {
755 err = post_one_recv(rq, wr);757 err = post_one_recv(rq, wr);
@@ -760,7 +762,7 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, · 4 unmodified lines
760 wr = wr->next;762 wr = wr->next;
761 }763 }
762 764
763 spin_unlock_irqrestore(&rq->producer_lock, flags);765 spin_unlock_bh(&rq->producer_lock);
764 766
765 if (qp->resp.state == QP_STATE_ERROR)767 if (qp->resp.state == QP_STATE_ERROR)
766 rxe_run_task(&qp->resp.task, 1);768 rxe_run_task(&qp->resp.task, 1);
@@ -841,9 +843,8 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) · 74 unmodified lines
841 int i;843 int i;
842 struct rxe_cq *cq = to_rcq(ibcq);844 struct rxe_cq *cq = to_rcq(ibcq);
843 struct rxe_cqe *cqe;845 struct rxe_cqe *cqe;
844 unsigned long flags;
845 846
846 spin_lock_irqsave(&cq->cq_lock, flags);847 spin_lock_bh(&cq->cq_lock);
847 for (i = 0; i < num_entries; i++) {848 for (i = 0; i < num_entries; i++) {
848 cqe = queue_head(cq->queue, QUEUE_TYPE_FROM_DRIVER);849 cqe = queue_head(cq->queue, QUEUE_TYPE_FROM_DRIVER);
849 if (!cqe)850 if (!cqe)
@@ -852,7 +853,7 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) · 2 unmodified lines
852 memcpy(wc++, &cqe->ibwc, sizeof(*wc));853 memcpy(wc++, &cqe->ibwc, sizeof(*wc));
853 queue_advance_consumer(cq->queue, QUEUE_TYPE_FROM_DRIVER);854 queue_advance_consumer(cq->queue, QUEUE_TYPE_FROM_DRIVER);
854 }855 }
855 spin_unlock_irqrestore(&cq->cq_lock, flags);856 spin_unlock_bh(&cq->cq_lock);
856 857
857 return i;858 return i;
858}859}
@@ -870,11 +871,10 @@ static int rxe_peek_cq(struct ib_cq *ibcq, int wc_cnt) · 11 unmodified lines
870static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)871static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
871{872{
872 struct rxe_cq *cq = to_rcq(ibcq);873 struct rxe_cq *cq = to_rcq(ibcq);
873 unsigned long irq_flags;
874 int ret = 0;874 int ret = 0;
875 int empty;875 int empty;
876 876
877 spin_lock_irqsave(&cq->cq_lock, irq_flags);877 spin_lock_bh(&cq->cq_lock);
878 if (cq->notify != IB_CQ_NEXT_COMP)878 if (cq->notify != IB_CQ_NEXT_COMP)
879 cq->notify = flags & IB_CQ_SOLICITED_MASK;879 cq->notify = flags & IB_CQ_SOLICITED_MASK;
880 880
@@ -883,7 +883,7 @@ static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) · 2 unmodified lines
883 if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty)883 if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty)
884 ret = 1;884 ret = 1;
885 885
886 spin_unlock_irqrestore(&cq->cq_lock, irq_flags);886 spin_unlock_bh(&cq->cq_lock);
887 887
888 return ret;888 return ret;
889}889}
drivers/infiniband/sw/rxe/rxe_verbs.h+11−13
@@ -35,17 +35,17 @@ static inline int psn_compare(u32 psn_a, u32 psn_b) · 34 unmodified lines
35 35
36struct rxe_ucontext {36struct rxe_ucontext {
37 struct ib_ucontext ibuc;37 struct ib_ucontext ibuc;
38 struct rxe_pool_entry pelem;38 struct rxe_pool_elem elem;
39};39};
40 40
41struct rxe_pd {41struct rxe_pd {
42 struct ib_pd ibpd;42 struct ib_pd ibpd;
43 struct rxe_pool_entry pelem;43 struct rxe_pool_elem elem;
44};44};
45 45
46struct rxe_ah {46struct rxe_ah {
47 struct ib_ah ibah;47 struct ib_ah ibah;
48 struct rxe_pool_entry pelem;48 struct rxe_pool_elem elem;
49 struct rxe_av av;49 struct rxe_av av;
50 bool is_user;50 bool is_user;
51 int ah_num;51 int ah_num;
@@ -60,7 +60,7 @@ struct rxe_cqe { · 8 unmodified lines
60 60
61struct rxe_cq {61struct rxe_cq {
62 struct ib_cq ibcq;62 struct ib_cq ibcq;
63 struct rxe_pool_entry pelem;63 struct rxe_pool_elem elem;
64 struct rxe_queue *queue;64 struct rxe_queue *queue;
65 spinlock_t cq_lock;65 spinlock_t cq_lock;
66 u8 notify;66 u8 notify;
@@ -95,7 +95,7 @@ struct rxe_rq { · 28 unmodified lines
95 95
96struct rxe_srq {96struct rxe_srq {
97 struct ib_srq ibsrq;97 struct ib_srq ibsrq;
98 struct rxe_pool_entry pelem;98 struct rxe_pool_elem elem;
99 struct rxe_pd *pd;99 struct rxe_pd *pd;
100 struct rxe_rq rq;100 struct rxe_rq rq;
101 u32 srq_num;101 u32 srq_num;
@@ -209,7 +209,7 @@ struct rxe_resp_info { · 107 unmodified lines
209 209
210struct rxe_qp {210struct rxe_qp {
211 struct ib_qp ibqp;211 struct ib_qp ibqp;
212 struct rxe_pool_entry pelem;212 struct rxe_pool_elem elem;
213 struct ib_qp_attr attr;213 struct ib_qp_attr attr;
214 unsigned int valid;214 unsigned int valid;
215 unsigned int mtu;215 unsigned int mtu;
@@ -309,7 +309,7 @@ static inline int rkey_is_mw(u32 rkey) · 93 unmodified lines
309}309}
310 310
311struct rxe_mr {311struct rxe_mr {
312 struct rxe_pool_entry pelem;312 struct rxe_pool_elem elem;
313 struct ib_mr ibmr;313 struct ib_mr ibmr;
314 314
315 struct ib_umem *umem;315 struct ib_umem *umem;
@@ -342,7 +342,7 @@ enum rxe_mw_state { · 26 unmodified lines
342 342
343struct rxe_mw {343struct rxe_mw {
344 struct ib_mw ibmw;344 struct ib_mw ibmw;
345 struct rxe_pool_entry pelem;345 struct rxe_pool_elem elem;
346 spinlock_t lock;346 spinlock_t lock;
347 enum rxe_mw_state state;347 enum rxe_mw_state state;
348 struct rxe_qp *qp; /* Type 2 only */348 struct rxe_qp *qp; /* Type 2 only */
@@ -354,7 +354,7 @@ struct rxe_mw { · 5 unmodified lines
354};354};
355 355
356struct rxe_mc_grp {356struct rxe_mc_grp {
357 struct rxe_pool_entry pelem;357 struct rxe_pool_elem elem;
358 spinlock_t mcg_lock; /* guard group */358 spinlock_t mcg_lock; /* guard group */
359 struct rxe_dev *rxe;359 struct rxe_dev *rxe;
360 struct list_head qp_list;360 struct list_head qp_list;
@@ -365,7 +365,7 @@ struct rxe_mc_grp { · 4 unmodified lines
365};365};
366 366
367struct rxe_mc_elem {367struct rxe_mc_elem {
368 struct rxe_pool_entry pelem;368 struct rxe_pool_elem elem;
369 struct list_head qp_list;369 struct list_head qp_list;
370 struct list_head grp_list;370 struct list_head grp_list;
371 struct rxe_qp *qp;371 struct rxe_qp *qp;
@@ -392,8 +392,6 @@ struct rxe_dev { · 20 unmodified lines
392 392
393 struct net_device *ndev;393 struct net_device *ndev;
394 394
395 int xmit_errors;
396
397 struct rxe_pool uc_pool;395 struct rxe_pool uc_pool;
398 struct rxe_pool pd_pool;396 struct rxe_pool pd_pool;
399 struct rxe_pool ah_pool;397 struct rxe_pool ah_pool;
@@ -484,6 +482,6 @@ static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw) · 84 unmodified lines
484 482
485int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);483int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
486 484
487void rxe_mc_cleanup(struct rxe_pool_entry *arg);485void rxe_mc_cleanup(struct rxe_pool_elem *elem);
488 486
489#endif /* RXE_VERBS_H */487#endif /* RXE_VERBS_H */
drivers/infiniband/sw/siw/siw_verbs.c+4−2
@@ -8,6 +8,7 @@ · 7 unmodified lines
8#include <linux/uaccess.h>8#include <linux/uaccess.h>
9#include <linux/vmalloc.h>9#include <linux/vmalloc.h>
10#include <linux/xarray.h>10#include <linux/xarray.h>
11#include <net/addrconf.h>
11 12
12#include <rdma/iw_cm.h>13#include <rdma/iw_cm.h>
13#include <rdma/ib_verbs.h>14#include <rdma/ib_verbs.h>
@@ -155,7 +156,8 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr, · 141 unmodified lines
155 attr->vendor_id = SIW_VENDOR_ID;156 attr->vendor_id = SIW_VENDOR_ID;
156 attr->vendor_part_id = sdev->vendor_part_id;157 attr->vendor_part_id = sdev->vendor_part_id;
157 158
158 memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6);159 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
160 sdev->netdev->dev_addr);
159 161
160 return 0;162 return 0;
161}163}
@@ -660,7 +662,7 @@ static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr, · 498 unmodified lines
660 kbuf += core_sge->length;662 kbuf += core_sge->length;
661 core_sge++;663 core_sge++;
662 }664 }
663 sqe->sge[0].length = bytes > 0 ? bytes : 0;665 sqe->sge[0].length = max(bytes, 0);
664 sqe->num_sge = bytes > 0 ? 1 : 0;666 sqe->num_sge = bytes > 0 ? 1 : 0;
665 667
666 return bytes;668 return bytes;
drivers/infiniband/ulp/iser/iscsi_iser.c+28−48
@@ -113,10 +113,6 @@ bool iser_pi_enable = false; · 112 unmodified lines
113module_param_named(pi_enable, iser_pi_enable, bool, S_IRUGO);113module_param_named(pi_enable, iser_pi_enable, bool, S_IRUGO);
114MODULE_PARM_DESC(pi_enable, "Enable T10-PI offload support (default:disabled)");114MODULE_PARM_DESC(pi_enable, "Enable T10-PI offload support (default:disabled)");
115 115
116int iser_pi_guard;
117module_param_named(pi_guard, iser_pi_guard, int, S_IRUGO);
118MODULE_PARM_DESC(pi_guard, "T10-PI guard_type [deprecated]");
119
120static int iscsi_iser_set(const char *val, const struct kernel_param *kp)116static int iscsi_iser_set(const char *val, const struct kernel_param *kp)
121{117{
122 int ret;118 int ret;
@@ -139,9 +135,8 @@ static int iscsi_iser_set(const char *val, const struct kernel_param *kp) · 16 unmodified lines
139 * Notes: In case of data length errors or iscsi PDU completion failures135 * Notes: In case of data length errors or iscsi PDU completion failures
140 * this routine will signal iscsi layer of connection failure.136 * this routine will signal iscsi layer of connection failure.
141 */137 */
142void138void iscsi_iser_recv(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
143iscsi_iser_recv(struct iscsi_conn *conn, struct iscsi_hdr *hdr,139 char *rx_data, int rx_data_len)
144 char *rx_data, int rx_data_len)
145{140{
146 int rc = 0;141 int rc = 0;
147 int datalen;142 int datalen;
@@ -176,8 +171,7 @@ error: · 28 unmodified lines
176 * Netes: This routine can't fail, just assign iscsi task171 * Netes: This routine can't fail, just assign iscsi task
177 * hdr and max hdr size.172 * hdr and max hdr size.
178 */173 */
179static int174static int iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode)
180iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode)
181{175{
182 struct iscsi_iser_task *iser_task = task->dd_data;176 struct iscsi_iser_task *iser_task = task->dd_data;
183 177
@@ -198,9 +192,8 @@ iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode) · 14 unmodified lines
198 * state mutex to avoid dereferencing the IB device which192 * state mutex to avoid dereferencing the IB device which
199 * may have already been terminated.193 * may have already been terminated.
200 */194 */
201int195int iser_initialize_task_headers(struct iscsi_task *task,
202iser_initialize_task_headers(struct iscsi_task *task,196 struct iser_tx_desc *tx_desc)
203 struct iser_tx_desc *tx_desc)
204{197{
205 struct iser_conn *iser_conn = task->conn->dd_data;198 struct iser_conn *iser_conn = task->conn->dd_data;
206 struct iser_device *device = iser_conn->ib_conn.device;199 struct iser_device *device = iser_conn->ib_conn.device;
@@ -237,8 +230,7 @@ iser_initialize_task_headers(struct iscsi_task *task, · 30 unmodified lines
237 * Return: Returns zero on success or -ENOMEM when failing230 * Return: Returns zero on success or -ENOMEM when failing
238 * to init task headers (dma mapping error).231 * to init task headers (dma mapping error).
239 */232 */
240static int233static int iscsi_iser_task_init(struct iscsi_task *task)
241iscsi_iser_task_init(struct iscsi_task *task)
242{234{
243 struct iscsi_iser_task *iser_task = task->dd_data;235 struct iscsi_iser_task *iser_task = task->dd_data;
244 int ret;236 int ret;
@@ -272,8 +264,8 @@ iscsi_iser_task_init(struct iscsi_task *task) · 27 unmodified lines
272 * xmit.264 * xmit.
273 *265 *
274 **/266 **/
275static int267static int iscsi_iser_mtask_xmit(struct iscsi_conn *conn,
276iscsi_iser_mtask_xmit(struct iscsi_conn *conn, struct iscsi_task *task)268 struct iscsi_task *task)
277{269{
278 int error = 0;270 int error = 0;
279 271
@@ -290,9 +282,8 @@ iscsi_iser_mtask_xmit(struct iscsi_conn *conn, struct iscsi_task *task) · 10 unmodified lines
290 return error;282 return error;
291}283}
292 284
293static int285static int iscsi_iser_task_xmit_unsol_data(struct iscsi_conn *conn,
294iscsi_iser_task_xmit_unsol_data(struct iscsi_conn *conn,286 struct iscsi_task *task)
295 struct iscsi_task *task)
296{287{
297 struct iscsi_r2t_info *r2t = &task->unsol_r2t;288 struct iscsi_r2t_info *r2t = &task->unsol_r2t;
298 struct iscsi_data hdr;289 struct iscsi_data hdr;
@@ -326,8 +317,7 @@ iscsi_iser_task_xmit_unsol_data_exit: · 27 unmodified lines
326 *317 *
327 * Return: zero on success or escalates $error on failure.318 * Return: zero on success or escalates $error on failure.
328 */319 */
329static int320static int iscsi_iser_task_xmit(struct iscsi_task *task)
330iscsi_iser_task_xmit(struct iscsi_task *task)
331{321{
332 struct iscsi_conn *conn = task->conn;322 struct iscsi_conn *conn = task->conn;
333 struct iscsi_iser_task *iser_task = task->dd_data;323 struct iscsi_iser_task *iser_task = task->dd_data;
@@ -410,8 +400,7 @@ static void iscsi_iser_cleanup_task(struct iscsi_task *task) · 76 unmodified lines
410 *400 *
411 * In addition the error sector is marked.401 * In addition the error sector is marked.
412 */402 */
413static u8403static u8 iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector)
414iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector)
415{404{
416 struct iscsi_iser_task *iser_task = task->dd_data;405 struct iscsi_iser_task *iser_task = task->dd_data;
417 enum iser_data_dir dir = iser_task->dir[ISER_DIR_IN] ?406 enum iser_data_dir dir = iser_task->dir[ISER_DIR_IN] ?
@@ -460,11 +449,9 @@ iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, · 42 unmodified lines
460 * -EINVAL in case end-point doesn't exsits anymore or iser connection449 * -EINVAL in case end-point doesn't exsits anymore or iser connection
461 * state is not UP (teardown already started).450 * state is not UP (teardown already started).
462 */451 */
463static int452static int iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
464iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,453 struct iscsi_cls_conn *cls_conn,
465 struct iscsi_cls_conn *cls_conn,454 uint64_t transport_eph, int is_leading)
466 uint64_t transport_eph,
467 int is_leading)
468{455{
469 struct iscsi_conn *conn = cls_conn->dd_data;456 struct iscsi_conn *conn = cls_conn->dd_data;
470 struct iser_conn *iser_conn;457 struct iser_conn *iser_conn;
@@ -519,8 +506,7 @@ out: · 48 unmodified lines
519 * from this point iscsi must call conn_stop in session/connection506 * from this point iscsi must call conn_stop in session/connection
520 * teardown so iser transport must wait for it.507 * teardown so iser transport must wait for it.
521 */508 */
522static int509static int iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn)
523iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn)
524{510{
525 struct iscsi_conn *iscsi_conn;511 struct iscsi_conn *iscsi_conn;
526 struct iser_conn *iser_conn;512 struct iser_conn *iser_conn;
@@ -542,8 +528,7 @@ iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn) · 15 unmodified lines
542 * handle, so we call it under iser the state lock to protect against528 * handle, so we call it under iser the state lock to protect against
543 * this kind of race.529 * this kind of race.
544 */530 */
545static void531static void iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
546iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
547{532{
548 struct iscsi_conn *conn = cls_conn->dd_data;533 struct iscsi_conn *conn = cls_conn->dd_data;
549 struct iser_conn *iser_conn = conn->dd_data;534 struct iser_conn *iser_conn = conn->dd_data;
@@ -578,8 +563,7 @@ iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) · 28 unmodified lines
578 *563 *
579 * Removes and free iscsi host.564 * Removes and free iscsi host.
580 */565 */
581static void566static void iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session)
582iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session)
583{567{
584 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);568 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
585 569
@@ -588,8 +572,7 @@ iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session) · 2 unmodified lines
588 iscsi_host_free(shost);572 iscsi_host_free(shost);
589}573}
590 574
591static inline unsigned int575static inline unsigned int iser_dif_prot_caps(int prot_caps)
592iser_dif_prot_caps(int prot_caps)
593{576{
594 int ret = 0;577 int ret = 0;
595 578
@@ -708,9 +691,8 @@ free_host: · 112 unmodified lines
708 return NULL;691 return NULL;
709}692}
710 693
711static int694static int iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn,
712iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn,695 enum iscsi_param param, char *buf, int buflen)
713 enum iscsi_param param, char *buf, int buflen)
714{696{
715 int value;697 int value;
716 698
@@ -760,8 +742,8 @@ iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn, · 43 unmodified lines
760 *742 *
761 * Output connection statistics.743 * Output connection statistics.
762 */744 */
763static void745static void iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn,
764iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)746 struct iscsi_stats *stats)
765{747{
766 struct iscsi_conn *conn = cls_conn->dd_data;748 struct iscsi_conn *conn = cls_conn->dd_data;
767 749
@@ -812,9 +794,9 @@ static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep, · 44 unmodified lines
812 * Return: iscsi_endpoint created by iscsi layer or ERR_PTR(error)794 * Return: iscsi_endpoint created by iscsi layer or ERR_PTR(error)
813 * if fails.795 * if fails.
814 */796 */
815static struct iscsi_endpoint *797static struct iscsi_endpoint *iscsi_iser_ep_connect(struct Scsi_Host *shost,
816iscsi_iser_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,798 struct sockaddr *dst_addr,
817 int non_blocking)799 int non_blocking)
818{800{
819 int err;801 int err;
820 struct iser_conn *iser_conn;802 struct iser_conn *iser_conn;
@@ -857,8 +839,7 @@ failure: · 36 unmodified lines
857 * or more likely iser connection state transitioned to TEMINATING or839 * or more likely iser connection state transitioned to TEMINATING or
858 * DOWN during the wait period.840 * DOWN during the wait period.
859 */841 */
860static int842static int iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
861iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
862{843{
863 struct iser_conn *iser_conn = ep->dd_data;844 struct iser_conn *iser_conn = ep->dd_data;
864 int rc;845 int rc;
@@ -893,8 +874,7 @@ iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) · 28 unmodified lines
893 * and cleanup or actually call it immediately in case we didn't pass874 * and cleanup or actually call it immediately in case we didn't pass
894 * iscsi conn bind/start stage, thus it is safe.875 * iscsi conn bind/start stage, thus it is safe.
895 */876 */
896static void877static void iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep)
897iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep)
898{878{
899 struct iser_conn *iser_conn = ep->dd_data;879 struct iser_conn *iser_conn = ep->dd_data;
900 880
drivers/infiniband/ulp/iser/iscsi_iser.h+3−20
@@ -119,8 +119,6 @@ · 118 unmodified lines
119 119
120#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)120#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)
121 121
122#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
123
124/* the max TX (send) WR supported by the iSER QP is defined by *122/* the max TX (send) WR supported by the iSER QP is defined by *
125 * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect *123 * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect *
126 * to have at max for SCSI command. The tx posting & completion handling code *124 * to have at max for SCSI command. The tx posting & completion handling code *
@@ -148,8 +146,6 @@ · 21 unmodified lines
148 - ISER_MAX_RX_MISC_PDUS) / \146 - ISER_MAX_RX_MISC_PDUS) / \
149 (1 + ISER_INFLIGHT_DATAOUTS))147 (1 + ISER_INFLIGHT_DATAOUTS))
150 148
151#define ISER_SIGNAL_CMD_COUNT 32
152
153/* Constant PDU lengths calculations */149/* Constant PDU lengths calculations */
154#define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr))150#define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr))
155 151
@@ -366,9 +362,6 @@ struct iser_fr_pool { · 210 unmodified lines
366 * @qp: Connection Queue-pair362 * @qp: Connection Queue-pair
367 * @cq: Connection completion queue363 * @cq: Connection completion queue
368 * @cq_size: The number of max outstanding completions364 * @cq_size: The number of max outstanding completions
369 * @post_recv_buf_count: post receive counter
370 * @sig_count: send work request signal count
371 * @rx_wr: receive work request for batch posts
372 * @device: reference to iser device365 * @device: reference to iser device
373 * @fr_pool: connection fast registration poool366 * @fr_pool: connection fast registration poool
374 * @pi_support: Indicate device T10-PI support367 * @pi_support: Indicate device T10-PI support
@@ -379,9 +372,6 @@ struct ib_conn { · 4 unmodified lines
379 struct ib_qp *qp;372 struct ib_qp *qp;
380 struct ib_cq *cq;373 struct ib_cq *cq;
381 u32 cq_size;374 u32 cq_size;
382 int post_recv_buf_count;
383 u8 sig_count;
384 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
385 struct iser_device *device;375 struct iser_device *device;
386 struct iser_fr_pool fr_pool;376 struct iser_fr_pool fr_pool;
387 bool pi_support;377 bool pi_support;
@@ -397,8 +387,6 @@ struct ib_conn { · 9 unmodified lines
397 * @state: connection logical state387 * @state: connection logical state
398 * @qp_max_recv_dtos: maximum number of data outs, corresponds388 * @qp_max_recv_dtos: maximum number of data outs, corresponds
399 * to max number of post recvs389 * to max number of post recvs
400 * @qp_max_recv_dtos_mask: (qp_max_recv_dtos - 1)
401 * @min_posted_rx: (qp_max_recv_dtos >> 2)
402 * @max_cmds: maximum cmds allowed for this connection390 * @max_cmds: maximum cmds allowed for this connection
403 * @name: connection peer portal391 * @name: connection peer portal
404 * @release_work: deffered work for release job392 * @release_work: deffered work for release job
@@ -409,7 +397,6 @@ struct ib_conn { · 4 unmodified lines
409 * (state is ISER_CONN_UP)397 * (state is ISER_CONN_UP)
410 * @conn_list: entry in ig conn list398 * @conn_list: entry in ig conn list
411 * @login_desc: login descriptor399 * @login_desc: login descriptor
412 * @rx_desc_head: head of rx_descs cyclic buffer
413 * @rx_descs: rx buffers array (cyclic buffer)400 * @rx_descs: rx buffers array (cyclic buffer)
414 * @num_rx_descs: number of rx descriptors401 * @num_rx_descs: number of rx descriptors
415 * @scsi_sg_tablesize: scsi host sg_tablesize402 * @scsi_sg_tablesize: scsi host sg_tablesize
@@ -422,8 +409,6 @@ struct iser_conn { · 6 unmodified lines
422 struct iscsi_endpoint *ep;409 struct iscsi_endpoint *ep;
423 enum iser_conn_state state;410 enum iser_conn_state state;
424 unsigned qp_max_recv_dtos;411 unsigned qp_max_recv_dtos;
425 unsigned qp_max_recv_dtos_mask;
426 unsigned min_posted_rx;
427 u16 max_cmds;412 u16 max_cmds;
428 char name[ISER_OBJECT_NAME_SIZE];413 char name[ISER_OBJECT_NAME_SIZE];
429 struct work_struct release_work;414 struct work_struct release_work;
@@ -433,7 +418,6 @@ struct iser_conn { · 3 unmodified lines
433 struct completion up_completion;418 struct completion up_completion;
434 struct list_head conn_list;419 struct list_head conn_list;
435 struct iser_login_desc login_desc;420 struct iser_login_desc login_desc;
436 unsigned int rx_desc_head;
437 struct iser_rx_desc *rx_descs;421 struct iser_rx_desc *rx_descs;
438 u32 num_rx_descs;422 u32 num_rx_descs;
439 unsigned short scsi_sg_tablesize;423 unsigned short scsi_sg_tablesize;
@@ -486,7 +470,6 @@ struct iser_global { · 46 unmodified lines
486extern struct iser_global ig;470extern struct iser_global ig;
487extern int iser_debug_level;471extern int iser_debug_level;
488extern bool iser_pi_enable;472extern bool iser_pi_enable;
489extern int iser_pi_guard;
490extern unsigned int iser_max_sectors;473extern unsigned int iser_max_sectors;
491extern bool iser_always_reg;474extern bool iser_always_reg;
492 475
@@ -543,9 +526,9 @@ int iser_connect(struct iser_conn *iser_conn, · 50 unmodified lines
543 int non_blocking);526 int non_blocking);
544 527
545int iser_post_recvl(struct iser_conn *iser_conn);528int iser_post_recvl(struct iser_conn *iser_conn);
546int iser_post_recvm(struct iser_conn *iser_conn, int count);529int iser_post_recvm(struct iser_conn *iser_conn,
547int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,530 struct iser_rx_desc *rx_desc);
548 bool signal);531int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc);
549 532
550int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,533int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
551 struct iser_data_buf *data,534 struct iser_data_buf *data,
drivers/infiniband/ulp/iser/iser_initiator.c+41−65
@@ -95,11 +95,8 @@ static int iser_prepare_read_cmd(struct iscsi_task *task) · 94 unmodified lines
95 * task->data[ISER_DIR_OUT].data_len, Protection size95 * task->data[ISER_DIR_OUT].data_len, Protection size
96 * is stored at task->prot[ISER_DIR_OUT].data_len96 * is stored at task->prot[ISER_DIR_OUT].data_len
97 */97 */
98static int98static int iser_prepare_write_cmd(struct iscsi_task *task, unsigned int imm_sz,
99iser_prepare_write_cmd(struct iscsi_task *task,99 unsigned int unsol_sz, unsigned int edtl)
100 unsigned int imm_sz,
101 unsigned int unsol_sz,
102 unsigned int edtl)
103{100{
104 struct iscsi_iser_task *iser_task = task->dd_data;101 struct iscsi_iser_task *iser_task = task->dd_data;
105 struct iser_mem_reg *mem_reg;102 struct iser_mem_reg *mem_reg;
@@ -160,8 +157,8 @@ iser_prepare_write_cmd(struct iscsi_task *task, · 54 unmodified lines
160}157}
161 158
162/* creates a new tx descriptor and adds header regd buffer */159/* creates a new tx descriptor and adds header regd buffer */
163static void iser_create_send_desc(struct iser_conn *iser_conn,160static void iser_create_send_desc(struct iser_conn *iser_conn,
164 struct iser_tx_desc *tx_desc)161 struct iser_tx_desc *tx_desc)
165{162{
166 struct iser_device *device = iser_conn->ib_conn.device;163 struct iser_device *device = iser_conn->ib_conn.device;
167 164
@@ -247,8 +244,6 @@ int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, · 79 unmodified lines
247 struct iser_device *device = ib_conn->device;244 struct iser_device *device = ib_conn->device;
248 245
249 iser_conn->qp_max_recv_dtos = session->cmds_max;246 iser_conn->qp_max_recv_dtos = session->cmds_max;
250 iser_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */
251 iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2;
252 247
253 if (iser_alloc_fastreg_pool(ib_conn, session->scsi_cmds_max,248 if (iser_alloc_fastreg_pool(ib_conn, session->scsi_cmds_max,
254 iser_conn->pages_per_mr))249 iser_conn->pages_per_mr))
@@ -280,7 +275,6 @@ int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, · 25 unmodified lines
280 rx_sg->lkey = device->pd->local_dma_lkey;275 rx_sg->lkey = device->pd->local_dma_lkey;
281 }276 }
282 277
283 iser_conn->rx_desc_head = 0;
284 return 0;278 return 0;
285 279
286rx_desc_dma_map_failed:280rx_desc_dma_map_failed:
@@ -322,37 +316,35 @@ void iser_free_rx_descriptors(struct iser_conn *iser_conn) · 35 unmodified lines
322static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req)316static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req)
323{317{
324 struct iser_conn *iser_conn = conn->dd_data;318 struct iser_conn *iser_conn = conn->dd_data;
325 struct ib_conn *ib_conn = &iser_conn->ib_conn;
326 struct iscsi_session *session = conn->session;319 struct iscsi_session *session = conn->session;
320 int err = 0;
321 int i;
327 322
328 iser_dbg("req op %x flags %x\n", req->opcode, req->flags);323 iser_dbg("req op %x flags %x\n", req->opcode, req->flags);
329 /* check if this is the last login - going to full feature phase */324 /* check if this is the last login - going to full feature phase */
330 if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE)325 if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE)
331 return 0;326 goto out;
332
333 /*
334 * Check that there is one posted recv buffer
335 * (for the last login response).
336 */
337 WARN_ON(ib_conn->post_recv_buf_count != 1);
338 327
339 if (session->discovery_sess) {328 if (session->discovery_sess) {
340 iser_info("Discovery session, re-using login RX buffer\n");329 iser_info("Discovery session, re-using login RX buffer\n");
341 return 0;330 goto out;
342 } else331 }
343 iser_info("Normal session, posting batch of RX %d buffers\n",
344 iser_conn->min_posted_rx);
345
346 /* Initial post receive buffers */
347 if (iser_post_recvm(iser_conn, iser_conn->min_posted_rx))
348 return -ENOMEM;
349 332
350 return 0;333 iser_info("Normal session, posting batch of RX %d buffers\n",
351}334 iser_conn->qp_max_recv_dtos - 1);
352 335
353static inline bool iser_signal_comp(u8 sig_count)336 /*
354{337 * Initial post receive buffers.
355 return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0);338 * There is one already posted recv buffer (for the last login
339 * response). Therefore, the first recv buffer is skipped here.
340 */
341 for (i = 1; i < iser_conn->qp_max_recv_dtos; i++) {
342 err = iser_post_recvm(iser_conn, &iser_conn->rx_descs[i]);
343 if (err)
344 goto out;
345 }
346out:
347 return err;
356}348}
357 349
358/**350/**
@@ -360,8 +352,7 @@ static inline bool iser_signal_comp(u8 sig_count) · 1 unmodified line
360 * @conn: link to matching iscsi connection352 * @conn: link to matching iscsi connection
361 * @task: SCSI command task353 * @task: SCSI command task
362 */354 */
363int iser_send_command(struct iscsi_conn *conn,355int iser_send_command(struct iscsi_conn *conn, struct iscsi_task *task)
364 struct iscsi_task *task)
365{356{
366 struct iser_conn *iser_conn = conn->dd_data;357 struct iser_conn *iser_conn = conn->dd_data;
367 struct iscsi_iser_task *iser_task = task->dd_data;358 struct iscsi_iser_task *iser_task = task->dd_data;
@@ -371,7 +362,6 @@ int iser_send_command(struct iscsi_conn *conn, · 3 unmodified lines
371 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr;362 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr;
372 struct scsi_cmnd *sc = task->sc;363 struct scsi_cmnd *sc = task->sc;
373 struct iser_tx_desc *tx_desc = &iser_task->desc;364 struct iser_tx_desc *tx_desc = &iser_task->desc;
374 u8 sig_count = ++iser_conn->ib_conn.sig_count;
375 365
376 edtl = ntohl(hdr->data_length);366 edtl = ntohl(hdr->data_length);
377 367
@@ -418,8 +408,7 @@ int iser_send_command(struct iscsi_conn *conn, · 40 unmodified lines
418 408
419 iser_task->status = ISER_TASK_STATUS_STARTED;409 iser_task->status = ISER_TASK_STATUS_STARTED;
420 410
421 err = iser_post_send(&iser_conn->ib_conn, tx_desc,411 err = iser_post_send(&iser_conn->ib_conn, tx_desc);
422 iser_signal_comp(sig_count));
423 if (!err)412 if (!err)
424 return 0;413 return 0;
425 414
@@ -434,8 +423,7 @@ send_command_error: · 8 unmodified lines
434 * @task: SCSI command task423 * @task: SCSI command task
435 * @hdr: pointer to the LLD's iSCSI message header424 * @hdr: pointer to the LLD's iSCSI message header
436 */425 */
437int iser_send_data_out(struct iscsi_conn *conn,426int iser_send_data_out(struct iscsi_conn *conn, struct iscsi_task *task,
438 struct iscsi_task *task,
439 struct iscsi_data *hdr)427 struct iscsi_data *hdr)
440{428{
441 struct iser_conn *iser_conn = conn->dd_data;429 struct iser_conn *iser_conn = conn->dd_data;
@@ -487,7 +475,7 @@ int iser_send_data_out(struct iscsi_conn *conn, · 45 unmodified lines
487 itt, buf_offset, data_seg_len);475 itt, buf_offset, data_seg_len);
488 476
489 477
490 err = iser_post_send(&iser_conn->ib_conn, tx_desc, true);478 err = iser_post_send(&iser_conn->ib_conn, tx_desc);
491 if (!err)479 if (!err)
492 return 0;480 return 0;
493 481
@@ -497,8 +485,7 @@ send_data_out_error: · 3 unmodified lines
497 return err;485 return err;
498}486}
499 487
500int iser_send_control(struct iscsi_conn *conn,488int iser_send_control(struct iscsi_conn *conn, struct iscsi_task *task)
501 struct iscsi_task *task)
502{489{
503 struct iser_conn *iser_conn = conn->dd_data;490 struct iser_conn *iser_conn = conn->dd_data;
504 struct iscsi_iser_task *iser_task = task->dd_data;491 struct iscsi_iser_task *iser_task = task->dd_data;
@@ -550,7 +537,7 @@ int iser_send_control(struct iscsi_conn *conn, · 45 unmodified lines
550 goto send_control_error;537 goto send_control_error;
551 }538 }
552 539
553 err = iser_post_send(&iser_conn->ib_conn, mdesc, true);540 err = iser_post_send(&iser_conn->ib_conn, mdesc);
554 if (!err)541 if (!err)
555 return 0;542 return 0;
556 543
@@ -590,11 +577,14 @@ void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc) · 33 unmodified lines
590 desc->rsp_dma, ISER_RX_LOGIN_SIZE,577 desc->rsp_dma, ISER_RX_LOGIN_SIZE,
591 DMA_FROM_DEVICE);578 DMA_FROM_DEVICE);
592 579
593 ib_conn->post_recv_buf_count--;580 if (iser_conn->iscsi_conn->session->discovery_sess)
581 return;
582
583 /* Post the first RX buffer that is skipped in iser_post_rx_bufs() */
584 iser_post_recvm(iser_conn, iser_conn->rx_descs);
594}585}
595 586
596static inline int587static inline int iser_inv_desc(struct iser_fr_desc *desc, u32 rkey)
597iser_inv_desc(struct iser_fr_desc *desc, u32 rkey)
598{588{
599 if (unlikely((!desc->sig_protected && rkey != desc->rsc.mr->rkey) ||589 if (unlikely((!desc->sig_protected && rkey != desc->rsc.mr->rkey) ||
600 (desc->sig_protected && rkey != desc->rsc.sig_mr->rkey))) {590 (desc->sig_protected && rkey != desc->rsc.sig_mr->rkey))) {
@@ -607,10 +597,8 @@ iser_inv_desc(struct iser_fr_desc *desc, u32 rkey) · 6 unmodified lines
607 return 0;597 return 0;
608}598}
609 599
610static int600static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc,
611iser_check_remote_inv(struct iser_conn *iser_conn,601 struct iscsi_hdr *hdr)
612 struct ib_wc *wc,
613 struct iscsi_hdr *hdr)
614{602{
615 if (wc->wc_flags & IB_WC_WITH_INVALIDATE) {603 if (wc->wc_flags & IB_WC_WITH_INVALIDATE) {
616 struct iscsi_task *task;604 struct iscsi_task *task;
@@ -657,8 +645,7 @@ void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc) · 40 unmodified lines
657 struct iser_conn *iser_conn = to_iser_conn(ib_conn);645 struct iser_conn *iser_conn = to_iser_conn(ib_conn);
658 struct iser_rx_desc *desc = iser_rx(wc->wr_cqe);646 struct iser_rx_desc *desc = iser_rx(wc->wr_cqe);
659 struct iscsi_hdr *hdr;647 struct iscsi_hdr *hdr;
660 int length;648 int length, err;
661 int outstanding, count, err;
662 649
663 if (unlikely(wc->status != IB_WC_SUCCESS)) {650 if (unlikely(wc->status != IB_WC_SUCCESS)) {
664 iser_err_comp(wc, "task_rsp");651 iser_err_comp(wc, "task_rsp");
@@ -687,20 +674,9 @@ void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc) · 22 unmodified lines
687 desc->dma_addr, ISER_RX_PAYLOAD_SIZE,674 desc->dma_addr, ISER_RX_PAYLOAD_SIZE,
688 DMA_FROM_DEVICE);675 DMA_FROM_DEVICE);
689 676
690 /* decrementing conn->post_recv_buf_count only --after-- freeing the *677 err = iser_post_recvm(iser_conn, desc);
691 * task eliminates the need to worry on tasks which are completed in *678 if (err)
692 * parallel to the execution of iser_conn_term. So the code that waits *679 iser_err("posting rx buffer err %d\n", err);
693 * for the posted rx bufs refcount to become zero handles everything */
694 ib_conn->post_recv_buf_count--;
695
696 outstanding = ib_conn->post_recv_buf_count;
697 if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) {
698 count = min(iser_conn->qp_max_recv_dtos - outstanding,
699 iser_conn->min_posted_rx);
700 err = iser_post_recvm(iser_conn, count);
701 if (err)
702 iser_err("posting %d rx bufs err %d\n", count, err);
703 }
704}680}
705 681
706void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc)682void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc)
drivers/infiniband/ulp/iser/iser_memory.c+24−34
@@ -44,8 +44,7 @@ void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc) · 43 unmodified lines
44 iser_err_comp(wc, "memreg");44 iser_err_comp(wc, "memreg");
45}45}
46 46
47static struct iser_fr_desc *47static struct iser_fr_desc *iser_reg_desc_get_fr(struct ib_conn *ib_conn)
48iser_reg_desc_get_fr(struct ib_conn *ib_conn)
49{48{
50 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;49 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
51 struct iser_fr_desc *desc;50 struct iser_fr_desc *desc;
@@ -60,9 +59,8 @@ iser_reg_desc_get_fr(struct ib_conn *ib_conn) · 8 unmodified lines
60 return desc;59 return desc;
61}60}
62 61
63static void62static void iser_reg_desc_put_fr(struct ib_conn *ib_conn,
64iser_reg_desc_put_fr(struct ib_conn *ib_conn,63 struct iser_fr_desc *desc)
65 struct iser_fr_desc *desc)
66{64{
67 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;65 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
68 unsigned long flags;66 unsigned long flags;
@@ -73,9 +71,9 @@ iser_reg_desc_put_fr(struct ib_conn *ib_conn, · 4 unmodified lines
73}71}
74 72
75int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,73int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
76 struct iser_data_buf *data,74 struct iser_data_buf *data,
77 enum iser_data_dir iser_dir,75 enum iser_data_dir iser_dir,
78 enum dma_data_direction dma_dir)76 enum dma_data_direction dma_dir)
79{77{
80 struct ib_device *dev;78 struct ib_device *dev;
81 79
@@ -100,9 +98,8 @@ void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, · 18 unmodified lines
100 ib_dma_unmap_sg(dev, data->sg, data->size, dir);98 ib_dma_unmap_sg(dev, data->sg, data->size, dir);
101}99}
102 100
103static int101static int iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
104iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,102 struct iser_mem_reg *reg)
105 struct iser_mem_reg *reg)
106{103{
107 struct scatterlist *sg = mem->sg;104 struct scatterlist *sg = mem->sg;
108 105
@@ -154,8 +151,8 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task, · 45 unmodified lines
154 reg->mem_h = NULL;151 reg->mem_h = NULL;
155}152}
156 153
157static void154static void iser_set_dif_domain(struct scsi_cmnd *sc,
158iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_domain *domain)155 struct ib_sig_domain *domain)
159{156{
160 domain->sig_type = IB_SIG_TYPE_T10_DIF;157 domain->sig_type = IB_SIG_TYPE_T10_DIF;
161 domain->sig.dif.pi_interval = scsi_prot_interval(sc);158 domain->sig.dif.pi_interval = scsi_prot_interval(sc);
@@ -171,8 +168,8 @@ iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_domain *domain) · 9 unmodified lines
171 domain->sig.dif.ref_remap = true;168 domain->sig.dif.ref_remap = true;
172}169}
173 170
174static int171static int iser_set_sig_attrs(struct scsi_cmnd *sc,
175iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs)172 struct ib_sig_attrs *sig_attrs)
176{173{
177 switch (scsi_get_prot_op(sc)) {174 switch (scsi_get_prot_op(sc)) {
178 case SCSI_PROT_WRITE_INSERT:175 case SCSI_PROT_WRITE_INSERT:
@@ -205,8 +202,7 @@ iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) · 26 unmodified lines
205 return 0;202 return 0;
206}203}
207 204
208static inline void205static inline void iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask)
209iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask)
210{206{
211 *mask = 0;207 *mask = 0;
212 if (sc->prot_flags & SCSI_PROT_REF_CHECK)208 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
@@ -215,11 +211,8 @@ iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) · 2 unmodified lines
215 *mask |= IB_SIG_CHECK_GUARD;211 *mask |= IB_SIG_CHECK_GUARD;
216}212}
217 213
218static inline void214static inline void iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr,
219iser_inv_rkey(struct ib_send_wr *inv_wr,215 struct ib_cqe *cqe, struct ib_send_wr *next_wr)
220 struct ib_mr *mr,
221 struct ib_cqe *cqe,
222 struct ib_send_wr *next_wr)
223{216{
224 inv_wr->opcode = IB_WR_LOCAL_INV;217 inv_wr->opcode = IB_WR_LOCAL_INV;
225 inv_wr->wr_cqe = cqe;218 inv_wr->wr_cqe = cqe;
@@ -229,12 +222,11 @@ iser_inv_rkey(struct ib_send_wr *inv_wr, · 3 unmodified lines
229 inv_wr->next = next_wr;222 inv_wr->next = next_wr;
230}223}
231 224
232static int225static int iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
233iser_reg_sig_mr(struct iscsi_iser_task *iser_task,226 struct iser_data_buf *mem,
234 struct iser_data_buf *mem,227 struct iser_data_buf *sig_mem,
235 struct iser_data_buf *sig_mem,228 struct iser_reg_resources *rsc,
236 struct iser_reg_resources *rsc,229 struct iser_mem_reg *sig_reg)
237 struct iser_mem_reg *sig_reg)
238{230{
239 struct iser_tx_desc *tx_desc = &iser_task->desc;231 struct iser_tx_desc *tx_desc = &iser_task->desc;
240 struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;232 struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;
@@ -335,12 +327,10 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, · 94 unmodified lines
335 return 0;327 return 0;
336}328}
337 329
338static int330static int iser_reg_data_sg(struct iscsi_iser_task *task,
339iser_reg_data_sg(struct iscsi_iser_task *task,331 struct iser_data_buf *mem,
340 struct iser_data_buf *mem,332 struct iser_fr_desc *desc, bool use_dma_key,
341 struct iser_fr_desc *desc,333 struct iser_mem_reg *reg)
342 bool use_dma_key,
343 struct iser_mem_reg *reg)
344{334{
345 struct iser_device *device = task->iser_conn->ib_conn.device;335 struct iser_device *device = task->iser_conn->ib_conn.device;
346 336
drivers/infiniband/ulp/iser/iser_verbs.c+57−81
@@ -265,14 +265,14 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) · 264 unmodified lines
265 memset(&init_attr, 0, sizeof(init_attr));265 memset(&init_attr, 0, sizeof(init_attr));
266 266
267 init_attr.event_handler = iser_qp_event_callback;267 init_attr.event_handler = iser_qp_event_callback;
268 init_attr.qp_context = (void *)ib_conn;268 init_attr.qp_context = (void *)ib_conn;
269 init_attr.send_cq = ib_conn->cq;269 init_attr.send_cq = ib_conn->cq;
270 init_attr.recv_cq = ib_conn->cq;270 init_attr.recv_cq = ib_conn->cq;
271 init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS;271 init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS;
272 init_attr.cap.max_send_sge = 2;272 init_attr.cap.max_send_sge = 2;
273 init_attr.cap.max_recv_sge = 1;273 init_attr.cap.max_recv_sge = 1;
274 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;274 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
275 init_attr.qp_type = IB_QPT_RC;275 init_attr.qp_type = IB_QPT_RC;
276 init_attr.cap.max_send_wr = max_send_wr;276 init_attr.cap.max_send_wr = max_send_wr;
277 if (ib_conn->pi_support)277 if (ib_conn->pi_support)
278 init_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;278 init_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
@@ -283,9 +283,8 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) · 4 unmodified lines
283 goto out_err;283 goto out_err;
284 284
285 ib_conn->qp = ib_conn->cma_id->qp;285 ib_conn->qp = ib_conn->cma_id->qp;
286 iser_info("setting conn %p cma_id %p qp %p max_send_wr %d\n",286 iser_info("setting conn %p cma_id %p qp %p max_send_wr %d\n", ib_conn,
287 ib_conn, ib_conn->cma_id,287 ib_conn->cma_id, ib_conn->cma_id->qp, max_send_wr);
288 ib_conn->cma_id->qp, max_send_wr);
289 return ret;288 return ret;
290 289
291out_err:290out_err:
@@ -313,7 +312,7 @@ struct iser_device *iser_device_find_by_ib_device(struct rdma_cm_id *cma_id) · 21 unmodified lines
313 goto inc_refcnt;312 goto inc_refcnt;
314 313
315 device = kzalloc(sizeof *device, GFP_KERNEL);314 device = kzalloc(sizeof *device, GFP_KERNEL);
316 if (device == NULL)315 if (!device)
317 goto out;316 goto out;
318 317
319 /* assign this device to the device */318 /* assign this device to the device */
@@ -392,8 +391,7 @@ void iser_release_work(struct work_struct *work) · 72 unmodified lines
392 * so the cm_id removal is out of here. It is Safe to391 * so the cm_id removal is out of here. It is Safe to
393 * be invoked multiple times.392 * be invoked multiple times.
394 */393 */
395static void iser_free_ib_conn_res(struct iser_conn *iser_conn,394static void iser_free_ib_conn_res(struct iser_conn *iser_conn, bool destroy)
396 bool destroy)
397{395{
398 struct ib_conn *ib_conn = &iser_conn->ib_conn;396 struct ib_conn *ib_conn = &iser_conn->ib_conn;
399 struct iser_device *device = ib_conn->device;397 struct iser_device *device = ib_conn->device;
@@ -401,7 +399,7 @@ static void iser_free_ib_conn_res(struct iser_conn *iser_conn, · 1 unmodified line
401 iser_info("freeing conn %p cma_id %p qp %p\n",399 iser_info("freeing conn %p cma_id %p qp %p\n",
402 iser_conn, ib_conn->cma_id, ib_conn->qp);400 iser_conn, ib_conn->cma_id, ib_conn->qp);
403 401
404 if (ib_conn->qp != NULL) {402 if (ib_conn->qp) {
405 rdma_destroy_qp(ib_conn->cma_id);403 rdma_destroy_qp(ib_conn->cma_id);
406 ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size);404 ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size);
407 ib_conn->qp = NULL;405 ib_conn->qp = NULL;
@@ -411,7 +409,7 @@ static void iser_free_ib_conn_res(struct iser_conn *iser_conn, · 3 unmodified lines
411 if (iser_conn->rx_descs)409 if (iser_conn->rx_descs)
412 iser_free_rx_descriptors(iser_conn);410 iser_free_rx_descriptors(iser_conn);
413 411
414 if (device != NULL) {412 if (device) {
415 iser_device_try_release(device);413 iser_device_try_release(device);
416 ib_conn->device = NULL;414 ib_conn->device = NULL;
417 }415 }
@@ -445,7 +443,7 @@ void iser_conn_release(struct iser_conn *iser_conn) · 27 unmodified lines
445 iser_free_ib_conn_res(iser_conn, true);443 iser_free_ib_conn_res(iser_conn, true);
446 mutex_unlock(&iser_conn->state_mutex);444 mutex_unlock(&iser_conn->state_mutex);
447 445
448 if (ib_conn->cma_id != NULL) {446 if (ib_conn->cma_id) {
449 rdma_destroy_id(ib_conn->cma_id);447 rdma_destroy_id(ib_conn->cma_id);
450 ib_conn->cma_id = NULL;448 ib_conn->cma_id = NULL;
451 }449 }
@@ -501,13 +499,12 @@ static void iser_connect_error(struct rdma_cm_id *cma_id) · 49 unmodified lines
501{499{
502 struct iser_conn *iser_conn;500 struct iser_conn *iser_conn;
503 501
504 iser_conn = (struct iser_conn *)cma_id->context;502 iser_conn = cma_id->context;
505 iser_conn->state = ISER_CONN_TERMINATING;503 iser_conn->state = ISER_CONN_TERMINATING;
506}504}
507 505
508static void506static void iser_calc_scsi_params(struct iser_conn *iser_conn,
509iser_calc_scsi_params(struct iser_conn *iser_conn,507 unsigned int max_sectors)
510 unsigned int max_sectors)
511{508{
512 struct iser_device *device = iser_conn->ib_conn.device;509 struct iser_device *device = iser_conn->ib_conn.device;
513 struct ib_device_attr *attr = &device->ib_device->attrs;510 struct ib_device_attr *attr = &device->ib_device->attrs;
@@ -545,11 +542,11 @@ iser_calc_scsi_params(struct iser_conn *iser_conn, · 31 unmodified lines
545static void iser_addr_handler(struct rdma_cm_id *cma_id)542static void iser_addr_handler(struct rdma_cm_id *cma_id)
546{543{
547 struct iser_device *device;544 struct iser_device *device;
548 struct iser_conn *iser_conn;545 struct iser_conn *iser_conn;
549 struct ib_conn *ib_conn;546 struct ib_conn *ib_conn;
550 int ret;547 int ret;
551 548
552 iser_conn = (struct iser_conn *)cma_id->context;549 iser_conn = cma_id->context;
553 if (iser_conn->state != ISER_CONN_PENDING)550 if (iser_conn->state != ISER_CONN_PENDING)
554 /* bailout */551 /* bailout */
555 return;552 return;
@@ -593,9 +590,9 @@ static void iser_addr_handler(struct rdma_cm_id *cma_id) · 37 unmodified lines
593static void iser_route_handler(struct rdma_cm_id *cma_id)590static void iser_route_handler(struct rdma_cm_id *cma_id)
594{591{
595 struct rdma_conn_param conn_param;592 struct rdma_conn_param conn_param;
596 int ret;593 int ret;
597 struct iser_cm_hdr req_hdr;594 struct iser_cm_hdr req_hdr;
598 struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;595 struct iser_conn *iser_conn = cma_id->context;
599 struct ib_conn *ib_conn = &iser_conn->ib_conn;596 struct ib_conn *ib_conn = &iser_conn->ib_conn;
600 struct ib_device *ib_dev = ib_conn->device->ib_device;597 struct ib_device *ib_dev = ib_conn->device->ib_device;
601 598
@@ -609,9 +606,9 @@ static void iser_route_handler(struct rdma_cm_id *cma_id) · 7 unmodified lines
609 606
610 memset(&conn_param, 0, sizeof conn_param);607 memset(&conn_param, 0, sizeof conn_param);
611 conn_param.responder_resources = ib_dev->attrs.max_qp_rd_atom;608 conn_param.responder_resources = ib_dev->attrs.max_qp_rd_atom;
612 conn_param.initiator_depth = 1;609 conn_param.initiator_depth = 1;
613 conn_param.retry_count = 7;610 conn_param.retry_count = 7;
614 conn_param.rnr_retry_count = 6;611 conn_param.rnr_retry_count = 6;
615 612
616 memset(&req_hdr, 0, sizeof(req_hdr));613 memset(&req_hdr, 0, sizeof(req_hdr));
617 req_hdr.flags = ISER_ZBVA_NOT_SUP;614 req_hdr.flags = ISER_ZBVA_NOT_SUP;
@@ -638,7 +635,7 @@ static void iser_connected_handler(struct rdma_cm_id *cma_id, · 20 unmodified lines
638 struct ib_qp_attr attr;635 struct ib_qp_attr attr;
639 struct ib_qp_init_attr init_attr;636 struct ib_qp_init_attr init_attr;
640 637
641 iser_conn = (struct iser_conn *)cma_id->context;638 iser_conn = cma_id->context;
642 if (iser_conn->state != ISER_CONN_PENDING)639 if (iser_conn->state != ISER_CONN_PENDING)
643 /* bailout */640 /* bailout */
644 return;641 return;
@@ -661,7 +658,7 @@ static void iser_connected_handler(struct rdma_cm_id *cma_id, · 16 unmodified lines
661 658
662static void iser_disconnected_handler(struct rdma_cm_id *cma_id)659static void iser_disconnected_handler(struct rdma_cm_id *cma_id)
663{660{
664 struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;661 struct iser_conn *iser_conn = cma_id->context;
665 662
666 if (iser_conn_terminate(iser_conn)) {663 if (iser_conn_terminate(iser_conn)) {
667 if (iser_conn->iscsi_conn)664 if (iser_conn->iscsi_conn)
@@ -675,7 +672,7 @@ static void iser_disconnected_handler(struct rdma_cm_id *cma_id) · 7 unmodified lines
675static void iser_cleanup_handler(struct rdma_cm_id *cma_id,672static void iser_cleanup_handler(struct rdma_cm_id *cma_id,
676 bool destroy)673 bool destroy)
677{674{
678 struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;675 struct iser_conn *iser_conn = cma_id->context;
679 676
680 /*677 /*
681 * We are not guaranteed that we visited disconnected_handler678 * We are not guaranteed that we visited disconnected_handler
@@ -687,12 +684,13 @@ static void iser_cleanup_handler(struct rdma_cm_id *cma_id, · 5 unmodified lines
687 complete(&iser_conn->ib_completion);684 complete(&iser_conn->ib_completion);
688}685}
689 686
690static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)687static int iser_cma_handler(struct rdma_cm_id *cma_id,
688 struct rdma_cm_event *event)
691{689{
692 struct iser_conn *iser_conn;690 struct iser_conn *iser_conn;
693 int ret = 0;691 int ret = 0;
694 692
695 iser_conn = (struct iser_conn *)cma_id->context;693 iser_conn = cma_id->context;
696 iser_info("%s (%d): status %d conn %p id %p\n",694 iser_info("%s (%d): status %d conn %p id %p\n",
697 rdma_event_msg(event->event), event->event,695 rdma_event_msg(event->event), event->event,
698 event->status, cma_id->context, cma_id);696 event->status, cma_id->context, cma_id);
@@ -757,7 +755,6 @@ void iser_conn_init(struct iser_conn *iser_conn) · 58 unmodified lines
757 INIT_LIST_HEAD(&iser_conn->conn_list);755 INIT_LIST_HEAD(&iser_conn->conn_list);
758 mutex_init(&iser_conn->state_mutex);756 mutex_init(&iser_conn->state_mutex);
759 757
760 ib_conn->post_recv_buf_count = 0;
761 ib_conn->reg_cqe.done = iser_reg_comp;758 ib_conn->reg_cqe.done = iser_reg_comp;
762}759}
763 760
@@ -765,10 +762,8 @@ void iser_conn_init(struct iser_conn *iser_conn) · 1 unmodified line
765 * starts the process of connecting to the target762 * starts the process of connecting to the target
766 * sleeps until the connection is established or rejected763 * sleeps until the connection is established or rejected
767 */764 */
768int iser_connect(struct iser_conn *iser_conn,765int iser_connect(struct iser_conn *iser_conn, struct sockaddr *src_addr,
769 struct sockaddr *src_addr,766 struct sockaddr *dst_addr, int non_blocking)
770 struct sockaddr *dst_addr,
771 int non_blocking)
772{767{
773 struct ib_conn *ib_conn = &iser_conn->ib_conn;768 struct ib_conn *ib_conn = &iser_conn->ib_conn;
774 int err = 0;769 int err = 0;
@@ -785,8 +780,7 @@ int iser_connect(struct iser_conn *iser_conn, · 10 unmodified lines
785 iser_conn->state = ISER_CONN_PENDING;780 iser_conn->state = ISER_CONN_PENDING;
786 781
787 ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler,782 ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler,
788 (void *)iser_conn,783 iser_conn, RDMA_PS_TCP, IB_QPT_RC);
789 RDMA_PS_TCP, IB_QPT_RC);
790 if (IS_ERR(ib_conn->cma_id)) {784 if (IS_ERR(ib_conn->cma_id)) {
791 err = PTR_ERR(ib_conn->cma_id);785 err = PTR_ERR(ib_conn->cma_id);
792 iser_err("rdma_create_id failed: %d\n", err);786 iser_err("rdma_create_id failed: %d\n", err);
@@ -829,7 +823,7 @@ int iser_post_recvl(struct iser_conn *iser_conn) · 36 unmodified lines
829 struct ib_conn *ib_conn = &iser_conn->ib_conn;823 struct ib_conn *ib_conn = &iser_conn->ib_conn;
830 struct iser_login_desc *desc = &iser_conn->login_desc;824 struct iser_login_desc *desc = &iser_conn->login_desc;
831 struct ib_recv_wr wr;825 struct ib_recv_wr wr;
832 int ib_ret;826 int ret;
833 827
834 desc->sge.addr = desc->rsp_dma;828 desc->sge.addr = desc->rsp_dma;
835 desc->sge.length = ISER_RX_LOGIN_SIZE;829 desc->sge.length = ISER_RX_LOGIN_SIZE;
@@ -841,46 +835,30 @@ int iser_post_recvl(struct iser_conn *iser_conn) · 5 unmodified lines
841 wr.num_sge = 1;835 wr.num_sge = 1;
842 wr.next = NULL;836 wr.next = NULL;
843 837
844 ib_conn->post_recv_buf_count++;838 ret = ib_post_recv(ib_conn->qp, &wr, NULL);
845 ib_ret = ib_post_recv(ib_conn->qp, &wr, NULL);839 if (unlikely(ret))
846 if (ib_ret) {840 iser_err("ib_post_recv login failed ret=%d\n", ret);
847 iser_err("ib_post_recv failed ret=%d\n", ib_ret);
848 ib_conn->post_recv_buf_count--;
849 }
850 841
851 return ib_ret;842 return ret;
852}843}
853 844
854int iser_post_recvm(struct iser_conn *iser_conn, int count)845int iser_post_recvm(struct iser_conn *iser_conn, struct iser_rx_desc *rx_desc)
855{846{
856 struct ib_conn *ib_conn = &iser_conn->ib_conn;847 struct ib_conn *ib_conn = &iser_conn->ib_conn;
857 unsigned int my_rx_head = iser_conn->rx_desc_head;848 struct ib_recv_wr wr;
858 struct iser_rx_desc *rx_desc;849 int ret;
859 struct ib_recv_wr *wr;
860 int i, ib_ret;
861
862 for (wr = ib_conn->rx_wr, i = 0; i < count; i++, wr++) {
863 rx_desc = &iser_conn->rx_descs[my_rx_head];
864 rx_desc->cqe.done = iser_task_rsp;
865 wr->wr_cqe = &rx_desc->cqe;
866 wr->sg_list = &rx_desc->rx_sg;
867 wr->num_sge = 1;
868 wr->next = wr + 1;
869 my_rx_head = (my_rx_head + 1) & iser_conn->qp_max_recv_dtos_mask;
870 }
871 850
872 wr--;851 rx_desc->cqe.done = iser_task_rsp;
873 wr->next = NULL; /* mark end of work requests list */852 wr.wr_cqe = &rx_desc->cqe;
853 wr.sg_list = &rx_desc->rx_sg;
854 wr.num_sge = 1;
855 wr.next = NULL;
874 856
875 ib_conn->post_recv_buf_count += count;857 ret = ib_post_recv(ib_conn->qp, &wr, NULL);
876 ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, NULL);858 if (unlikely(ret))
877 if (unlikely(ib_ret)) {859 iser_err("ib_post_recv failed ret=%d\n", ret);
878 iser_err("ib_post_recv failed ret=%d\n", ib_ret);
879 ib_conn->post_recv_buf_count -= count;
880 } else
881 iser_conn->rx_desc_head = my_rx_head;
882 860
883 return ib_ret;861 return ret;
884}862}
885 863
886 864
@@ -888,16 +866,14 @@ int iser_post_recvm(struct iser_conn *iser_conn, int count) · 1 unmodified line
888 * iser_post_send - Initiate a Send DTO operation866 * iser_post_send - Initiate a Send DTO operation
889 * @ib_conn: connection RDMA resources867 * @ib_conn: connection RDMA resources
890 * @tx_desc: iSER TX descriptor868 * @tx_desc: iSER TX descriptor
891 * @signal: true to send work request as SIGNALED
892 *869 *
893 * Return: 0 on success, -1 on failure870 * Return: 0 on success, -1 on failure
894 */871 */
895int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,872int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc)
896 bool signal)
897{873{
898 struct ib_send_wr *wr = &tx_desc->send_wr;874 struct ib_send_wr *wr = &tx_desc->send_wr;
899 struct ib_send_wr *first_wr;875 struct ib_send_wr *first_wr;
900 int ib_ret;876 int ret;
901 877
902 ib_dma_sync_single_for_device(ib_conn->device->ib_device,878 ib_dma_sync_single_for_device(ib_conn->device->ib_device,
903 tx_desc->dma_addr, ISER_HEADERS_LEN,879 tx_desc->dma_addr, ISER_HEADERS_LEN,
@@ -908,7 +884,7 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, · 4 unmodified lines
908 wr->sg_list = tx_desc->tx_sg;884 wr->sg_list = tx_desc->tx_sg;
909 wr->num_sge = tx_desc->num_sge;885 wr->num_sge = tx_desc->num_sge;
910 wr->opcode = IB_WR_SEND;886 wr->opcode = IB_WR_SEND;
911 wr->send_flags = signal ? IB_SEND_SIGNALED : 0;887 wr->send_flags = IB_SEND_SIGNALED;
912 888
913 if (tx_desc->inv_wr.next)889 if (tx_desc->inv_wr.next)
914 first_wr = &tx_desc->inv_wr;890 first_wr = &tx_desc->inv_wr;
@@ -917,12 +893,12 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, · 2 unmodified lines
917 else893 else
918 first_wr = wr;894 first_wr = wr;
919 895
920 ib_ret = ib_post_send(ib_conn->qp, first_wr, NULL);896 ret = ib_post_send(ib_conn->qp, first_wr, NULL);
921 if (unlikely(ib_ret))897 if (unlikely(ret))
922 iser_err("ib_post_send failed, ret:%d opcode:%d\n",898 iser_err("ib_post_send failed, ret:%d opcode:%d\n",
923 ib_ret, wr->opcode);899 ret, wr->opcode);
924 900
925 return ib_ret;901 return ret;
926}902}
927 903
928u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,904u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c+4−4
@@ -13,8 +13,8 @@ · 12 unmodified lines
13 13
14void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con)14void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con)
15{15{
16 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);16 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
17 struct rtrs_clt_stats *stats = sess->stats;17 struct rtrs_clt_stats *stats = clt_path->stats;
18 struct rtrs_clt_stats_pcpu *s;18 struct rtrs_clt_stats_pcpu *s;
19 int cpu;19 int cpu;
20 20
@@ -180,8 +180,8 @@ static inline void rtrs_clt_update_rdma_stats(struct rtrs_clt_stats *stats, · 159 unmodified lines
180void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir)180void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir)
181{181{
182 struct rtrs_clt_con *con = req->con;182 struct rtrs_clt_con *con = req->con;
183 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);183 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
184 struct rtrs_clt_stats *stats = sess->stats;184 struct rtrs_clt_stats *stats = clt_path->stats;
185 unsigned int len;185 unsigned int len;
186 186
187 len = req->usr_len + req->data_len;187 len = req->usr_len + req->data_len;
drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c+74−71
@@ -16,21 +16,21 @@ · 15 unmodified lines
16#define MIN_MAX_RECONN_ATT -116#define MIN_MAX_RECONN_ATT -1
17#define MAX_MAX_RECONN_ATT 999917#define MAX_MAX_RECONN_ATT 9999
18 18
19static void rtrs_clt_sess_release(struct kobject *kobj)19static void rtrs_clt_path_release(struct kobject *kobj)
20{20{
21 struct rtrs_clt_sess *sess;21 struct rtrs_clt_path *clt_path;
22 22
23 sess = container_of(kobj, struct rtrs_clt_sess, kobj);23 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
24 24
25 free_sess(sess);25 free_path(clt_path);
26}26}
27 27
28static struct kobj_type ktype_sess = {28static struct kobj_type ktype_sess = {
29 .sysfs_ops = &kobj_sysfs_ops,29 .sysfs_ops = &kobj_sysfs_ops,
30 .release = rtrs_clt_sess_release30 .release = rtrs_clt_path_release
31};31};
32 32
33static void rtrs_clt_sess_stats_release(struct kobject *kobj)33static void rtrs_clt_path_stats_release(struct kobject *kobj)
34{34{
35 struct rtrs_clt_stats *stats;35 struct rtrs_clt_stats *stats;
36 36
@@ -43,14 +43,15 @@ static void rtrs_clt_sess_stats_release(struct kobject *kobj) · 6 unmodified lines
43 43
44static struct kobj_type ktype_stats = {44static struct kobj_type ktype_stats = {
45 .sysfs_ops = &kobj_sysfs_ops,45 .sysfs_ops = &kobj_sysfs_ops,
46 .release = rtrs_clt_sess_stats_release,46 .release = rtrs_clt_path_stats_release,
47};47};
48 48
49static ssize_t max_reconnect_attempts_show(struct device *dev,49static ssize_t max_reconnect_attempts_show(struct device *dev,
50 struct device_attribute *attr,50 struct device_attribute *attr,
51 char *page)51 char *page)
52{52{
53 struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);53 struct rtrs_clt_sess *clt = container_of(dev, struct rtrs_clt_sess,
54 dev);
54 55
55 return sysfs_emit(page, "%d\n",56 return sysfs_emit(page, "%d\n",
56 rtrs_clt_get_max_reconnect_attempts(clt));57 rtrs_clt_get_max_reconnect_attempts(clt));
@@ -63,7 +64,8 @@ static ssize_t max_reconnect_attempts_store(struct device *dev, · 6 unmodified lines
63{64{
64 int value;65 int value;
65 int ret;66 int ret;
66 struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);67 struct rtrs_clt_sess *clt = container_of(dev, struct rtrs_clt_sess,
68 dev);
67 69
68 ret = kstrtoint(buf, 10, &value);70 ret = kstrtoint(buf, 10, &value);
69 if (ret) {71 if (ret) {
@@ -90,9 +92,9 @@ static ssize_t mpath_policy_show(struct device *dev, · 20 unmodified lines
90 struct device_attribute *attr,92 struct device_attribute *attr,
91 char *page)93 char *page)
92{94{
93 struct rtrs_clt *clt;95 struct rtrs_clt_sess *clt;
94 96
95 clt = container_of(dev, struct rtrs_clt, dev);97 clt = container_of(dev, struct rtrs_clt_sess, dev);
96 98
97 switch (clt->mp_policy) {99 switch (clt->mp_policy) {
98 case MP_POLICY_RR:100 case MP_POLICY_RR:
@@ -114,12 +116,12 @@ static ssize_t mpath_policy_store(struct device *dev, · 15 unmodified lines
114 const char *buf,116 const char *buf,
115 size_t count)117 size_t count)
116{118{
117 struct rtrs_clt *clt;119 struct rtrs_clt_sess *clt;
118 int value;120 int value;
119 int ret;121 int ret;
120 size_t len = 0;122 size_t len = 0;
121 123
122 clt = container_of(dev, struct rtrs_clt, dev);124 clt = container_of(dev, struct rtrs_clt_sess, dev);
123 125
124 ret = kstrtoint(buf, 10, &value);126 ret = kstrtoint(buf, 10, &value);
125 if (!ret && (value == MP_POLICY_RR ||127 if (!ret && (value == MP_POLICY_RR ||
@@ -169,12 +171,12 @@ static ssize_t add_path_store(struct device *dev, · 43 unmodified lines
169 .src = &srcaddr,171 .src = &srcaddr,
170 .dst = &dstaddr172 .dst = &dstaddr
171 };173 };
172 struct rtrs_clt *clt;174 struct rtrs_clt_sess *clt;
173 const char *nl;175 const char *nl;
174 size_t len;176 size_t len;
175 int err;177 int err;
176 178
177 clt = container_of(dev, struct rtrs_clt, dev);179 clt = container_of(dev, struct rtrs_clt_sess, dev);
178 180
179 nl = strchr(buf, '\n');181 nl = strchr(buf, '\n');
180 if (nl)182 if (nl)
@@ -197,10 +199,10 @@ static DEVICE_ATTR_RW(add_path); · 16 unmodified lines
197static ssize_t rtrs_clt_state_show(struct kobject *kobj,199static ssize_t rtrs_clt_state_show(struct kobject *kobj,
198 struct kobj_attribute *attr, char *page)200 struct kobj_attribute *attr, char *page)
199{201{
200 struct rtrs_clt_sess *sess;202 struct rtrs_clt_path *clt_path;
201 203
202 sess = container_of(kobj, struct rtrs_clt_sess, kobj);204 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
203 if (sess->state == RTRS_CLT_CONNECTED)205 if (clt_path->state == RTRS_CLT_CONNECTED)
204 return sysfs_emit(page, "connected\n");206 return sysfs_emit(page, "connected\n");
205 207
206 return sysfs_emit(page, "disconnected\n");208 return sysfs_emit(page, "disconnected\n");
@@ -219,16 +221,16 @@ static ssize_t rtrs_clt_reconnect_store(struct kobject *kobj, · 12 unmodified lines
219 struct kobj_attribute *attr,221 struct kobj_attribute *attr,
220 const char *buf, size_t count)222 const char *buf, size_t count)
221{223{
222 struct rtrs_clt_sess *sess;224 struct rtrs_clt_path *clt_path;
223 int ret;225 int ret;
224 226
225 sess = container_of(kobj, struct rtrs_clt_sess, kobj);227 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
226 if (!sysfs_streq(buf, "1")) {228 if (!sysfs_streq(buf, "1")) {
227 rtrs_err(sess->clt, "%s: unknown value: '%s'\n",229 rtrs_err(clt_path->clt, "%s: unknown value: '%s'\n",
228 attr->attr.name, buf);230 attr->attr.name, buf);
229 return -EINVAL;231 return -EINVAL;
230 }232 }
231 ret = rtrs_clt_reconnect_from_sysfs(sess);233 ret = rtrs_clt_reconnect_from_sysfs(clt_path);
232 if (ret)234 if (ret)
233 return ret;235 return ret;
234 236
@@ -249,15 +251,15 @@ static ssize_t rtrs_clt_disconnect_store(struct kobject *kobj, · 14 unmodified lines
249 struct kobj_attribute *attr,251 struct kobj_attribute *attr,
250 const char *buf, size_t count)252 const char *buf, size_t count)
251{253{
252 struct rtrs_clt_sess *sess;254 struct rtrs_clt_path *clt_path;
253 255
254 sess = container_of(kobj, struct rtrs_clt_sess, kobj);256 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
255 if (!sysfs_streq(buf, "1")) {257 if (!sysfs_streq(buf, "1")) {
256 rtrs_err(sess->clt, "%s: unknown value: '%s'\n",258 rtrs_err(clt_path->clt, "%s: unknown value: '%s'\n",
257 attr->attr.name, buf);259 attr->attr.name, buf);
258 return -EINVAL;260 return -EINVAL;
259 }261 }
260 rtrs_clt_close_conns(sess, true);262 rtrs_clt_close_conns(clt_path, true);
261 263
262 return count;264 return count;
263}265}
@@ -276,16 +278,16 @@ static ssize_t rtrs_clt_remove_path_store(struct kobject *kobj, · 12 unmodified lines
276 struct kobj_attribute *attr,278 struct kobj_attribute *attr,
277 const char *buf, size_t count)279 const char *buf, size_t count)
278{280{
279 struct rtrs_clt_sess *sess;281 struct rtrs_clt_path *clt_path;
280 int ret;282 int ret;
281 283
282 sess = container_of(kobj, struct rtrs_clt_sess, kobj);284 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
283 if (!sysfs_streq(buf, "1")) {285 if (!sysfs_streq(buf, "1")) {
284 rtrs_err(sess->clt, "%s: unknown value: '%s'\n",286 rtrs_err(clt_path->clt, "%s: unknown value: '%s'\n",
285 attr->attr.name, buf);287 attr->attr.name, buf);
286 return -EINVAL;288 return -EINVAL;
287 }289 }
288 ret = rtrs_clt_remove_path_from_sysfs(sess, &attr->attr);290 ret = rtrs_clt_remove_path_from_sysfs(clt_path, &attr->attr);
289 if (ret)291 if (ret)
290 return ret;292 return ret;
291 293
@@ -333,11 +335,11 @@ static ssize_t rtrs_clt_hca_port_show(struct kobject *kobj, · 41 unmodified lines
333 struct kobj_attribute *attr,335 struct kobj_attribute *attr,
334 char *page)336 char *page)
335{337{
336 struct rtrs_clt_sess *sess;338 struct rtrs_clt_path *clt_path;
337 339
338 sess = container_of(kobj, typeof(*sess), kobj);340 clt_path = container_of(kobj, typeof(*clt_path), kobj);
339 341
340 return sysfs_emit(page, "%u\n", sess->hca_port);342 return sysfs_emit(page, "%u\n", clt_path->hca_port);
341}343}
342 344
343static struct kobj_attribute rtrs_clt_hca_port_attr =345static struct kobj_attribute rtrs_clt_hca_port_attr =
@@ -347,11 +349,11 @@ static ssize_t rtrs_clt_hca_name_show(struct kobject *kobj, · 3 unmodified lines
347 struct kobj_attribute *attr,349 struct kobj_attribute *attr,
348 char *page)350 char *page)
349{351{
350 struct rtrs_clt_sess *sess;352 struct rtrs_clt_path *clt_path;
351 353
352 sess = container_of(kobj, struct rtrs_clt_sess, kobj);354 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
353 355
354 return sysfs_emit(page, "%s\n", sess->hca_name);356 return sysfs_emit(page, "%s\n", clt_path->hca_name);
355}357}
356 358
357static struct kobj_attribute rtrs_clt_hca_name_attr =359static struct kobj_attribute rtrs_clt_hca_name_attr =
@@ -361,12 +363,12 @@ static ssize_t rtrs_clt_cur_latency_show(struct kobject *kobj, · 3 unmodified lines
361 struct kobj_attribute *attr,363 struct kobj_attribute *attr,
362 char *page)364 char *page)
363{365{
364 struct rtrs_clt_sess *sess;366 struct rtrs_clt_path *clt_path;
365 367
366 sess = container_of(kobj, struct rtrs_clt_sess, kobj);368 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
367 369
368 return sysfs_emit(page, "%lld ns\n",370 return sysfs_emit(page, "%lld ns\n",
369 ktime_to_ns(sess->s.hb_cur_latency));371 ktime_to_ns(clt_path->s.hb_cur_latency));
370}372}
371 373
372static struct kobj_attribute rtrs_clt_cur_latency_attr =374static struct kobj_attribute rtrs_clt_cur_latency_attr =
@@ -376,11 +378,11 @@ static ssize_t rtrs_clt_src_addr_show(struct kobject *kobj, · 3 unmodified lines
376 struct kobj_attribute *attr,378 struct kobj_attribute *attr,
377 char *page)379 char *page)
378{380{
379 struct rtrs_clt_sess *sess;381 struct rtrs_clt_path *clt_path;
380 int len;382 int len;
381 383
382 sess = container_of(kobj, struct rtrs_clt_sess, kobj);384 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
383 len = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr, page,385 len = sockaddr_to_str((struct sockaddr *)&clt_path->s.src_addr, page,
384 PAGE_SIZE);386 PAGE_SIZE);
385 len += sysfs_emit_at(page, len, "\n");387 len += sysfs_emit_at(page, len, "\n");
386 return len;388 return len;
@@ -393,11 +395,11 @@ static ssize_t rtrs_clt_dst_addr_show(struct kobject *kobj, · 6 unmodified lines
393 struct kobj_attribute *attr,395 struct kobj_attribute *attr,
394 char *page)396 char *page)
395{397{
396 struct rtrs_clt_sess *sess;398 struct rtrs_clt_path *clt_path;
397 int len;399 int len;
398 400
399 sess = container_of(kobj, struct rtrs_clt_sess, kobj);401 clt_path = container_of(kobj, struct rtrs_clt_path, kobj);
400 len = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr, page,402 len = sockaddr_to_str((struct sockaddr *)&clt_path->s.dst_addr, page,
401 PAGE_SIZE);403 PAGE_SIZE);
402 len += sysfs_emit_at(page, len, "\n");404 len += sysfs_emit_at(page, len, "\n");
403 return len;405 return len;
@@ -406,7 +408,7 @@ static ssize_t rtrs_clt_dst_addr_show(struct kobject *kobj, · 2 unmodified lines
406static struct kobj_attribute rtrs_clt_dst_addr_attr =408static struct kobj_attribute rtrs_clt_dst_addr_attr =
407 __ATTR(dst_addr, 0444, rtrs_clt_dst_addr_show, NULL);409 __ATTR(dst_addr, 0444, rtrs_clt_dst_addr_show, NULL);
408 410
409static struct attribute *rtrs_clt_sess_attrs[] = {411static struct attribute *rtrs_clt_path_attrs[] = {
410 &rtrs_clt_hca_name_attr.attr,412 &rtrs_clt_hca_name_attr.attr,
411 &rtrs_clt_hca_port_attr.attr,413 &rtrs_clt_hca_port_attr.attr,
412 &rtrs_clt_src_addr_attr.attr,414 &rtrs_clt_src_addr_attr.attr,
@@ -419,42 +421,43 @@ static struct attribute *rtrs_clt_sess_attrs[] = { · 6 unmodified lines
419 NULL,421 NULL,
420};422};
421 423
422static const struct attribute_group rtrs_clt_sess_attr_group = {424static const struct attribute_group rtrs_clt_path_attr_group = {
423 .attrs = rtrs_clt_sess_attrs,425 .attrs = rtrs_clt_path_attrs,
424};426};
425 427
426int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess)428int rtrs_clt_create_path_files(struct rtrs_clt_path *clt_path)
427{429{
428 struct rtrs_clt *clt = sess->clt;430 struct rtrs_clt_sess *clt = clt_path->clt;
429 char str[NAME_MAX];431 char str[NAME_MAX];
430 int err;432 int err;
431 struct rtrs_addr path = {433 struct rtrs_addr path = {
432 .src = &sess->s.src_addr,434 .src = &clt_path->s.src_addr,
433 .dst = &sess->s.dst_addr,435 .dst = &clt_path->s.dst_addr,
434 };436 };
435 437
436 rtrs_addr_to_str(&path, str, sizeof(str));438 rtrs_addr_to_str(&path, str, sizeof(str));
437 err = kobject_init_and_add(&sess->kobj, &ktype_sess, clt->kobj_paths,439 err = kobject_init_and_add(&clt_path->kobj, &ktype_sess,
440 clt->kobj_paths,
438 "%s", str);441 "%s", str);
439 if (err) {442 if (err) {
440 pr_err("kobject_init_and_add: %d\n", err);443 pr_err("kobject_init_and_add: %d\n", err);
441 kobject_put(&sess->kobj);444 kobject_put(&clt_path->kobj);
442 return err;445 return err;
443 }446 }
444 err = sysfs_create_group(&sess->kobj, &rtrs_clt_sess_attr_group);447 err = sysfs_create_group(&clt_path->kobj, &rtrs_clt_path_attr_group);
445 if (err) {448 if (err) {
446 pr_err("sysfs_create_group(): %d\n", err);449 pr_err("sysfs_create_group(): %d\n", err);
447 goto put_kobj;450 goto put_kobj;
448 }451 }
449 err = kobject_init_and_add(&sess->stats->kobj_stats, &ktype_stats,452 err = kobject_init_and_add(&clt_path->stats->kobj_stats, &ktype_stats,
450 &sess->kobj, "stats");453 &clt_path->kobj, "stats");
451 if (err) {454 if (err) {
452 pr_err("kobject_init_and_add: %d\n", err);455 pr_err("kobject_init_and_add: %d\n", err);
453 kobject_put(&sess->stats->kobj_stats);456 kobject_put(&clt_path->stats->kobj_stats);
454 goto remove_group;457 goto remove_group;
455 }458 }
456 459
457 err = sysfs_create_group(&sess->stats->kobj_stats,460 err = sysfs_create_group(&clt_path->stats->kobj_stats,
458 &rtrs_clt_stats_attr_group);461 &rtrs_clt_stats_attr_group);
459 if (err) {462 if (err) {
460 pr_err("failed to create stats sysfs group, err: %d\n", err);463 pr_err("failed to create stats sysfs group, err: %d\n", err);
@@ -464,25 +467,25 @@ int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess) · 3 unmodified lines
464 return 0;467 return 0;
465 468
466put_kobj_stats:469put_kobj_stats:
467 kobject_del(&sess->stats->kobj_stats);470 kobject_del(&clt_path->stats->kobj_stats);
468 kobject_put(&sess->stats->kobj_stats);471 kobject_put(&clt_path->stats->kobj_stats);
469remove_group:472remove_group:
470 sysfs_remove_group(&sess->kobj, &rtrs_clt_sess_attr_group);473 sysfs_remove_group(&clt_path->kobj, &rtrs_clt_path_attr_group);
471put_kobj:474put_kobj:
472 kobject_del(&sess->kobj);475 kobject_del(&clt_path->kobj);
473 kobject_put(&sess->kobj);476 kobject_put(&clt_path->kobj);
474 477
475 return err;478 return err;
476}479}
477 480
478void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess,481void rtrs_clt_destroy_path_files(struct rtrs_clt_path *clt_path,
479 const struct attribute *sysfs_self)482 const struct attribute *sysfs_self)
480{483{
481 kobject_del(&sess->stats->kobj_stats);484 kobject_del(&clt_path->stats->kobj_stats);
482 kobject_put(&sess->stats->kobj_stats);485 kobject_put(&clt_path->stats->kobj_stats);
483 if (sysfs_self)486 if (sysfs_self)
484 sysfs_remove_file_self(&sess->kobj, sysfs_self);487 sysfs_remove_file_self(&clt_path->kobj, sysfs_self);
485 kobject_del(&sess->kobj);488 kobject_del(&clt_path->kobj);
486}489}
487 490
488static struct attribute *rtrs_clt_attrs[] = {491static struct attribute *rtrs_clt_attrs[] = {
@@ -496,12 +499,12 @@ static const struct attribute_group rtrs_clt_attr_group = { · 7 unmodified lines
496 .attrs = rtrs_clt_attrs,499 .attrs = rtrs_clt_attrs,
497};500};
498 501
499int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt)502int rtrs_clt_create_sysfs_root_files(struct rtrs_clt_sess *clt)
500{503{
501 return sysfs_create_group(&clt->dev.kobj, &rtrs_clt_attr_group);504 return sysfs_create_group(&clt->dev.kobj, &rtrs_clt_attr_group);
502}505}
503 506
504void rtrs_clt_destroy_sysfs_root(struct rtrs_clt *clt)507void rtrs_clt_destroy_sysfs_root(struct rtrs_clt_sess *clt)
505{508{
506 sysfs_remove_group(&clt->dev.kobj, &rtrs_clt_attr_group);509 sysfs_remove_group(&clt->dev.kobj, &rtrs_clt_attr_group);
507 510
drivers/infiniband/ulp/rtrs/rtrs-clt.c+554−535
@@ -46,21 +46,21 @@ static struct rtrs_rdma_dev_pd dev_pd = { · 45 unmodified lines
46static struct workqueue_struct *rtrs_wq;46static struct workqueue_struct *rtrs_wq;
47static struct class *rtrs_clt_dev_class;47static struct class *rtrs_clt_dev_class;
48 48
49static inline bool rtrs_clt_is_connected(const struct rtrs_clt *clt)49static inline bool rtrs_clt_is_connected(const struct rtrs_clt_sess *clt)
50{50{
51 struct rtrs_clt_sess *sess;51 struct rtrs_clt_path *clt_path;
52 bool connected = false;52 bool connected = false;
53 53
54 rcu_read_lock();54 rcu_read_lock();
55 list_for_each_entry_rcu(sess, &clt->paths_list, s.entry)55 list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry)
56 connected |= READ_ONCE(sess->state) == RTRS_CLT_CONNECTED;56 connected |= READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED;
57 rcu_read_unlock();57 rcu_read_unlock();
58 58
59 return connected;59 return connected;
60}60}
61 61
62static struct rtrs_permit *62static struct rtrs_permit *
63__rtrs_get_permit(struct rtrs_clt *clt, enum rtrs_clt_con_type con_type)63__rtrs_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type)
64{64{
65 size_t max_depth = clt->queue_depth;65 size_t max_depth = clt->queue_depth;
66 struct rtrs_permit *permit;66 struct rtrs_permit *permit;
@@ -87,7 +87,7 @@ __rtrs_get_permit(struct rtrs_clt *clt, enum rtrs_clt_con_type con_type) · 20 unmodified lines
87 return permit;87 return permit;
88}88}
89 89
90static inline void __rtrs_put_permit(struct rtrs_clt *clt,90static inline void __rtrs_put_permit(struct rtrs_clt_sess *clt,
91 struct rtrs_permit *permit)91 struct rtrs_permit *permit)
92{92{
93 clear_bit_unlock(permit->mem_id, clt->permits_map);93 clear_bit_unlock(permit->mem_id, clt->permits_map);
@@ -107,7 +107,7 @@ static inline void __rtrs_put_permit(struct rtrs_clt *clt, · 13 unmodified lines
107 * Context:107 * Context:
108 * Can sleep if @wait == RTRS_PERMIT_WAIT108 * Can sleep if @wait == RTRS_PERMIT_WAIT
109 */109 */
110struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt *clt,110struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt_sess *clt,
111 enum rtrs_clt_con_type con_type,111 enum rtrs_clt_con_type con_type,
112 enum wait_type can_wait)112 enum wait_type can_wait)
113{113{
@@ -142,7 +142,8 @@ EXPORT_SYMBOL(rtrs_clt_get_permit); · 28 unmodified lines
142 * Context:142 * Context:
143 * Does not matter143 * Does not matter
144 */144 */
145void rtrs_clt_put_permit(struct rtrs_clt *clt, struct rtrs_permit *permit)145void rtrs_clt_put_permit(struct rtrs_clt_sess *clt,
146 struct rtrs_permit *permit)
146{147{
147 if (WARN_ON(!test_bit(permit->mem_id, clt->permits_map)))148 if (WARN_ON(!test_bit(permit->mem_id, clt->permits_map)))
148 return;149 return;
@@ -163,29 +164,29 @@ EXPORT_SYMBOL(rtrs_clt_put_permit); · 14 unmodified lines
163 164
164/**165/**
165 * rtrs_permit_to_clt_con() - returns RDMA connection pointer by the permit166 * rtrs_permit_to_clt_con() - returns RDMA connection pointer by the permit
166 * @sess: client session pointer167 * @clt_path: client path pointer
167 * @permit: permit for the allocation of the RDMA buffer168 * @permit: permit for the allocation of the RDMA buffer
168 * Note:169 * Note:
169 * IO connection starts from 1.170 * IO connection starts from 1.
170 * 0 connection is for user messages.171 * 0 connection is for user messages.
171 */172 */
172static173static
173struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_sess *sess,174struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_path *clt_path,
174 struct rtrs_permit *permit)175 struct rtrs_permit *permit)
175{176{
176 int id = 0;177 int id = 0;
177 178
178 if (permit->con_type == RTRS_IO_CON)179 if (permit->con_type == RTRS_IO_CON)
179 id = (permit->cpu_id % (sess->s.irq_con_num - 1)) + 1;180 id = (permit->cpu_id % (clt_path->s.irq_con_num - 1)) + 1;
180 181
181 return to_clt_con(sess->s.con[id]);182 return to_clt_con(clt_path->s.con[id]);
182}183}
183 184
184/**185/**
185 * rtrs_clt_change_state() - change the session state through session state186 * rtrs_clt_change_state() - change the session state through session state
186 * machine.187 * machine.
187 *188 *
188 * @sess: client session to change the state of.189 * @clt_path: client path to change the state of.
189 * @new_state: state to change to.190 * @new_state: state to change to.
190 *191 *
191 * returns true if sess's state is changed to new state, otherwise return false.192 * returns true if sess's state is changed to new state, otherwise return false.
@@ -193,15 +194,15 @@ struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_sess *sess, · 1 unmodified line
193 * Locks:194 * Locks:
194 * state_wq lock must be hold.195 * state_wq lock must be hold.
195 */196 */
196static bool rtrs_clt_change_state(struct rtrs_clt_sess *sess,197static bool rtrs_clt_change_state(struct rtrs_clt_path *clt_path,
197 enum rtrs_clt_state new_state)198 enum rtrs_clt_state new_state)
198{199{
199 enum rtrs_clt_state old_state;200 enum rtrs_clt_state old_state;
200 bool changed = false;201 bool changed = false;
201 202
202 lockdep_assert_held(&sess->state_wq.lock);203 lockdep_assert_held(&clt_path->state_wq.lock);
203 204
204 old_state = sess->state;205 old_state = clt_path->state;
205 switch (new_state) {206 switch (new_state) {
206 case RTRS_CLT_CONNECTING:207 case RTRS_CLT_CONNECTING:
207 switch (old_state) {208 switch (old_state) {
@@ -275,42 +276,42 @@ static bool rtrs_clt_change_state(struct rtrs_clt_sess *sess, · 67 unmodified lines
275 break;276 break;
276 }277 }
277 if (changed) {278 if (changed) {
278 sess->state = new_state;279 clt_path->state = new_state;
279 wake_up_locked(&sess->state_wq);280 wake_up_locked(&clt_path->state_wq);
280 }281 }
281 282
282 return changed;283 return changed;
283}284}
284 285
285static bool rtrs_clt_change_state_from_to(struct rtrs_clt_sess *sess,286static bool rtrs_clt_change_state_from_to(struct rtrs_clt_path *clt_path,
286 enum rtrs_clt_state old_state,287 enum rtrs_clt_state old_state,
287 enum rtrs_clt_state new_state)288 enum rtrs_clt_state new_state)
288{289{
289 bool changed = false;290 bool changed = false;
290 291
291 spin_lock_irq(&sess->state_wq.lock);292 spin_lock_irq(&clt_path->state_wq.lock);
292 if (sess->state == old_state)293 if (clt_path->state == old_state)
293 changed = rtrs_clt_change_state(sess, new_state);294 changed = rtrs_clt_change_state(clt_path, new_state);
294 spin_unlock_irq(&sess->state_wq.lock);295 spin_unlock_irq(&clt_path->state_wq.lock);
295 296
296 return changed;297 return changed;
297}298}
298 299
299static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con)300static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con)
300{301{
301 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);302 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
302 303
303 if (rtrs_clt_change_state_from_to(sess,304 if (rtrs_clt_change_state_from_to(clt_path,
304 RTRS_CLT_CONNECTED,305 RTRS_CLT_CONNECTED,
305 RTRS_CLT_RECONNECTING)) {306 RTRS_CLT_RECONNECTING)) {
306 struct rtrs_clt *clt = sess->clt;307 struct rtrs_clt_sess *clt = clt_path->clt;
307 unsigned int delay_ms;308 unsigned int delay_ms;
308 309
309 /*310 /*
310 * Normal scenario, reconnect if we were successfully connected311 * Normal scenario, reconnect if we were successfully connected
311 */312 */
312 delay_ms = clt->reconnect_delay_sec * 1000;313 delay_ms = clt->reconnect_delay_sec * 1000;
313 queue_delayed_work(rtrs_wq, &sess->reconnect_dwork,314 queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork,
314 msecs_to_jiffies(delay_ms +315 msecs_to_jiffies(delay_ms +
315 prandom_u32() % RTRS_RECONNECT_SEED));316 prandom_u32() % RTRS_RECONNECT_SEED));
316 } else {317 } else {
@@ -319,7 +320,7 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con) · 2 unmodified lines
319 * so notify waiter with error state, waiter is responsible320 * so notify waiter with error state, waiter is responsible
320 * for cleaning the rest and reconnect if needed.321 * for cleaning the rest and reconnect if needed.
321 */322 */
322 rtrs_clt_change_state_from_to(sess,323 rtrs_clt_change_state_from_to(clt_path,
323 RTRS_CLT_CONNECTING,324 RTRS_CLT_CONNECTING,
324 RTRS_CLT_CONNECTING_ERR);325 RTRS_CLT_CONNECTING_ERR);
325 }326 }
@@ -330,7 +331,7 @@ static void rtrs_clt_fast_reg_done(struct ib_cq *cq, struct ib_wc *wc) · 4 unmodified lines
330 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);331 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
331 332
332 if (wc->status != IB_WC_SUCCESS) {333 if (wc->status != IB_WC_SUCCESS) {
333 rtrs_err(con->c.sess, "Failed IB_WR_REG_MR: %s\n",334 rtrs_err(con->c.path, "Failed IB_WR_REG_MR: %s\n",
334 ib_wc_status_msg(wc->status));335 ib_wc_status_msg(wc->status));
335 rtrs_rdma_error_recovery(con);336 rtrs_rdma_error_recovery(con);
336 }337 }
@@ -350,7 +351,7 @@ static void rtrs_clt_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) · 13 unmodified lines
350 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);351 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
351 352
352 if (wc->status != IB_WC_SUCCESS) {353 if (wc->status != IB_WC_SUCCESS) {
353 rtrs_err(con->c.sess, "Failed IB_WR_LOCAL_INV: %s\n",354 rtrs_err(con->c.path, "Failed IB_WR_LOCAL_INV: %s\n",
354 ib_wc_status_msg(wc->status));355 ib_wc_status_msg(wc->status));
355 rtrs_rdma_error_recovery(con);356 rtrs_rdma_error_recovery(con);
356 }357 }
@@ -380,14 +381,14 @@ static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno, · 23 unmodified lines
380 bool notify, bool can_wait)381 bool notify, bool can_wait)
381{382{
382 struct rtrs_clt_con *con = req->con;383 struct rtrs_clt_con *con = req->con;
383 struct rtrs_clt_sess *sess;384 struct rtrs_clt_path *clt_path;
384 int err;385 int err;
385 386
386 if (WARN_ON(!req->in_use))387 if (WARN_ON(!req->in_use))
387 return;388 return;
388 if (WARN_ON(!req->con))389 if (WARN_ON(!req->con))
389 return;390 return;
390 sess = to_clt_sess(con->c.sess);391 clt_path = to_clt_path(con->c.path);
391 392
392 if (req->sg_cnt) {393 if (req->sg_cnt) {
393 if (req->dir == DMA_FROM_DEVICE && req->need_inv) {394 if (req->dir == DMA_FROM_DEVICE && req->need_inv) {
@@ -417,7 +418,7 @@ static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno, · 23 unmodified lines
417 refcount_inc(&req->ref);418 refcount_inc(&req->ref);
418 err = rtrs_inv_rkey(req);419 err = rtrs_inv_rkey(req);
419 if (err) {420 if (err) {
420 rtrs_err(con->c.sess, "Send INV WR key=%#x: %d\n",421 rtrs_err(con->c.path, "Send INV WR key=%#x: %d\n",
421 req->mr->rkey, err);422 req->mr->rkey, err);
422 } else if (can_wait) {423 } else if (can_wait) {
423 wait_for_completion(&req->inv_comp);424 wait_for_completion(&req->inv_comp);
@@ -433,21 +434,21 @@ static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno, · 9 unmodified lines
433 if (!refcount_dec_and_test(&req->ref))434 if (!refcount_dec_and_test(&req->ref))
434 return;435 return;
435 }436 }
436 ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist,437 ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist,
437 req->sg_cnt, req->dir);438 req->sg_cnt, req->dir);
438 }439 }
439 if (!refcount_dec_and_test(&req->ref))440 if (!refcount_dec_and_test(&req->ref))
440 return;441 return;
441 if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)442 if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)
442 atomic_dec(&sess->stats->inflight);443 atomic_dec(&clt_path->stats->inflight);
443 444
444 req->in_use = false;445 req->in_use = false;
445 req->con = NULL;446 req->con = NULL;
446 447
447 if (errno) {448 if (errno) {
448 rtrs_err_rl(con->c.sess, "IO request failed: error=%d path=%s [%s:%u] notify=%d\n",449 rtrs_err_rl(con->c.path, "IO request failed: error=%d path=%s [%s:%u] notify=%d\n",
449 errno, kobject_name(&sess->kobj), sess->hca_name,450 errno, kobject_name(&clt_path->kobj), clt_path->hca_name,
450 sess->hca_port, notify);451 clt_path->hca_port, notify);
451 }452 }
452 453
453 if (notify)454 if (notify)
@@ -459,12 +460,12 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con, · 5 unmodified lines
459 struct rtrs_rbuf *rbuf, u32 off,460 struct rtrs_rbuf *rbuf, u32 off,
460 u32 imm, struct ib_send_wr *wr)461 u32 imm, struct ib_send_wr *wr)
461{462{
462 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);463 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
463 enum ib_send_flags flags;464 enum ib_send_flags flags;
464 struct ib_sge sge;465 struct ib_sge sge;
465 466
466 if (!req->sg_size) {467 if (!req->sg_size) {
467 rtrs_wrn(con->c.sess,468 rtrs_wrn(con->c.path,
468 "Doing RDMA Write failed, no data supplied\n");469 "Doing RDMA Write failed, no data supplied\n");
469 return -EINVAL;470 return -EINVAL;
470 }471 }
@@ -472,16 +473,17 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con, · 1 unmodified line
472 /* user data and user message in the first list element */473 /* user data and user message in the first list element */
473 sge.addr = req->iu->dma_addr;474 sge.addr = req->iu->dma_addr;
474 sge.length = req->sg_size;475 sge.length = req->sg_size;
475 sge.lkey = sess->s.dev->ib_pd->local_dma_lkey;476 sge.lkey = clt_path->s.dev->ib_pd->local_dma_lkey;
476 477
477 /*478 /*
478 * From time to time we have to post signalled sends,479 * From time to time we have to post signalled sends,
479 * or send queue will fill up and only QP reset can help.480 * or send queue will fill up and only QP reset can help.
480 */481 */
481 flags = atomic_inc_return(&con->c.wr_cnt) % sess->s.signal_interval ?482 flags = atomic_inc_return(&con->c.wr_cnt) % clt_path->s.signal_interval ?
482 0 : IB_SEND_SIGNALED;483 0 : IB_SEND_SIGNALED;
483 484
484 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, req->iu->dma_addr,485 ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev,
486 req->iu->dma_addr,
485 req->sg_size, DMA_TO_DEVICE);487 req->sg_size, DMA_TO_DEVICE);
486 488
487 return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, &sge, 1,489 return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, &sge, 1,
@@ -489,15 +491,15 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con, · 1 unmodified line
489 imm, flags, wr, NULL);491 imm, flags, wr, NULL);
490}492}
491 493
492static void process_io_rsp(struct rtrs_clt_sess *sess, u32 msg_id,494static void process_io_rsp(struct rtrs_clt_path *clt_path, u32 msg_id,
493 s16 errno, bool w_inval)495 s16 errno, bool w_inval)
494{496{
495 struct rtrs_clt_io_req *req;497 struct rtrs_clt_io_req *req;
496 498
497 if (WARN_ON(msg_id >= sess->queue_depth))499 if (WARN_ON(msg_id >= clt_path->queue_depth))
498 return;500 return;
499 501
500 req = &sess->reqs[msg_id];502 req = &clt_path->reqs[msg_id];
501 /* Drop need_inv if server responded with send with invalidation */503 /* Drop need_inv if server responded with send with invalidation */
502 req->need_inv &= !w_inval;504 req->need_inv &= !w_inval;
503 complete_rdma_req(req, errno, true, false);505 complete_rdma_req(req, errno, true, false);
@@ -507,21 +509,21 @@ static void rtrs_clt_recv_done(struct rtrs_clt_con *con, struct ib_wc *wc) · 3 unmodified lines
507{509{
508 struct rtrs_iu *iu;510 struct rtrs_iu *iu;
509 int err;511 int err;
510 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);512 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
511 513
512 WARN_ON((sess->flags & RTRS_MSG_NEW_RKEY_F) == 0);514 WARN_ON((clt_path->flags & RTRS_MSG_NEW_RKEY_F) == 0);
513 iu = container_of(wc->wr_cqe, struct rtrs_iu,515 iu = container_of(wc->wr_cqe, struct rtrs_iu,
514 cqe);516 cqe);
515 err = rtrs_iu_post_recv(&con->c, iu);517 err = rtrs_iu_post_recv(&con->c, iu);
516 if (err) {518 if (err) {
517 rtrs_err(con->c.sess, "post iu failed %d\n", err);519 rtrs_err(con->c.path, "post iu failed %d\n", err);
518 rtrs_rdma_error_recovery(con);520 rtrs_rdma_error_recovery(con);
519 }521 }
520}522}
521 523
522static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc)524static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc)
523{525{
524 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);526 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
525 struct rtrs_msg_rkey_rsp *msg;527 struct rtrs_msg_rkey_rsp *msg;
526 u32 imm_type, imm_payload;528 u32 imm_type, imm_payload;
527 bool w_inval = false;529 bool w_inval = false;
@@ -529,25 +531,26 @@ static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc) · 1 unmodified line
529 u32 buf_id;531 u32 buf_id;
530 int err;532 int err;
531 533
532 WARN_ON((sess->flags & RTRS_MSG_NEW_RKEY_F) == 0);534 WARN_ON((clt_path->flags & RTRS_MSG_NEW_RKEY_F) == 0);
533 535
534 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);536 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
535 537
536 if (wc->byte_len < sizeof(*msg)) {538 if (wc->byte_len < sizeof(*msg)) {
537 rtrs_err(con->c.sess, "rkey response is malformed: size %d\n",539 rtrs_err(con->c.path, "rkey response is malformed: size %d\n",
538 wc->byte_len);540 wc->byte_len);
539 goto out;541 goto out;
540 }542 }
541 ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr,543 ib_dma_sync_single_for_cpu(clt_path->s.dev->ib_dev, iu->dma_addr,
542 iu->size, DMA_FROM_DEVICE);544 iu->size, DMA_FROM_DEVICE);
543 msg = iu->buf;545 msg = iu->buf;
544 if (le16_to_cpu(msg->type) != RTRS_MSG_RKEY_RSP) {546 if (le16_to_cpu(msg->type) != RTRS_MSG_RKEY_RSP) {
545 rtrs_err(sess->clt, "rkey response is malformed: type %d\n",547 rtrs_err(clt_path->clt,
548 "rkey response is malformed: type %d\n",
546 le16_to_cpu(msg->type));549 le16_to_cpu(msg->type));
547 goto out;550 goto out;
548 }551 }
549 buf_id = le16_to_cpu(msg->buf_id);552 buf_id = le16_to_cpu(msg->buf_id);
550 if (WARN_ON(buf_id >= sess->queue_depth))553 if (WARN_ON(buf_id >= clt_path->queue_depth))
551 goto out;554 goto out;
552 555
553 rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), &imm_type, &imm_payload);556 rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), &imm_type, &imm_payload);
@@ -560,10 +563,10 @@ static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc) · 6 unmodified lines
560 563
561 if (WARN_ON(buf_id != msg_id))564 if (WARN_ON(buf_id != msg_id))
562 goto out;565 goto out;
563 sess->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey);566 clt_path->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey);
564 process_io_rsp(sess, msg_id, err, w_inval);567 process_io_rsp(clt_path, msg_id, err, w_inval);
565 }568 }
566 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, iu->dma_addr,569 ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev, iu->dma_addr,
567 iu->size, DMA_FROM_DEVICE);570 iu->size, DMA_FROM_DEVICE);
568 return rtrs_clt_recv_done(con, wc);571 return rtrs_clt_recv_done(con, wc);
569out:572out:
@@ -600,14 +603,14 @@ static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe) · 30 unmodified lines
600static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc)603static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc)
601{604{
602 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);605 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
603 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);606 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
604 u32 imm_type, imm_payload;607 u32 imm_type, imm_payload;
605 bool w_inval = false;608 bool w_inval = false;
606 int err;609 int err;
607 610
608 if (wc->status != IB_WC_SUCCESS) {611 if (wc->status != IB_WC_SUCCESS) {
609 if (wc->status != IB_WC_WR_FLUSH_ERR) {612 if (wc->status != IB_WC_WR_FLUSH_ERR) {
610 rtrs_err(sess->clt, "RDMA failed: %s\n",613 rtrs_err(clt_path->clt, "RDMA failed: %s\n",
611 ib_wc_status_msg(wc->status));614 ib_wc_status_msg(wc->status));
612 rtrs_rdma_error_recovery(con);615 rtrs_rdma_error_recovery(con);
613 }616 }
@@ -632,21 +635,21 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 18 unmodified lines
632 w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM);635 w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM);
633 rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err);636 rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err);
634 637
635 process_io_rsp(sess, msg_id, err, w_inval);638 process_io_rsp(clt_path, msg_id, err, w_inval);
636 } else if (imm_type == RTRS_HB_MSG_IMM) {639 } else if (imm_type == RTRS_HB_MSG_IMM) {
637 WARN_ON(con->c.cid);640 WARN_ON(con->c.cid);
638 rtrs_send_hb_ack(&sess->s);641 rtrs_send_hb_ack(&clt_path->s);
639 if (sess->flags & RTRS_MSG_NEW_RKEY_F)642 if (clt_path->flags & RTRS_MSG_NEW_RKEY_F)
640 return rtrs_clt_recv_done(con, wc);643 return rtrs_clt_recv_done(con, wc);
641 } else if (imm_type == RTRS_HB_ACK_IMM) {644 } else if (imm_type == RTRS_HB_ACK_IMM) {
642 WARN_ON(con->c.cid);645 WARN_ON(con->c.cid);
643 sess->s.hb_missed_cnt = 0;646 clt_path->s.hb_missed_cnt = 0;
644 sess->s.hb_cur_latency =647 clt_path->s.hb_cur_latency =
645 ktime_sub(ktime_get(), sess->s.hb_last_sent);648 ktime_sub(ktime_get(), clt_path->s.hb_last_sent);
646 if (sess->flags & RTRS_MSG_NEW_RKEY_F)649 if (clt_path->flags & RTRS_MSG_NEW_RKEY_F)
647 return rtrs_clt_recv_done(con, wc);650 return rtrs_clt_recv_done(con, wc);
648 } else {651 } else {
649 rtrs_wrn(con->c.sess, "Unknown IMM type %u\n",652 rtrs_wrn(con->c.path, "Unknown IMM type %u\n",
650 imm_type);653 imm_type);
651 }654 }
652 if (w_inval)655 if (w_inval)
@@ -658,7 +661,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 5 unmodified lines
658 else661 else
659 err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);662 err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);
660 if (err) {663 if (err) {
661 rtrs_err(con->c.sess, "rtrs_post_recv_empty(): %d\n",664 rtrs_err(con->c.path, "rtrs_post_recv_empty(): %d\n",
662 err);665 err);
663 rtrs_rdma_error_recovery(con);666 rtrs_rdma_error_recovery(con);
664 }667 }
@@ -670,7 +673,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 5 unmodified lines
670 WARN_ON(!(wc->wc_flags & IB_WC_WITH_INVALIDATE ||673 WARN_ON(!(wc->wc_flags & IB_WC_WITH_INVALIDATE ||
671 wc->wc_flags & IB_WC_WITH_IMM));674 wc->wc_flags & IB_WC_WITH_IMM));
672 WARN_ON(wc->wr_cqe->done != rtrs_clt_rdma_done);675 WARN_ON(wc->wr_cqe->done != rtrs_clt_rdma_done);
673 if (sess->flags & RTRS_MSG_NEW_RKEY_F) {676 if (clt_path->flags & RTRS_MSG_NEW_RKEY_F) {
674 if (wc->wc_flags & IB_WC_WITH_INVALIDATE)677 if (wc->wc_flags & IB_WC_WITH_INVALIDATE)
675 return rtrs_clt_recv_done(con, wc);678 return rtrs_clt_recv_done(con, wc);
676 679
@@ -685,7 +688,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 8 unmodified lines
685 break;688 break;
686 689
687 default:690 default:
688 rtrs_wrn(sess->clt, "Unexpected WC type: %d\n", wc->opcode);691 rtrs_wrn(clt_path->clt, "Unexpected WC type: %d\n", wc->opcode);
689 return;692 return;
690 }693 }
691}694}
@@ -693,10 +696,10 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 1 unmodified line
693static int post_recv_io(struct rtrs_clt_con *con, size_t q_size)696static int post_recv_io(struct rtrs_clt_con *con, size_t q_size)
694{697{
695 int err, i;698 int err, i;
696 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);699 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
697 700
698 for (i = 0; i < q_size; i++) {701 for (i = 0; i < q_size; i++) {
699 if (sess->flags & RTRS_MSG_NEW_RKEY_F) {702 if (clt_path->flags & RTRS_MSG_NEW_RKEY_F) {
700 struct rtrs_iu *iu = &con->rsp_ius[i];703 struct rtrs_iu *iu = &con->rsp_ius[i];
701 704
702 err = rtrs_iu_post_recv(&con->c, iu);705 err = rtrs_iu_post_recv(&con->c, iu);
@@ -710,16 +713,16 @@ static int post_recv_io(struct rtrs_clt_con *con, size_t q_size) · 7 unmodified lines
710 return 0;713 return 0;
711}714}
712 715
713static int post_recv_sess(struct rtrs_clt_sess *sess)716static int post_recv_path(struct rtrs_clt_path *clt_path)
714{717{
715 size_t q_size = 0;718 size_t q_size = 0;
716 int err, cid;719 int err, cid;
717 720
718 for (cid = 0; cid < sess->s.con_num; cid++) {721 for (cid = 0; cid < clt_path->s.con_num; cid++) {
719 if (cid == 0)722 if (cid == 0)
720 q_size = SERVICE_CON_QUEUE_DEPTH;723 q_size = SERVICE_CON_QUEUE_DEPTH;
721 else724 else
722 q_size = sess->queue_depth;725 q_size = clt_path->queue_depth;
723 726
724 /*727 /*
725 * x2 for RDMA read responses + FR key invalidations,728 * x2 for RDMA read responses + FR key invalidations,
@@ -727,9 +730,10 @@ static int post_recv_sess(struct rtrs_clt_sess *sess) · 1 unmodified line
727 */730 */
728 q_size *= 2;731 q_size *= 2;
729 732
730 err = post_recv_io(to_clt_con(sess->s.con[cid]), q_size);733 err = post_recv_io(to_clt_con(clt_path->s.con[cid]), q_size);
731 if (err) {734 if (err) {
732 rtrs_err(sess->clt, "post_recv_io(), err: %d\n", err);735 rtrs_err(clt_path->clt, "post_recv_io(), err: %d\n",
736 err);
733 return err;737 return err;
734 }738 }
735 }739 }
@@ -740,8 +744,8 @@ static int post_recv_sess(struct rtrs_clt_sess *sess) · 4 unmodified lines
740struct path_it {744struct path_it {
741 int i;745 int i;
742 struct list_head skip_list;746 struct list_head skip_list;
743 struct rtrs_clt *clt;747 struct rtrs_clt_sess *clt;
744 struct rtrs_clt_sess *(*next_path)(struct path_it *it);748 struct rtrs_clt_path *(*next_path)(struct path_it *it);
745};749};
746 750
747/**751/**
@@ -773,11 +777,11 @@ struct path_it { · 25 unmodified lines
773 * Locks:777 * Locks:
774 * rcu_read_lock() must be hold.778 * rcu_read_lock() must be hold.
775 */779 */
776static struct rtrs_clt_sess *get_next_path_rr(struct path_it *it)780static struct rtrs_clt_path *get_next_path_rr(struct path_it *it)
777{781{
778 struct rtrs_clt_sess __rcu **ppcpu_path;782 struct rtrs_clt_path __rcu **ppcpu_path;
779 struct rtrs_clt_sess *path;783 struct rtrs_clt_path *path;
780 struct rtrs_clt *clt;784 struct rtrs_clt_sess *clt;
781 785
782 clt = it->clt;786 clt = it->clt;
783 787
@@ -811,26 +815,26 @@ static struct rtrs_clt_sess *get_next_path_rr(struct path_it *it) · 27 unmodified lines
811 * Locks:815 * Locks:
812 * rcu_read_lock() must be hold.816 * rcu_read_lock() must be hold.
813 */817 */
814static struct rtrs_clt_sess *get_next_path_min_inflight(struct path_it *it)818static struct rtrs_clt_path *get_next_path_min_inflight(struct path_it *it)
815{819{
816 struct rtrs_clt_sess *min_path = NULL;820 struct rtrs_clt_path *min_path = NULL;
817 struct rtrs_clt *clt = it->clt;821 struct rtrs_clt_sess *clt = it->clt;
818 struct rtrs_clt_sess *sess;822 struct rtrs_clt_path *clt_path;
819 int min_inflight = INT_MAX;823 int min_inflight = INT_MAX;
820 int inflight;824 int inflight;
821 825
822 list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) {826 list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry) {
823 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED)827 if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED)
824 continue;828 continue;
825 829
826 if (!list_empty(raw_cpu_ptr(sess->mp_skip_entry)))830 if (!list_empty(raw_cpu_ptr(clt_path->mp_skip_entry)))
827 continue;831 continue;
828 832
829 inflight = atomic_read(&sess->stats->inflight);833 inflight = atomic_read(&clt_path->stats->inflight);
830 834
831 if (inflight < min_inflight) {835 if (inflight < min_inflight) {
832 min_inflight = inflight;836 min_inflight = inflight;
833 min_path = sess;837 min_path = clt_path;
834 }838 }
835 }839 }
836 840
@@ -862,26 +866,26 @@ static struct rtrs_clt_sess *get_next_path_min_inflight(struct path_it *it) · 25 unmodified lines
862 * Therefore the caller MUST check the returned866 * Therefore the caller MUST check the returned
863 * path is NULL and trigger the IO error.867 * path is NULL and trigger the IO error.
864 */868 */
865static struct rtrs_clt_sess *get_next_path_min_latency(struct path_it *it)869static struct rtrs_clt_path *get_next_path_min_latency(struct path_it *it)
866{870{
867 struct rtrs_clt_sess *min_path = NULL;871 struct rtrs_clt_path *min_path = NULL;
868 struct rtrs_clt *clt = it->clt;872 struct rtrs_clt_sess *clt = it->clt;
869 struct rtrs_clt_sess *sess;873 struct rtrs_clt_path *clt_path;
870 ktime_t min_latency = INT_MAX;874 ktime_t min_latency = KTIME_MAX;
871 ktime_t latency;875 ktime_t latency;
872 876
873 list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) {877 list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry) {
874 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED)878 if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED)
875 continue;879 continue;
876 880
877 if (!list_empty(raw_cpu_ptr(sess->mp_skip_entry)))881 if (!list_empty(raw_cpu_ptr(clt_path->mp_skip_entry)))
878 continue;882 continue;
879 883
880 latency = sess->s.hb_cur_latency;884 latency = clt_path->s.hb_cur_latency;
881 885
882 if (latency < min_latency) {886 if (latency < min_latency) {
883 min_latency = latency;887 min_latency = latency;
884 min_path = sess;888 min_path = clt_path;
885 }889 }
886 }890 }
887 891
@@ -895,7 +899,7 @@ static struct rtrs_clt_sess *get_next_path_min_latency(struct path_it *it) · 7 unmodified lines
895 return min_path;899 return min_path;
896}900}
897 901
898static inline void path_it_init(struct path_it *it, struct rtrs_clt *clt)902static inline void path_it_init(struct path_it *it, struct rtrs_clt_sess *clt)
899{903{
900 INIT_LIST_HEAD(&it->skip_list);904 INIT_LIST_HEAD(&it->skip_list);
901 it->clt = clt;905 it->clt = clt;
@@ -928,7 +932,7 @@ static inline void path_it_deinit(struct path_it *it) · 26 unmodified lines
928 * the corresponding buffer of rtrs_iu (req->iu->buf), which later on will932 * the corresponding buffer of rtrs_iu (req->iu->buf), which later on will
929 * also hold the control message of rtrs.933 * also hold the control message of rtrs.
930 * @req: an io request holding information about IO.934 * @req: an io request holding information about IO.
931 * @sess: client session935 * @clt_path: client path
932 * @conf: conformation callback function to notify upper layer.936 * @conf: conformation callback function to notify upper layer.
933 * @permit: permit for allocation of RDMA remote buffer937 * @permit: permit for allocation of RDMA remote buffer
934 * @priv: private pointer938 * @priv: private pointer
@@ -940,7 +944,7 @@ static inline void path_it_deinit(struct path_it *it) · 5 unmodified lines
940 * @dir: direction of the IO.944 * @dir: direction of the IO.
941 */945 */
942static void rtrs_clt_init_req(struct rtrs_clt_io_req *req,946static void rtrs_clt_init_req(struct rtrs_clt_io_req *req,
943 struct rtrs_clt_sess *sess,947 struct rtrs_clt_path *clt_path,
944 void (*conf)(void *priv, int errno),948 void (*conf)(void *priv, int errno),
945 struct rtrs_permit *permit, void *priv,949 struct rtrs_permit *permit, void *priv,
946 const struct kvec *vec, size_t usr_len,950 const struct kvec *vec, size_t usr_len,
@@ -958,13 +962,13 @@ static void rtrs_clt_init_req(struct rtrs_clt_io_req *req, · 11 unmodified lines
958 req->sg_cnt = sg_cnt;962 req->sg_cnt = sg_cnt;
959 req->priv = priv;963 req->priv = priv;
960 req->dir = dir;964 req->dir = dir;
961 req->con = rtrs_permit_to_clt_con(sess, permit);965 req->con = rtrs_permit_to_clt_con(clt_path, permit);
962 req->conf = conf;966 req->conf = conf;
963 req->need_inv = false;967 req->need_inv = false;
964 req->need_inv_comp = false;968 req->need_inv_comp = false;
965 req->inv_errno = 0;969 req->inv_errno = 0;
966 refcount_set(&req->ref, 1);970 refcount_set(&req->ref, 1);
967 req->mp_policy = sess->clt->mp_policy;971 req->mp_policy = clt_path->clt->mp_policy;
968 972
969 iov_iter_kvec(&iter, READ, vec, 1, usr_len);973 iov_iter_kvec(&iter, READ, vec, 1, usr_len);
970 len = _copy_from_iter(req->iu->buf, usr_len, &iter);974 len = _copy_from_iter(req->iu->buf, usr_len, &iter);
@@ -974,7 +978,7 @@ static void rtrs_clt_init_req(struct rtrs_clt_io_req *req, · 3 unmodified lines
974}978}
975 979
976static struct rtrs_clt_io_req *980static struct rtrs_clt_io_req *
977rtrs_clt_get_req(struct rtrs_clt_sess *sess,981rtrs_clt_get_req(struct rtrs_clt_path *clt_path,
978 void (*conf)(void *priv, int errno),982 void (*conf)(void *priv, int errno),
979 struct rtrs_permit *permit, void *priv,983 struct rtrs_permit *permit, void *priv,
980 const struct kvec *vec, size_t usr_len,984 const struct kvec *vec, size_t usr_len,
@@ -983,14 +987,14 @@ rtrs_clt_get_req(struct rtrs_clt_sess *sess, · 2 unmodified lines
983{987{
984 struct rtrs_clt_io_req *req;988 struct rtrs_clt_io_req *req;
985 989
986 req = &sess->reqs[permit->mem_id];990 req = &clt_path->reqs[permit->mem_id];
987 rtrs_clt_init_req(req, sess, conf, permit, priv, vec, usr_len,991 rtrs_clt_init_req(req, clt_path, conf, permit, priv, vec, usr_len,
988 sg, sg_cnt, data_len, dir);992 sg, sg_cnt, data_len, dir);
989 return req;993 return req;
990}994}
991 995
992static struct rtrs_clt_io_req *996static struct rtrs_clt_io_req *
993rtrs_clt_get_copy_req(struct rtrs_clt_sess *alive_sess,997rtrs_clt_get_copy_req(struct rtrs_clt_path *alive_path,
994 struct rtrs_clt_io_req *fail_req)998 struct rtrs_clt_io_req *fail_req)
995{999{
996 struct rtrs_clt_io_req *req;1000 struct rtrs_clt_io_req *req;
@@ -999,8 +1003,8 @@ rtrs_clt_get_copy_req(struct rtrs_clt_sess *alive_sess, · 2 unmodified lines
999 .iov_len = fail_req->usr_len1003 .iov_len = fail_req->usr_len
1000 };1004 };
1001 1005
1002 req = &alive_sess->reqs[fail_req->permit->mem_id];1006 req = &alive_path->reqs[fail_req->permit->mem_id];
1003 rtrs_clt_init_req(req, alive_sess, fail_req->conf, fail_req->permit,1007 rtrs_clt_init_req(req, alive_path, fail_req->conf, fail_req->permit,
1004 fail_req->priv, &vec, fail_req->usr_len,1008 fail_req->priv, &vec, fail_req->usr_len,
1005 fail_req->sglist, fail_req->sg_cnt,1009 fail_req->sglist, fail_req->sg_cnt,
1006 fail_req->data_len, fail_req->dir);1010 fail_req->data_len, fail_req->dir);
@@ -1013,7 +1017,7 @@ static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con, · 6 unmodified lines
1013 u32 size, u32 imm, struct ib_send_wr *wr,1017 u32 size, u32 imm, struct ib_send_wr *wr,
1014 struct ib_send_wr *tail)1018 struct ib_send_wr *tail)
1015{1019{
1016 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);1020 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1017 struct ib_sge *sge = req->sge;1021 struct ib_sge *sge = req->sge;
1018 enum ib_send_flags flags;1022 enum ib_send_flags flags;
1019 struct scatterlist *sg;1023 struct scatterlist *sg;
@@ -1033,22 +1037,23 @@ static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con, · 13 unmodified lines
1033 for_each_sg(req->sglist, sg, req->sg_cnt, i) {1037 for_each_sg(req->sglist, sg, req->sg_cnt, i) {
1034 sge[i].addr = sg_dma_address(sg);1038 sge[i].addr = sg_dma_address(sg);
1035 sge[i].length = sg_dma_len(sg);1039 sge[i].length = sg_dma_len(sg);
1036 sge[i].lkey = sess->s.dev->ib_pd->local_dma_lkey;1040 sge[i].lkey = clt_path->s.dev->ib_pd->local_dma_lkey;
1037 }1041 }
1038 num_sge = 1 + req->sg_cnt;1042 num_sge = 1 + req->sg_cnt;
1039 }1043 }
1040 sge[i].addr = req->iu->dma_addr;1044 sge[i].addr = req->iu->dma_addr;
1041 sge[i].length = size;1045 sge[i].length = size;
1042 sge[i].lkey = sess->s.dev->ib_pd->local_dma_lkey;1046 sge[i].lkey = clt_path->s.dev->ib_pd->local_dma_lkey;
1043 1047
1044 /*1048 /*
1045 * From time to time we have to post signalled sends,1049 * From time to time we have to post signalled sends,
1046 * or send queue will fill up and only QP reset can help.1050 * or send queue will fill up and only QP reset can help.
1047 */1051 */
1048 flags = atomic_inc_return(&con->c.wr_cnt) % sess->s.signal_interval ?1052 flags = atomic_inc_return(&con->c.wr_cnt) % clt_path->s.signal_interval ?
1049 0 : IB_SEND_SIGNALED;1053 0 : IB_SEND_SIGNALED;
1050 1054
1051 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, req->iu->dma_addr,1055 ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev,
1056 req->iu->dma_addr,
1052 size, DMA_TO_DEVICE);1057 size, DMA_TO_DEVICE);
1053 1058
1054 return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, sge, num_sge,1059 return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, sge, num_sge,
@@ -1074,8 +1079,8 @@ static int rtrs_map_sg_fr(struct rtrs_clt_io_req *req, size_t count) · 19 unmodified lines
1074static int rtrs_clt_write_req(struct rtrs_clt_io_req *req)1079static int rtrs_clt_write_req(struct rtrs_clt_io_req *req)
1075{1080{
1076 struct rtrs_clt_con *con = req->con;1081 struct rtrs_clt_con *con = req->con;
1077 struct rtrs_sess *s = con->c.sess;1082 struct rtrs_path *s = con->c.path;
1078 struct rtrs_clt_sess *sess = to_clt_sess(s);1083 struct rtrs_clt_path *clt_path = to_clt_path(s);
1079 struct rtrs_msg_rdma_write *msg;1084 struct rtrs_msg_rdma_write *msg;
1080 1085
1081 struct rtrs_rbuf *rbuf;1086 struct rtrs_rbuf *rbuf;
@@ -1088,13 +1093,13 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) · 6 unmodified lines
1088 1093
1089 const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len;1094 const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len;
1090 1095
1091 if (tsize > sess->chunk_size) {1096 if (tsize > clt_path->chunk_size) {
1092 rtrs_wrn(s, "Write request failed, size too big %zu > %d\n",1097 rtrs_wrn(s, "Write request failed, size too big %zu > %d\n",
1093 tsize, sess->chunk_size);1098 tsize, clt_path->chunk_size);
1094 return -EMSGSIZE;1099 return -EMSGSIZE;
1095 }1100 }
1096 if (req->sg_cnt) {1101 if (req->sg_cnt) {
1097 count = ib_dma_map_sg(sess->s.dev->ib_dev, req->sglist,1102 count = ib_dma_map_sg(clt_path->s.dev->ib_dev, req->sglist,
1098 req->sg_cnt, req->dir);1103 req->sg_cnt, req->dir);
1099 if (!count) {1104 if (!count) {
1100 rtrs_wrn(s, "Write request failed, map failed\n");1105 rtrs_wrn(s, "Write request failed, map failed\n");
@@ -1111,7 +1116,7 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) · 10 unmodified lines
1111 imm = rtrs_to_io_req_imm(imm);1116 imm = rtrs_to_io_req_imm(imm);
1112 buf_id = req->permit->mem_id;1117 buf_id = req->permit->mem_id;
1113 req->sg_size = tsize;1118 req->sg_size = tsize;
1114 rbuf = &sess->rbufs[buf_id];1119 rbuf = &clt_path->rbufs[buf_id];
1115 1120
1116 if (count) {1121 if (count) {
1117 ret = rtrs_map_sg_fr(req, count);1122 ret = rtrs_map_sg_fr(req, count);
@@ -1119,7 +1124,7 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) · 1 unmodified line
1119 rtrs_err_rl(s,1124 rtrs_err_rl(s,
1120 "Write request failed, failed to map fast reg. data, err: %d\n",1125 "Write request failed, failed to map fast reg. data, err: %d\n",
1121 ret);1126 ret);
1122 ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist,1127 ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist,
1123 req->sg_cnt, req->dir);1128 req->sg_cnt, req->dir);
1124 return ret;1129 return ret;
1125 }1130 }
@@ -1153,12 +1158,12 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) · 27 unmodified lines
1153 if (ret) {1158 if (ret) {
1154 rtrs_err_rl(s,1159 rtrs_err_rl(s,
1155 "Write request failed: error=%d path=%s [%s:%u]\n",1160 "Write request failed: error=%d path=%s [%s:%u]\n",
1156 ret, kobject_name(&sess->kobj), sess->hca_name,1161 ret, kobject_name(&clt_path->kobj), clt_path->hca_name,
1157 sess->hca_port);1162 clt_path->hca_port);
1158 if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)1163 if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)
1159 atomic_dec(&sess->stats->inflight);1164 atomic_dec(&clt_path->stats->inflight);
1160 if (req->sg_cnt)1165 if (req->sg_cnt)
1161 ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist,1166 ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist,
1162 req->sg_cnt, req->dir);1167 req->sg_cnt, req->dir);
1163 }1168 }
1164 1169
@@ -1168,10 +1173,10 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) · 3 unmodified lines
1168static int rtrs_clt_read_req(struct rtrs_clt_io_req *req)1173static int rtrs_clt_read_req(struct rtrs_clt_io_req *req)
1169{1174{
1170 struct rtrs_clt_con *con = req->con;1175 struct rtrs_clt_con *con = req->con;
1171 struct rtrs_sess *s = con->c.sess;1176 struct rtrs_path *s = con->c.path;
1172 struct rtrs_clt_sess *sess = to_clt_sess(s);1177 struct rtrs_clt_path *clt_path = to_clt_path(s);
1173 struct rtrs_msg_rdma_read *msg;1178 struct rtrs_msg_rdma_read *msg;
1174 struct rtrs_ib_dev *dev = sess->s.dev;1179 struct rtrs_ib_dev *dev = clt_path->s.dev;
1175 1180
1176 struct ib_reg_wr rwr;1181 struct ib_reg_wr rwr;
1177 struct ib_send_wr *wr = NULL;1182 struct ib_send_wr *wr = NULL;
@@ -1181,10 +1186,10 @@ static int rtrs_clt_read_req(struct rtrs_clt_io_req *req) · 3 unmodified lines
1181 1186
1182 const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len;1187 const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len;
1183 1188
1184 if (tsize > sess->chunk_size) {1189 if (tsize > clt_path->chunk_size) {
1185 rtrs_wrn(s,1190 rtrs_wrn(s,
1186 "Read request failed, message size is %zu, bigger than CHUNK_SIZE %d\n",1191 "Read request failed, message size is %zu, bigger than CHUNK_SIZE %d\n",
1187 tsize, sess->chunk_size);1192 tsize, clt_path->chunk_size);
1188 return -EMSGSIZE;1193 return -EMSGSIZE;
1189 }1194 }
1190 1195
@@ -1254,15 +1259,15 @@ static int rtrs_clt_read_req(struct rtrs_clt_io_req *req) · 63 unmodified lines
1254 */1259 */
1255 rtrs_clt_update_all_stats(req, READ);1260 rtrs_clt_update_all_stats(req, READ);
1256 1261
1257 ret = rtrs_post_send_rdma(req->con, req, &sess->rbufs[buf_id],1262 ret = rtrs_post_send_rdma(req->con, req, &clt_path->rbufs[buf_id],
1258 req->data_len, imm, wr);1263 req->data_len, imm, wr);
1259 if (ret) {1264 if (ret) {
1260 rtrs_err_rl(s,1265 rtrs_err_rl(s,
1261 "Read request failed: error=%d path=%s [%s:%u]\n",1266 "Read request failed: error=%d path=%s [%s:%u]\n",
1262 ret, kobject_name(&sess->kobj), sess->hca_name,1267 ret, kobject_name(&clt_path->kobj), clt_path->hca_name,
1263 sess->hca_port);1268 clt_path->hca_port);
1264 if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)1269 if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)
1265 atomic_dec(&sess->stats->inflight);1270 atomic_dec(&clt_path->stats->inflight);
1266 req->need_inv = false;1271 req->need_inv = false;
1267 if (req->sg_cnt)1272 if (req->sg_cnt)
1268 ib_dma_unmap_sg(dev->ib_dev, req->sglist,1273 ib_dma_unmap_sg(dev->ib_dev, req->sglist,
@@ -1277,21 +1282,21 @@ static int rtrs_clt_read_req(struct rtrs_clt_io_req *req) · 8 unmodified lines
1277 * @clt: clt context1282 * @clt: clt context
1278 * @fail_req: a failed io request.1283 * @fail_req: a failed io request.
1279 */1284 */
1280static int rtrs_clt_failover_req(struct rtrs_clt *clt,1285static int rtrs_clt_failover_req(struct rtrs_clt_sess *clt,
1281 struct rtrs_clt_io_req *fail_req)1286 struct rtrs_clt_io_req *fail_req)
1282{1287{
1283 struct rtrs_clt_sess *alive_sess;1288 struct rtrs_clt_path *alive_path;
1284 struct rtrs_clt_io_req *req;1289 struct rtrs_clt_io_req *req;
1285 int err = -ECONNABORTED;1290 int err = -ECONNABORTED;
1286 struct path_it it;1291 struct path_it it;
1287 1292
1288 rcu_read_lock();1293 rcu_read_lock();
1289 for (path_it_init(&it, clt);1294 for (path_it_init(&it, clt);
1290 (alive_sess = it.next_path(&it)) && it.i < it.clt->paths_num;1295 (alive_path = it.next_path(&it)) && it.i < it.clt->paths_num;
1291 it.i++) {1296 it.i++) {
1292 if (READ_ONCE(alive_sess->state) != RTRS_CLT_CONNECTED)1297 if (READ_ONCE(alive_path->state) != RTRS_CLT_CONNECTED)
1293 continue;1298 continue;
1294 req = rtrs_clt_get_copy_req(alive_sess, fail_req);1299 req = rtrs_clt_get_copy_req(alive_path, fail_req);
1295 if (req->dir == DMA_TO_DEVICE)1300 if (req->dir == DMA_TO_DEVICE)
1296 err = rtrs_clt_write_req(req);1301 err = rtrs_clt_write_req(req);
1297 else1302 else
@@ -1301,7 +1306,7 @@ static int rtrs_clt_failover_req(struct rtrs_clt *clt, · 3 unmodified lines
1301 continue;1306 continue;
1302 }1307 }
1303 /* Success path */1308 /* Success path */
1304 rtrs_clt_inc_failover_cnt(alive_sess->stats);1309 rtrs_clt_inc_failover_cnt(alive_path->stats);
1305 break;1310 break;
1306 }1311 }
1307 path_it_deinit(&it);1312 path_it_deinit(&it);
@@ -1310,16 +1315,16 @@ static int rtrs_clt_failover_req(struct rtrs_clt *clt, · 2 unmodified lines
1310 return err;1315 return err;
1311}1316}
1312 1317
1313static void fail_all_outstanding_reqs(struct rtrs_clt_sess *sess)1318static void fail_all_outstanding_reqs(struct rtrs_clt_path *clt_path)
1314{1319{
1315 struct rtrs_clt *clt = sess->clt;1320 struct rtrs_clt_sess *clt = clt_path->clt;
1316 struct rtrs_clt_io_req *req;1321 struct rtrs_clt_io_req *req;
1317 int i, err;1322 int i, err;
1318 1323
1319 if (!sess->reqs)1324 if (!clt_path->reqs)
1320 return;1325 return;
1321 for (i = 0; i < sess->queue_depth; ++i) {1326 for (i = 0; i < clt_path->queue_depth; ++i) {
1322 req = &sess->reqs[i];1327 req = &clt_path->reqs[i];
1323 if (!req->in_use)1328 if (!req->in_use)
1324 continue;1329 continue;
1325 1330
@@ -1337,38 +1342,39 @@ static void fail_all_outstanding_reqs(struct rtrs_clt_sess *sess) · 11 unmodified lines
1337 }1342 }
1338}1343}
1339 1344
1340static void free_sess_reqs(struct rtrs_clt_sess *sess)1345static void free_path_reqs(struct rtrs_clt_path *clt_path)
1341{1346{
1342 struct rtrs_clt_io_req *req;1347 struct rtrs_clt_io_req *req;
1343 int i;1348 int i;
1344 1349
1345 if (!sess->reqs)1350 if (!clt_path->reqs)
1346 return;1351 return;
1347 for (i = 0; i < sess->queue_depth; ++i) {1352 for (i = 0; i < clt_path->queue_depth; ++i) {
1348 req = &sess->reqs[i];1353 req = &clt_path->reqs[i];
1349 if (req->mr)1354 if (req->mr)
1350 ib_dereg_mr(req->mr);1355 ib_dereg_mr(req->mr);
1351 kfree(req->sge);1356 kfree(req->sge);
1352 rtrs_iu_free(req->iu, sess->s.dev->ib_dev, 1);1357 rtrs_iu_free(req->iu, clt_path->s.dev->ib_dev, 1);
1353 }1358 }
1354 kfree(sess->reqs);1359 kfree(clt_path->reqs);
1355 sess->reqs = NULL;1360 clt_path->reqs = NULL;
1356}1361}
1357 1362
1358static int alloc_sess_reqs(struct rtrs_clt_sess *sess)1363static int alloc_path_reqs(struct rtrs_clt_path *clt_path)
1359{1364{
1360 struct rtrs_clt_io_req *req;1365 struct rtrs_clt_io_req *req;
1361 int i, err = -ENOMEM;1366 int i, err = -ENOMEM;
1362 1367
1363 sess->reqs = kcalloc(sess->queue_depth, sizeof(*sess->reqs),1368 clt_path->reqs = kcalloc(clt_path->queue_depth,
1364 GFP_KERNEL);1369 sizeof(*clt_path->reqs),
1365 if (!sess->reqs)1370 GFP_KERNEL);
1371 if (!clt_path->reqs)
1366 return -ENOMEM;1372 return -ENOMEM;
1367 1373
1368 for (i = 0; i < sess->queue_depth; ++i) {1374 for (i = 0; i < clt_path->queue_depth; ++i) {
1369 req = &sess->reqs[i];1375 req = &clt_path->reqs[i];
1370 req->iu = rtrs_iu_alloc(1, sess->max_hdr_size, GFP_KERNEL,1376 req->iu = rtrs_iu_alloc(1, clt_path->max_hdr_size, GFP_KERNEL,
1371 sess->s.dev->ib_dev,1377 clt_path->s.dev->ib_dev,
1372 DMA_TO_DEVICE,1378 DMA_TO_DEVICE,
1373 rtrs_clt_rdma_done);1379 rtrs_clt_rdma_done);
1374 if (!req->iu)1380 if (!req->iu)
@@ -1378,13 +1384,14 @@ static int alloc_sess_reqs(struct rtrs_clt_sess *sess) · 3 unmodified lines
1378 if (!req->sge)1384 if (!req->sge)
1379 goto out;1385 goto out;
1380 1386
1381 req->mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG,1387 req->mr = ib_alloc_mr(clt_path->s.dev->ib_pd,
1382 sess->max_pages_per_mr);1388 IB_MR_TYPE_MEM_REG,
1389 clt_path->max_pages_per_mr);
1383 if (IS_ERR(req->mr)) {1390 if (IS_ERR(req->mr)) {
1384 err = PTR_ERR(req->mr);1391 err = PTR_ERR(req->mr);
1385 req->mr = NULL;1392 req->mr = NULL;
1386 pr_err("Failed to alloc sess->max_pages_per_mr %d\n",1393 pr_err("Failed to alloc clt_path->max_pages_per_mr %d\n",
1387 sess->max_pages_per_mr);1394 clt_path->max_pages_per_mr);
1388 goto out;1395 goto out;
1389 }1396 }
1390 1397
@@ -1394,12 +1401,12 @@ static int alloc_sess_reqs(struct rtrs_clt_sess *sess) · 3 unmodified lines
1394 return 0;1401 return 0;
1395 1402
1396out:1403out:
1397 free_sess_reqs(sess);1404 free_path_reqs(clt_path);
1398 1405
1399 return err;1406 return err;
1400}1407}
1401 1408
1402static int alloc_permits(struct rtrs_clt *clt)1409static int alloc_permits(struct rtrs_clt_sess *clt)
1403{1410{
1404 unsigned int chunk_bits;1411 unsigned int chunk_bits;
1405 int err, i;1412 int err, i;
@@ -1433,7 +1440,7 @@ out_err: · 27 unmodified lines
1433 return err;1440 return err;
1434}1441}
1435 1442
1436static void free_permits(struct rtrs_clt *clt)1443static void free_permits(struct rtrs_clt_sess *clt)
1437{1444{
1438 if (clt->permits_map) {1445 if (clt->permits_map) {
1439 size_t sz = clt->queue_depth;1446 size_t sz = clt->queue_depth;
@@ -1447,13 +1454,13 @@ static void free_permits(struct rtrs_clt *clt) · 7 unmodified lines
1447 clt->permits = NULL;1454 clt->permits = NULL;
1448}1455}
1449 1456
1450static void query_fast_reg_mode(struct rtrs_clt_sess *sess)1457static void query_fast_reg_mode(struct rtrs_clt_path *clt_path)
1451{1458{
1452 struct ib_device *ib_dev;1459 struct ib_device *ib_dev;
1453 u64 max_pages_per_mr;1460 u64 max_pages_per_mr;
1454 int mr_page_shift;1461 int mr_page_shift;
1455 1462
1456 ib_dev = sess->s.dev->ib_dev;1463 ib_dev = clt_path->s.dev->ib_dev;
1457 1464
1458 /*1465 /*
1459 * Use the smallest page size supported by the HCA, down to a1466 * Use the smallest page size supported by the HCA, down to a
@@ -1463,24 +1470,24 @@ static void query_fast_reg_mode(struct rtrs_clt_sess *sess) · 3 unmodified lines
1463 mr_page_shift = max(12, ffs(ib_dev->attrs.page_size_cap) - 1);1470 mr_page_shift = max(12, ffs(ib_dev->attrs.page_size_cap) - 1);
1464 max_pages_per_mr = ib_dev->attrs.max_mr_size;1471 max_pages_per_mr = ib_dev->attrs.max_mr_size;
1465 do_div(max_pages_per_mr, (1ull << mr_page_shift));1472 do_div(max_pages_per_mr, (1ull << mr_page_shift));
1466 sess->max_pages_per_mr =1473 clt_path->max_pages_per_mr =
1467 min3(sess->max_pages_per_mr, (u32)max_pages_per_mr,1474 min3(clt_path->max_pages_per_mr, (u32)max_pages_per_mr,
1468 ib_dev->attrs.max_fast_reg_page_list_len);1475 ib_dev->attrs.max_fast_reg_page_list_len);
1469 sess->clt->max_segments =1476 clt_path->clt->max_segments =
1470 min(sess->max_pages_per_mr, sess->clt->max_segments);1477 min(clt_path->max_pages_per_mr, clt_path->clt->max_segments);
1471}1478}
1472 1479
1473static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess,1480static bool rtrs_clt_change_state_get_old(struct rtrs_clt_path *clt_path,
1474 enum rtrs_clt_state new_state,1481 enum rtrs_clt_state new_state,
1475 enum rtrs_clt_state *old_state)1482 enum rtrs_clt_state *old_state)
1476{1483{
1477 bool changed;1484 bool changed;
1478 1485
1479 spin_lock_irq(&sess->state_wq.lock);1486 spin_lock_irq(&clt_path->state_wq.lock);
1480 if (old_state)1487 if (old_state)
1481 *old_state = sess->state;1488 *old_state = clt_path->state;
1482 changed = rtrs_clt_change_state(sess, new_state);1489 changed = rtrs_clt_change_state(clt_path, new_state);
1483 spin_unlock_irq(&sess->state_wq.lock);1490 spin_unlock_irq(&clt_path->state_wq.lock);
1484 1491
1485 return changed;1492 return changed;
1486}1493}
@@ -1492,9 +1499,9 @@ static void rtrs_clt_hb_err_handler(struct rtrs_con *c) · 5 unmodified lines
1492 rtrs_rdma_error_recovery(con);1499 rtrs_rdma_error_recovery(con);
1493}1500}
1494 1501
1495static void rtrs_clt_init_hb(struct rtrs_clt_sess *sess)1502static void rtrs_clt_init_hb(struct rtrs_clt_path *clt_path)
1496{1503{
1497 rtrs_init_hb(&sess->s, &io_comp_cqe,1504 rtrs_init_hb(&clt_path->s, &io_comp_cqe,
1498 RTRS_HB_INTERVAL_MS,1505 RTRS_HB_INTERVAL_MS,
1499 RTRS_HB_MISSED_MAX,1506 RTRS_HB_MISSED_MAX,
1500 rtrs_clt_hb_err_handler,1507 rtrs_clt_hb_err_handler,
@@ -1504,17 +1511,17 @@ static void rtrs_clt_init_hb(struct rtrs_clt_sess *sess) · 3 unmodified lines
1504static void rtrs_clt_reconnect_work(struct work_struct *work);1511static void rtrs_clt_reconnect_work(struct work_struct *work);
1505static void rtrs_clt_close_work(struct work_struct *work);1512static void rtrs_clt_close_work(struct work_struct *work);
1506 1513
1507static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt,1514static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt,
1508 const struct rtrs_addr *path,1515 const struct rtrs_addr *path,
1509 size_t con_num, u32 nr_poll_queues)1516 size_t con_num, u32 nr_poll_queues)
1510{1517{
1511 struct rtrs_clt_sess *sess;1518 struct rtrs_clt_path *clt_path;
1512 int err = -ENOMEM;1519 int err = -ENOMEM;
1513 int cpu;1520 int cpu;
1514 size_t total_con;1521 size_t total_con;
1515 1522
1516 sess = kzalloc(sizeof(*sess), GFP_KERNEL);1523 clt_path = kzalloc(sizeof(*clt_path), GFP_KERNEL);
1517 if (!sess)1524 if (!clt_path)
1518 goto err;1525 goto err;
1519 1526
1520 /*1527 /*
@@ -1522,20 +1529,21 @@ static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt, · 1 unmodified line
1522 * +1: Extra connection for user messages1529 * +1: Extra connection for user messages
1523 */1530 */
1524 total_con = con_num + nr_poll_queues + 1;1531 total_con = con_num + nr_poll_queues + 1;
1525 sess->s.con = kcalloc(total_con, sizeof(*sess->s.con), GFP_KERNEL);1532 clt_path->s.con = kcalloc(total_con, sizeof(*clt_path->s.con),
1526 if (!sess->s.con)1533 GFP_KERNEL);
1527 goto err_free_sess;1534 if (!clt_path->s.con)
1535 goto err_free_path;
1528 1536
1529 sess->s.con_num = total_con;1537 clt_path->s.con_num = total_con;
1530 sess->s.irq_con_num = con_num + 1;1538 clt_path->s.irq_con_num = con_num + 1;
1531 1539
1532 sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL);1540 clt_path->stats = kzalloc(sizeof(*clt_path->stats), GFP_KERNEL);
1533 if (!sess->stats)1541 if (!clt_path->stats)
1534 goto err_free_con;1542 goto err_free_con;
1535 1543
1536 mutex_init(&sess->init_mutex);1544 mutex_init(&clt_path->init_mutex);
1537 uuid_gen(&sess->s.uuid);1545 uuid_gen(&clt_path->s.uuid);
1538 memcpy(&sess->s.dst_addr, path->dst,1546 memcpy(&clt_path->s.dst_addr, path->dst,
1539 rdma_addr_size((struct sockaddr *)path->dst));1547 rdma_addr_size((struct sockaddr *)path->dst));
1540 1548
1541 /*1549 /*
@@ -1544,53 +1552,54 @@ static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt, · 2 unmodified lines
1544 * the sess->src_addr will contain only zeros, which is then fine.1552 * the sess->src_addr will contain only zeros, which is then fine.
1545 */1553 */
1546 if (path->src)1554 if (path->src)
1547 memcpy(&sess->s.src_addr, path->src,1555 memcpy(&clt_path->s.src_addr, path->src,
1548 rdma_addr_size((struct sockaddr *)path->src));1556 rdma_addr_size((struct sockaddr *)path->src));
1549 strscpy(sess->s.sessname, clt->sessname, sizeof(sess->s.sessname));1557 strscpy(clt_path->s.sessname, clt->sessname,
1550 sess->clt = clt;1558 sizeof(clt_path->s.sessname));
1551 sess->max_pages_per_mr = RTRS_MAX_SEGMENTS;1559 clt_path->clt = clt;
1552 init_waitqueue_head(&sess->state_wq);1560 clt_path->max_pages_per_mr = RTRS_MAX_SEGMENTS;
1553 sess->state = RTRS_CLT_CONNECTING;1561 init_waitqueue_head(&clt_path->state_wq);
1554 atomic_set(&sess->connected_cnt, 0);1562 clt_path->state = RTRS_CLT_CONNECTING;
1555 INIT_WORK(&sess->close_work, rtrs_clt_close_work);1563 atomic_set(&clt_path->connected_cnt, 0);
1556 INIT_DELAYED_WORK(&sess->reconnect_dwork, rtrs_clt_reconnect_work);1564 INIT_WORK(&clt_path->close_work, rtrs_clt_close_work);
1557 rtrs_clt_init_hb(sess);1565 INIT_DELAYED_WORK(&clt_path->reconnect_dwork, rtrs_clt_reconnect_work);
1558 1566 rtrs_clt_init_hb(clt_path);
1559 sess->mp_skip_entry = alloc_percpu(typeof(*sess->mp_skip_entry));1567
1560 if (!sess->mp_skip_entry)1568 clt_path->mp_skip_entry = alloc_percpu(typeof(*clt_path->mp_skip_entry));
1569 if (!clt_path->mp_skip_entry)
1561 goto err_free_stats;1570 goto err_free_stats;
1562 1571
1563 for_each_possible_cpu(cpu)1572 for_each_possible_cpu(cpu)
1564 INIT_LIST_HEAD(per_cpu_ptr(sess->mp_skip_entry, cpu));1573 INIT_LIST_HEAD(per_cpu_ptr(clt_path->mp_skip_entry, cpu));
1565 1574
1566 err = rtrs_clt_init_stats(sess->stats);1575 err = rtrs_clt_init_stats(clt_path->stats);
1567 if (err)1576 if (err)
1568 goto err_free_percpu;1577 goto err_free_percpu;
1569 1578
1570 return sess;1579 return clt_path;
1571 1580
1572err_free_percpu:1581err_free_percpu:
1573 free_percpu(sess->mp_skip_entry);1582 free_percpu(clt_path->mp_skip_entry);
1574err_free_stats:1583err_free_stats:
1575 kfree(sess->stats);1584 kfree(clt_path->stats);
1576err_free_con:1585err_free_con:
1577 kfree(sess->s.con);1586 kfree(clt_path->s.con);
1578err_free_sess:1587err_free_path:
1579 kfree(sess);1588 kfree(clt_path);
1580err:1589err:
1581 return ERR_PTR(err);1590 return ERR_PTR(err);
1582}1591}
1583 1592
1584void free_sess(struct rtrs_clt_sess *sess)1593void free_path(struct rtrs_clt_path *clt_path)
1585{1594{
1586 free_percpu(sess->mp_skip_entry);1595 free_percpu(clt_path->mp_skip_entry);
1587 mutex_destroy(&sess->init_mutex);1596 mutex_destroy(&clt_path->init_mutex);
1588 kfree(sess->s.con);1597 kfree(clt_path->s.con);
1589 kfree(sess->rbufs);1598 kfree(clt_path->rbufs);
1590 kfree(sess);1599 kfree(clt_path);
1591}1600}
1592 1601
1593static int create_con(struct rtrs_clt_sess *sess, unsigned int cid)1602static int create_con(struct rtrs_clt_path *clt_path, unsigned int cid)
1594{1603{
1595 struct rtrs_clt_con *con;1604 struct rtrs_clt_con *con;
1596 1605
@@ -1601,28 +1610,28 @@ static int create_con(struct rtrs_clt_sess *sess, unsigned int cid) · 4 unmodified lines
1601 /* Map first two connections to the first CPU */1610 /* Map first two connections to the first CPU */
1602 con->cpu = (cid ? cid - 1 : 0) % nr_cpu_ids;1611 con->cpu = (cid ? cid - 1 : 0) % nr_cpu_ids;
1603 con->c.cid = cid;1612 con->c.cid = cid;
1604 con->c.sess = &sess->s;1613 con->c.path = &clt_path->s;
1605 /* Align with srv, init as 1 */1614 /* Align with srv, init as 1 */
1606 atomic_set(&con->c.wr_cnt, 1);1615 atomic_set(&con->c.wr_cnt, 1);
1607 mutex_init(&con->con_mutex);1616 mutex_init(&con->con_mutex);
1608 1617
1609 sess->s.con[cid] = &con->c;1618 clt_path->s.con[cid] = &con->c;
1610 1619
1611 return 0;1620 return 0;
1612}1621}
1613 1622
1614static void destroy_con(struct rtrs_clt_con *con)1623static void destroy_con(struct rtrs_clt_con *con)
1615{1624{
1616 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);1625 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1617 1626
1618 sess->s.con[con->c.cid] = NULL;1627 clt_path->s.con[con->c.cid] = NULL;
1619 mutex_destroy(&con->con_mutex);1628 mutex_destroy(&con->con_mutex);
1620 kfree(con);1629 kfree(con);
1621}1630}
1622 1631
1623static int create_con_cq_qp(struct rtrs_clt_con *con)1632static int create_con_cq_qp(struct rtrs_clt_con *con)
1624{1633{
1625 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);1634 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1626 u32 max_send_wr, max_recv_wr, cq_num, max_send_sge, wr_limit;1635 u32 max_send_wr, max_recv_wr, cq_num, max_send_sge, wr_limit;
1627 int err, cq_vector;1636 int err, cq_vector;
1628 struct rtrs_msg_rkey_rsp *rsp;1637 struct rtrs_msg_rkey_rsp *rsp;
@@ -1631,7 +1640,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) · 2 unmodified lines
1631 if (con->c.cid == 0) {1640 if (con->c.cid == 0) {
1632 max_send_sge = 1;1641 max_send_sge = 1;
1633 /* We must be the first here */1642 /* We must be the first here */
1634 if (WARN_ON(sess->s.dev))1643 if (WARN_ON(clt_path->s.dev))
1635 return -EINVAL;1644 return -EINVAL;
1636 1645
1637 /*1646 /*
@@ -1639,16 +1648,16 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) · 1 unmodified line
1639 * Be careful not to close user connection before ib dev1648 * Be careful not to close user connection before ib dev
1640 * is gracefully put.1649 * is gracefully put.
1641 */1650 */
1642 sess->s.dev = rtrs_ib_dev_find_or_add(con->c.cm_id->device,1651 clt_path->s.dev = rtrs_ib_dev_find_or_add(con->c.cm_id->device,
1643 &dev_pd);1652 &dev_pd);
1644 if (!sess->s.dev) {1653 if (!clt_path->s.dev) {
1645 rtrs_wrn(sess->clt,1654 rtrs_wrn(clt_path->clt,
1646 "rtrs_ib_dev_find_get_or_add(): no memory\n");1655 "rtrs_ib_dev_find_get_or_add(): no memory\n");
1647 return -ENOMEM;1656 return -ENOMEM;
1648 }1657 }
1649 sess->s.dev_ref = 1;1658 clt_path->s.dev_ref = 1;
1650 query_fast_reg_mode(sess);1659 query_fast_reg_mode(clt_path);
1651 wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr;1660 wr_limit = clt_path->s.dev->ib_dev->attrs.max_qp_wr;
1652 /*1661 /*
1653 * Two (request + registration) completion for send1662 * Two (request + registration) completion for send
1654 * Two for recv if always_invalidate is set on server1663 * Two for recv if always_invalidate is set on server
@@ -1665,27 +1674,28 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) · 10 unmodified lines
1665 * This is always true if user connection (cid == 0) is1674 * This is always true if user connection (cid == 0) is
1666 * established first.1675 * established first.
1667 */1676 */
1668 if (WARN_ON(!sess->s.dev))1677 if (WARN_ON(!clt_path->s.dev))
1669 return -EINVAL;1678 return -EINVAL;
1670 if (WARN_ON(!sess->queue_depth))1679 if (WARN_ON(!clt_path->queue_depth))
1671 return -EINVAL;1680 return -EINVAL;
1672 1681
1673 wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr;1682 wr_limit = clt_path->s.dev->ib_dev->attrs.max_qp_wr;
1674 /* Shared between connections */1683 /* Shared between connections */
1675 sess->s.dev_ref++;1684 clt_path->s.dev_ref++;
1676 max_send_wr = min_t(int, wr_limit,1685 max_send_wr = min_t(int, wr_limit,
1677 /* QD * (REQ + RSP + FR REGS or INVS) + drain */1686 /* QD * (REQ + RSP + FR REGS or INVS) + drain */
1678 sess->queue_depth * 3 + 1);1687 clt_path->queue_depth * 3 + 1);
1679 max_recv_wr = min_t(int, wr_limit,1688 max_recv_wr = min_t(int, wr_limit,
1680 sess->queue_depth * 3 + 1);1689 clt_path->queue_depth * 3 + 1);
1681 max_send_sge = 2;1690 max_send_sge = 2;
1682 }1691 }
1683 atomic_set(&con->c.sq_wr_avail, max_send_wr);1692 atomic_set(&con->c.sq_wr_avail, max_send_wr);
1684 cq_num = max_send_wr + max_recv_wr;1693 cq_num = max_send_wr + max_recv_wr;
1685 /* alloc iu to recv new rkey reply when server reports flags set */1694 /* alloc iu to recv new rkey reply when server reports flags set */
1686 if (sess->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) {1695 if (clt_path->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) {
1687 con->rsp_ius = rtrs_iu_alloc(cq_num, sizeof(*rsp),1696 con->rsp_ius = rtrs_iu_alloc(cq_num, sizeof(*rsp),
1688 GFP_KERNEL, sess->s.dev->ib_dev,1697 GFP_KERNEL,
1698 clt_path->s.dev->ib_dev,
1689 DMA_FROM_DEVICE,1699 DMA_FROM_DEVICE,
1690 rtrs_clt_rdma_done);1700 rtrs_clt_rdma_done);
1691 if (!con->rsp_ius)1701 if (!con->rsp_ius)
@@ -1693,13 +1703,13 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) · 1 unmodified line
1693 con->queue_num = cq_num;1703 con->queue_num = cq_num;
1694 }1704 }
1695 cq_num = max_send_wr + max_recv_wr;1705 cq_num = max_send_wr + max_recv_wr;
1696 cq_vector = con->cpu % sess->s.dev->ib_dev->num_comp_vectors;1706 cq_vector = con->cpu % clt_path->s.dev->ib_dev->num_comp_vectors;
1697 if (con->c.cid >= sess->s.irq_con_num)1707 if (con->c.cid >= clt_path->s.irq_con_num)
1698 err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge,1708 err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge,
1699 cq_vector, cq_num, max_send_wr,1709 cq_vector, cq_num, max_send_wr,
1700 max_recv_wr, IB_POLL_DIRECT);1710 max_recv_wr, IB_POLL_DIRECT);
1701 else1711 else
1702 err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge,1712 err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge,
1703 cq_vector, cq_num, max_send_wr,1713 cq_vector, cq_num, max_send_wr,
1704 max_recv_wr, IB_POLL_SOFTIRQ);1714 max_recv_wr, IB_POLL_SOFTIRQ);
1705 /*1715 /*
@@ -1711,7 +1721,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) · 5 unmodified lines
1711 1721
1712static void destroy_con_cq_qp(struct rtrs_clt_con *con)1722static void destroy_con_cq_qp(struct rtrs_clt_con *con)
1713{1723{
1714 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);1724 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1715 1725
1716 /*1726 /*
1717 * Be careful here: destroy_con_cq_qp() can be called even1727 * Be careful here: destroy_con_cq_qp() can be called even
@@ -1720,13 +1730,14 @@ static void destroy_con_cq_qp(struct rtrs_clt_con *con) · 2 unmodified lines
1720 lockdep_assert_held(&con->con_mutex);1730 lockdep_assert_held(&con->con_mutex);
1721 rtrs_cq_qp_destroy(&con->c);1731 rtrs_cq_qp_destroy(&con->c);
1722 if (con->rsp_ius) {1732 if (con->rsp_ius) {
1723 rtrs_iu_free(con->rsp_ius, sess->s.dev->ib_dev, con->queue_num);1733 rtrs_iu_free(con->rsp_ius, clt_path->s.dev->ib_dev,
1734 con->queue_num);
1724 con->rsp_ius = NULL;1735 con->rsp_ius = NULL;
1725 con->queue_num = 0;1736 con->queue_num = 0;
1726 }1737 }
1727 if (sess->s.dev_ref && !--sess->s.dev_ref) {1738 if (clt_path->s.dev_ref && !--clt_path->s.dev_ref) {
1728 rtrs_ib_dev_put(sess->s.dev);1739 rtrs_ib_dev_put(clt_path->s.dev);
1729 sess->s.dev = NULL;1740 clt_path->s.dev = NULL;
1730 }1741 }
1731}1742}
1732 1743
@@ -1745,7 +1756,7 @@ static void destroy_cm(struct rtrs_clt_con *con) · 12 unmodified lines
1745 1756
1746static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con)1757static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con)
1747{1758{
1748 struct rtrs_sess *s = con->c.sess;1759 struct rtrs_path *s = con->c.path;
1749 int err;1760 int err;
1750 1761
1751 mutex_lock(&con->con_mutex);1762 mutex_lock(&con->con_mutex);
@@ -1764,8 +1775,8 @@ static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con) · 12 unmodified lines
1764 1775
1765static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con)1776static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con)
1766{1777{
1767 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);1778 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1768 struct rtrs_clt *clt = sess->clt;1779 struct rtrs_clt_sess *clt = clt_path->clt;
1769 struct rtrs_msg_conn_req msg;1780 struct rtrs_msg_conn_req msg;
1770 struct rdma_conn_param param;1781 struct rdma_conn_param param;
1771 1782
@@ -1782,11 +1793,11 @@ static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con) · 10 unmodified lines
1782 .magic = cpu_to_le16(RTRS_MAGIC),1793 .magic = cpu_to_le16(RTRS_MAGIC),
1783 .version = cpu_to_le16(RTRS_PROTO_VER),1794 .version = cpu_to_le16(RTRS_PROTO_VER),
1784 .cid = cpu_to_le16(con->c.cid),1795 .cid = cpu_to_le16(con->c.cid),
1785 .cid_num = cpu_to_le16(sess->s.con_num),1796 .cid_num = cpu_to_le16(clt_path->s.con_num),
1786 .recon_cnt = cpu_to_le16(sess->s.recon_cnt),1797 .recon_cnt = cpu_to_le16(clt_path->s.recon_cnt),
1787 };1798 };
1788 msg.first_conn = sess->for_new_clt ? FIRST_CONN : 0;1799 msg.first_conn = clt_path->for_new_clt ? FIRST_CONN : 0;
1789 uuid_copy(&msg.sess_uuid, &sess->s.uuid);1800 uuid_copy(&msg.sess_uuid, &clt_path->s.uuid);
1790 uuid_copy(&msg.paths_uuid, &clt->paths_uuid);1801 uuid_copy(&msg.paths_uuid, &clt->paths_uuid);
1791 1802
1792 err = rdma_connect_locked(con->c.cm_id, &param);1803 err = rdma_connect_locked(con->c.cm_id, &param);
@@ -1799,8 +1810,8 @@ static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con) · 6 unmodified lines
1799static int rtrs_rdma_conn_established(struct rtrs_clt_con *con,1810static int rtrs_rdma_conn_established(struct rtrs_clt_con *con,
1800 struct rdma_cm_event *ev)1811 struct rdma_cm_event *ev)
1801{1812{
1802 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);1813 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1803 struct rtrs_clt *clt = sess->clt;1814 struct rtrs_clt_sess *clt = clt_path->clt;
1804 const struct rtrs_msg_conn_rsp *msg;1815 const struct rtrs_msg_conn_rsp *msg;
1805 u16 version, queue_depth;1816 u16 version, queue_depth;
1806 int errno;1817 int errno;
@@ -1831,31 +1842,32 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, · 24 unmodified lines
1831 if (con->c.cid == 0) {1842 if (con->c.cid == 0) {
1832 queue_depth = le16_to_cpu(msg->queue_depth);1843 queue_depth = le16_to_cpu(msg->queue_depth);
1833 1844
1834 if (sess->queue_depth > 0 && queue_depth != sess->queue_depth) {1845 if (clt_path->queue_depth > 0 && queue_depth != clt_path->queue_depth) {
1835 rtrs_err(clt, "Error: queue depth changed\n");1846 rtrs_err(clt, "Error: queue depth changed\n");
1836 1847
1837 /*1848 /*
1838 * Stop any more reconnection attempts1849 * Stop any more reconnection attempts
1839 */1850 */
1840 sess->reconnect_attempts = -1;1851 clt_path->reconnect_attempts = -1;
1841 rtrs_err(clt,1852 rtrs_err(clt,
1842 "Disabling auto-reconnect. Trigger a manual reconnect after issue is resolved\n");1853 "Disabling auto-reconnect. Trigger a manual reconnect after issue is resolved\n");
1843 return -ECONNRESET;1854 return -ECONNRESET;
1844 }1855 }
1845 1856
1846 if (!sess->rbufs) {1857 if (!clt_path->rbufs) {
1847 sess->rbufs = kcalloc(queue_depth, sizeof(*sess->rbufs),1858 clt_path->rbufs = kcalloc(queue_depth,
1848 GFP_KERNEL);1859 sizeof(*clt_path->rbufs),
1849 if (!sess->rbufs)1860 GFP_KERNEL);
1861 if (!clt_path->rbufs)
1850 return -ENOMEM;1862 return -ENOMEM;
1851 }1863 }
1852 sess->queue_depth = queue_depth;1864 clt_path->queue_depth = queue_depth;
1853 sess->s.signal_interval = min_not_zero(queue_depth,1865 clt_path->s.signal_interval = min_not_zero(queue_depth,
1854 (unsigned short) SERVICE_CON_QUEUE_DEPTH);1866 (unsigned short) SERVICE_CON_QUEUE_DEPTH);
1855 sess->max_hdr_size = le32_to_cpu(msg->max_hdr_size);1867 clt_path->max_hdr_size = le32_to_cpu(msg->max_hdr_size);
1856 sess->max_io_size = le32_to_cpu(msg->max_io_size);1868 clt_path->max_io_size = le32_to_cpu(msg->max_io_size);
1857 sess->flags = le32_to_cpu(msg->flags);1869 clt_path->flags = le32_to_cpu(msg->flags);
1858 sess->chunk_size = sess->max_io_size + sess->max_hdr_size;1870 clt_path->chunk_size = clt_path->max_io_size + clt_path->max_hdr_size;
1859 1871
1860 /*1872 /*
1861 * Global IO size is always a minimum.1873 * Global IO size is always a minimum.
@@ -1866,20 +1878,20 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, · 4 unmodified lines
1866 * connections in parallel, use lock.1878 * connections in parallel, use lock.
1867 */1879 */
1868 mutex_lock(&clt->paths_mutex);1880 mutex_lock(&clt->paths_mutex);
1869 clt->queue_depth = sess->queue_depth;1881 clt->queue_depth = clt_path->queue_depth;
1870 clt->max_io_size = min_not_zero(sess->max_io_size,1882 clt->max_io_size = min_not_zero(clt_path->max_io_size,
1871 clt->max_io_size);1883 clt->max_io_size);
1872 mutex_unlock(&clt->paths_mutex);1884 mutex_unlock(&clt->paths_mutex);
1873 1885
1874 /*1886 /*
1875 * Cache the hca_port and hca_name for sysfs1887 * Cache the hca_port and hca_name for sysfs
1876 */1888 */
1877 sess->hca_port = con->c.cm_id->port_num;1889 clt_path->hca_port = con->c.cm_id->port_num;
1878 scnprintf(sess->hca_name, sizeof(sess->hca_name),1890 scnprintf(clt_path->hca_name, sizeof(clt_path->hca_name),
1879 sess->s.dev->ib_dev->name);1891 clt_path->s.dev->ib_dev->name);
1880 sess->s.src_addr = con->c.cm_id->route.addr.src_addr;1892 clt_path->s.src_addr = con->c.cm_id->route.addr.src_addr;
1881 /* set for_new_clt, to allow future reconnect on any path */1893 /* set for_new_clt, to allow future reconnect on any path */
1882 sess->for_new_clt = 1;1894 clt_path->for_new_clt = 1;
1883 }1895 }
1884 1896
1885 return 0;1897 return 0;
@@ -1887,16 +1899,16 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, · 1 unmodified line
1887 1899
1888static inline void flag_success_on_conn(struct rtrs_clt_con *con)1900static inline void flag_success_on_conn(struct rtrs_clt_con *con)
1889{1901{
1890 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);1902 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1891 1903
1892 atomic_inc(&sess->connected_cnt);1904 atomic_inc(&clt_path->connected_cnt);
1893 con->cm_err = 1;1905 con->cm_err = 1;
1894}1906}
1895 1907
1896static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con,1908static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con,
1897 struct rdma_cm_event *ev)1909 struct rdma_cm_event *ev)
1898{1910{
1899 struct rtrs_sess *s = con->c.sess;1911 struct rtrs_path *s = con->c.path;
1900 const struct rtrs_msg_conn_rsp *msg;1912 const struct rtrs_msg_conn_rsp *msg;
1901 const char *rej_msg;1913 const char *rej_msg;
1902 int status, errno;1914 int status, errno;
@@ -1924,23 +1936,23 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, · 21 unmodified lines
1924 return -ECONNRESET;1936 return -ECONNRESET;
1925}1937}
1926 1938
1927void rtrs_clt_close_conns(struct rtrs_clt_sess *sess, bool wait)1939void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait)
1928{1940{
1929 if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_CLOSING, NULL))1941 if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSING, NULL))
1930 queue_work(rtrs_wq, &sess->close_work);1942 queue_work(rtrs_wq, &clt_path->close_work);
1931 if (wait)1943 if (wait)
1932 flush_work(&sess->close_work);1944 flush_work(&clt_path->close_work);
1933}1945}
1934 1946
1935static inline void flag_error_on_conn(struct rtrs_clt_con *con, int cm_err)1947static inline void flag_error_on_conn(struct rtrs_clt_con *con, int cm_err)
1936{1948{
1937 if (con->cm_err == 1) {1949 if (con->cm_err == 1) {
1938 struct rtrs_clt_sess *sess;1950 struct rtrs_clt_path *clt_path;
1939 1951
1940 sess = to_clt_sess(con->c.sess);1952 clt_path = to_clt_path(con->c.path);
1941 if (atomic_dec_and_test(&sess->connected_cnt))1953 if (atomic_dec_and_test(&clt_path->connected_cnt))
1942 1954
1943 wake_up(&sess->state_wq);1955 wake_up(&clt_path->state_wq);
1944 }1956 }
1945 con->cm_err = cm_err;1957 con->cm_err = cm_err;
1946}1958}
@@ -1949,8 +1961,8 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, · 2 unmodified lines
1949 struct rdma_cm_event *ev)1961 struct rdma_cm_event *ev)
1950{1962{
1951 struct rtrs_clt_con *con = cm_id->context;1963 struct rtrs_clt_con *con = cm_id->context;
1952 struct rtrs_sess *s = con->c.sess;1964 struct rtrs_path *s = con->c.path;
1953 struct rtrs_clt_sess *sess = to_clt_sess(s);1965 struct rtrs_clt_path *clt_path = to_clt_path(s);
1954 int cm_err = 0;1966 int cm_err = 0;
1955 1967
1956 switch (ev->event) {1968 switch (ev->event) {
@@ -1968,7 +1980,7 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, · 11 unmodified lines
1968 * i.e. wake up without state change, but we set cm_err.1980 * i.e. wake up without state change, but we set cm_err.
1969 */1981 */
1970 flag_success_on_conn(con);1982 flag_success_on_conn(con);
1971 wake_up(&sess->state_wq);1983 wake_up(&clt_path->state_wq);
1972 return 0;1984 return 0;
1973 }1985 }
1974 break;1986 break;
@@ -1997,7 +2009,7 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, · 22 unmodified lines
1997 /*2009 /*
1998 * Device removal is a special case. Queue close and return 0.2010 * Device removal is a special case. Queue close and return 0.
1999 */2011 */
2000 rtrs_clt_close_conns(sess, false);2012 rtrs_clt_close_conns(clt_path, false);
2001 return 0;2013 return 0;
2002 default:2014 default:
2003 rtrs_err(s, "Unexpected RDMA CM error (CM event: %s, err: %d)\n",2015 rtrs_err(s, "Unexpected RDMA CM error (CM event: %s, err: %d)\n",
@@ -2020,13 +2032,13 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, · 16 unmodified lines
2020 2032
2021static int create_cm(struct rtrs_clt_con *con)2033static int create_cm(struct rtrs_clt_con *con)
2022{2034{
2023 struct rtrs_sess *s = con->c.sess;2035 struct rtrs_path *s = con->c.path;
2024 struct rtrs_clt_sess *sess = to_clt_sess(s);2036 struct rtrs_clt_path *clt_path = to_clt_path(s);
2025 struct rdma_cm_id *cm_id;2037 struct rdma_cm_id *cm_id;
2026 int err;2038 int err;
2027 2039
2028 cm_id = rdma_create_id(&init_net, rtrs_clt_rdma_cm_handler, con,2040 cm_id = rdma_create_id(&init_net, rtrs_clt_rdma_cm_handler, con,
2029 sess->s.dst_addr.ss_family == AF_IB ?2041 clt_path->s.dst_addr.ss_family == AF_IB ?
2030 RDMA_PS_IB : RDMA_PS_TCP, IB_QPT_RC);2042 RDMA_PS_IB : RDMA_PS_TCP, IB_QPT_RC);
2031 if (IS_ERR(cm_id)) {2043 if (IS_ERR(cm_id)) {
2032 err = PTR_ERR(cm_id);2044 err = PTR_ERR(cm_id);
@@ -2042,8 +2054,8 @@ static int create_cm(struct rtrs_clt_con *con) · 9 unmodified lines
2042 rtrs_err(s, "Set address reuse failed, err: %d\n", err);2054 rtrs_err(s, "Set address reuse failed, err: %d\n", err);
2043 goto destroy_cm;2055 goto destroy_cm;
2044 }2056 }
2045 err = rdma_resolve_addr(cm_id, (struct sockaddr *)&sess->s.src_addr,2057 err = rdma_resolve_addr(cm_id, (struct sockaddr *)&clt_path->s.src_addr,
2046 (struct sockaddr *)&sess->s.dst_addr,2058 (struct sockaddr *)&clt_path->s.dst_addr,
2047 RTRS_CONNECT_TIMEOUT_MS);2059 RTRS_CONNECT_TIMEOUT_MS);
2048 if (err) {2060 if (err) {
2049 rtrs_err(s, "Failed to resolve address, err: %d\n", err);2061 rtrs_err(s, "Failed to resolve address, err: %d\n", err);
@@ -2055,8 +2067,8 @@ static int create_cm(struct rtrs_clt_con *con) · 5 unmodified lines
2055 * or session state was really changed to error by device removal.2067 * or session state was really changed to error by device removal.
2056 */2068 */
2057 err = wait_event_interruptible_timeout(2069 err = wait_event_interruptible_timeout(
2058 sess->state_wq,2070 clt_path->state_wq,
2059 con->cm_err || sess->state != RTRS_CLT_CONNECTING,2071 con->cm_err || clt_path->state != RTRS_CLT_CONNECTING,
2060 msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS));2072 msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS));
2061 if (err == 0 || err == -ERESTARTSYS) {2073 if (err == 0 || err == -ERESTARTSYS) {
2062 if (err == 0)2074 if (err == 0)
@@ -2068,7 +2080,7 @@ static int create_cm(struct rtrs_clt_con *con) · 5 unmodified lines
2068 err = con->cm_err;2080 err = con->cm_err;
2069 goto errr;2081 goto errr;
2070 }2082 }
2071 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTING) {2083 if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTING) {
2072 /* Device removal */2084 /* Device removal */
2073 err = -ECONNABORTED;2085 err = -ECONNABORTED;
2074 goto errr;2086 goto errr;
@@ -2087,9 +2099,9 @@ destroy_cm: · 12 unmodified lines
2087 return err;2099 return err;
2088}2100}
2089 2101
2090static void rtrs_clt_sess_up(struct rtrs_clt_sess *sess)2102static void rtrs_clt_path_up(struct rtrs_clt_path *clt_path)
2091{2103{
2092 struct rtrs_clt *clt = sess->clt;2104 struct rtrs_clt_sess *clt = clt_path->clt;
2093 int up;2105 int up;
2094 2106
2095 /*2107 /*
@@ -2113,19 +2125,19 @@ static void rtrs_clt_sess_up(struct rtrs_clt_sess *sess) · 17 unmodified lines
2113 mutex_unlock(&clt->paths_ev_mutex);2125 mutex_unlock(&clt->paths_ev_mutex);
2114 2126
2115 /* Mark session as established */2127 /* Mark session as established */
2116 sess->established = true;2128 clt_path->established = true;
2117 sess->reconnect_attempts = 0;2129 clt_path->reconnect_attempts = 0;
2118 sess->stats->reconnects.successful_cnt++;2130 clt_path->stats->reconnects.successful_cnt++;
2119}2131}
2120 2132
2121static void rtrs_clt_sess_down(struct rtrs_clt_sess *sess)2133static void rtrs_clt_path_down(struct rtrs_clt_path *clt_path)
2122{2134{
2123 struct rtrs_clt *clt = sess->clt;2135 struct rtrs_clt_sess *clt = clt_path->clt;
2124 2136
2125 if (!sess->established)2137 if (!clt_path->established)
2126 return;2138 return;
2127 2139
2128 sess->established = false;2140 clt_path->established = false;
2129 mutex_lock(&clt->paths_ev_mutex);2141 mutex_lock(&clt->paths_ev_mutex);
2130 WARN_ON(!clt->paths_up);2142 WARN_ON(!clt->paths_up);
2131 if (--clt->paths_up == 0)2143 if (--clt->paths_up == 0)
@@ -2133,19 +2145,19 @@ static void rtrs_clt_sess_down(struct rtrs_clt_sess *sess) · 1 unmodified line
2133 mutex_unlock(&clt->paths_ev_mutex);2145 mutex_unlock(&clt->paths_ev_mutex);
2134}2146}
2135 2147
2136static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess)2148static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path)
2137{2149{
2138 struct rtrs_clt_con *con;2150 struct rtrs_clt_con *con;
2139 unsigned int cid;2151 unsigned int cid;
2140 2152
2141 WARN_ON(READ_ONCE(sess->state) == RTRS_CLT_CONNECTED);2153 WARN_ON(READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED);
2142 2154
2143 /*2155 /*
2144 * Possible race with rtrs_clt_open(), when DEVICE_REMOVAL comes2156 * Possible race with rtrs_clt_open(), when DEVICE_REMOVAL comes
2145 * exactly in between. Start destroying after it finishes.2157 * exactly in between. Start destroying after it finishes.
2146 */2158 */
2147 mutex_lock(&sess->init_mutex);2159 mutex_lock(&clt_path->init_mutex);
2148 mutex_unlock(&sess->init_mutex);2160 mutex_unlock(&clt_path->init_mutex);
2149 2161
2150 /*2162 /*
2151 * All IO paths must observe !CONNECTED state before we2163 * All IO paths must observe !CONNECTED state before we
@@ -2153,7 +2165,7 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) · 1 unmodified line
2153 */2165 */
2154 synchronize_rcu();2166 synchronize_rcu();
2155 2167
2156 rtrs_stop_hb(&sess->s);2168 rtrs_stop_hb(&clt_path->s);
2157 2169
2158 /*2170 /*
2159 * The order it utterly crucial: firstly disconnect and complete all2171 * The order it utterly crucial: firstly disconnect and complete all
@@ -2162,15 +2174,15 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) · 2 unmodified lines
2162 * eventually notify upper layer about session disconnection.2174 * eventually notify upper layer about session disconnection.
2163 */2175 */
2164 2176
2165 for (cid = 0; cid < sess->s.con_num; cid++) {2177 for (cid = 0; cid < clt_path->s.con_num; cid++) {
2166 if (!sess->s.con[cid])2178 if (!clt_path->s.con[cid])
2167 break;2179 break;
2168 con = to_clt_con(sess->s.con[cid]);2180 con = to_clt_con(clt_path->s.con[cid]);
2169 stop_cm(con);2181 stop_cm(con);
2170 }2182 }
2171 fail_all_outstanding_reqs(sess);2183 fail_all_outstanding_reqs(clt_path);
2172 free_sess_reqs(sess);2184 free_path_reqs(clt_path);
2173 rtrs_clt_sess_down(sess);2185 rtrs_clt_path_down(clt_path);
2174 2186
2175 /*2187 /*
2176 * Wait for graceful shutdown, namely when peer side invokes2188 * Wait for graceful shutdown, namely when peer side invokes
@@ -2180,13 +2192,14 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) · 3 unmodified lines
2180 * since CM does not fire anything. That is fine, we are not in2192 * since CM does not fire anything. That is fine, we are not in
2181 * hurry.2193 * hurry.
2182 */2194 */
2183 wait_event_timeout(sess->state_wq, !atomic_read(&sess->connected_cnt),2195 wait_event_timeout(clt_path->state_wq,
2196 !atomic_read(&clt_path->connected_cnt),
2184 msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS));2197 msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS));
2185 2198
2186 for (cid = 0; cid < sess->s.con_num; cid++) {2199 for (cid = 0; cid < clt_path->s.con_num; cid++) {
2187 if (!sess->s.con[cid])2200 if (!clt_path->s.con[cid])
2188 break;2201 break;
2189 con = to_clt_con(sess->s.con[cid]);2202 con = to_clt_con(clt_path->s.con[cid]);
2190 mutex_lock(&con->con_mutex);2203 mutex_lock(&con->con_mutex);
2191 destroy_con_cq_qp(con);2204 destroy_con_cq_qp(con);
2192 mutex_unlock(&con->con_mutex);2205 mutex_unlock(&con->con_mutex);
@@ -2195,26 +2208,26 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) · 2 unmodified lines
2195 }2208 }
2196}2209}
2197 2210
2198static inline bool xchg_sessions(struct rtrs_clt_sess __rcu **rcu_ppcpu_path,2211static inline bool xchg_paths(struct rtrs_clt_path __rcu **rcu_ppcpu_path,
2199 struct rtrs_clt_sess *sess,2212 struct rtrs_clt_path *clt_path,
2200 struct rtrs_clt_sess *next)2213 struct rtrs_clt_path *next)
2201{2214{
2202 struct rtrs_clt_sess **ppcpu_path;2215 struct rtrs_clt_path **ppcpu_path;
2203 2216
2204 /* Call cmpxchg() without sparse warnings */2217 /* Call cmpxchg() without sparse warnings */
2205 ppcpu_path = (typeof(ppcpu_path))rcu_ppcpu_path;2218 ppcpu_path = (typeof(ppcpu_path))rcu_ppcpu_path;
2206 return sess == cmpxchg(ppcpu_path, sess, next);2219 return clt_path == cmpxchg(ppcpu_path, clt_path, next);
2207}2220}
2208 2221
2209static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess)2222static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_path *clt_path)
2210{2223{
2211 struct rtrs_clt *clt = sess->clt;2224 struct rtrs_clt_sess *clt = clt_path->clt;
2212 struct rtrs_clt_sess *next;2225 struct rtrs_clt_path *next;
2213 bool wait_for_grace = false;2226 bool wait_for_grace = false;
2214 int cpu;2227 int cpu;
2215 2228
2216 mutex_lock(&clt->paths_mutex);2229 mutex_lock(&clt->paths_mutex);
2217 list_del_rcu(&sess->s.entry);2230 list_del_rcu(&clt_path->s.entry);
2218 2231
2219 /* Make sure everybody observes path removal. */2232 /* Make sure everybody observes path removal. */
2220 synchronize_rcu();2233 synchronize_rcu();
@@ -2255,7 +2268,7 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) · 34 unmodified lines
2255 * removed. If @sess is the last element, then @next is NULL.2268 * removed. If @sess is the last element, then @next is NULL.
2256 */2269 */
2257 rcu_read_lock();2270 rcu_read_lock();
2258 next = list_next_or_null_rr_rcu(&clt->paths_list, &sess->s.entry,2271 next = list_next_or_null_rr_rcu(&clt->paths_list, &clt_path->s.entry,
2259 typeof(*next), s.entry);2272 typeof(*next), s.entry);
2260 rcu_read_unlock();2273 rcu_read_unlock();
2261 2274
@@ -2264,11 +2277,11 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) · 2 unmodified lines
2264 * removed, so change the pointer manually.2277 * removed, so change the pointer manually.
2265 */2278 */
2266 for_each_possible_cpu(cpu) {2279 for_each_possible_cpu(cpu) {
2267 struct rtrs_clt_sess __rcu **ppcpu_path;2280 struct rtrs_clt_path __rcu **ppcpu_path;
2268 2281
2269 ppcpu_path = per_cpu_ptr(clt->pcpu_path, cpu);2282 ppcpu_path = per_cpu_ptr(clt->pcpu_path, cpu);
2270 if (rcu_dereference_protected(*ppcpu_path,2283 if (rcu_dereference_protected(*ppcpu_path,
2271 lockdep_is_held(&clt->paths_mutex)) != sess)2284 lockdep_is_held(&clt->paths_mutex)) != clt_path)
2272 /*2285 /*
2273 * synchronize_rcu() was called just after deleting2286 * synchronize_rcu() was called just after deleting
2274 * entry from the list, thus IO code path cannot2287 * entry from the list, thus IO code path cannot
@@ -2281,7 +2294,7 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) · 6 unmodified lines
2281 * We race with IO code path, which also changes pointer,2294 * We race with IO code path, which also changes pointer,
2282 * thus we have to be careful not to overwrite it.2295 * thus we have to be careful not to overwrite it.
2283 */2296 */
2284 if (xchg_sessions(ppcpu_path, sess, next))2297 if (xchg_paths(ppcpu_path, clt_path, next))
2285 /*2298 /*
2286 * @ppcpu_path was successfully replaced with @next,2299 * @ppcpu_path was successfully replaced with @next,
2287 * that means that someone could also pick up the2300 * that means that someone could also pick up the
@@ -2296,29 +2309,29 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) · 8 unmodified lines
2296 mutex_unlock(&clt->paths_mutex);2309 mutex_unlock(&clt->paths_mutex);
2297}2310}
2298 2311
2299static void rtrs_clt_add_path_to_arr(struct rtrs_clt_sess *sess)2312static void rtrs_clt_add_path_to_arr(struct rtrs_clt_path *clt_path)
2300{2313{
2301 struct rtrs_clt *clt = sess->clt;2314 struct rtrs_clt_sess *clt = clt_path->clt;
2302 2315
2303 mutex_lock(&clt->paths_mutex);2316 mutex_lock(&clt->paths_mutex);
2304 clt->paths_num++;2317 clt->paths_num++;
2305 2318
2306 list_add_tail_rcu(&sess->s.entry, &clt->paths_list);2319 list_add_tail_rcu(&clt_path->s.entry, &clt->paths_list);
2307 mutex_unlock(&clt->paths_mutex);2320 mutex_unlock(&clt->paths_mutex);
2308}2321}
2309 2322
2310static void rtrs_clt_close_work(struct work_struct *work)2323static void rtrs_clt_close_work(struct work_struct *work)
2311{2324{
2312 struct rtrs_clt_sess *sess;2325 struct rtrs_clt_path *clt_path;
2313 2326
2314 sess = container_of(work, struct rtrs_clt_sess, close_work);2327 clt_path = container_of(work, struct rtrs_clt_path, close_work);
2315 2328
2316 cancel_delayed_work_sync(&sess->reconnect_dwork);2329 cancel_delayed_work_sync(&clt_path->reconnect_dwork);
2317 rtrs_clt_stop_and_destroy_conns(sess);2330 rtrs_clt_stop_and_destroy_conns(clt_path);
2318 rtrs_clt_change_state_get_old(sess, RTRS_CLT_CLOSED, NULL);2331 rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSED, NULL);
2319}2332}
2320 2333
2321static int init_conns(struct rtrs_clt_sess *sess)2334static int init_conns(struct rtrs_clt_path *clt_path)
2322{2335{
2323 unsigned int cid;2336 unsigned int cid;
2324 int err;2337 int err;
@@ -2328,31 +2341,31 @@ static int init_conns(struct rtrs_clt_sess *sess) · 3 unmodified lines
2328 * to avoid clashes with previous sessions not yet closed2341 * to avoid clashes with previous sessions not yet closed
2329 * sessions on a server side.2342 * sessions on a server side.
2330 */2343 */
2331 sess->s.recon_cnt++;2344 clt_path->s.recon_cnt++;
2332 2345
2333 /* Establish all RDMA connections */2346 /* Establish all RDMA connections */
2334 for (cid = 0; cid < sess->s.con_num; cid++) {2347 for (cid = 0; cid < clt_path->s.con_num; cid++) {
2335 err = create_con(sess, cid);2348 err = create_con(clt_path, cid);
2336 if (err)2349 if (err)
2337 goto destroy;2350 goto destroy;
2338 2351
2339 err = create_cm(to_clt_con(sess->s.con[cid]));2352 err = create_cm(to_clt_con(clt_path->s.con[cid]));
2340 if (err) {2353 if (err) {
2341 destroy_con(to_clt_con(sess->s.con[cid]));2354 destroy_con(to_clt_con(clt_path->s.con[cid]));
2342 goto destroy;2355 goto destroy;
2343 }2356 }
2344 }2357 }
2345 err = alloc_sess_reqs(sess);2358 err = alloc_path_reqs(clt_path);
2346 if (err)2359 if (err)
2347 goto destroy;2360 goto destroy;
2348 2361
2349 rtrs_start_hb(&sess->s);2362 rtrs_start_hb(&clt_path->s);
2350 2363
2351 return 0;2364 return 0;
2352 2365
2353destroy:2366destroy:
2354 while (cid--) {2367 while (cid--) {
2355 struct rtrs_clt_con *con = to_clt_con(sess->s.con[cid]);2368 struct rtrs_clt_con *con = to_clt_con(clt_path->s.con[cid]);
2356 2369
2357 stop_cm(con);2370 stop_cm(con);
2358 2371
@@ -2367,7 +2380,7 @@ destroy: · 8 unmodified lines
2367 * doing rdma_resolve_addr(), switch to CONNECTION_ERR state2380 * doing rdma_resolve_addr(), switch to CONNECTION_ERR state
2368 * manually to keep reconnecting.2381 * manually to keep reconnecting.
2369 */2382 */
2370 rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL);2383 rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING_ERR, NULL);
2371 2384
2372 return err;2385 return err;
2373}2386}
@@ -2375,31 +2388,32 @@ destroy: · 1 unmodified line
2375static void rtrs_clt_info_req_done(struct ib_cq *cq, struct ib_wc *wc)2388static void rtrs_clt_info_req_done(struct ib_cq *cq, struct ib_wc *wc)
2376{2389{
2377 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);2390 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
2378 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);2391 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
2379 struct rtrs_iu *iu;2392 struct rtrs_iu *iu;
2380 2393
2381 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);2394 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
2382 rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);2395 rtrs_iu_free(iu, clt_path->s.dev->ib_dev, 1);
2383 2396
2384 if (wc->status != IB_WC_SUCCESS) {2397 if (wc->status != IB_WC_SUCCESS) {
2385 rtrs_err(sess->clt, "Sess info request send failed: %s\n",2398 rtrs_err(clt_path->clt, "Path info request send failed: %s\n",
2386 ib_wc_status_msg(wc->status));2399 ib_wc_status_msg(wc->status));
2387 rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL);2400 rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING_ERR, NULL);
2388 return;2401 return;
2389 }2402 }
2390 2403
2391 rtrs_clt_update_wc_stats(con);2404 rtrs_clt_update_wc_stats(con);
2392}2405}
2393 2406
2394static int process_info_rsp(struct rtrs_clt_sess *sess,2407static int process_info_rsp(struct rtrs_clt_path *clt_path,
2395 const struct rtrs_msg_info_rsp *msg)2408 const struct rtrs_msg_info_rsp *msg)
2396{2409{
2397 unsigned int sg_cnt, total_len;2410 unsigned int sg_cnt, total_len;
2398 int i, sgi;2411 int i, sgi;
2399 2412
2400 sg_cnt = le16_to_cpu(msg->sg_cnt);2413 sg_cnt = le16_to_cpu(msg->sg_cnt);
2401 if (!sg_cnt || (sess->queue_depth % sg_cnt)) {2414 if (!sg_cnt || (clt_path->queue_depth % sg_cnt)) {
2402 rtrs_err(sess->clt, "Incorrect sg_cnt %d, is not multiple\n",2415 rtrs_err(clt_path->clt,
2416 "Incorrect sg_cnt %d, is not multiple\n",
2403 sg_cnt);2417 sg_cnt);
2404 return -EINVAL;2418 return -EINVAL;
2405 }2419 }
@@ -2408,15 +2422,15 @@ static int process_info_rsp(struct rtrs_clt_sess *sess, · 2 unmodified lines
2408 * Check if IB immediate data size is enough to hold the mem_id and2422 * Check if IB immediate data size is enough to hold the mem_id and
2409 * the offset inside the memory chunk.2423 * the offset inside the memory chunk.
2410 */2424 */
2411 if ((ilog2(sg_cnt - 1) + 1) + (ilog2(sess->chunk_size - 1) + 1) >2425 if ((ilog2(sg_cnt - 1) + 1) + (ilog2(clt_path->chunk_size - 1) + 1) >
2412 MAX_IMM_PAYL_BITS) {2426 MAX_IMM_PAYL_BITS) {
2413 rtrs_err(sess->clt,2427 rtrs_err(clt_path->clt,
2414 "RDMA immediate size (%db) not enough to encode %d buffers of size %dB\n",2428 "RDMA immediate size (%db) not enough to encode %d buffers of size %dB\n",
2415 MAX_IMM_PAYL_BITS, sg_cnt, sess->chunk_size);2429 MAX_IMM_PAYL_BITS, sg_cnt, clt_path->chunk_size);
2416 return -EINVAL;2430 return -EINVAL;
2417 }2431 }
2418 total_len = 0;2432 total_len = 0;
2419 for (sgi = 0, i = 0; sgi < sg_cnt && i < sess->queue_depth; sgi++) {2433 for (sgi = 0, i = 0; sgi < sg_cnt && i < clt_path->queue_depth; sgi++) {
2420 const struct rtrs_sg_desc *desc = &msg->desc[sgi];2434 const struct rtrs_sg_desc *desc = &msg->desc[sgi];
2421 u32 len, rkey;2435 u32 len, rkey;
2422 u64 addr;2436 u64 addr;
@@ -2427,26 +2441,28 @@ static int process_info_rsp(struct rtrs_clt_sess *sess, · 4 unmodified lines
2427 2441
2428 total_len += len;2442 total_len += len;
2429 2443
2430 if (!len || (len % sess->chunk_size)) {2444 if (!len || (len % clt_path->chunk_size)) {
2431 rtrs_err(sess->clt, "Incorrect [%d].len %d\n", sgi,2445 rtrs_err(clt_path->clt, "Incorrect [%d].len %d\n",
2446 sgi,
2432 len);2447 len);
2433 return -EINVAL;2448 return -EINVAL;
2434 }2449 }
2435 for ( ; len && i < sess->queue_depth; i++) {2450 for ( ; len && i < clt_path->queue_depth; i++) {
2436 sess->rbufs[i].addr = addr;2451 clt_path->rbufs[i].addr = addr;
2437 sess->rbufs[i].rkey = rkey;2452 clt_path->rbufs[i].rkey = rkey;
2438 2453
2439 len -= sess->chunk_size;2454 len -= clt_path->chunk_size;
2440 addr += sess->chunk_size;2455 addr += clt_path->chunk_size;
2441 }2456 }
2442 }2457 }
2443 /* Sanity check */2458 /* Sanity check */
2444 if (sgi != sg_cnt || i != sess->queue_depth) {2459 if (sgi != sg_cnt || i != clt_path->queue_depth) {
2445 rtrs_err(sess->clt, "Incorrect sg vector, not fully mapped\n");2460 rtrs_err(clt_path->clt,
2461 "Incorrect sg vector, not fully mapped\n");
2446 return -EINVAL;2462 return -EINVAL;
2447 }2463 }
2448 if (total_len != sess->chunk_size * sess->queue_depth) {2464 if (total_len != clt_path->chunk_size * clt_path->queue_depth) {
2449 rtrs_err(sess->clt, "Incorrect total_len %d\n", total_len);2465 rtrs_err(clt_path->clt, "Incorrect total_len %d\n", total_len);
2450 return -EINVAL;2466 return -EINVAL;
2451 }2467 }
2452 2468
@@ -2456,7 +2472,7 @@ static int process_info_rsp(struct rtrs_clt_sess *sess, · 3 unmodified lines
2456static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc)2472static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc)
2457{2473{
2458 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);2474 struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
2459 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);2475 struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
2460 struct rtrs_msg_info_rsp *msg;2476 struct rtrs_msg_info_rsp *msg;
2461 enum rtrs_clt_state state;2477 enum rtrs_clt_state state;
2462 struct rtrs_iu *iu;2478 struct rtrs_iu *iu;
@@ -2468,37 +2484,37 @@ static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) · 5 unmodified lines
2468 WARN_ON(con->c.cid);2484 WARN_ON(con->c.cid);
2469 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);2485 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
2470 if (wc->status != IB_WC_SUCCESS) {2486 if (wc->status != IB_WC_SUCCESS) {
2471 rtrs_err(sess->clt, "Sess info response recv failed: %s\n",2487 rtrs_err(clt_path->clt, "Path info response recv failed: %s\n",
2472 ib_wc_status_msg(wc->status));2488 ib_wc_status_msg(wc->status));
2473 goto out;2489 goto out;
2474 }2490 }
2475 WARN_ON(wc->opcode != IB_WC_RECV);2491 WARN_ON(wc->opcode != IB_WC_RECV);
2476 2492
2477 if (wc->byte_len < sizeof(*msg)) {2493 if (wc->byte_len < sizeof(*msg)) {
2478 rtrs_err(sess->clt, "Sess info response is malformed: size %d\n",2494 rtrs_err(clt_path->clt, "Path info response is malformed: size %d\n",
2479 wc->byte_len);2495 wc->byte_len);
2480 goto out;2496 goto out;
2481 }2497 }
2482 ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr,2498 ib_dma_sync_single_for_cpu(clt_path->s.dev->ib_dev, iu->dma_addr,
2483 iu->size, DMA_FROM_DEVICE);2499 iu->size, DMA_FROM_DEVICE);
2484 msg = iu->buf;2500 msg = iu->buf;
2485 if (le16_to_cpu(msg->type) != RTRS_MSG_INFO_RSP) {2501 if (le16_to_cpu(msg->type) != RTRS_MSG_INFO_RSP) {
2486 rtrs_err(sess->clt, "Sess info response is malformed: type %d\n",2502 rtrs_err(clt_path->clt, "Path info response is malformed: type %d\n",
2487 le16_to_cpu(msg->type));2503 le16_to_cpu(msg->type));
2488 goto out;2504 goto out;
2489 }2505 }
2490 rx_sz = sizeof(*msg);2506 rx_sz = sizeof(*msg);
2491 rx_sz += sizeof(msg->desc[0]) * le16_to_cpu(msg->sg_cnt);2507 rx_sz += sizeof(msg->desc[0]) * le16_to_cpu(msg->sg_cnt);
2492 if (wc->byte_len < rx_sz) {2508 if (wc->byte_len < rx_sz) {
2493 rtrs_err(sess->clt, "Sess info response is malformed: size %d\n",2509 rtrs_err(clt_path->clt, "Path info response is malformed: size %d\n",
2494 wc->byte_len);2510 wc->byte_len);
2495 goto out;2511 goto out;
2496 }2512 }
2497 err = process_info_rsp(sess, msg);2513 err = process_info_rsp(clt_path, msg);
2498 if (err)2514 if (err)
2499 goto out;2515 goto out;
2500 2516
2501 err = post_recv_sess(sess);2517 err = post_recv_path(clt_path);
2502 if (err)2518 if (err)
2503 goto out;2519 goto out;
2504 2520
@@ -2506,25 +2522,25 @@ static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) · 1 unmodified line
2506 2522
2507out:2523out:
2508 rtrs_clt_update_wc_stats(con);2524 rtrs_clt_update_wc_stats(con);
2509 rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);2525 rtrs_iu_free(iu, clt_path->s.dev->ib_dev, 1);
2510 rtrs_clt_change_state_get_old(sess, state, NULL);2526 rtrs_clt_change_state_get_old(clt_path, state, NULL);
2511}2527}
2512 2528
2513static int rtrs_send_sess_info(struct rtrs_clt_sess *sess)2529static int rtrs_send_path_info(struct rtrs_clt_path *clt_path)
2514{2530{
2515 struct rtrs_clt_con *usr_con = to_clt_con(sess->s.con[0]);2531 struct rtrs_clt_con *usr_con = to_clt_con(clt_path->s.con[0]);
2516 struct rtrs_msg_info_req *msg;2532 struct rtrs_msg_info_req *msg;
2517 struct rtrs_iu *tx_iu, *rx_iu;2533 struct rtrs_iu *tx_iu, *rx_iu;
2518 size_t rx_sz;2534 size_t rx_sz;
2519 int err;2535 int err;
2520 2536
2521 rx_sz = sizeof(struct rtrs_msg_info_rsp);2537 rx_sz = sizeof(struct rtrs_msg_info_rsp);
2522 rx_sz += sizeof(struct rtrs_sg_desc) * sess->queue_depth;2538 rx_sz += sizeof(struct rtrs_sg_desc) * clt_path->queue_depth;
2523 2539
2524 tx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), GFP_KERNEL,2540 tx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), GFP_KERNEL,
2525 sess->s.dev->ib_dev, DMA_TO_DEVICE,2541 clt_path->s.dev->ib_dev, DMA_TO_DEVICE,
2526 rtrs_clt_info_req_done);2542 rtrs_clt_info_req_done);
2527 rx_iu = rtrs_iu_alloc(1, rx_sz, GFP_KERNEL, sess->s.dev->ib_dev,2543 rx_iu = rtrs_iu_alloc(1, rx_sz, GFP_KERNEL, clt_path->s.dev->ib_dev,
2528 DMA_FROM_DEVICE, rtrs_clt_info_rsp_done);2544 DMA_FROM_DEVICE, rtrs_clt_info_rsp_done);
2529 if (!tx_iu || !rx_iu) {2545 if (!tx_iu || !rx_iu) {
2530 err = -ENOMEM;2546 err = -ENOMEM;
@@ -2533,33 +2549,34 @@ static int rtrs_send_sess_info(struct rtrs_clt_sess *sess) · 2 unmodified lines
2533 /* Prepare for getting info response */2549 /* Prepare for getting info response */
2534 err = rtrs_iu_post_recv(&usr_con->c, rx_iu);2550 err = rtrs_iu_post_recv(&usr_con->c, rx_iu);
2535 if (err) {2551 if (err) {
2536 rtrs_err(sess->clt, "rtrs_iu_post_recv(), err: %d\n", err);2552 rtrs_err(clt_path->clt, "rtrs_iu_post_recv(), err: %d\n", err);
2537 goto out;2553 goto out;
2538 }2554 }
2539 rx_iu = NULL;2555 rx_iu = NULL;
2540 2556
2541 msg = tx_iu->buf;2557 msg = tx_iu->buf;
2542 msg->type = cpu_to_le16(RTRS_MSG_INFO_REQ);2558 msg->type = cpu_to_le16(RTRS_MSG_INFO_REQ);
2543 memcpy(msg->sessname, sess->s.sessname, sizeof(msg->sessname));2559 memcpy(msg->pathname, clt_path->s.sessname, sizeof(msg->pathname));
2544 2560
2545 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr,2561 ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev,
2562 tx_iu->dma_addr,
2546 tx_iu->size, DMA_TO_DEVICE);2563 tx_iu->size, DMA_TO_DEVICE);
2547 2564
2548 /* Send info request */2565 /* Send info request */
2549 err = rtrs_iu_post_send(&usr_con->c, tx_iu, sizeof(*msg), NULL);2566 err = rtrs_iu_post_send(&usr_con->c, tx_iu, sizeof(*msg), NULL);
2550 if (err) {2567 if (err) {
2551 rtrs_err(sess->clt, "rtrs_iu_post_send(), err: %d\n", err);2568 rtrs_err(clt_path->clt, "rtrs_iu_post_send(), err: %d\n", err);
2552 goto out;2569 goto out;
2553 }2570 }
2554 tx_iu = NULL;2571 tx_iu = NULL;
2555 2572
2556 /* Wait for state change */2573 /* Wait for state change */
2557 wait_event_interruptible_timeout(sess->state_wq,2574 wait_event_interruptible_timeout(clt_path->state_wq,
2558 sess->state != RTRS_CLT_CONNECTING,2575 clt_path->state != RTRS_CLT_CONNECTING,
2559 msecs_to_jiffies(2576 msecs_to_jiffies(
2560 RTRS_CONNECT_TIMEOUT_MS));2577 RTRS_CONNECT_TIMEOUT_MS));
2561 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) {2578 if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) {
2562 if (READ_ONCE(sess->state) == RTRS_CLT_CONNECTING_ERR)2579 if (READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTING_ERR)
2563 err = -ECONNRESET;2580 err = -ECONNRESET;
2564 else2581 else
2565 err = -ETIMEDOUT;2582 err = -ETIMEDOUT;
@@ -2567,82 +2584,82 @@ static int rtrs_send_sess_info(struct rtrs_clt_sess *sess) · 1 unmodified line
2567 2584
2568out:2585out:
2569 if (tx_iu)2586 if (tx_iu)
2570 rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1);2587 rtrs_iu_free(tx_iu, clt_path->s.dev->ib_dev, 1);
2571 if (rx_iu)2588 if (rx_iu)
2572 rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1);2589 rtrs_iu_free(rx_iu, clt_path->s.dev->ib_dev, 1);
2573 if (err)2590 if (err)
2574 /* If we've never taken async path because of malloc problems */2591 /* If we've never taken async path because of malloc problems */
2575 rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL);2592 rtrs_clt_change_state_get_old(clt_path,
2593 RTRS_CLT_CONNECTING_ERR, NULL);
2576 2594
2577 return err;2595 return err;
2578}2596}
2579 2597
2580/**2598/**
2581 * init_sess() - establishes all session connections and does handshake2599 * init_path() - establishes all path connections and does handshake
2582 * @sess: client session.2600 * @clt_path: client path.
2583 * In case of error full close or reconnect procedure should be taken,2601 * In case of error full close or reconnect procedure should be taken,
2584 * because reconnect or close async works can be started.2602 * because reconnect or close async works can be started.
2585 */2603 */
2586static int init_sess(struct rtrs_clt_sess *sess)2604static int init_path(struct rtrs_clt_path *clt_path)
2587{2605{
2588 int err;2606 int err;
2589 char str[NAME_MAX];2607 char str[NAME_MAX];
2590 struct rtrs_addr path = {2608 struct rtrs_addr path = {
2591 .src = &sess->s.src_addr,2609 .src = &clt_path->s.src_addr,
2592 .dst = &sess->s.dst_addr,2610 .dst = &clt_path->s.dst_addr,
2593 };2611 };
2594 2612
2595 rtrs_addr_to_str(&path, str, sizeof(str));2613 rtrs_addr_to_str(&path, str, sizeof(str));
2596 2614
2597 mutex_lock(&sess->init_mutex);2615 mutex_lock(&clt_path->init_mutex);
2598 err = init_conns(sess);2616 err = init_conns(clt_path);
2599 if (err) {2617 if (err) {
2600 rtrs_err(sess->clt,2618 rtrs_err(clt_path->clt,
2601 "init_conns() failed: err=%d path=%s [%s:%u]\n", err,2619 "init_conns() failed: err=%d path=%s [%s:%u]\n", err,
2602 str, sess->hca_name, sess->hca_port);2620 str, clt_path->hca_name, clt_path->hca_port);
2603 goto out;2621 goto out;
2604 }2622 }
2605 err = rtrs_send_sess_info(sess);2623 err = rtrs_send_path_info(clt_path);
2606 if (err) {2624 if (err) {
2607 rtrs_err(2625 rtrs_err(clt_path->clt,
2608 sess->clt,2626 "rtrs_send_path_info() failed: err=%d path=%s [%s:%u]\n",
2609 "rtrs_send_sess_info() failed: err=%d path=%s [%s:%u]\n",2627 err, str, clt_path->hca_name, clt_path->hca_port);
2610 err, str, sess->hca_name, sess->hca_port);
2611 goto out;2628 goto out;
2612 }2629 }
2613 rtrs_clt_sess_up(sess);2630 rtrs_clt_path_up(clt_path);
2614out:2631out:
2615 mutex_unlock(&sess->init_mutex);2632 mutex_unlock(&clt_path->init_mutex);
2616 2633
2617 return err;2634 return err;
2618}2635}
2619 2636
2620static void rtrs_clt_reconnect_work(struct work_struct *work)2637static void rtrs_clt_reconnect_work(struct work_struct *work)
2621{2638{
2622 struct rtrs_clt_sess *sess;2639 struct rtrs_clt_path *clt_path;
2623 struct rtrs_clt *clt;2640 struct rtrs_clt_sess *clt;
2624 unsigned int delay_ms;2641 unsigned int delay_ms;
2625 int err;2642 int err;
2626 2643
2627 sess = container_of(to_delayed_work(work), struct rtrs_clt_sess,2644 clt_path = container_of(to_delayed_work(work), struct rtrs_clt_path,
2628 reconnect_dwork);2645 reconnect_dwork);
2629 clt = sess->clt;2646 clt = clt_path->clt;
2630 2647
2631 if (READ_ONCE(sess->state) != RTRS_CLT_RECONNECTING)2648 if (READ_ONCE(clt_path->state) != RTRS_CLT_RECONNECTING)
2632 return;2649 return;
2633 2650
2634 if (sess->reconnect_attempts >= clt->max_reconnect_attempts) {2651 if (clt_path->reconnect_attempts >= clt->max_reconnect_attempts) {
2635 /* Close a session completely if max attempts is reached */2652 /* Close a path completely if max attempts is reached */
2636 rtrs_clt_close_conns(sess, false);2653 rtrs_clt_close_conns(clt_path, false);
2637 return;2654 return;
2638 }2655 }
2639 sess->reconnect_attempts++;2656 clt_path->reconnect_attempts++;
2640 2657
2641 /* Stop everything */2658 /* Stop everything */
2642 rtrs_clt_stop_and_destroy_conns(sess);2659 rtrs_clt_stop_and_destroy_conns(clt_path);
2643 msleep(RTRS_RECONNECT_BACKOFF);2660 msleep(RTRS_RECONNECT_BACKOFF);
2644 if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING, NULL)) {2661 if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING, NULL)) {
2645 err = init_sess(sess);2662 err = init_path(clt_path);
2646 if (err)2663 if (err)
2647 goto reconnect_again;2664 goto reconnect_again;
2648 }2665 }
@@ -2650,10 +2667,10 @@ static void rtrs_clt_reconnect_work(struct work_struct *work) · 1 unmodified line
2650 return;2667 return;
2651 2668
2652reconnect_again:2669reconnect_again:
2653 if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_RECONNECTING, NULL)) {2670 if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_RECONNECTING, NULL)) {
2654 sess->stats->reconnects.fail_cnt++;2671 clt_path->stats->reconnects.fail_cnt++;
2655 delay_ms = clt->reconnect_delay_sec * 1000;2672 delay_ms = clt->reconnect_delay_sec * 1000;
2656 queue_delayed_work(rtrs_wq, &sess->reconnect_dwork,2673 queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork,
2657 msecs_to_jiffies(delay_ms +2674 msecs_to_jiffies(delay_ms +
2658 prandom_u32() %2675 prandom_u32() %
2659 RTRS_RECONNECT_SEED));2676 RTRS_RECONNECT_SEED));
@@ -2662,19 +2679,20 @@ reconnect_again: · 2 unmodified lines
2662 2679
2663static void rtrs_clt_dev_release(struct device *dev)2680static void rtrs_clt_dev_release(struct device *dev)
2664{2681{
2665 struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);2682 struct rtrs_clt_sess *clt = container_of(dev, struct rtrs_clt_sess,
2683 dev);
2666 2684
2667 kfree(clt);2685 kfree(clt);
2668}2686}
2669 2687
2670static struct rtrs_clt *alloc_clt(const char *sessname, size_t paths_num,2688static struct rtrs_clt_sess *alloc_clt(const char *sessname, size_t paths_num,
2671 u16 port, size_t pdu_sz, void *priv,2689 u16 port, size_t pdu_sz, void *priv,
2672 void (*link_ev)(void *priv,2690 void (*link_ev)(void *priv,
2673 enum rtrs_clt_link_ev ev),2691 enum rtrs_clt_link_ev ev),
2674 unsigned int reconnect_delay_sec,2692 unsigned int reconnect_delay_sec,
2675 unsigned int max_reconnect_attempts)2693 unsigned int max_reconnect_attempts)
2676{2694{
2677 struct rtrs_clt *clt;2695 struct rtrs_clt_sess *clt;
2678 int err;2696 int err;
2679 2697
2680 if (!paths_num || paths_num > MAX_PATHS_NUM)2698 if (!paths_num || paths_num > MAX_PATHS_NUM)
@@ -2749,7 +2767,7 @@ err: · 68 unmodified lines
2749 return ERR_PTR(err);2767 return ERR_PTR(err);
2750}2768}
2751 2769
2752static void free_clt(struct rtrs_clt *clt)2770static void free_clt(struct rtrs_clt_sess *clt)
2753{2771{
2754 free_permits(clt);2772 free_permits(clt);
2755 free_percpu(clt->pcpu_path);2773 free_percpu(clt->pcpu_path);
@@ -2760,7 +2778,7 @@ static void free_clt(struct rtrs_clt *clt) · 4 unmodified lines
2760}2778}
2761 2779
2762/**2780/**
2763 * rtrs_clt_open() - Open a session to an RTRS server2781 * rtrs_clt_open() - Open a path to an RTRS server
2764 * @ops: holds the link event callback and the private pointer.2782 * @ops: holds the link event callback and the private pointer.
2765 * @sessname: name of the session2783 * @sessname: name of the session
2766 * @paths: Paths to be established defined by their src and dst addresses2784 * @paths: Paths to be established defined by their src and dst addresses
@@ -2777,24 +2795,24 @@ static void free_clt(struct rtrs_clt *clt) · 10 unmodified lines
2777 *2795 *
2778 * Return a valid pointer on success otherwise PTR_ERR.2796 * Return a valid pointer on success otherwise PTR_ERR.
2779 */2797 */
2780struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,2798struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops,
2781 const char *sessname,2799 const char *pathname,
2782 const struct rtrs_addr *paths,2800 const struct rtrs_addr *paths,
2783 size_t paths_num, u16 port,2801 size_t paths_num, u16 port,
2784 size_t pdu_sz, u8 reconnect_delay_sec,2802 size_t pdu_sz, u8 reconnect_delay_sec,
2785 s16 max_reconnect_attempts, u32 nr_poll_queues)2803 s16 max_reconnect_attempts, u32 nr_poll_queues)
2786{2804{
2787 struct rtrs_clt_sess *sess, *tmp;2805 struct rtrs_clt_path *clt_path, *tmp;
2788 struct rtrs_clt *clt;2806 struct rtrs_clt_sess *clt;
2789 int err, i;2807 int err, i;
2790 2808
2791 if (strchr(sessname, '/') || strchr(sessname, '.')) {2809 if (strchr(pathname, '/') || strchr(pathname, '.')) {
2792 pr_err("sessname cannot contain / and .\n");2810 pr_err("pathname cannot contain / and .\n");
2793 err = -EINVAL;2811 err = -EINVAL;
2794 goto out;2812 goto out;
2795 }2813 }
2796 2814
2797 clt = alloc_clt(sessname, paths_num, port, pdu_sz, ops->priv,2815 clt = alloc_clt(pathname, paths_num, port, pdu_sz, ops->priv,
2798 ops->link_ev,2816 ops->link_ev,
2799 reconnect_delay_sec,2817 reconnect_delay_sec,
2800 max_reconnect_attempts);2818 max_reconnect_attempts);
@@ -2803,49 +2821,49 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops, · 2 unmodified lines
2803 goto out;2821 goto out;
2804 }2822 }
2805 for (i = 0; i < paths_num; i++) {2823 for (i = 0; i < paths_num; i++) {
2806 struct rtrs_clt_sess *sess;2824 struct rtrs_clt_path *clt_path;
2807 2825
2808 sess = alloc_sess(clt, &paths[i], nr_cpu_ids,2826 clt_path = alloc_path(clt, &paths[i], nr_cpu_ids,
2809 nr_poll_queues);2827 nr_poll_queues);
2810 if (IS_ERR(sess)) {2828 if (IS_ERR(clt_path)) {
2811 err = PTR_ERR(sess);2829 err = PTR_ERR(clt_path);
2812 goto close_all_sess;2830 goto close_all_path;
2813 }2831 }
2814 if (!i)2832 if (!i)
2815 sess->for_new_clt = 1;2833 clt_path->for_new_clt = 1;
2816 list_add_tail_rcu(&sess->s.entry, &clt->paths_list);2834 list_add_tail_rcu(&clt_path->s.entry, &clt->paths_list);
2817 2835
2818 err = init_sess(sess);2836 err = init_path(clt_path);
2819 if (err) {2837 if (err) {
2820 list_del_rcu(&sess->s.entry);2838 list_del_rcu(&clt_path->s.entry);
2821 rtrs_clt_close_conns(sess, true);2839 rtrs_clt_close_conns(clt_path, true);
2822 free_percpu(sess->stats->pcpu_stats);2840 free_percpu(clt_path->stats->pcpu_stats);
2823 kfree(sess->stats);2841 kfree(clt_path->stats);
2824 free_sess(sess);2842 free_path(clt_path);
2825 goto close_all_sess;2843 goto close_all_path;
2826 }2844 }
2827 2845
2828 err = rtrs_clt_create_sess_files(sess);2846 err = rtrs_clt_create_path_files(clt_path);
2829 if (err) {2847 if (err) {
2830 list_del_rcu(&sess->s.entry);2848 list_del_rcu(&clt_path->s.entry);
2831 rtrs_clt_close_conns(sess, true);2849 rtrs_clt_close_conns(clt_path, true);
2832 free_percpu(sess->stats->pcpu_stats);2850 free_percpu(clt_path->stats->pcpu_stats);
2833 kfree(sess->stats);2851 kfree(clt_path->stats);
2834 free_sess(sess);2852 free_path(clt_path);
2835 goto close_all_sess;2853 goto close_all_path;
2836 }2854 }
2837 }2855 }
2838 err = alloc_permits(clt);2856 err = alloc_permits(clt);
2839 if (err)2857 if (err)
2840 goto close_all_sess;2858 goto close_all_path;
2841 2859
2842 return clt;2860 return clt;
2843 2861
2844close_all_sess:2862close_all_path:
2845 list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) {2863 list_for_each_entry_safe(clt_path, tmp, &clt->paths_list, s.entry) {
2846 rtrs_clt_destroy_sess_files(sess, NULL);2864 rtrs_clt_destroy_path_files(clt_path, NULL);
2847 rtrs_clt_close_conns(sess, true);2865 rtrs_clt_close_conns(clt_path, true);
2848 kobject_put(&sess->kobj);2866 kobject_put(&clt_path->kobj);
2849 }2867 }
2850 rtrs_clt_destroy_sysfs_root(clt);2868 rtrs_clt_destroy_sysfs_root(clt);
2851 free_clt(clt);2869 free_clt(clt);
@@ -2856,37 +2874,38 @@ out: · 4 unmodified lines
2856EXPORT_SYMBOL(rtrs_clt_open);2874EXPORT_SYMBOL(rtrs_clt_open);
2857 2875
2858/**2876/**
2859 * rtrs_clt_close() - Close a session2877 * rtrs_clt_close() - Close a path
2860 * @clt: Session handle. Session is freed upon return.2878 * @clt: Session handle. Session is freed upon return.
2861 */2879 */
2862void rtrs_clt_close(struct rtrs_clt *clt)2880void rtrs_clt_close(struct rtrs_clt_sess *clt)
2863{2881{
2864 struct rtrs_clt_sess *sess, *tmp;2882 struct rtrs_clt_path *clt_path, *tmp;
2865 2883
2866 /* Firstly forbid sysfs access */2884 /* Firstly forbid sysfs access */
2867 rtrs_clt_destroy_sysfs_root(clt);2885 rtrs_clt_destroy_sysfs_root(clt);
2868 2886
2869 /* Now it is safe to iterate over all paths without locks */2887 /* Now it is safe to iterate over all paths without locks */
2870 list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) {2888 list_for_each_entry_safe(clt_path, tmp, &clt->paths_list, s.entry) {
2871 rtrs_clt_close_conns(sess, true);2889 rtrs_clt_close_conns(clt_path, true);
2872 rtrs_clt_destroy_sess_files(sess, NULL);2890 rtrs_clt_destroy_path_files(clt_path, NULL);
2873 kobject_put(&sess->kobj);2891 kobject_put(&clt_path->kobj);
2874 }2892 }
2875 free_clt(clt);2893 free_clt(clt);
2876}2894}
2877EXPORT_SYMBOL(rtrs_clt_close);2895EXPORT_SYMBOL(rtrs_clt_close);
2878 2896
2879int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess)2897int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_path *clt_path)
2880{2898{
2881 enum rtrs_clt_state old_state;2899 enum rtrs_clt_state old_state;
2882 int err = -EBUSY;2900 int err = -EBUSY;
2883 bool changed;2901 bool changed;
2884 2902
2885 changed = rtrs_clt_change_state_get_old(sess, RTRS_CLT_RECONNECTING,2903 changed = rtrs_clt_change_state_get_old(clt_path,
2904 RTRS_CLT_RECONNECTING,
2886 &old_state);2905 &old_state);
2887 if (changed) {2906 if (changed) {
2888 sess->reconnect_attempts = 0;2907 clt_path->reconnect_attempts = 0;
2889 queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, 0);2908 queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork, 0);
2890 }2909 }
2891 if (changed || old_state == RTRS_CLT_RECONNECTING) {2910 if (changed || old_state == RTRS_CLT_RECONNECTING) {
2892 /*2911 /*
@@ -2894,15 +2913,15 @@ int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess) · 1 unmodified line
2894 * execution, so do the flush if we have queued something2913 * execution, so do the flush if we have queued something
2895 * right now or work is pending.2914 * right now or work is pending.
2896 */2915 */
2897 flush_delayed_work(&sess->reconnect_dwork);2916 flush_delayed_work(&clt_path->reconnect_dwork);
2898 err = (READ_ONCE(sess->state) ==2917 err = (READ_ONCE(clt_path->state) ==
2899 RTRS_CLT_CONNECTED ? 0 : -ENOTCONN);2918 RTRS_CLT_CONNECTED ? 0 : -ENOTCONN);
2900 }2919 }
2901 2920
2902 return err;2921 return err;
2903}2922}
2904 2923
2905int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess,2924int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_path *clt_path,
2906 const struct attribute *sysfs_self)2925 const struct attribute *sysfs_self)
2907{2926{
2908 enum rtrs_clt_state old_state;2927 enum rtrs_clt_state old_state;
@@ -2918,27 +2937,27 @@ int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess, · 9 unmodified lines
2918 * removing the path.2937 * removing the path.
2919 */2938 */
2920 do {2939 do {
2921 rtrs_clt_close_conns(sess, true);2940 rtrs_clt_close_conns(clt_path, true);
2922 changed = rtrs_clt_change_state_get_old(sess,2941 changed = rtrs_clt_change_state_get_old(clt_path,
2923 RTRS_CLT_DEAD,2942 RTRS_CLT_DEAD,
2924 &old_state);2943 &old_state);
2925 } while (!changed && old_state != RTRS_CLT_DEAD);2944 } while (!changed && old_state != RTRS_CLT_DEAD);
2926 2945
2927 if (changed) {2946 if (changed) {
2928 rtrs_clt_remove_path_from_arr(sess);2947 rtrs_clt_remove_path_from_arr(clt_path);
2929 rtrs_clt_destroy_sess_files(sess, sysfs_self);2948 rtrs_clt_destroy_path_files(clt_path, sysfs_self);
2930 kobject_put(&sess->kobj);2949 kobject_put(&clt_path->kobj);
2931 }2950 }
2932 2951
2933 return 0;2952 return 0;
2934}2953}
2935 2954
2936void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value)2955void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt_sess *clt, int value)
2937{2956{
2938 clt->max_reconnect_attempts = (unsigned int)value;2957 clt->max_reconnect_attempts = (unsigned int)value;
2939}2958}
2940 2959
2941int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt)2960int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt_sess *clt)
2942{2961{
2943 return (int)clt->max_reconnect_attempts;2962 return (int)clt->max_reconnect_attempts;
2944}2963}
@@ -2968,12 +2987,12 @@ int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt) · 23 unmodified lines
2968 * On dir=WRITE rtrs client will rdma write data in sg to server side.2987 * On dir=WRITE rtrs client will rdma write data in sg to server side.
2969 */2988 */
2970int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,2989int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,
2971 struct rtrs_clt *clt, struct rtrs_permit *permit,2990 struct rtrs_clt_sess *clt, struct rtrs_permit *permit,
2972 const struct kvec *vec, size_t nr, size_t data_len,2991 const struct kvec *vec, size_t nr, size_t data_len,
2973 struct scatterlist *sg, unsigned int sg_cnt)2992 struct scatterlist *sg, unsigned int sg_cnt)
2974{2993{
2975 struct rtrs_clt_io_req *req;2994 struct rtrs_clt_io_req *req;
2976 struct rtrs_clt_sess *sess;2995 struct rtrs_clt_path *clt_path;
2977 2996
2978 enum dma_data_direction dma_dir;2997 enum dma_data_direction dma_dir;
2979 int err = -ECONNABORTED, i;2998 int err = -ECONNABORTED, i;
@@ -2995,19 +3014,19 @@ int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops, · 15 unmodified lines
2995 3014
2996 rcu_read_lock();3015 rcu_read_lock();
2997 for (path_it_init(&it, clt);3016 for (path_it_init(&it, clt);
2998 (sess = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) {3017 (clt_path = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) {
2999 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED)3018 if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED)
3000 continue;3019 continue;
3001 3020
3002 if (usr_len + hdr_len > sess->max_hdr_size) {3021 if (usr_len + hdr_len > clt_path->max_hdr_size) {
3003 rtrs_wrn_rl(sess->clt,3022 rtrs_wrn_rl(clt_path->clt,
3004 "%s request failed, user message size is %zu and header length %zu, but max size is %u\n",3023 "%s request failed, user message size is %zu and header length %zu, but max size is %u\n",
3005 dir == READ ? "Read" : "Write",3024 dir == READ ? "Read" : "Write",
3006 usr_len, hdr_len, sess->max_hdr_size);3025 usr_len, hdr_len, clt_path->max_hdr_size);
3007 err = -EMSGSIZE;3026 err = -EMSGSIZE;
3008 break;3027 break;
3009 }3028 }
3010 req = rtrs_clt_get_req(sess, ops->conf_fn, permit, ops->priv,3029 req = rtrs_clt_get_req(clt_path, ops->conf_fn, permit, ops->priv,
3011 vec, usr_len, sg, sg_cnt, data_len,3030 vec, usr_len, sg, sg_cnt, data_len,
3012 dma_dir);3031 dma_dir);
3013 if (dir == READ)3032 if (dir == READ)
@@ -3028,21 +3047,21 @@ int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops, · 14 unmodified lines
3028}3047}
3029EXPORT_SYMBOL(rtrs_clt_request);3048EXPORT_SYMBOL(rtrs_clt_request);
3030 3049
3031int rtrs_clt_rdma_cq_direct(struct rtrs_clt *clt, unsigned int index)3050int rtrs_clt_rdma_cq_direct(struct rtrs_clt_sess *clt, unsigned int index)
3032{3051{
3033 /* If no path, return -1 for block layer not to try again */3052 /* If no path, return -1 for block layer not to try again */
3034 int cnt = -1;3053 int cnt = -1;
3035 struct rtrs_con *con;3054 struct rtrs_con *con;
3036 struct rtrs_clt_sess *sess;3055 struct rtrs_clt_path *clt_path;
3037 struct path_it it;3056 struct path_it it;
3038 3057
3039 rcu_read_lock();3058 rcu_read_lock();
3040 for (path_it_init(&it, clt);3059 for (path_it_init(&it, clt);
3041 (sess = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) {3060 (clt_path = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) {
3042 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED)3061 if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED)
3043 continue;3062 continue;
3044 3063
3045 con = sess->s.con[index + 1];3064 con = clt_path->s.con[index + 1];
3046 cnt = ib_process_cq_direct(con->cq, -1);3065 cnt = ib_process_cq_direct(con->cq, -1);
3047 if (cnt)3066 if (cnt)
3048 break;3067 break;
@@ -3062,7 +3081,7 @@ EXPORT_SYMBOL(rtrs_clt_rdma_cq_direct); · 13 unmodified lines
3062 * 0 on success3081 * 0 on success
3063 * -ECOMM no connection to the server3082 * -ECOMM no connection to the server
3064 */3083 */
3065int rtrs_clt_query(struct rtrs_clt *clt, struct rtrs_attrs *attr)3084int rtrs_clt_query(struct rtrs_clt_sess *clt, struct rtrs_attrs *attr)
3066{3085{
3067 if (!rtrs_clt_is_connected(clt))3086 if (!rtrs_clt_is_connected(clt))
3068 return -ECOMM;3087 return -ECOMM;
@@ -3077,15 +3096,15 @@ int rtrs_clt_query(struct rtrs_clt *clt, struct rtrs_attrs *attr) · 8 unmodified lines
3077}3096}
3078EXPORT_SYMBOL(rtrs_clt_query);3097EXPORT_SYMBOL(rtrs_clt_query);
3079 3098
3080int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,3099int rtrs_clt_create_path_from_sysfs(struct rtrs_clt_sess *clt,
3081 struct rtrs_addr *addr)3100 struct rtrs_addr *addr)
3082{3101{
3083 struct rtrs_clt_sess *sess;3102 struct rtrs_clt_path *clt_path;
3084 int err;3103 int err;
3085 3104
3086 sess = alloc_sess(clt, addr, nr_cpu_ids, 0);3105 clt_path = alloc_path(clt, addr, nr_cpu_ids, 0);
3087 if (IS_ERR(sess))3106 if (IS_ERR(clt_path))
3088 return PTR_ERR(sess);3107 return PTR_ERR(clt_path);
3089 3108
3090 mutex_lock(&clt->paths_mutex);3109 mutex_lock(&clt->paths_mutex);
3091 if (clt->paths_num == 0) {3110 if (clt->paths_num == 0) {
@@ -3094,7 +3113,7 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt, · 2 unmodified lines
3094 * the addition of the first path is like a new session for3113 * the addition of the first path is like a new session for
3095 * the storage server3114 * the storage server
3096 */3115 */
3097 sess->for_new_clt = 1;3116 clt_path->for_new_clt = 1;
3098 }3117 }
3099 3118
3100 mutex_unlock(&clt->paths_mutex);3119 mutex_unlock(&clt->paths_mutex);
@@ -3104,24 +3123,24 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt, · 3 unmodified lines
3104 * IO will never grab it. Also it is very important to add3123 * IO will never grab it. Also it is very important to add
3105 * path before init, since init fires LINK_CONNECTED event.3124 * path before init, since init fires LINK_CONNECTED event.
3106 */3125 */
3107 rtrs_clt_add_path_to_arr(sess);3126 rtrs_clt_add_path_to_arr(clt_path);
3108 3127
3109 err = init_sess(sess);3128 err = init_path(clt_path);
3110 if (err)3129 if (err)
3111 goto close_sess;3130 goto close_path;
3112 3131
3113 err = rtrs_clt_create_sess_files(sess);3132 err = rtrs_clt_create_path_files(clt_path);
3114 if (err)3133 if (err)
3115 goto close_sess;3134 goto close_path;
3116 3135
3117 return 0;3136 return 0;
3118 3137
3119close_sess:3138close_path:
3120 rtrs_clt_remove_path_from_arr(sess);3139 rtrs_clt_remove_path_from_arr(clt_path);
3121 rtrs_clt_close_conns(sess, true);3140 rtrs_clt_close_conns(clt_path, true);
3122 free_percpu(sess->stats->pcpu_stats);3141 free_percpu(clt_path->stats->pcpu_stats);
3123 kfree(sess->stats);3142 kfree(clt_path->stats);
3124 free_sess(sess);3143 free_path(clt_path);
3125 3144
3126 return err;3145 return err;
3127}3146}
drivers/infiniband/ulp/rtrs/rtrs-clt.h+21−20
@@ -124,9 +124,9 @@ struct rtrs_rbuf { · 123 unmodified lines
124 u32 rkey;124 u32 rkey;
125};125};
126 126
127struct rtrs_clt_sess {127struct rtrs_clt_path {
128 struct rtrs_sess s;128 struct rtrs_path s;
129 struct rtrs_clt *clt;129 struct rtrs_clt_sess *clt;
130 wait_queue_head_t state_wq;130 wait_queue_head_t state_wq;
131 enum rtrs_clt_state state;131 enum rtrs_clt_state state;
132 atomic_t connected_cnt;132 atomic_t connected_cnt;
@@ -153,10 +153,10 @@ struct rtrs_clt_sess { · 20 unmodified lines
153 *mp_skip_entry;153 *mp_skip_entry;
154};154};
155 155
156struct rtrs_clt {156struct rtrs_clt_sess {
157 struct list_head paths_list; /* rcu protected list */157 struct list_head paths_list; /* rcu protected list */
158 size_t paths_num;158 size_t paths_num;
159 struct rtrs_clt_sess159 struct rtrs_clt_path
160 __rcu * __percpu *pcpu_path;160 __rcu * __percpu *pcpu_path;
161 uuid_t paths_uuid;161 uuid_t paths_uuid;
162 int paths_up;162 int paths_up;
@@ -186,31 +186,32 @@ static inline struct rtrs_clt_con *to_clt_con(struct rtrs_con *c) · 23 unmodified lines
186 return container_of(c, struct rtrs_clt_con, c);186 return container_of(c, struct rtrs_clt_con, c);
187}187}
188 188
189static inline struct rtrs_clt_sess *to_clt_sess(struct rtrs_sess *s)189static inline struct rtrs_clt_path *to_clt_path(struct rtrs_path *s)
190{190{
191 return container_of(s, struct rtrs_clt_sess, s);191 return container_of(s, struct rtrs_clt_path, s);
192}192}
193 193
194static inline int permit_size(struct rtrs_clt *clt)194static inline int permit_size(struct rtrs_clt_sess *clt)
195{195{
196 return sizeof(struct rtrs_permit) + clt->pdu_sz;196 return sizeof(struct rtrs_permit) + clt->pdu_sz;
197}197}
198 198
199static inline struct rtrs_permit *get_permit(struct rtrs_clt *clt, int idx)199static inline struct rtrs_permit *get_permit(struct rtrs_clt_sess *clt,
200 int idx)
200{201{
201 return (struct rtrs_permit *)(clt->permits + permit_size(clt) * idx);202 return (struct rtrs_permit *)(clt->permits + permit_size(clt) * idx);
202}203}
203 204
204int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess);205int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_path *path);
205void rtrs_clt_close_conns(struct rtrs_clt_sess *sess, bool wait);206void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait);
206int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,207int rtrs_clt_create_path_from_sysfs(struct rtrs_clt_sess *clt,
207 struct rtrs_addr *addr);208 struct rtrs_addr *addr);
208int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess,209int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_path *path,
209 const struct attribute *sysfs_self);210 const struct attribute *sysfs_self);
210 211
211void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value);212void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt_sess *clt, int value);
212int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt);213int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt_sess *clt);
213void free_sess(struct rtrs_clt_sess *sess);214void free_path(struct rtrs_clt_path *clt_path);
214 215
215/* rtrs-clt-stats.c */216/* rtrs-clt-stats.c */
216 217
@@ -239,11 +240,11 @@ ssize_t rtrs_clt_reset_all_help(struct rtrs_clt_stats *stats, · 22 unmodified lines
239 240
240/* rtrs-clt-sysfs.c */241/* rtrs-clt-sysfs.c */
241 242
242int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt);243int rtrs_clt_create_sysfs_root_files(struct rtrs_clt_sess *clt);
243void rtrs_clt_destroy_sysfs_root(struct rtrs_clt *clt);244void rtrs_clt_destroy_sysfs_root(struct rtrs_clt_sess *clt);
244 245
245int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess);246int rtrs_clt_create_path_files(struct rtrs_clt_path *clt_path);
246void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess,247void rtrs_clt_destroy_path_files(struct rtrs_clt_path *clt_path,
247 const struct attribute *sysfs_self);248 const struct attribute *sysfs_self);
248 249
249#endif /* RTRS_CLT_H */250#endif /* RTRS_CLT_H */
drivers/infiniband/ulp/rtrs/rtrs-pri.h+9−9
@@ -90,7 +90,7 @@ struct rtrs_ib_dev { · 89 unmodified lines
90};90};
91 91
92struct rtrs_con {92struct rtrs_con {
93 struct rtrs_sess *sess;93 struct rtrs_path *path;
94 struct ib_qp *qp;94 struct ib_qp *qp;
95 struct ib_cq *cq;95 struct ib_cq *cq;
96 struct rdma_cm_id *cm_id;96 struct rdma_cm_id *cm_id;
@@ -100,7 +100,7 @@ struct rtrs_con { · 3 unmodified lines
100 atomic_t sq_wr_avail;100 atomic_t sq_wr_avail;
101};101};
102 102
103struct rtrs_sess {103struct rtrs_path {
104 struct list_head entry;104 struct list_head entry;
105 struct sockaddr_storage dst_addr;105 struct sockaddr_storage dst_addr;
106 struct sockaddr_storage src_addr;106 struct sockaddr_storage src_addr;
@@ -229,11 +229,11 @@ struct rtrs_msg_conn_rsp { · 122 unmodified lines
229/**229/**
230 * struct rtrs_msg_info_req230 * struct rtrs_msg_info_req
231 * @type: @RTRS_MSG_INFO_REQ231 * @type: @RTRS_MSG_INFO_REQ
232 * @sessname: Session name chosen by client232 * @pathname: Path name chosen by client
233 */233 */
234struct rtrs_msg_info_req {234struct rtrs_msg_info_req {
235 __le16 type;235 __le16 type;
236 u8 sessname[NAME_MAX];236 u8 pathname[NAME_MAX];
237 u8 reserved[15];237 u8 reserved[15];
238};238};
239 239
@@ -313,19 +313,19 @@ int rtrs_iu_post_rdma_write_imm(struct rtrs_con *con, struct rtrs_iu *iu, · 73 unmodified lines
313 313
314int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe);314int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe);
315 315
316int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con,316int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con,
317 u32 max_send_sge, int cq_vector, int nr_cqe,317 u32 max_send_sge, int cq_vector, int nr_cqe,
318 u32 max_send_wr, u32 max_recv_wr,318 u32 max_send_wr, u32 max_recv_wr,
319 enum ib_poll_context poll_ctx);319 enum ib_poll_context poll_ctx);
320void rtrs_cq_qp_destroy(struct rtrs_con *con);320void rtrs_cq_qp_destroy(struct rtrs_con *con);
321 321
322void rtrs_init_hb(struct rtrs_sess *sess, struct ib_cqe *cqe,322void rtrs_init_hb(struct rtrs_path *path, struct ib_cqe *cqe,
323 unsigned int interval_ms, unsigned int missed_max,323 unsigned int interval_ms, unsigned int missed_max,
324 void (*err_handler)(struct rtrs_con *con),324 void (*err_handler)(struct rtrs_con *con),
325 struct workqueue_struct *wq);325 struct workqueue_struct *wq);
326void rtrs_start_hb(struct rtrs_sess *sess);326void rtrs_start_hb(struct rtrs_path *path);
327void rtrs_stop_hb(struct rtrs_sess *sess);327void rtrs_stop_hb(struct rtrs_path *path);
328void rtrs_send_hb_ack(struct rtrs_sess *sess);328void rtrs_send_hb_ack(struct rtrs_path *path);
329 329
330void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,330void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,
331 struct rtrs_rdma_dev_pd *pool);331 struct rtrs_rdma_dev_pd *pool);
drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c+61−60
@@ -15,10 +15,10 @@ · 14 unmodified lines
15 15
16static void rtrs_srv_release(struct kobject *kobj)16static void rtrs_srv_release(struct kobject *kobj)
17{17{
18 struct rtrs_srv_sess *sess;18 struct rtrs_srv_path *srv_path;
19 19
20 sess = container_of(kobj, struct rtrs_srv_sess, kobj);20 srv_path = container_of(kobj, struct rtrs_srv_path, kobj);
21 kfree(sess);21 kfree(srv_path);
22}22}
23 23
24static struct kobj_type ktype = {24static struct kobj_type ktype = {
@@ -36,24 +36,25 @@ static ssize_t rtrs_srv_disconnect_store(struct kobject *kobj, · 11 unmodified lines
36 struct kobj_attribute *attr,36 struct kobj_attribute *attr,
37 const char *buf, size_t count)37 const char *buf, size_t count)
38{38{
39 struct rtrs_srv_sess *sess;39 struct rtrs_srv_path *srv_path;
40 struct rtrs_sess *s;40 struct rtrs_path *s;
41 char str[MAXHOSTNAMELEN];41 char str[MAXHOSTNAMELEN];
42 42
43 sess = container_of(kobj, struct rtrs_srv_sess, kobj);43 srv_path = container_of(kobj, struct rtrs_srv_path, kobj);
44 s = &sess->s;44 s = &srv_path->s;
45 if (!sysfs_streq(buf, "1")) {45 if (!sysfs_streq(buf, "1")) {
46 rtrs_err(s, "%s: invalid value: '%s'\n",46 rtrs_err(s, "%s: invalid value: '%s'\n",
47 attr->attr.name, buf);47 attr->attr.name, buf);
48 return -EINVAL;48 return -EINVAL;
49 }49 }
50 50
51 sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr, str, sizeof(str));51 sockaddr_to_str((struct sockaddr *)&srv_path->s.dst_addr, str,
52 sizeof(str));
52 53
53 rtrs_info(s, "disconnect for path %s requested\n", str);54 rtrs_info(s, "disconnect for path %s requested\n", str);
54 /* first remove sysfs itself to avoid deadlock */55 /* first remove sysfs itself to avoid deadlock */
55 sysfs_remove_file_self(&sess->kobj, &attr->attr);56 sysfs_remove_file_self(&srv_path->kobj, &attr->attr);
56 close_sess(sess);57 close_path(srv_path);
57 58
58 return count;59 return count;
59}60}
@@ -66,11 +67,11 @@ static ssize_t rtrs_srv_hca_port_show(struct kobject *kobj, · 6 unmodified lines
66 struct kobj_attribute *attr,67 struct kobj_attribute *attr,
67 char *page)68 char *page)
68{69{
69 struct rtrs_srv_sess *sess;70 struct rtrs_srv_path *srv_path;
70 struct rtrs_con *usr_con;71 struct rtrs_con *usr_con;
71 72
72 sess = container_of(kobj, typeof(*sess), kobj);73 srv_path = container_of(kobj, typeof(*srv_path), kobj);
73 usr_con = sess->s.con[0];74 usr_con = srv_path->s.con[0];
74 75
75 return sysfs_emit(page, "%u\n", usr_con->cm_id->port_num);76 return sysfs_emit(page, "%u\n", usr_con->cm_id->port_num);
76}77}
@@ -82,11 +83,11 @@ static ssize_t rtrs_srv_hca_name_show(struct kobject *kobj, · 5 unmodified lines
82 struct kobj_attribute *attr,83 struct kobj_attribute *attr,
83 char *page)84 char *page)
84{85{
85 struct rtrs_srv_sess *sess;86 struct rtrs_srv_path *srv_path;
86 87
87 sess = container_of(kobj, struct rtrs_srv_sess, kobj);88 srv_path = container_of(kobj, struct rtrs_srv_path, kobj);
88 89
89 return sysfs_emit(page, "%s\n", sess->s.dev->ib_dev->name);90 return sysfs_emit(page, "%s\n", srv_path->s.dev->ib_dev->name);
90}91}
91 92
92static struct kobj_attribute rtrs_srv_hca_name_attr =93static struct kobj_attribute rtrs_srv_hca_name_attr =
@@ -96,11 +97,11 @@ static ssize_t rtrs_srv_src_addr_show(struct kobject *kobj, · 3 unmodified lines
96 struct kobj_attribute *attr,97 struct kobj_attribute *attr,
97 char *page)98 char *page)
98{99{
99 struct rtrs_srv_sess *sess;100 struct rtrs_srv_path *srv_path;
100 int cnt;101 int cnt;
101 102
102 sess = container_of(kobj, struct rtrs_srv_sess, kobj);103 srv_path = container_of(kobj, struct rtrs_srv_path, kobj);
103 cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,104 cnt = sockaddr_to_str((struct sockaddr *)&srv_path->s.dst_addr,
104 page, PAGE_SIZE);105 page, PAGE_SIZE);
105 return cnt + sysfs_emit_at(page, cnt, "\n");106 return cnt + sysfs_emit_at(page, cnt, "\n");
106}107}
@@ -112,11 +113,11 @@ static ssize_t rtrs_srv_dst_addr_show(struct kobject *kobj, · 5 unmodified lines
112 struct kobj_attribute *attr,113 struct kobj_attribute *attr,
113 char *page)114 char *page)
114{115{
115 struct rtrs_srv_sess *sess;116 struct rtrs_srv_path *srv_path;
116 int len;117 int len;
117 118
118 sess = container_of(kobj, struct rtrs_srv_sess, kobj);119 srv_path = container_of(kobj, struct rtrs_srv_path, kobj);
119 len = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr, page,120 len = sockaddr_to_str((struct sockaddr *)&srv_path->s.src_addr, page,
120 PAGE_SIZE);121 PAGE_SIZE);
121 len += sysfs_emit_at(page, len, "\n");122 len += sysfs_emit_at(page, len, "\n");
122 return len;123 return len;
@@ -125,7 +126,7 @@ static ssize_t rtrs_srv_dst_addr_show(struct kobject *kobj, · 2 unmodified lines
125static struct kobj_attribute rtrs_srv_dst_addr_attr =126static struct kobj_attribute rtrs_srv_dst_addr_attr =
126 __ATTR(dst_addr, 0444, rtrs_srv_dst_addr_show, NULL);127 __ATTR(dst_addr, 0444, rtrs_srv_dst_addr_show, NULL);
127 128
128static struct attribute *rtrs_srv_sess_attrs[] = {129static struct attribute *rtrs_srv_path_attrs[] = {
129 &rtrs_srv_hca_name_attr.attr,130 &rtrs_srv_hca_name_attr.attr,
130 &rtrs_srv_hca_port_attr.attr,131 &rtrs_srv_hca_port_attr.attr,
131 &rtrs_srv_src_addr_attr.attr,132 &rtrs_srv_src_addr_attr.attr,
@@ -134,8 +135,8 @@ static struct attribute *rtrs_srv_sess_attrs[] = { · 2 unmodified lines
134 NULL,135 NULL,
135};136};
136 137
137static const struct attribute_group rtrs_srv_sess_attr_group = {138static const struct attribute_group rtrs_srv_path_attr_group = {
138 .attrs = rtrs_srv_sess_attrs,139 .attrs = rtrs_srv_path_attrs,
139};140};
140 141
141STAT_ATTR(struct rtrs_srv_stats, rdma,142STAT_ATTR(struct rtrs_srv_stats, rdma,
@@ -151,9 +152,9 @@ static const struct attribute_group rtrs_srv_stats_attr_group = { · 9 unmodified lines
151 .attrs = rtrs_srv_stats_attrs,152 .attrs = rtrs_srv_stats_attrs,
152};153};
153 154
154static int rtrs_srv_create_once_sysfs_root_folders(struct rtrs_srv_sess *sess)155static int rtrs_srv_create_once_sysfs_root_folders(struct rtrs_srv_path *srv_path)
155{156{
156 struct rtrs_srv *srv = sess->srv;157 struct rtrs_srv_sess *srv = srv_path->srv;
157 int err = 0;158 int err = 0;
158 159
159 mutex_lock(&srv->paths_mutex);160 mutex_lock(&srv->paths_mutex);
@@ -164,7 +165,7 @@ static int rtrs_srv_create_once_sysfs_root_folders(struct rtrs_srv_sess *sess) · 4 unmodified lines
164 goto unlock;165 goto unlock;
165 }166 }
166 srv->dev.class = rtrs_dev_class;167 srv->dev.class = rtrs_dev_class;
167 err = dev_set_name(&srv->dev, "%s", sess->s.sessname);168 err = dev_set_name(&srv->dev, "%s", srv_path->s.sessname);
168 if (err)169 if (err)
169 goto unlock;170 goto unlock;
170 171
@@ -196,9 +197,9 @@ unlock: · 25 unmodified lines
196}197}
197 198
198static void199static void
199rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_sess *sess)200rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_path *srv_path)
200{201{
201 struct rtrs_srv *srv = sess->srv;202 struct rtrs_srv_sess *srv = srv_path->srv;
202 203
203 mutex_lock(&srv->paths_mutex);204 mutex_lock(&srv->paths_mutex);
204 if (!--srv->dev_ref) {205 if (!--srv->dev_ref) {
@@ -213,7 +214,7 @@ rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_sess *sess) · 8 unmodified lines
213 }214 }
214}215}
215 216
216static void rtrs_srv_sess_stats_release(struct kobject *kobj)217static void rtrs_srv_path_stats_release(struct kobject *kobj)
217{218{
218 struct rtrs_srv_stats *stats;219 struct rtrs_srv_stats *stats;
219 220
@@ -224,22 +225,22 @@ static void rtrs_srv_sess_stats_release(struct kobject *kobj) · 4 unmodified lines
224 225
225static struct kobj_type ktype_stats = {226static struct kobj_type ktype_stats = {
226 .sysfs_ops = &kobj_sysfs_ops,227 .sysfs_ops = &kobj_sysfs_ops,
227 .release = rtrs_srv_sess_stats_release,228 .release = rtrs_srv_path_stats_release,
228};229};
229 230
230static int rtrs_srv_create_stats_files(struct rtrs_srv_sess *sess)231static int rtrs_srv_create_stats_files(struct rtrs_srv_path *srv_path)
231{232{
232 int err;233 int err;
233 struct rtrs_sess *s = &sess->s;234 struct rtrs_path *s = &srv_path->s;
234 235
235 err = kobject_init_and_add(&sess->stats->kobj_stats, &ktype_stats,236 err = kobject_init_and_add(&srv_path->stats->kobj_stats, &ktype_stats,
236 &sess->kobj, "stats");237 &srv_path->kobj, "stats");
237 if (err) {238 if (err) {
238 rtrs_err(s, "kobject_init_and_add(): %d\n", err);239 rtrs_err(s, "kobject_init_and_add(): %d\n", err);
239 kobject_put(&sess->stats->kobj_stats);240 kobject_put(&srv_path->stats->kobj_stats);
240 return err;241 return err;
241 }242 }
242 err = sysfs_create_group(&sess->stats->kobj_stats,243 err = sysfs_create_group(&srv_path->stats->kobj_stats,
243 &rtrs_srv_stats_attr_group);244 &rtrs_srv_stats_attr_group);
244 if (err) {245 if (err) {
245 rtrs_err(s, "sysfs_create_group(): %d\n", err);246 rtrs_err(s, "sysfs_create_group(): %d\n", err);
@@ -249,64 +250,64 @@ static int rtrs_srv_create_stats_files(struct rtrs_srv_sess *sess) · 3 unmodified lines
249 return 0;250 return 0;
250 251
251err:252err:
252 kobject_del(&sess->stats->kobj_stats);253 kobject_del(&srv_path->stats->kobj_stats);
253 kobject_put(&sess->stats->kobj_stats);254 kobject_put(&srv_path->stats->kobj_stats);
254 255
255 return err;256 return err;
256}257}
257 258
258int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess)259int rtrs_srv_create_path_files(struct rtrs_srv_path *srv_path)
259{260{
260 struct rtrs_srv *srv = sess->srv;261 struct rtrs_srv_sess *srv = srv_path->srv;
261 struct rtrs_sess *s = &sess->s;262 struct rtrs_path *s = &srv_path->s;
262 char str[NAME_MAX];263 char str[NAME_MAX];
263 int err;264 int err;
264 struct rtrs_addr path = {265 struct rtrs_addr path = {
265 .src = &sess->s.dst_addr,266 .src = &srv_path->s.dst_addr,
266 .dst = &sess->s.src_addr,267 .dst = &srv_path->s.src_addr,
267 };268 };
268 269
269 rtrs_addr_to_str(&path, str, sizeof(str));270 rtrs_addr_to_str(&path, str, sizeof(str));
270 err = rtrs_srv_create_once_sysfs_root_folders(sess);271 err = rtrs_srv_create_once_sysfs_root_folders(srv_path);
271 if (err)272 if (err)
272 return err;273 return err;
273 274
274 err = kobject_init_and_add(&sess->kobj, &ktype, srv->kobj_paths,275 err = kobject_init_and_add(&srv_path->kobj, &ktype, srv->kobj_paths,
275 "%s", str);276 "%s", str);
276 if (err) {277 if (err) {
277 rtrs_err(s, "kobject_init_and_add(): %d\n", err);278 rtrs_err(s, "kobject_init_and_add(): %d\n", err);
278 goto destroy_root;279 goto destroy_root;
279 }280 }
280 err = sysfs_create_group(&sess->kobj, &rtrs_srv_sess_attr_group);281 err = sysfs_create_group(&srv_path->kobj, &rtrs_srv_path_attr_group);
281 if (err) {282 if (err) {
282 rtrs_err(s, "sysfs_create_group(): %d\n", err);283 rtrs_err(s, "sysfs_create_group(): %d\n", err);
283 goto put_kobj;284 goto put_kobj;
284 }285 }
285 err = rtrs_srv_create_stats_files(sess);286 err = rtrs_srv_create_stats_files(srv_path);
286 if (err)287 if (err)
287 goto remove_group;288 goto remove_group;
288 289
289 return 0;290 return 0;
290 291
291remove_group:292remove_group:
292 sysfs_remove_group(&sess->kobj, &rtrs_srv_sess_attr_group);293 sysfs_remove_group(&srv_path->kobj, &rtrs_srv_path_attr_group);
293put_kobj:294put_kobj:
294 kobject_del(&sess->kobj);295 kobject_del(&srv_path->kobj);
295destroy_root:296destroy_root:
296 kobject_put(&sess->kobj);297 kobject_put(&srv_path->kobj);
297 rtrs_srv_destroy_once_sysfs_root_folders(sess);298 rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
298 299
299 return err;300 return err;
300}301}
301 302
302void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess)303void rtrs_srv_destroy_path_files(struct rtrs_srv_path *srv_path)
303{304{
304 if (sess->kobj.state_in_sysfs) {305 if (srv_path->kobj.state_in_sysfs) {
305 kobject_del(&sess->stats->kobj_stats);306 kobject_del(&srv_path->stats->kobj_stats);
306 kobject_put(&sess->stats->kobj_stats);307 kobject_put(&srv_path->stats->kobj_stats);
307 sysfs_remove_group(&sess->kobj, &rtrs_srv_sess_attr_group);308 sysfs_remove_group(&srv_path->kobj, &rtrs_srv_path_attr_group);
308 kobject_put(&sess->kobj);309 kobject_put(&srv_path->kobj);
309 310
310 rtrs_srv_destroy_once_sysfs_root_folders(sess);311 rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
311 }312 }
312}313}
drivers/infiniband/ulp/rtrs/rtrs-srv.c+347−337
@@ -62,19 +62,19 @@ static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c) · 61 unmodified lines
62 return container_of(c, struct rtrs_srv_con, c);62 return container_of(c, struct rtrs_srv_con, c);
63}63}
64 64
65static inline struct rtrs_srv_sess *to_srv_sess(struct rtrs_sess *s)65static inline struct rtrs_srv_path *to_srv_path(struct rtrs_path *s)
66{66{
67 return container_of(s, struct rtrs_srv_sess, s);67 return container_of(s, struct rtrs_srv_path, s);
68}68}
69 69
70static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess,70static bool rtrs_srv_change_state(struct rtrs_srv_path *srv_path,
71 enum rtrs_srv_state new_state)71 enum rtrs_srv_state new_state)
72{72{
73 enum rtrs_srv_state old_state;73 enum rtrs_srv_state old_state;
74 bool changed = false;74 bool changed = false;
75 75
76 spin_lock_irq(&sess->state_lock);76 spin_lock_irq(&srv_path->state_lock);
77 old_state = sess->state;77 old_state = srv_path->state;
78 switch (new_state) {78 switch (new_state) {
79 case RTRS_SRV_CONNECTED:79 case RTRS_SRV_CONNECTED:
80 if (old_state == RTRS_SRV_CONNECTING)80 if (old_state == RTRS_SRV_CONNECTING)
@@ -93,8 +93,8 @@ static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess, · 12 unmodified lines
93 break;93 break;
94 }94 }
95 if (changed)95 if (changed)
96 sess->state = new_state;96 srv_path->state = new_state;
97 spin_unlock_irq(&sess->state_lock);97 spin_unlock_irq(&srv_path->state_lock);
98 98
99 return changed;99 return changed;
100}100}
@@ -106,16 +106,16 @@ static void free_id(struct rtrs_srv_op *id) · 5 unmodified lines
106 kfree(id);106 kfree(id);
107}107}
108 108
109static void rtrs_srv_free_ops_ids(struct rtrs_srv_sess *sess)109static void rtrs_srv_free_ops_ids(struct rtrs_srv_path *srv_path)
110{110{
111 struct rtrs_srv *srv = sess->srv;111 struct rtrs_srv_sess *srv = srv_path->srv;
112 int i;112 int i;
113 113
114 if (sess->ops_ids) {114 if (srv_path->ops_ids) {
115 for (i = 0; i < srv->queue_depth; i++)115 for (i = 0; i < srv->queue_depth; i++)
116 free_id(sess->ops_ids[i]);116 free_id(srv_path->ops_ids[i]);
117 kfree(sess->ops_ids);117 kfree(srv_path->ops_ids);
118 sess->ops_ids = NULL;118 srv_path->ops_ids = NULL;
119 }119 }
120}120}
121 121
@@ -127,21 +127,24 @@ static struct ib_cqe io_comp_cqe = { · 5 unmodified lines
127 127
128static inline void rtrs_srv_inflight_ref_release(struct percpu_ref *ref)128static inline void rtrs_srv_inflight_ref_release(struct percpu_ref *ref)
129{129{
130 struct rtrs_srv_sess *sess = container_of(ref, struct rtrs_srv_sess, ids_inflight_ref);130 struct rtrs_srv_path *srv_path = container_of(ref,
131 struct rtrs_srv_path,
132 ids_inflight_ref);
131 133
132 percpu_ref_exit(&sess->ids_inflight_ref);134 percpu_ref_exit(&srv_path->ids_inflight_ref);
133 complete(&sess->complete_done);135 complete(&srv_path->complete_done);
134}136}
135 137
136static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess)138static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_path *srv_path)
137{139{
138 struct rtrs_srv *srv = sess->srv;140 struct rtrs_srv_sess *srv = srv_path->srv;
139 struct rtrs_srv_op *id;141 struct rtrs_srv_op *id;
140 int i, ret;142 int i, ret;
141 143
142 sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids),144 srv_path->ops_ids = kcalloc(srv->queue_depth,
143 GFP_KERNEL);145 sizeof(*srv_path->ops_ids),
144 if (!sess->ops_ids)146 GFP_KERNEL);
147 if (!srv_path->ops_ids)
145 goto err;148 goto err;
146 149
147 for (i = 0; i < srv->queue_depth; ++i) {150 for (i = 0; i < srv->queue_depth; ++i) {
@@ -149,44 +152,44 @@ static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess) · 1 unmodified line
149 if (!id)152 if (!id)
150 goto err;153 goto err;
151 154
152 sess->ops_ids[i] = id;155 srv_path->ops_ids[i] = id;
153 }156 }
154 157
155 ret = percpu_ref_init(&sess->ids_inflight_ref,158 ret = percpu_ref_init(&srv_path->ids_inflight_ref,
156 rtrs_srv_inflight_ref_release, 0, GFP_KERNEL);159 rtrs_srv_inflight_ref_release, 0, GFP_KERNEL);
157 if (ret) {160 if (ret) {
158 pr_err("Percpu reference init failed\n");161 pr_err("Percpu reference init failed\n");
159 goto err;162 goto err;
160 }163 }
161 init_completion(&sess->complete_done);164 init_completion(&srv_path->complete_done);
162 165
163 return 0;166 return 0;
164 167
165err:168err:
166 rtrs_srv_free_ops_ids(sess);169 rtrs_srv_free_ops_ids(srv_path);
167 return -ENOMEM;170 return -ENOMEM;
168}171}
169 172
170static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_sess *sess)173static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_path *srv_path)
171{174{
172 percpu_ref_get(&sess->ids_inflight_ref);175 percpu_ref_get(&srv_path->ids_inflight_ref);
173}176}
174 177
175static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_sess *sess)178static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_path *srv_path)
176{179{
177 percpu_ref_put(&sess->ids_inflight_ref);180 percpu_ref_put(&srv_path->ids_inflight_ref);
178}181}
179 182
180static void rtrs_srv_reg_mr_done(struct ib_cq *cq, struct ib_wc *wc)183static void rtrs_srv_reg_mr_done(struct ib_cq *cq, struct ib_wc *wc)
181{184{
182 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);185 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);
183 struct rtrs_sess *s = con->c.sess;186 struct rtrs_path *s = con->c.path;
184 struct rtrs_srv_sess *sess = to_srv_sess(s);187 struct rtrs_srv_path *srv_path = to_srv_path(s);
185 188
186 if (wc->status != IB_WC_SUCCESS) {189 if (wc->status != IB_WC_SUCCESS) {
187 rtrs_err(s, "REG MR failed: %s\n",190 rtrs_err(s, "REG MR failed: %s\n",
188 ib_wc_status_msg(wc->status));191 ib_wc_status_msg(wc->status));
189 close_sess(sess);192 close_path(srv_path);
190 return;193 return;
191 }194 }
192}195}
@@ -197,9 +200,9 @@ static struct ib_cqe local_reg_cqe = { · 4 unmodified lines
197 200
198static int rdma_write_sg(struct rtrs_srv_op *id)201static int rdma_write_sg(struct rtrs_srv_op *id)
199{202{
200 struct rtrs_sess *s = id->con->c.sess;203 struct rtrs_path *s = id->con->c.path;
201 struct rtrs_srv_sess *sess = to_srv_sess(s);204 struct rtrs_srv_path *srv_path = to_srv_path(s);
202 dma_addr_t dma_addr = sess->dma_addr[id->msg_id];205 dma_addr_t dma_addr = srv_path->dma_addr[id->msg_id];
203 struct rtrs_srv_mr *srv_mr;206 struct rtrs_srv_mr *srv_mr;
204 struct ib_send_wr inv_wr;207 struct ib_send_wr inv_wr;
205 struct ib_rdma_wr imm_wr;208 struct ib_rdma_wr imm_wr;
@@ -233,7 +236,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) · 27 unmodified lines
233 return -EINVAL;236 return -EINVAL;
234 }237 }
235 238
236 plist->lkey = sess->s.dev->ib_pd->local_dma_lkey;239 plist->lkey = srv_path->s.dev->ib_pd->local_dma_lkey;
237 offset += plist->length;240 offset += plist->length;
238 241
239 wr->wr.sg_list = plist;242 wr->wr.sg_list = plist;
@@ -284,7 +287,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) · 44 unmodified lines
284 if (always_invalidate) {287 if (always_invalidate) {
285 struct rtrs_msg_rkey_rsp *msg;288 struct rtrs_msg_rkey_rsp *msg;
286 289
287 srv_mr = &sess->mrs[id->msg_id];290 srv_mr = &srv_path->mrs[id->msg_id];
288 rwr.wr.opcode = IB_WR_REG_MR;291 rwr.wr.opcode = IB_WR_REG_MR;
289 rwr.wr.wr_cqe = &local_reg_cqe;292 rwr.wr.wr_cqe = &local_reg_cqe;
290 rwr.wr.num_sge = 0;293 rwr.wr.num_sge = 0;
@@ -300,11 +303,11 @@ static int rdma_write_sg(struct rtrs_srv_op *id) · 9 unmodified lines
300 303
301 list.addr = srv_mr->iu->dma_addr;304 list.addr = srv_mr->iu->dma_addr;
302 list.length = sizeof(*msg);305 list.length = sizeof(*msg);
303 list.lkey = sess->s.dev->ib_pd->local_dma_lkey;306 list.lkey = srv_path->s.dev->ib_pd->local_dma_lkey;
304 imm_wr.wr.sg_list = &list;307 imm_wr.wr.sg_list = &list;
305 imm_wr.wr.num_sge = 1;308 imm_wr.wr.num_sge = 1;
306 imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM;309 imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM;
307 ib_dma_sync_single_for_device(sess->s.dev->ib_dev,310 ib_dma_sync_single_for_device(srv_path->s.dev->ib_dev,
308 srv_mr->iu->dma_addr,311 srv_mr->iu->dma_addr,
309 srv_mr->iu->size, DMA_TO_DEVICE);312 srv_mr->iu->size, DMA_TO_DEVICE);
310 } else {313 } else {
@@ -317,7 +320,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) · 6 unmodified lines
317 0, need_inval));320 0, need_inval));
318 321
319 imm_wr.wr.wr_cqe = &io_comp_cqe;322 imm_wr.wr.wr_cqe = &io_comp_cqe;
320 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, dma_addr,323 ib_dma_sync_single_for_device(srv_path->s.dev->ib_dev, dma_addr,
321 offset, DMA_BIDIRECTIONAL);324 offset, DMA_BIDIRECTIONAL);
322 325
323 err = ib_post_send(id->con->c.qp, &id->tx_wr.wr, NULL);326 err = ib_post_send(id->con->c.qp, &id->tx_wr.wr, NULL);
@@ -341,8 +344,8 @@ static int rdma_write_sg(struct rtrs_srv_op *id) · 17 unmodified lines
341static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id,344static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id,
342 int errno)345 int errno)
343{346{
344 struct rtrs_sess *s = con->c.sess;347 struct rtrs_path *s = con->c.path;
345 struct rtrs_srv_sess *sess = to_srv_sess(s);348 struct rtrs_srv_path *srv_path = to_srv_path(s);
346 struct ib_send_wr inv_wr, *wr = NULL;349 struct ib_send_wr inv_wr, *wr = NULL;
347 struct ib_rdma_wr imm_wr;350 struct ib_rdma_wr imm_wr;
348 struct ib_reg_wr rwr;351 struct ib_reg_wr rwr;
@@ -402,7 +405,7 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, · 53 unmodified lines
402 struct ib_sge list;405 struct ib_sge list;
403 struct rtrs_msg_rkey_rsp *msg;406 struct rtrs_msg_rkey_rsp *msg;
404 407
405 srv_mr = &sess->mrs[id->msg_id];408 srv_mr = &srv_path->mrs[id->msg_id];
406 rwr.wr.next = &imm_wr.wr;409 rwr.wr.next = &imm_wr.wr;
407 rwr.wr.opcode = IB_WR_REG_MR;410 rwr.wr.opcode = IB_WR_REG_MR;
408 rwr.wr.wr_cqe = &local_reg_cqe;411 rwr.wr.wr_cqe = &local_reg_cqe;
@@ -419,11 +422,11 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, · 10 unmodified lines
419 422
420 list.addr = srv_mr->iu->dma_addr;423 list.addr = srv_mr->iu->dma_addr;
421 list.length = sizeof(*msg);424 list.length = sizeof(*msg);
422 list.lkey = sess->s.dev->ib_pd->local_dma_lkey;425 list.lkey = srv_path->s.dev->ib_pd->local_dma_lkey;
423 imm_wr.wr.sg_list = &list;426 imm_wr.wr.sg_list = &list;
424 imm_wr.wr.num_sge = 1;427 imm_wr.wr.num_sge = 1;
425 imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM;428 imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM;
426 ib_dma_sync_single_for_device(sess->s.dev->ib_dev,429 ib_dma_sync_single_for_device(srv_path->s.dev->ib_dev,
427 srv_mr->iu->dma_addr,430 srv_mr->iu->dma_addr,
428 srv_mr->iu->size, DMA_TO_DEVICE);431 srv_mr->iu->size, DMA_TO_DEVICE);
429 } else {432 } else {
@@ -444,11 +447,11 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, · 14 unmodified lines
444 return err;447 return err;
445}448}
446 449
447void close_sess(struct rtrs_srv_sess *sess)450void close_path(struct rtrs_srv_path *srv_path)
448{451{
449 if (rtrs_srv_change_state(sess, RTRS_SRV_CLOSING))452 if (rtrs_srv_change_state(srv_path, RTRS_SRV_CLOSING))
450 queue_work(rtrs_wq, &sess->close_work);453 queue_work(rtrs_wq, &srv_path->close_work);
451 WARN_ON(sess->state != RTRS_SRV_CLOSING);454 WARN_ON(srv_path->state != RTRS_SRV_CLOSING);
452}455}
453 456
454static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state)457static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state)
@@ -480,35 +483,35 @@ static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state) · 25 unmodified lines
480 */483 */
481bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status)484bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status)
482{485{
483 struct rtrs_srv_sess *sess;486 struct rtrs_srv_path *srv_path;
484 struct rtrs_srv_con *con;487 struct rtrs_srv_con *con;
485 struct rtrs_sess *s;488 struct rtrs_path *s;
486 int err;489 int err;
487 490
488 if (WARN_ON(!id))491 if (WARN_ON(!id))
489 return true;492 return true;
490 493
491 con = id->con;494 con = id->con;
492 s = con->c.sess;495 s = con->c.path;
493 sess = to_srv_sess(s);496 srv_path = to_srv_path(s);
494 497
495 id->status = status;498 id->status = status;
496 499
497 if (sess->state != RTRS_SRV_CONNECTED) {500 if (srv_path->state != RTRS_SRV_CONNECTED) {
498 rtrs_err_rl(s,501 rtrs_err_rl(s,
499 "Sending I/O response failed, session %s is disconnected, sess state %s\n",502 "Sending I/O response failed, server path %s is disconnected, path state %s\n",
500 kobject_name(&sess->kobj),503 kobject_name(&srv_path->kobj),
501 rtrs_srv_state_str(sess->state));504 rtrs_srv_state_str(srv_path->state));
502 goto out;505 goto out;
503 }506 }
504 if (always_invalidate) {507 if (always_invalidate) {
505 struct rtrs_srv_mr *mr = &sess->mrs[id->msg_id];508 struct rtrs_srv_mr *mr = &srv_path->mrs[id->msg_id];
506 509
507 ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey));510 ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey));
508 }511 }
509 if (atomic_sub_return(1, &con->c.sq_wr_avail) < 0) {512 if (atomic_sub_return(1, &con->c.sq_wr_avail) < 0) {
510 rtrs_err(s, "IB send queue full: sess=%s cid=%d\n",513 rtrs_err(s, "IB send queue full: srv_path=%s cid=%d\n",
511 kobject_name(&sess->kobj),514 kobject_name(&srv_path->kobj),
512 con->c.cid);515 con->c.cid);
513 atomic_add(1, &con->c.sq_wr_avail);516 atomic_add(1, &con->c.sq_wr_avail);
514 spin_lock(&con->rsp_wr_wait_lock);517 spin_lock(&con->rsp_wr_wait_lock);
@@ -523,12 +526,12 @@ bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status) · 8 unmodified lines
523 err = rdma_write_sg(id);526 err = rdma_write_sg(id);
524 527
525 if (err) {528 if (err) {
526 rtrs_err_rl(s, "IO response failed: %d: sess=%s\n", err,529 rtrs_err_rl(s, "IO response failed: %d: srv_path=%s\n", err,
527 kobject_name(&sess->kobj));530 kobject_name(&srv_path->kobj));
528 close_sess(sess);531 close_path(srv_path);
529 }532 }
530out:533out:
531 rtrs_srv_put_ops_ids(sess);534 rtrs_srv_put_ops_ids(srv_path);
532 return true;535 return true;
533}536}
534EXPORT_SYMBOL(rtrs_srv_resp_rdma);537EXPORT_SYMBOL(rtrs_srv_resp_rdma);
@@ -538,33 +541,33 @@ EXPORT_SYMBOL(rtrs_srv_resp_rdma); · 3 unmodified lines
538 * @srv: Session pointer541 * @srv: Session pointer
539 * @priv: The private pointer that is associated with the session.542 * @priv: The private pointer that is associated with the session.
540 */543 */
541void rtrs_srv_set_sess_priv(struct rtrs_srv *srv, void *priv)544void rtrs_srv_set_sess_priv(struct rtrs_srv_sess *srv, void *priv)
542{545{
543 srv->priv = priv;546 srv->priv = priv;
544}547}
545EXPORT_SYMBOL(rtrs_srv_set_sess_priv);548EXPORT_SYMBOL(rtrs_srv_set_sess_priv);
546 549
547static void unmap_cont_bufs(struct rtrs_srv_sess *sess)550static void unmap_cont_bufs(struct rtrs_srv_path *srv_path)
548{551{
549 int i;552 int i;
550 553
551 for (i = 0; i < sess->mrs_num; i++) {554 for (i = 0; i < srv_path->mrs_num; i++) {
552 struct rtrs_srv_mr *srv_mr;555 struct rtrs_srv_mr *srv_mr;
553 556
554 srv_mr = &sess->mrs[i];557 srv_mr = &srv_path->mrs[i];
555 rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1);558 rtrs_iu_free(srv_mr->iu, srv_path->s.dev->ib_dev, 1);
556 ib_dereg_mr(srv_mr->mr);559 ib_dereg_mr(srv_mr->mr);
557 ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl,560 ib_dma_unmap_sg(srv_path->s.dev->ib_dev, srv_mr->sgt.sgl,
558 srv_mr->sgt.nents, DMA_BIDIRECTIONAL);561 srv_mr->sgt.nents, DMA_BIDIRECTIONAL);
559 sg_free_table(&srv_mr->sgt);562 sg_free_table(&srv_mr->sgt);
560 }563 }
561 kfree(sess->mrs);564 kfree(srv_path->mrs);
562}565}
563 566
564static int map_cont_bufs(struct rtrs_srv_sess *sess)567static int map_cont_bufs(struct rtrs_srv_path *srv_path)
565{568{
566 struct rtrs_srv *srv = sess->srv;569 struct rtrs_srv_sess *srv = srv_path->srv;
567 struct rtrs_sess *ss = &sess->s;570 struct rtrs_path *ss = &srv_path->s;
568 int i, mri, err, mrs_num;571 int i, mri, err, mrs_num;
569 unsigned int chunk_bits;572 unsigned int chunk_bits;
570 int chunks_per_mr = 1;573 int chunks_per_mr = 1;
@@ -581,19 +584,19 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) · 10 unmodified lines
581 mrs_num = srv->queue_depth;584 mrs_num = srv->queue_depth;
582 } else {585 } else {
583 chunks_per_mr =586 chunks_per_mr =
584 sess->s.dev->ib_dev->attrs.max_fast_reg_page_list_len;587 srv_path->s.dev->ib_dev->attrs.max_fast_reg_page_list_len;
585 mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr);588 mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr);
586 chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num);589 chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num);
587 }590 }
588 591
589 sess->mrs = kcalloc(mrs_num, sizeof(*sess->mrs), GFP_KERNEL);592 srv_path->mrs = kcalloc(mrs_num, sizeof(*srv_path->mrs), GFP_KERNEL);
590 if (!sess->mrs)593 if (!srv_path->mrs)
591 return -ENOMEM;594 return -ENOMEM;
592 595
593 sess->mrs_num = mrs_num;596 srv_path->mrs_num = mrs_num;
594 597
595 for (mri = 0; mri < mrs_num; mri++) {598 for (mri = 0; mri < mrs_num; mri++) {
596 struct rtrs_srv_mr *srv_mr = &sess->mrs[mri];599 struct rtrs_srv_mr *srv_mr = &srv_path->mrs[mri];
597 struct sg_table *sgt = &srv_mr->sgt;600 struct sg_table *sgt = &srv_mr->sgt;
598 struct scatterlist *s;601 struct scatterlist *s;
599 struct ib_mr *mr;602 struct ib_mr *mr;
@@ -612,13 +615,13 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) · 12 unmodified lines
612 sg_set_page(s, srv->chunks[chunks + i],615 sg_set_page(s, srv->chunks[chunks + i],
613 max_chunk_size, 0);616 max_chunk_size, 0);
614 617
615 nr = ib_dma_map_sg(sess->s.dev->ib_dev, sgt->sgl,618 nr = ib_dma_map_sg(srv_path->s.dev->ib_dev, sgt->sgl,
616 sgt->nents, DMA_BIDIRECTIONAL);619 sgt->nents, DMA_BIDIRECTIONAL);
617 if (nr < sgt->nents) {620 if (nr < sgt->nents) {
618 err = nr < 0 ? nr : -EINVAL;621 err = nr < 0 ? nr : -EINVAL;
619 goto free_sg;622 goto free_sg;
620 }623 }
621 mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG,624 mr = ib_alloc_mr(srv_path->s.dev->ib_pd, IB_MR_TYPE_MEM_REG,
622 sgt->nents);625 sgt->nents);
623 if (IS_ERR(mr)) {626 if (IS_ERR(mr)) {
624 err = PTR_ERR(mr);627 err = PTR_ERR(mr);
@@ -634,7 +637,7 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) · 9 unmodified lines
634 if (always_invalidate) {637 if (always_invalidate) {
635 srv_mr->iu = rtrs_iu_alloc(1,638 srv_mr->iu = rtrs_iu_alloc(1,
636 sizeof(struct rtrs_msg_rkey_rsp),639 sizeof(struct rtrs_msg_rkey_rsp),
637 GFP_KERNEL, sess->s.dev->ib_dev,640 GFP_KERNEL, srv_path->s.dev->ib_dev,
638 DMA_TO_DEVICE, rtrs_srv_rdma_done);641 DMA_TO_DEVICE, rtrs_srv_rdma_done);
639 if (!srv_mr->iu) {642 if (!srv_mr->iu) {
640 err = -ENOMEM;643 err = -ENOMEM;
@@ -644,7 +647,7 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) · 3 unmodified lines
644 }647 }
645 /* Eventually dma addr for each chunk can be cached */648 /* Eventually dma addr for each chunk can be cached */
646 for_each_sg(sgt->sgl, s, sgt->orig_nents, i)649 for_each_sg(sgt->sgl, s, sgt->orig_nents, i)
647 sess->dma_addr[chunks + i] = sg_dma_address(s);650 srv_path->dma_addr[chunks + i] = sg_dma_address(s);
648 651
649 ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));652 ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
650 srv_mr->mr = mr;653 srv_mr->mr = mr;
@@ -652,75 +655,75 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) · 1 unmodified line
652 continue;655 continue;
653err:656err:
654 while (mri--) {657 while (mri--) {
655 srv_mr = &sess->mrs[mri];658 srv_mr = &srv_path->mrs[mri];
656 sgt = &srv_mr->sgt;659 sgt = &srv_mr->sgt;
657 mr = srv_mr->mr;660 mr = srv_mr->mr;
658 rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1);661 rtrs_iu_free(srv_mr->iu, srv_path->s.dev->ib_dev, 1);
659dereg_mr:662dereg_mr:
660 ib_dereg_mr(mr);663 ib_dereg_mr(mr);
661unmap_sg:664unmap_sg:
662 ib_dma_unmap_sg(sess->s.dev->ib_dev, sgt->sgl,665 ib_dma_unmap_sg(srv_path->s.dev->ib_dev, sgt->sgl,
663 sgt->nents, DMA_BIDIRECTIONAL);666 sgt->nents, DMA_BIDIRECTIONAL);
664free_sg:667free_sg:
665 sg_free_table(sgt);668 sg_free_table(sgt);
666 }669 }
667 kfree(sess->mrs);670 kfree(srv_path->mrs);
668 671
669 return err;672 return err;
670 }673 }
671 674
672 chunk_bits = ilog2(srv->queue_depth - 1) + 1;675 chunk_bits = ilog2(srv->queue_depth - 1) + 1;
673 sess->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits);676 srv_path->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits);
674 677
675 return 0;678 return 0;
676}679}
677 680
678static void rtrs_srv_hb_err_handler(struct rtrs_con *c)681static void rtrs_srv_hb_err_handler(struct rtrs_con *c)
679{682{
680 close_sess(to_srv_sess(c->sess));683 close_path(to_srv_path(c->path));
681}684}
682 685
683static void rtrs_srv_init_hb(struct rtrs_srv_sess *sess)686static void rtrs_srv_init_hb(struct rtrs_srv_path *srv_path)
684{687{
685 rtrs_init_hb(&sess->s, &io_comp_cqe,688 rtrs_init_hb(&srv_path->s, &io_comp_cqe,
686 RTRS_HB_INTERVAL_MS,689 RTRS_HB_INTERVAL_MS,
687 RTRS_HB_MISSED_MAX,690 RTRS_HB_MISSED_MAX,
688 rtrs_srv_hb_err_handler,691 rtrs_srv_hb_err_handler,
689 rtrs_wq);692 rtrs_wq);
690}693}
691 694
692static void rtrs_srv_start_hb(struct rtrs_srv_sess *sess)695static void rtrs_srv_start_hb(struct rtrs_srv_path *srv_path)
693{696{
694 rtrs_start_hb(&sess->s);697 rtrs_start_hb(&srv_path->s);
695}698}
696 699
697static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess)700static void rtrs_srv_stop_hb(struct rtrs_srv_path *srv_path)
698{701{
699 rtrs_stop_hb(&sess->s);702 rtrs_stop_hb(&srv_path->s);
700}703}
701 704
702static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc)705static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc)
703{706{
704 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);707 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);
705 struct rtrs_sess *s = con->c.sess;708 struct rtrs_path *s = con->c.path;
706 struct rtrs_srv_sess *sess = to_srv_sess(s);709 struct rtrs_srv_path *srv_path = to_srv_path(s);
707 struct rtrs_iu *iu;710 struct rtrs_iu *iu;
708 711
709 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);712 iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
710 rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);713 rtrs_iu_free(iu, srv_path->s.dev->ib_dev, 1);
711 714
712 if (wc->status != IB_WC_SUCCESS) {715 if (wc->status != IB_WC_SUCCESS) {
713 rtrs_err(s, "Sess info response send failed: %s\n",716 rtrs_err(s, "Sess info response send failed: %s\n",
714 ib_wc_status_msg(wc->status));717 ib_wc_status_msg(wc->status));
715 close_sess(sess);718 close_path(srv_path);
716 return;719 return;
717 }720 }
718 WARN_ON(wc->opcode != IB_WC_SEND);721 WARN_ON(wc->opcode != IB_WC_SEND);
719}722}
720 723
721static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess)724static void rtrs_srv_path_up(struct rtrs_srv_path *srv_path)
722{725{
723 struct rtrs_srv *srv = sess->srv;726 struct rtrs_srv_sess *srv = srv_path->srv;
724 struct rtrs_srv_ctx *ctx = srv->ctx;727 struct rtrs_srv_ctx *ctx = srv->ctx;
725 int up;728 int up;
726 729
@@ -731,18 +734,18 @@ static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess) · 4 unmodified lines
731 mutex_unlock(&srv->paths_ev_mutex);734 mutex_unlock(&srv->paths_ev_mutex);
732 735
733 /* Mark session as established */736 /* Mark session as established */
734 sess->established = true;737 srv_path->established = true;
735}738}
736 739
737static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess)740static void rtrs_srv_path_down(struct rtrs_srv_path *srv_path)
738{741{
739 struct rtrs_srv *srv = sess->srv;742 struct rtrs_srv_sess *srv = srv_path->srv;
740 struct rtrs_srv_ctx *ctx = srv->ctx;743 struct rtrs_srv_ctx *ctx = srv->ctx;
741 744
742 if (!sess->established)745 if (!srv_path->established)
743 return;746 return;
744 747
745 sess->established = false;748 srv_path->established = false;
746 mutex_lock(&srv->paths_ev_mutex);749 mutex_lock(&srv->paths_ev_mutex);
747 WARN_ON(!srv->paths_up);750 WARN_ON(!srv->paths_up);
748 if (--srv->paths_up == 0)751 if (--srv->paths_up == 0)
@@ -750,11 +753,11 @@ static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess) · 1 unmodified line
750 mutex_unlock(&srv->paths_ev_mutex);753 mutex_unlock(&srv->paths_ev_mutex);
751}754}
752 755
753static bool exist_sessname(struct rtrs_srv_ctx *ctx,756static bool exist_pathname(struct rtrs_srv_ctx *ctx,
754 const char *sessname, const uuid_t *path_uuid)757 const char *pathname, const uuid_t *path_uuid)
755{758{
756 struct rtrs_srv *srv;759 struct rtrs_srv_sess *srv;
757 struct rtrs_srv_sess *sess;760 struct rtrs_srv_path *srv_path;
758 bool found = false;761 bool found = false;
759 762
760 mutex_lock(&ctx->srv_mutex);763 mutex_lock(&ctx->srv_mutex);
@@ -767,9 +770,9 @@ static bool exist_sessname(struct rtrs_srv_ctx *ctx, · 6 unmodified lines
767 continue;770 continue;
768 }771 }
769 772
770 list_for_each_entry(sess, &srv->paths_list, s.entry) {773 list_for_each_entry(srv_path, &srv->paths_list, s.entry) {
771 if (strlen(sess->s.sessname) == strlen(sessname) &&774 if (strlen(srv_path->s.sessname) == strlen(pathname) &&
772 !strcmp(sess->s.sessname, sessname)) {775 !strcmp(srv_path->s.sessname, pathname)) {
773 found = true;776 found = true;
774 break;777 break;
775 }778 }
@@ -782,14 +785,14 @@ static bool exist_sessname(struct rtrs_srv_ctx *ctx, · 6 unmodified lines
782 return found;785 return found;
783}786}
784 787
785static int post_recv_sess(struct rtrs_srv_sess *sess);788static int post_recv_path(struct rtrs_srv_path *srv_path);
786static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno);789static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno);
787 790
788static int process_info_req(struct rtrs_srv_con *con,791static int process_info_req(struct rtrs_srv_con *con,
789 struct rtrs_msg_info_req *msg)792 struct rtrs_msg_info_req *msg)
790{793{
791 struct rtrs_sess *s = con->c.sess;794 struct rtrs_path *s = con->c.path;
792 struct rtrs_srv_sess *sess = to_srv_sess(s);795 struct rtrs_srv_path *srv_path = to_srv_path(s);
793 struct ib_send_wr *reg_wr = NULL;796 struct ib_send_wr *reg_wr = NULL;
794 struct rtrs_msg_info_rsp *rsp;797 struct rtrs_msg_info_rsp *rsp;
795 struct rtrs_iu *tx_iu;798 struct rtrs_iu *tx_iu;
@@ -797,31 +800,32 @@ static int process_info_req(struct rtrs_srv_con *con, · 1 unmodified line
797 int mri, err;800 int mri, err;
798 size_t tx_sz;801 size_t tx_sz;
799 802
800 err = post_recv_sess(sess);803 err = post_recv_path(srv_path);
801 if (err) {804 if (err) {
802 rtrs_err(s, "post_recv_sess(), err: %d\n", err);805 rtrs_err(s, "post_recv_path(), err: %d\n", err);
803 return err;806 return err;
804 }807 }
805 808
806 if (strchr(msg->sessname, '/') || strchr(msg->sessname, '.')) {809 if (strchr(msg->pathname, '/') || strchr(msg->pathname, '.')) {
807 rtrs_err(s, "sessname cannot contain / and .\n");810 rtrs_err(s, "pathname cannot contain / and .\n");
808 return -EINVAL;811 return -EINVAL;
809 }812 }
810 813
811 if (exist_sessname(sess->srv->ctx,814 if (exist_pathname(srv_path->srv->ctx,
812 msg->sessname, &sess->srv->paths_uuid)) {815 msg->pathname, &srv_path->srv->paths_uuid)) {
813 rtrs_err(s, "sessname is duplicated: %s\n", msg->sessname);816 rtrs_err(s, "pathname is duplicated: %s\n", msg->pathname);
814 return -EPERM;817 return -EPERM;
815 }818 }
816 strscpy(sess->s.sessname, msg->sessname, sizeof(sess->s.sessname));819 strscpy(srv_path->s.sessname, msg->pathname,
820 sizeof(srv_path->s.sessname));
817 821
818 rwr = kcalloc(sess->mrs_num, sizeof(*rwr), GFP_KERNEL);822 rwr = kcalloc(srv_path->mrs_num, sizeof(*rwr), GFP_KERNEL);
819 if (!rwr)823 if (!rwr)
820 return -ENOMEM;824 return -ENOMEM;
821 825
822 tx_sz = sizeof(*rsp);826 tx_sz = sizeof(*rsp);
823 tx_sz += sizeof(rsp->desc[0]) * sess->mrs_num;827 tx_sz += sizeof(rsp->desc[0]) * srv_path->mrs_num;
824 tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, sess->s.dev->ib_dev,828 tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, srv_path->s.dev->ib_dev,
825 DMA_TO_DEVICE, rtrs_srv_info_rsp_done);829 DMA_TO_DEVICE, rtrs_srv_info_rsp_done);
826 if (!tx_iu) {830 if (!tx_iu) {
827 err = -ENOMEM;831 err = -ENOMEM;
@@ -830,10 +834,10 @@ static int process_info_req(struct rtrs_srv_con *con, · 2 unmodified lines
830 834
831 rsp = tx_iu->buf;835 rsp = tx_iu->buf;
832 rsp->type = cpu_to_le16(RTRS_MSG_INFO_RSP);836 rsp->type = cpu_to_le16(RTRS_MSG_INFO_RSP);
833 rsp->sg_cnt = cpu_to_le16(sess->mrs_num);837 rsp->sg_cnt = cpu_to_le16(srv_path->mrs_num);
834 838
835 for (mri = 0; mri < sess->mrs_num; mri++) {839 for (mri = 0; mri < srv_path->mrs_num; mri++) {
836 struct ib_mr *mr = sess->mrs[mri].mr;840 struct ib_mr *mr = srv_path->mrs[mri].mr;
837 841
838 rsp->desc[mri].addr = cpu_to_le64(mr->iova);842 rsp->desc[mri].addr = cpu_to_le64(mr->iova);
839 rsp->desc[mri].key = cpu_to_le32(mr->rkey);843 rsp->desc[mri].key = cpu_to_le32(mr->rkey);
@@ -854,13 +858,13 @@ static int process_info_req(struct rtrs_srv_con *con, · 14 unmodified lines
854 reg_wr = &rwr[mri].wr;858 reg_wr = &rwr[mri].wr;
855 }859 }
856 860
857 err = rtrs_srv_create_sess_files(sess);861 err = rtrs_srv_create_path_files(srv_path);
858 if (err)862 if (err)
859 goto iu_free;863 goto iu_free;
860 kobject_get(&sess->kobj);864 kobject_get(&srv_path->kobj);
861 get_device(&sess->srv->dev);865 get_device(&srv_path->srv->dev);
862 rtrs_srv_change_state(sess, RTRS_SRV_CONNECTED);866 rtrs_srv_change_state(srv_path, RTRS_SRV_CONNECTED);
863 rtrs_srv_start_hb(sess);867 rtrs_srv_start_hb(srv_path);
864 868
865 /*869 /*
866 * We do not account number of established connections at the current870 * We do not account number of established connections at the current
@@ -868,9 +872,10 @@ static int process_info_req(struct rtrs_srv_con *con, · 1 unmodified line
868 * all connections are successfully established. Thus, simply notify872 * all connections are successfully established. Thus, simply notify
869 * listener with a proper event if we are the first path.873 * listener with a proper event if we are the first path.
870 */874 */
871 rtrs_srv_sess_up(sess);875 rtrs_srv_path_up(srv_path);
872 876
873 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr,877 ib_dma_sync_single_for_device(srv_path->s.dev->ib_dev,
878 tx_iu->dma_addr,
874 tx_iu->size, DMA_TO_DEVICE);879 tx_iu->size, DMA_TO_DEVICE);
875 880
876 /* Send info response */881 /* Send info response */
@@ -878,7 +883,7 @@ static int process_info_req(struct rtrs_srv_con *con, · 1 unmodified line
878 if (err) {883 if (err) {
879 rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err);884 rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err);
880iu_free:885iu_free:
881 rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1);886 rtrs_iu_free(tx_iu, srv_path->s.dev->ib_dev, 1);
882 }887 }
883rwr_free:888rwr_free:
884 kfree(rwr);889 kfree(rwr);
@@ -889,8 +894,8 @@ rwr_free: · 4 unmodified lines
889static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc)894static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc)
890{895{
891 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);896 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);
892 struct rtrs_sess *s = con->c.sess;897 struct rtrs_path *s = con->c.path;
893 struct rtrs_srv_sess *sess = to_srv_sess(s);898 struct rtrs_srv_path *srv_path = to_srv_path(s);
894 struct rtrs_msg_info_req *msg;899 struct rtrs_msg_info_req *msg;
895 struct rtrs_iu *iu;900 struct rtrs_iu *iu;
896 int err;901 int err;
@@ -910,7 +915,7 @@ static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) · 13 unmodified lines
910 wc->byte_len);915 wc->byte_len);
911 goto close;916 goto close;
912 }917 }
913 ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr,918 ib_dma_sync_single_for_cpu(srv_path->s.dev->ib_dev, iu->dma_addr,
914 iu->size, DMA_FROM_DEVICE);919 iu->size, DMA_FROM_DEVICE);
915 msg = iu->buf;920 msg = iu->buf;
916 if (le16_to_cpu(msg->type) != RTRS_MSG_INFO_REQ) {921 if (le16_to_cpu(msg->type) != RTRS_MSG_INFO_REQ) {
@@ -923,22 +928,22 @@ static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) · 6 unmodified lines
923 goto close;928 goto close;
924 929
925out:930out:
926 rtrs_iu_free(iu, sess->s.dev->ib_dev, 1);931 rtrs_iu_free(iu, srv_path->s.dev->ib_dev, 1);
927 return;932 return;
928close:933close:
929 close_sess(sess);934 close_path(srv_path);
930 goto out;935 goto out;
931}936}
932 937
933static int post_recv_info_req(struct rtrs_srv_con *con)938static int post_recv_info_req(struct rtrs_srv_con *con)
934{939{
935 struct rtrs_sess *s = con->c.sess;940 struct rtrs_path *s = con->c.path;
936 struct rtrs_srv_sess *sess = to_srv_sess(s);941 struct rtrs_srv_path *srv_path = to_srv_path(s);
937 struct rtrs_iu *rx_iu;942 struct rtrs_iu *rx_iu;
938 int err;943 int err;
939 944
940 rx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req),945 rx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req),
941 GFP_KERNEL, sess->s.dev->ib_dev,946 GFP_KERNEL, srv_path->s.dev->ib_dev,
942 DMA_FROM_DEVICE, rtrs_srv_info_req_done);947 DMA_FROM_DEVICE, rtrs_srv_info_req_done);
943 if (!rx_iu)948 if (!rx_iu)
944 return -ENOMEM;949 return -ENOMEM;
@@ -946,7 +951,7 @@ static int post_recv_info_req(struct rtrs_srv_con *con) · 1 unmodified line
946 err = rtrs_iu_post_recv(&con->c, rx_iu);951 err = rtrs_iu_post_recv(&con->c, rx_iu);
947 if (err) {952 if (err) {
948 rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err);953 rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err);
949 rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1);954 rtrs_iu_free(rx_iu, srv_path->s.dev->ib_dev, 1);
950 return err;955 return err;
951 }956 }
952 957
@@ -966,20 +971,20 @@ static int post_recv_io(struct rtrs_srv_con *con, size_t q_size) · 13 unmodified lines
966 return 0;971 return 0;
967}972}
968 973
969static int post_recv_sess(struct rtrs_srv_sess *sess)974static int post_recv_path(struct rtrs_srv_path *srv_path)
970{975{
971 struct rtrs_srv *srv = sess->srv;976 struct rtrs_srv_sess *srv = srv_path->srv;
972 struct rtrs_sess *s = &sess->s;977 struct rtrs_path *s = &srv_path->s;
973 size_t q_size;978 size_t q_size;
974 int err, cid;979 int err, cid;
975 980
976 for (cid = 0; cid < sess->s.con_num; cid++) {981 for (cid = 0; cid < srv_path->s.con_num; cid++) {
977 if (cid == 0)982 if (cid == 0)
978 q_size = SERVICE_CON_QUEUE_DEPTH;983 q_size = SERVICE_CON_QUEUE_DEPTH;
979 else984 else
980 q_size = srv->queue_depth;985 q_size = srv->queue_depth;
981 986
982 err = post_recv_io(to_srv_con(sess->s.con[cid]), q_size);987 err = post_recv_io(to_srv_con(srv_path->s.con[cid]), q_size);
983 if (err) {988 if (err) {
984 rtrs_err(s, "post_recv_io(), err: %d\n", err);989 rtrs_err(s, "post_recv_io(), err: %d\n", err);
985 return err;990 return err;
@@ -993,9 +998,9 @@ static void process_read(struct rtrs_srv_con *con, · 7 unmodified lines
993 struct rtrs_msg_rdma_read *msg,998 struct rtrs_msg_rdma_read *msg,
994 u32 buf_id, u32 off)999 u32 buf_id, u32 off)
995{1000{
996 struct rtrs_sess *s = con->c.sess;1001 struct rtrs_path *s = con->c.path;
997 struct rtrs_srv_sess *sess = to_srv_sess(s);1002 struct rtrs_srv_path *srv_path = to_srv_path(s);
998 struct rtrs_srv *srv = sess->srv;1003 struct rtrs_srv_sess *srv = srv_path->srv;
999 struct rtrs_srv_ctx *ctx = srv->ctx;1004 struct rtrs_srv_ctx *ctx = srv->ctx;
1000 struct rtrs_srv_op *id;1005 struct rtrs_srv_op *id;
1001 1006
@@ -1003,10 +1008,10 @@ static void process_read(struct rtrs_srv_con *con, · 1 unmodified line
1003 void *data;1008 void *data;
1004 int ret;1009 int ret;
1005 1010
1006 if (sess->state != RTRS_SRV_CONNECTED) {1011 if (srv_path->state != RTRS_SRV_CONNECTED) {
1007 rtrs_err_rl(s,1012 rtrs_err_rl(s,
1008 "Processing read request failed, session is disconnected, sess state %s\n",1013 "Processing read request failed, session is disconnected, sess state %s\n",
1009 rtrs_srv_state_str(sess->state));1014 rtrs_srv_state_str(srv_path->state));
1010 return;1015 return;
1011 }1016 }
1012 if (msg->sg_cnt != 1 && msg->sg_cnt != 0) {1017 if (msg->sg_cnt != 1 && msg->sg_cnt != 0) {
@@ -1014,9 +1019,9 @@ static void process_read(struct rtrs_srv_con *con, · 1 unmodified line
1014 "Processing read request failed, invalid message\n");1019 "Processing read request failed, invalid message\n");
1015 return;1020 return;
1016 }1021 }
1017 rtrs_srv_get_ops_ids(sess);1022 rtrs_srv_get_ops_ids(srv_path);
1018 rtrs_srv_update_rdma_stats(sess->stats, off, READ);1023 rtrs_srv_update_rdma_stats(srv_path->stats, off, READ);
1019 id = sess->ops_ids[buf_id];1024 id = srv_path->ops_ids[buf_id];
1020 id->con = con;1025 id->con = con;
1021 id->dir = READ;1026 id->dir = READ;
1022 id->msg_id = buf_id;1027 id->msg_id = buf_id;
@@ -1042,18 +1047,18 @@ send_err_msg: · 19 unmodified lines
1042 rtrs_err_rl(s,1047 rtrs_err_rl(s,
1043 "Sending err msg for failed RDMA-Write-Req failed, msg_id %d, err: %d\n",1048 "Sending err msg for failed RDMA-Write-Req failed, msg_id %d, err: %d\n",
1044 buf_id, ret);1049 buf_id, ret);
1045 close_sess(sess);1050 close_path(srv_path);
1046 }1051 }
1047 rtrs_srv_put_ops_ids(sess);1052 rtrs_srv_put_ops_ids(srv_path);
1048}1053}
1049 1054
1050static void process_write(struct rtrs_srv_con *con,1055static void process_write(struct rtrs_srv_con *con,
1051 struct rtrs_msg_rdma_write *req,1056 struct rtrs_msg_rdma_write *req,
1052 u32 buf_id, u32 off)1057 u32 buf_id, u32 off)
1053{1058{
1054 struct rtrs_sess *s = con->c.sess;1059 struct rtrs_path *s = con->c.path;
1055 struct rtrs_srv_sess *sess = to_srv_sess(s);1060 struct rtrs_srv_path *srv_path = to_srv_path(s);
1056 struct rtrs_srv *srv = sess->srv;1061 struct rtrs_srv_sess *srv = srv_path->srv;
1057 struct rtrs_srv_ctx *ctx = srv->ctx;1062 struct rtrs_srv_ctx *ctx = srv->ctx;
1058 struct rtrs_srv_op *id;1063 struct rtrs_srv_op *id;
1059 1064
@@ -1061,15 +1066,15 @@ static void process_write(struct rtrs_srv_con *con, · 1 unmodified line
1061 void *data;1066 void *data;
1062 int ret;1067 int ret;
1063 1068
1064 if (sess->state != RTRS_SRV_CONNECTED) {1069 if (srv_path->state != RTRS_SRV_CONNECTED) {
1065 rtrs_err_rl(s,1070 rtrs_err_rl(s,
1066 "Processing write request failed, session is disconnected, sess state %s\n",1071 "Processing write request failed, session is disconnected, sess state %s\n",
1067 rtrs_srv_state_str(sess->state));1072 rtrs_srv_state_str(srv_path->state));
1068 return;1073 return;
1069 }1074 }
1070 rtrs_srv_get_ops_ids(sess);1075 rtrs_srv_get_ops_ids(srv_path);
1071 rtrs_srv_update_rdma_stats(sess->stats, off, WRITE);1076 rtrs_srv_update_rdma_stats(srv_path->stats, off, WRITE);
1072 id = sess->ops_ids[buf_id];1077 id = srv_path->ops_ids[buf_id];
1073 id->con = con;1078 id->con = con;
1074 id->dir = WRITE;1079 id->dir = WRITE;
1075 id->msg_id = buf_id;1080 id->msg_id = buf_id;
@@ -1094,20 +1099,21 @@ send_err_msg: · 18 unmodified lines
1094 rtrs_err_rl(s,1099 rtrs_err_rl(s,
1095 "Processing write request failed, sending I/O response failed, msg_id %d, err: %d\n",1100 "Processing write request failed, sending I/O response failed, msg_id %d, err: %d\n",
1096 buf_id, ret);1101 buf_id, ret);
1097 close_sess(sess);1102 close_path(srv_path);
1098 }1103 }
1099 rtrs_srv_put_ops_ids(sess);1104 rtrs_srv_put_ops_ids(srv_path);
1100}1105}
1101 1106
1102static void process_io_req(struct rtrs_srv_con *con, void *msg,1107static void process_io_req(struct rtrs_srv_con *con, void *msg,
1103 u32 id, u32 off)1108 u32 id, u32 off)
1104{1109{
1105 struct rtrs_sess *s = con->c.sess;1110 struct rtrs_path *s = con->c.path;
1106 struct rtrs_srv_sess *sess = to_srv_sess(s);1111 struct rtrs_srv_path *srv_path = to_srv_path(s);
1107 struct rtrs_msg_rdma_hdr *hdr;1112 struct rtrs_msg_rdma_hdr *hdr;
1108 unsigned int type;1113 unsigned int type;
1109 1114
1110 ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, sess->dma_addr[id],1115 ib_dma_sync_single_for_cpu(srv_path->s.dev->ib_dev,
1116 srv_path->dma_addr[id],
1111 max_chunk_size, DMA_BIDIRECTIONAL);1117 max_chunk_size, DMA_BIDIRECTIONAL);
1112 hdr = msg;1118 hdr = msg;
1113 type = le16_to_cpu(hdr->type);1119 type = le16_to_cpu(hdr->type);
@@ -1129,7 +1135,7 @@ static void process_io_req(struct rtrs_srv_con *con, void *msg, · 15 unmodified lines
1129 return;1135 return;
1130 1136
1131err:1137err:
1132 close_sess(sess);1138 close_path(srv_path);
1133}1139}
1134 1140
1135static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)1141static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
@@ -1137,16 +1143,16 @@ static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) · 1 unmodified line
1137 struct rtrs_srv_mr *mr =1143 struct rtrs_srv_mr *mr =
1138 container_of(wc->wr_cqe, typeof(*mr), inv_cqe);1144 container_of(wc->wr_cqe, typeof(*mr), inv_cqe);
1139 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);1145 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);
1140 struct rtrs_sess *s = con->c.sess;1146 struct rtrs_path *s = con->c.path;
1141 struct rtrs_srv_sess *sess = to_srv_sess(s);1147 struct rtrs_srv_path *srv_path = to_srv_path(s);
1142 struct rtrs_srv *srv = sess->srv;1148 struct rtrs_srv_sess *srv = srv_path->srv;
1143 u32 msg_id, off;1149 u32 msg_id, off;
1144 void *data;1150 void *data;
1145 1151
1146 if (wc->status != IB_WC_SUCCESS) {1152 if (wc->status != IB_WC_SUCCESS) {
1147 rtrs_err(s, "Failed IB_WR_LOCAL_INV: %s\n",1153 rtrs_err(s, "Failed IB_WR_LOCAL_INV: %s\n",
1148 ib_wc_status_msg(wc->status));1154 ib_wc_status_msg(wc->status));
1149 close_sess(sess);1155 close_path(srv_path);
1150 }1156 }
1151 msg_id = mr->msg_id;1157 msg_id = mr->msg_id;
1152 off = mr->msg_off;1158 off = mr->msg_off;
@@ -1194,9 +1200,9 @@ static void rtrs_rdma_process_wr_wait_list(struct rtrs_srv_con *con) · 41 unmodified lines
1194static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc)1200static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc)
1195{1201{
1196 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);1202 struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context);
1197 struct rtrs_sess *s = con->c.sess;1203 struct rtrs_path *s = con->c.path;
1198 struct rtrs_srv_sess *sess = to_srv_sess(s);1204 struct rtrs_srv_path *srv_path = to_srv_path(s);
1199 struct rtrs_srv *srv = sess->srv;1205 struct rtrs_srv_sess *srv = srv_path->srv;
1200 u32 imm_type, imm_payload;1206 u32 imm_type, imm_payload;
1201 int err;1207 int err;
1202 1208
@@ -1206,7 +1212,7 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 3 unmodified lines
1206 "%s (wr_cqe: %p, type: %d, vendor_err: 0x%x, len: %u)\n",1212 "%s (wr_cqe: %p, type: %d, vendor_err: 0x%x, len: %u)\n",
1207 ib_wc_status_msg(wc->status), wc->wr_cqe,1213 ib_wc_status_msg(wc->status), wc->wr_cqe,
1208 wc->opcode, wc->vendor_err, wc->byte_len);1214 wc->opcode, wc->vendor_err, wc->byte_len);
1209 close_sess(sess);1215 close_path(srv_path);
1210 }1216 }
1211 return;1217 return;
1212 }1218 }
@@ -1222,7 +1228,7 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 9 unmodified lines
1222 err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);1228 err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);
1223 if (err) {1229 if (err) {
1224 rtrs_err(s, "rtrs_post_recv(), err: %d\n", err);1230 rtrs_err(s, "rtrs_post_recv(), err: %d\n", err);
1225 close_sess(sess);1231 close_path(srv_path);
1226 break;1232 break;
1227 }1233 }
1228 rtrs_from_imm(be32_to_cpu(wc->ex.imm_data),1234 rtrs_from_imm(be32_to_cpu(wc->ex.imm_data),
@@ -1231,16 +1237,16 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 2 unmodified lines
1231 u32 msg_id, off;1237 u32 msg_id, off;
1232 void *data;1238 void *data;
1233 1239
1234 msg_id = imm_payload >> sess->mem_bits;1240 msg_id = imm_payload >> srv_path->mem_bits;
1235 off = imm_payload & ((1 << sess->mem_bits) - 1);1241 off = imm_payload & ((1 << srv_path->mem_bits) - 1);
1236 if (msg_id >= srv->queue_depth || off >= max_chunk_size) {1242 if (msg_id >= srv->queue_depth || off >= max_chunk_size) {
1237 rtrs_err(s, "Wrong msg_id %u, off %u\n",1243 rtrs_err(s, "Wrong msg_id %u, off %u\n",
1238 msg_id, off);1244 msg_id, off);
1239 close_sess(sess);1245 close_path(srv_path);
1240 return;1246 return;
1241 }1247 }
1242 if (always_invalidate) {1248 if (always_invalidate) {
1243 struct rtrs_srv_mr *mr = &sess->mrs[msg_id];1249 struct rtrs_srv_mr *mr = &srv_path->mrs[msg_id];
1244 1250
1245 mr->msg_off = off;1251 mr->msg_off = off;
1246 mr->msg_id = msg_id;1252 mr->msg_id = msg_id;
@@ -1248,7 +1254,7 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 1 unmodified line
1248 if (err) {1254 if (err) {
1249 rtrs_err(s, "rtrs_post_recv(), err: %d\n",1255 rtrs_err(s, "rtrs_post_recv(), err: %d\n",
1250 err);1256 err);
1251 close_sess(sess);1257 close_path(srv_path);
1252 break;1258 break;
1253 }1259 }
1254 } else {1260 } else {
@@ -1257,10 +1263,10 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 2 unmodified lines
1257 }1263 }
1258 } else if (imm_type == RTRS_HB_MSG_IMM) {1264 } else if (imm_type == RTRS_HB_MSG_IMM) {
1259 WARN_ON(con->c.cid);1265 WARN_ON(con->c.cid);
1260 rtrs_send_hb_ack(&sess->s);1266 rtrs_send_hb_ack(&srv_path->s);
1261 } else if (imm_type == RTRS_HB_ACK_IMM) {1267 } else if (imm_type == RTRS_HB_ACK_IMM) {
1262 WARN_ON(con->c.cid);1268 WARN_ON(con->c.cid);
1263 sess->s.hb_missed_cnt = 0;1269 srv_path->s.hb_missed_cnt = 0;
1264 } else {1270 } else {
1265 rtrs_wrn(s, "Unknown IMM type %u\n", imm_type);1271 rtrs_wrn(s, "Unknown IMM type %u\n", imm_type);
1266 }1272 }
@@ -1284,22 +1290,23 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) · 17 unmodified lines
1284}1290}
1285 1291
1286/**1292/**
1287 * rtrs_srv_get_sess_name() - Get rtrs_srv peer hostname.1293 * rtrs_srv_get_path_name() - Get rtrs_srv peer hostname.
1288 * @srv: Session1294 * @srv: Session
1289 * @sessname: Sessname buffer1295 * @pathname: Pathname buffer
1290 * @len: Length of sessname buffer1296 * @len: Length of sessname buffer
1291 */1297 */
1292int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len)1298int rtrs_srv_get_path_name(struct rtrs_srv_sess *srv, char *pathname,
1299 size_t len)
1293{1300{
1294 struct rtrs_srv_sess *sess;1301 struct rtrs_srv_path *srv_path;
1295 int err = -ENOTCONN;1302 int err = -ENOTCONN;
1296 1303
1297 mutex_lock(&srv->paths_mutex);1304 mutex_lock(&srv->paths_mutex);
1298 list_for_each_entry(sess, &srv->paths_list, s.entry) {1305 list_for_each_entry(srv_path, &srv->paths_list, s.entry) {
1299 if (sess->state != RTRS_SRV_CONNECTED)1306 if (srv_path->state != RTRS_SRV_CONNECTED)
1300 continue;1307 continue;
1301 strscpy(sessname, sess->s.sessname,1308 strscpy(pathname, srv_path->s.sessname,
1302 min_t(size_t, sizeof(sess->s.sessname), len));1309 min_t(size_t, sizeof(srv_path->s.sessname), len));
1303 err = 0;1310 err = 0;
1304 break;1311 break;
1305 }1312 }
@@ -1307,44 +1314,45 @@ int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len) · 1 unmodified line
1307 1314
1308 return err;1315 return err;
1309}1316}
1310EXPORT_SYMBOL(rtrs_srv_get_sess_name);1317EXPORT_SYMBOL(rtrs_srv_get_path_name);
1311 1318
1312/**1319/**
1313 * rtrs_srv_get_queue_depth() - Get rtrs_srv qdepth.1320 * rtrs_srv_get_queue_depth() - Get rtrs_srv qdepth.
1314 * @srv: Session1321 * @srv: Session
1315 */1322 */
1316int rtrs_srv_get_queue_depth(struct rtrs_srv *srv)1323int rtrs_srv_get_queue_depth(struct rtrs_srv_sess *srv)
1317{1324{
1318 return srv->queue_depth;1325 return srv->queue_depth;
1319}1326}
1320EXPORT_SYMBOL(rtrs_srv_get_queue_depth);1327EXPORT_SYMBOL(rtrs_srv_get_queue_depth);
1321 1328
1322static int find_next_bit_ring(struct rtrs_srv_sess *sess)1329static int find_next_bit_ring(struct rtrs_srv_path *srv_path)
1323{1330{
1324 struct ib_device *ib_dev = sess->s.dev->ib_dev;1331 struct ib_device *ib_dev = srv_path->s.dev->ib_dev;
1325 int v;1332 int v;
1326 1333
1327 v = cpumask_next(sess->cur_cq_vector, &cq_affinity_mask);1334 v = cpumask_next(srv_path->cur_cq_vector, &cq_affinity_mask);
1328 if (v >= nr_cpu_ids || v >= ib_dev->num_comp_vectors)1335 if (v >= nr_cpu_ids || v >= ib_dev->num_comp_vectors)
1329 v = cpumask_first(&cq_affinity_mask);1336 v = cpumask_first(&cq_affinity_mask);
1330 return v;1337 return v;
1331}1338}
1332 1339
1333static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_sess *sess)1340static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_path *srv_path)
1334{1341{
1335 sess->cur_cq_vector = find_next_bit_ring(sess);1342 srv_path->cur_cq_vector = find_next_bit_ring(srv_path);
1336 1343
1337 return sess->cur_cq_vector;1344 return srv_path->cur_cq_vector;
1338}1345}
1339 1346
1340static void rtrs_srv_dev_release(struct device *dev)1347static void rtrs_srv_dev_release(struct device *dev)
1341{1348{
1342 struct rtrs_srv *srv = container_of(dev, struct rtrs_srv, dev);1349 struct rtrs_srv_sess *srv = container_of(dev, struct rtrs_srv_sess,
1350 dev);
1343 1351
1344 kfree(srv);1352 kfree(srv);
1345}1353}
1346 1354
1347static void free_srv(struct rtrs_srv *srv)1355static void free_srv(struct rtrs_srv_sess *srv)
1348{1356{
1349 int i;1357 int i;
1350 1358
@@ -1358,11 +1366,11 @@ static void free_srv(struct rtrs_srv *srv) · 7 unmodified lines
1358 put_device(&srv->dev);1366 put_device(&srv->dev);
1359}1367}
1360 1368
1361static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx,1369static struct rtrs_srv_sess *get_or_create_srv(struct rtrs_srv_ctx *ctx,
1362 const uuid_t *paths_uuid,1370 const uuid_t *paths_uuid,
1363 bool first_conn)1371 bool first_conn)
1364{1372{
1365 struct rtrs_srv *srv;1373 struct rtrs_srv_sess *srv;
1366 int i;1374 int i;
1367 1375
1368 mutex_lock(&ctx->srv_mutex);1376 mutex_lock(&ctx->srv_mutex);
@@ -1424,7 +1432,7 @@ err_free_srv: · 55 unmodified lines
1424 return ERR_PTR(-ENOMEM);1432 return ERR_PTR(-ENOMEM);
1425}1433}
1426 1434
1427static void put_srv(struct rtrs_srv *srv)1435static void put_srv(struct rtrs_srv_sess *srv)
1428{1436{
1429 if (refcount_dec_and_test(&srv->refcount)) {1437 if (refcount_dec_and_test(&srv->refcount)) {
1430 struct rtrs_srv_ctx *ctx = srv->ctx;1438 struct rtrs_srv_ctx *ctx = srv->ctx;
@@ -1438,23 +1446,23 @@ static void put_srv(struct rtrs_srv *srv) · 7 unmodified lines
1438 }1446 }
1439}1447}
1440 1448
1441static void __add_path_to_srv(struct rtrs_srv *srv,1449static void __add_path_to_srv(struct rtrs_srv_sess *srv,
1442 struct rtrs_srv_sess *sess)1450 struct rtrs_srv_path *srv_path)
1443{1451{
1444 list_add_tail(&sess->s.entry, &srv->paths_list);1452 list_add_tail(&srv_path->s.entry, &srv->paths_list);
1445 srv->paths_num++;1453 srv->paths_num++;
1446 WARN_ON(srv->paths_num >= MAX_PATHS_NUM);1454 WARN_ON(srv->paths_num >= MAX_PATHS_NUM);
1447}1455}
1448 1456
1449static void del_path_from_srv(struct rtrs_srv_sess *sess)1457static void del_path_from_srv(struct rtrs_srv_path *srv_path)
1450{1458{
1451 struct rtrs_srv *srv = sess->srv;1459 struct rtrs_srv_sess *srv = srv_path->srv;
1452 1460
1453 if (WARN_ON(!srv))1461 if (WARN_ON(!srv))
1454 return;1462 return;
1455 1463
1456 mutex_lock(&srv->paths_mutex);1464 mutex_lock(&srv->paths_mutex);
1457 list_del(&sess->s.entry);1465 list_del(&srv_path->s.entry);
1458 WARN_ON(!srv->paths_num);1466 WARN_ON(!srv->paths_num);
1459 srv->paths_num--;1467 srv->paths_num--;
1460 mutex_unlock(&srv->paths_mutex);1468 mutex_unlock(&srv->paths_mutex);
@@ -1484,47 +1492,47 @@ static int sockaddr_cmp(const struct sockaddr *a, const struct sockaddr *b) · 23 unmodified lines
1484 }1492 }
1485}1493}
1486 1494
1487static bool __is_path_w_addr_exists(struct rtrs_srv *srv,1495static bool __is_path_w_addr_exists(struct rtrs_srv_sess *srv,
1488 struct rdma_addr *addr)1496 struct rdma_addr *addr)
1489{1497{
1490 struct rtrs_srv_sess *sess;1498 struct rtrs_srv_path *srv_path;
1491 1499
1492 list_for_each_entry(sess, &srv->paths_list, s.entry)1500 list_for_each_entry(srv_path, &srv->paths_list, s.entry)
1493 if (!sockaddr_cmp((struct sockaddr *)&sess->s.dst_addr,1501 if (!sockaddr_cmp((struct sockaddr *)&srv_path->s.dst_addr,
1494 (struct sockaddr *)&addr->dst_addr) &&1502 (struct sockaddr *)&addr->dst_addr) &&
1495 !sockaddr_cmp((struct sockaddr *)&sess->s.src_addr,1503 !sockaddr_cmp((struct sockaddr *)&srv_path->s.src_addr,
1496 (struct sockaddr *)&addr->src_addr))1504 (struct sockaddr *)&addr->src_addr))
1497 return true;1505 return true;
1498 1506
1499 return false;1507 return false;
1500}1508}
1501 1509
1502static void free_sess(struct rtrs_srv_sess *sess)1510static void free_path(struct rtrs_srv_path *srv_path)
1503{1511{
1504 if (sess->kobj.state_in_sysfs) {1512 if (srv_path->kobj.state_in_sysfs) {
1505 kobject_del(&sess->kobj);1513 kobject_del(&srv_path->kobj);
1506 kobject_put(&sess->kobj);1514 kobject_put(&srv_path->kobj);
1507 } else {1515 } else {
1508 kfree(sess->stats);1516 kfree(srv_path->stats);
1509 kfree(sess);1517 kfree(srv_path);
1510 }1518 }
1511}1519}
1512 1520
1513static void rtrs_srv_close_work(struct work_struct *work)1521static void rtrs_srv_close_work(struct work_struct *work)
1514{1522{
1515 struct rtrs_srv_sess *sess;1523 struct rtrs_srv_path *srv_path;
1516 struct rtrs_srv_con *con;1524 struct rtrs_srv_con *con;
1517 int i;1525 int i;
1518 1526
1519 sess = container_of(work, typeof(*sess), close_work);1527 srv_path = container_of(work, typeof(*srv_path), close_work);
1520 1528
1521 rtrs_srv_destroy_sess_files(sess);1529 rtrs_srv_destroy_path_files(srv_path);
1522 rtrs_srv_stop_hb(sess);1530 rtrs_srv_stop_hb(srv_path);
1523 1531
1524 for (i = 0; i < sess->s.con_num; i++) {1532 for (i = 0; i < srv_path->s.con_num; i++) {
1525 if (!sess->s.con[i])1533 if (!srv_path->s.con[i])
1526 continue;1534 continue;
1527 con = to_srv_con(sess->s.con[i]);1535 con = to_srv_con(srv_path->s.con[i]);
1528 rdma_disconnect(con->c.cm_id);1536 rdma_disconnect(con->c.cm_id);
1529 ib_drain_qp(con->c.qp);1537 ib_drain_qp(con->c.qp);
1530 }1538 }
@@ -1533,41 +1541,41 @@ static void rtrs_srv_close_work(struct work_struct *work) · 2 unmodified lines
1533 * Degrade ref count to the usual model with a single shared1541 * Degrade ref count to the usual model with a single shared
1534 * atomic_t counter1542 * atomic_t counter
1535 */1543 */
1536 percpu_ref_kill(&sess->ids_inflight_ref);1544 percpu_ref_kill(&srv_path->ids_inflight_ref);
1537 1545
1538 /* Wait for all completion */1546 /* Wait for all completion */
1539 wait_for_completion(&sess->complete_done);1547 wait_for_completion(&srv_path->complete_done);
1540 1548
1541 /* Notify upper layer if we are the last path */1549 /* Notify upper layer if we are the last path */
1542 rtrs_srv_sess_down(sess);1550 rtrs_srv_path_down(srv_path);
1543 1551
1544 unmap_cont_bufs(sess);1552 unmap_cont_bufs(srv_path);
1545 rtrs_srv_free_ops_ids(sess);1553 rtrs_srv_free_ops_ids(srv_path);
1546 1554
1547 for (i = 0; i < sess->s.con_num; i++) {1555 for (i = 0; i < srv_path->s.con_num; i++) {
1548 if (!sess->s.con[i])1556 if (!srv_path->s.con[i])
1549 continue;1557 continue;
1550 con = to_srv_con(sess->s.con[i]);1558 con = to_srv_con(srv_path->s.con[i]);
1551 rtrs_cq_qp_destroy(&con->c);1559 rtrs_cq_qp_destroy(&con->c);
1552 rdma_destroy_id(con->c.cm_id);1560 rdma_destroy_id(con->c.cm_id);
1553 kfree(con);1561 kfree(con);
1554 }1562 }
1555 rtrs_ib_dev_put(sess->s.dev);1563 rtrs_ib_dev_put(srv_path->s.dev);
1556 1564
1557 del_path_from_srv(sess);1565 del_path_from_srv(srv_path);
1558 put_srv(sess->srv);1566 put_srv(srv_path->srv);
1559 sess->srv = NULL;1567 srv_path->srv = NULL;
1560 rtrs_srv_change_state(sess, RTRS_SRV_CLOSED);1568 rtrs_srv_change_state(srv_path, RTRS_SRV_CLOSED);
1561 1569
1562 kfree(sess->dma_addr);1570 kfree(srv_path->dma_addr);
1563 kfree(sess->s.con);1571 kfree(srv_path->s.con);
1564 free_sess(sess);1572 free_path(srv_path);
1565}1573}
1566 1574
1567static int rtrs_rdma_do_accept(struct rtrs_srv_sess *sess,1575static int rtrs_rdma_do_accept(struct rtrs_srv_path *srv_path,
1568 struct rdma_cm_id *cm_id)1576 struct rdma_cm_id *cm_id)
1569{1577{
1570 struct rtrs_srv *srv = sess->srv;1578 struct rtrs_srv_sess *srv = srv_path->srv;
1571 struct rtrs_msg_conn_rsp msg;1579 struct rtrs_msg_conn_rsp msg;
1572 struct rdma_conn_param param;1580 struct rdma_conn_param param;
1573 int err;1581 int err;
@@ -1615,25 +1623,25 @@ static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno) · 41 unmodified lines
1615 return errno;1623 return errno;
1616}1624}
1617 1625
1618static struct rtrs_srv_sess *1626static struct rtrs_srv_path *
1619__find_sess(struct rtrs_srv *srv, const uuid_t *sess_uuid)1627__find_path(struct rtrs_srv_sess *srv, const uuid_t *sess_uuid)
1620{1628{
1621 struct rtrs_srv_sess *sess;1629 struct rtrs_srv_path *srv_path;
1622 1630
1623 list_for_each_entry(sess, &srv->paths_list, s.entry) {1631 list_for_each_entry(srv_path, &srv->paths_list, s.entry) {
1624 if (uuid_equal(&sess->s.uuid, sess_uuid))1632 if (uuid_equal(&srv_path->s.uuid, sess_uuid))
1625 return sess;1633 return srv_path;
1626 }1634 }
1627 1635
1628 return NULL;1636 return NULL;
1629}1637}
1630 1638
1631static int create_con(struct rtrs_srv_sess *sess,1639static int create_con(struct rtrs_srv_path *srv_path,
1632 struct rdma_cm_id *cm_id,1640 struct rdma_cm_id *cm_id,
1633 unsigned int cid)1641 unsigned int cid)
1634{1642{
1635 struct rtrs_srv *srv = sess->srv;1643 struct rtrs_srv_sess *srv = srv_path->srv;
1636 struct rtrs_sess *s = &sess->s;1644 struct rtrs_path *s = &srv_path->s;
1637 struct rtrs_srv_con *con;1645 struct rtrs_srv_con *con;
1638 1646
1639 u32 cq_num, max_send_wr, max_recv_wr, wr_limit;1647 u32 cq_num, max_send_wr, max_recv_wr, wr_limit;
@@ -1648,10 +1656,10 @@ static int create_con(struct rtrs_srv_sess *sess, · 8 unmodified lines
1648 spin_lock_init(&con->rsp_wr_wait_lock);1656 spin_lock_init(&con->rsp_wr_wait_lock);
1649 INIT_LIST_HEAD(&con->rsp_wr_wait_list);1657 INIT_LIST_HEAD(&con->rsp_wr_wait_list);
1650 con->c.cm_id = cm_id;1658 con->c.cm_id = cm_id;
1651 con->c.sess = &sess->s;1659 con->c.path = &srv_path->s;
1652 con->c.cid = cid;1660 con->c.cid = cid;
1653 atomic_set(&con->c.wr_cnt, 1);1661 atomic_set(&con->c.wr_cnt, 1);
1654 wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr;1662 wr_limit = srv_path->s.dev->ib_dev->attrs.max_qp_wr;
1655 1663
1656 if (con->c.cid == 0) {1664 if (con->c.cid == 0) {
1657 /*1665 /*
@@ -1684,10 +1692,10 @@ static int create_con(struct rtrs_srv_sess *sess, · 26 unmodified lines
1684 }1692 }
1685 cq_num = max_send_wr + max_recv_wr;1693 cq_num = max_send_wr + max_recv_wr;
1686 atomic_set(&con->c.sq_wr_avail, max_send_wr);1694 atomic_set(&con->c.sq_wr_avail, max_send_wr);
1687 cq_vector = rtrs_srv_get_next_cq_vector(sess);1695 cq_vector = rtrs_srv_get_next_cq_vector(srv_path);
1688 1696
1689 /* TODO: SOFTIRQ can be faster, but be careful with softirq context */1697 /* TODO: SOFTIRQ can be faster, but be careful with softirq context */
1690 err = rtrs_cq_qp_create(&sess->s, &con->c, 1, cq_vector, cq_num,1698 err = rtrs_cq_qp_create(&srv_path->s, &con->c, 1, cq_vector, cq_num,
1691 max_send_wr, max_recv_wr,1699 max_send_wr, max_recv_wr,
1692 IB_POLL_WORKQUEUE);1700 IB_POLL_WORKQUEUE);
1693 if (err) {1701 if (err) {
@@ -1699,8 +1707,8 @@ static int create_con(struct rtrs_srv_sess *sess, · 5 unmodified lines
1699 if (err)1707 if (err)
1700 goto free_cqqp;1708 goto free_cqqp;
1701 }1709 }
1702 WARN_ON(sess->s.con[cid]);1710 WARN_ON(srv_path->s.con[cid]);
1703 sess->s.con[cid] = &con->c;1711 srv_path->s.con[cid] = &con->c;
1704 1712
1705 /*1713 /*
1706 * Change context from server to current connection. The other1714 * Change context from server to current connection. The other
@@ -1719,13 +1727,13 @@ err: · 12 unmodified lines
1719 return err;1727 return err;
1720}1728}
1721 1729
1722static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv,1730static struct rtrs_srv_path *__alloc_path(struct rtrs_srv_sess *srv,
1723 struct rdma_cm_id *cm_id,1731 struct rdma_cm_id *cm_id,
1724 unsigned int con_num,1732 unsigned int con_num,
1725 unsigned int recon_cnt,1733 unsigned int recon_cnt,
1726 const uuid_t *uuid)1734 const uuid_t *uuid)
1727{1735{
1728 struct rtrs_srv_sess *sess;1736 struct rtrs_srv_path *srv_path;
1729 int err = -ENOMEM;1737 int err = -ENOMEM;
1730 char str[NAME_MAX];1738 char str[NAME_MAX];
1731 struct rtrs_addr path;1739 struct rtrs_addr path;
@@ -1739,74 +1747,76 @@ static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv, · 7 unmodified lines
1739 pr_err("Path with same addr exists\n");1747 pr_err("Path with same addr exists\n");
1740 goto err;1748 goto err;
1741 }1749 }
1742 sess = kzalloc(sizeof(*sess), GFP_KERNEL);1750 srv_path = kzalloc(sizeof(*srv_path), GFP_KERNEL);
1743 if (!sess)1751 if (!srv_path)
1744 goto err;1752 goto err;
1745 1753
1746 sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL);1754 srv_path->stats = kzalloc(sizeof(*srv_path->stats), GFP_KERNEL);
1747 if (!sess->stats)1755 if (!srv_path->stats)
1748 goto err_free_sess;1756 goto err_free_sess;
1749 1757
1750 sess->stats->sess = sess;1758 srv_path->stats->srv_path = srv_path;
1751 1759
1752 sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr),1760 srv_path->dma_addr = kcalloc(srv->queue_depth,
1753 GFP_KERNEL);1761 sizeof(*srv_path->dma_addr),
1754 if (!sess->dma_addr)1762 GFP_KERNEL);
1763 if (!srv_path->dma_addr)
1755 goto err_free_stats;1764 goto err_free_stats;
1756 1765
1757 sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL);1766 srv_path->s.con = kcalloc(con_num, sizeof(*srv_path->s.con),
1758 if (!sess->s.con)1767 GFP_KERNEL);
1768 if (!srv_path->s.con)
1759 goto err_free_dma_addr;1769 goto err_free_dma_addr;
1760 1770
1761 sess->state = RTRS_SRV_CONNECTING;1771 srv_path->state = RTRS_SRV_CONNECTING;
1762 sess->srv = srv;1772 srv_path->srv = srv;
1763 sess->cur_cq_vector = -1;1773 srv_path->cur_cq_vector = -1;
1764 sess->s.dst_addr = cm_id->route.addr.dst_addr;1774 srv_path->s.dst_addr = cm_id->route.addr.dst_addr;
1765 sess->s.src_addr = cm_id->route.addr.src_addr;1775 srv_path->s.src_addr = cm_id->route.addr.src_addr;
1766 1776
1767 /* temporary until receiving session-name from client */1777 /* temporary until receiving session-name from client */
1768 path.src = &sess->s.src_addr;1778 path.src = &srv_path->s.src_addr;
1769 path.dst = &sess->s.dst_addr;1779 path.dst = &srv_path->s.dst_addr;
1770 rtrs_addr_to_str(&path, str, sizeof(str));1780 rtrs_addr_to_str(&path, str, sizeof(str));
1771 strscpy(sess->s.sessname, str, sizeof(sess->s.sessname));1781 strscpy(srv_path->s.sessname, str, sizeof(srv_path->s.sessname));
1772 1782
1773 sess->s.con_num = con_num;1783 srv_path->s.con_num = con_num;
1774 sess->s.irq_con_num = con_num;1784 srv_path->s.irq_con_num = con_num;
1775 sess->s.recon_cnt = recon_cnt;1785 srv_path->s.recon_cnt = recon_cnt;
1776 uuid_copy(&sess->s.uuid, uuid);1786 uuid_copy(&srv_path->s.uuid, uuid);
1777 spin_lock_init(&sess->state_lock);1787 spin_lock_init(&srv_path->state_lock);
1778 INIT_WORK(&sess->close_work, rtrs_srv_close_work);1788 INIT_WORK(&srv_path->close_work, rtrs_srv_close_work);
1779 rtrs_srv_init_hb(sess);1789 rtrs_srv_init_hb(srv_path);
1780 1790
1781 sess->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd);1791 srv_path->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd);
1782 if (!sess->s.dev) {1792 if (!srv_path->s.dev) {
1783 err = -ENOMEM;1793 err = -ENOMEM;
1784 goto err_free_con;1794 goto err_free_con;
1785 }1795 }
1786 err = map_cont_bufs(sess);1796 err = map_cont_bufs(srv_path);
1787 if (err)1797 if (err)
1788 goto err_put_dev;1798 goto err_put_dev;
1789 1799
1790 err = rtrs_srv_alloc_ops_ids(sess);1800 err = rtrs_srv_alloc_ops_ids(srv_path);
1791 if (err)1801 if (err)
1792 goto err_unmap_bufs;1802 goto err_unmap_bufs;
1793 1803
1794 __add_path_to_srv(srv, sess);1804 __add_path_to_srv(srv, srv_path);
1795 1805
1796 return sess;1806 return srv_path;
1797 1807
1798err_unmap_bufs:1808err_unmap_bufs:
1799 unmap_cont_bufs(sess);1809 unmap_cont_bufs(srv_path);
1800err_put_dev:1810err_put_dev:
1801 rtrs_ib_dev_put(sess->s.dev);1811 rtrs_ib_dev_put(srv_path->s.dev);
1802err_free_con:1812err_free_con:
1803 kfree(sess->s.con);1813 kfree(srv_path->s.con);
1804err_free_dma_addr:1814err_free_dma_addr:
1805 kfree(sess->dma_addr);1815 kfree(srv_path->dma_addr);
1806err_free_stats:1816err_free_stats:
1807 kfree(sess->stats);1817 kfree(srv_path->stats);
1808err_free_sess:1818err_free_sess:
1809 kfree(sess);1819 kfree(srv_path);
1810err:1820err:
1811 return ERR_PTR(err);1821 return ERR_PTR(err);
1812}1822}
@@ -1816,8 +1826,8 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, · 3 unmodified lines
1816 size_t len)1826 size_t len)
1817{1827{
1818 struct rtrs_srv_ctx *ctx = cm_id->context;1828 struct rtrs_srv_ctx *ctx = cm_id->context;
1819 struct rtrs_srv_sess *sess;1829 struct rtrs_srv_path *srv_path;
1820 struct rtrs_srv *srv;1830 struct rtrs_srv_sess *srv;
1821 1831
1822 u16 version, con_num, cid;1832 u16 version, con_num, cid;
1823 u16 recon_cnt;1833 u16 recon_cnt;
@@ -1857,16 +1867,16 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, · 33 unmodified lines
1857 goto reject_w_err;1867 goto reject_w_err;
1858 }1868 }
1859 mutex_lock(&srv->paths_mutex);1869 mutex_lock(&srv->paths_mutex);
1860 sess = __find_sess(srv, &msg->sess_uuid);1870 srv_path = __find_path(srv, &msg->sess_uuid);
1861 if (sess) {1871 if (srv_path) {
1862 struct rtrs_sess *s = &sess->s;1872 struct rtrs_path *s = &srv_path->s;
1863 1873
1864 /* Session already holds a reference */1874 /* Session already holds a reference */
1865 put_srv(srv);1875 put_srv(srv);
1866 1876
1867 if (sess->state != RTRS_SRV_CONNECTING) {1877 if (srv_path->state != RTRS_SRV_CONNECTING) {
1868 rtrs_err(s, "Session in wrong state: %s\n",1878 rtrs_err(s, "Session in wrong state: %s\n",
1869 rtrs_srv_state_str(sess->state));1879 rtrs_srv_state_str(srv_path->state));
1870 mutex_unlock(&srv->paths_mutex);1880 mutex_unlock(&srv->paths_mutex);
1871 goto reject_w_err;1881 goto reject_w_err;
1872 }1882 }
@@ -1886,19 +1896,19 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, · 13 unmodified lines
1886 goto reject_w_err;1896 goto reject_w_err;
1887 }1897 }
1888 } else {1898 } else {
1889 sess = __alloc_sess(srv, cm_id, con_num, recon_cnt,1899 srv_path = __alloc_path(srv, cm_id, con_num, recon_cnt,
1890 &msg->sess_uuid);1900 &msg->sess_uuid);
1891 if (IS_ERR(sess)) {1901 if (IS_ERR(srv_path)) {
1892 mutex_unlock(&srv->paths_mutex);1902 mutex_unlock(&srv->paths_mutex);
1893 put_srv(srv);1903 put_srv(srv);
1894 err = PTR_ERR(sess);1904 err = PTR_ERR(srv_path);
1895 pr_err("RTRS server session allocation failed: %d\n", err);1905 pr_err("RTRS server session allocation failed: %d\n", err);
1896 goto reject_w_err;1906 goto reject_w_err;
1897 }1907 }
1898 }1908 }
1899 err = create_con(sess, cm_id, cid);1909 err = create_con(srv_path, cm_id, cid);
1900 if (err) {1910 if (err) {
1901 rtrs_err((&sess->s), "create_con(), error %d\n", err);1911 rtrs_err((&srv_path->s), "create_con(), error %d\n", err);
1902 rtrs_rdma_do_reject(cm_id, err);1912 rtrs_rdma_do_reject(cm_id, err);
1903 /*1913 /*
1904 * Since session has other connections we follow normal way1914 * Since session has other connections we follow normal way
@@ -1907,9 +1917,9 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, · 2 unmodified lines
1907 */1917 */
1908 goto close_and_return_err;1918 goto close_and_return_err;
1909 }1919 }
1910 err = rtrs_rdma_do_accept(sess, cm_id);1920 err = rtrs_rdma_do_accept(srv_path, cm_id);
1911 if (err) {1921 if (err) {
1912 rtrs_err((&sess->s), "rtrs_rdma_do_accept(), error %d\n", err);1922 rtrs_err((&srv_path->s), "rtrs_rdma_do_accept(), error %d\n", err);
1913 rtrs_rdma_do_reject(cm_id, err);1923 rtrs_rdma_do_reject(cm_id, err);
1914 /*1924 /*
1915 * Since current connection was successfully added to the1925 * Since current connection was successfully added to the
@@ -1929,7 +1939,7 @@ reject_w_err: · 13 unmodified lines
1929 1939
1930close_and_return_err:1940close_and_return_err:
1931 mutex_unlock(&srv->paths_mutex);1941 mutex_unlock(&srv->paths_mutex);
1932 close_sess(sess);1942 close_path(srv_path);
1933 1943
1934 return err;1944 return err;
1935}1945}
@@ -1937,14 +1947,14 @@ close_and_return_err: · 1 unmodified line
1937static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id,1947static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id,
1938 struct rdma_cm_event *ev)1948 struct rdma_cm_event *ev)
1939{1949{
1940 struct rtrs_srv_sess *sess = NULL;1950 struct rtrs_srv_path *srv_path = NULL;
1941 struct rtrs_sess *s = NULL;1951 struct rtrs_path *s = NULL;
1942 1952
1943 if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) {1953 if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
1944 struct rtrs_con *c = cm_id->context;1954 struct rtrs_con *c = cm_id->context;
1945 1955
1946 s = c->sess;1956 s = c->path;
1947 sess = to_srv_sess(s);1957 srv_path = to_srv_path(s);
1948 }1958 }
1949 1959
1950 switch (ev->event) {1960 switch (ev->event) {
@@ -1968,7 +1978,7 @@ static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, · 17 unmodified lines
1968 case RDMA_CM_EVENT_ADDR_CHANGE:1978 case RDMA_CM_EVENT_ADDR_CHANGE:
1969 case RDMA_CM_EVENT_TIMEWAIT_EXIT:1979 case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1970 case RDMA_CM_EVENT_DEVICE_REMOVAL:1980 case RDMA_CM_EVENT_DEVICE_REMOVAL:
1971 close_sess(sess);1981 close_path(srv_path);
1972 break;1982 break;
1973 default:1983 default:
1974 pr_err("Ignoring unexpected CM event %s, err %d\n",1984 pr_err("Ignoring unexpected CM event %s, err %d\n",
@@ -2176,23 +2186,23 @@ struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port) · 201 unmodified lines
2176}2186}
2177EXPORT_SYMBOL(rtrs_srv_open);2187EXPORT_SYMBOL(rtrs_srv_open);
2178 2188
2179static void close_sessions(struct rtrs_srv *srv)2189static void close_paths(struct rtrs_srv_sess *srv)
2180{2190{
2181 struct rtrs_srv_sess *sess;2191 struct rtrs_srv_path *srv_path;
2182 2192
2183 mutex_lock(&srv->paths_mutex);2193 mutex_lock(&srv->paths_mutex);
2184 list_for_each_entry(sess, &srv->paths_list, s.entry)2194 list_for_each_entry(srv_path, &srv->paths_list, s.entry)
2185 close_sess(sess);2195 close_path(srv_path);
2186 mutex_unlock(&srv->paths_mutex);2196 mutex_unlock(&srv->paths_mutex);
2187}2197}
2188 2198
2189static void close_ctx(struct rtrs_srv_ctx *ctx)2199static void close_ctx(struct rtrs_srv_ctx *ctx)
2190{2200{
2191 struct rtrs_srv *srv;2201 struct rtrs_srv_sess *srv;
2192 2202
2193 mutex_lock(&ctx->srv_mutex);2203 mutex_lock(&ctx->srv_mutex);
2194 list_for_each_entry(srv, &ctx->srv_list, ctx_list)2204 list_for_each_entry(srv, &ctx->srv_list, ctx_list)
2195 close_sessions(srv);2205 close_paths(srv);
2196 mutex_unlock(&ctx->srv_mutex);2206 mutex_unlock(&ctx->srv_mutex);
2197 flush_workqueue(rtrs_wq);2207 flush_workqueue(rtrs_wq);
2198}2208}
drivers/infiniband/ulp/rtrs/rtrs-srv.h+8−8
@@ -37,7 +37,7 @@ struct rtrs_srv_stats_rdma_stats { · 36 unmodified lines
37struct rtrs_srv_stats {37struct rtrs_srv_stats {
38 struct kobject kobj_stats;38 struct kobject kobj_stats;
39 struct rtrs_srv_stats_rdma_stats rdma_stats;39 struct rtrs_srv_stats_rdma_stats rdma_stats;
40 struct rtrs_srv_sess *sess;40 struct rtrs_srv_path *srv_path;
41};41};
42 42
43struct rtrs_srv_con {43struct rtrs_srv_con {
@@ -71,9 +71,9 @@ struct rtrs_srv_mr { · 27 unmodified lines
71 struct rtrs_iu *iu; /* send buffer for new rkey msg */71 struct rtrs_iu *iu; /* send buffer for new rkey msg */
72};72};
73 73
74struct rtrs_srv_sess {74struct rtrs_srv_path {
75 struct rtrs_sess s;75 struct rtrs_path s;
76 struct rtrs_srv *srv;76 struct rtrs_srv_sess *srv;
77 struct work_struct close_work;77 struct work_struct close_work;
78 enum rtrs_srv_state state;78 enum rtrs_srv_state state;
79 spinlock_t state_lock;79 spinlock_t state_lock;
@@ -90,7 +90,7 @@ struct rtrs_srv_sess { · 10 unmodified lines
90 struct rtrs_srv_stats *stats;90 struct rtrs_srv_stats *stats;
91};91};
92 92
93struct rtrs_srv {93struct rtrs_srv_sess {
94 struct list_head paths_list;94 struct list_head paths_list;
95 int paths_up;95 int paths_up;
96 struct mutex paths_ev_mutex;96 struct mutex paths_ev_mutex;
@@ -125,7 +125,7 @@ struct rtrs_srv_ib_ctx { · 28 unmodified lines
125 125
126extern struct class *rtrs_dev_class;126extern struct class *rtrs_dev_class;
127 127
128void close_sess(struct rtrs_srv_sess *sess);128void close_path(struct rtrs_srv_path *srv_path);
129 129
130static inline void rtrs_srv_update_rdma_stats(struct rtrs_srv_stats *s,130static inline void rtrs_srv_update_rdma_stats(struct rtrs_srv_stats *s,
131 size_t size, int d)131 size_t size, int d)
@@ -142,7 +142,7 @@ ssize_t rtrs_srv_reset_all_help(struct rtrs_srv_stats *stats, · 10 unmodified lines
142 char *page, size_t len);142 char *page, size_t len);
143 143
144/* functions which are implemented in rtrs-srv-sysfs.c */144/* functions which are implemented in rtrs-srv-sysfs.c */
145int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess);145int rtrs_srv_create_path_files(struct rtrs_srv_path *srv_path);
146void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess);146void rtrs_srv_destroy_path_files(struct rtrs_srv_path *srv_path);
147 147
148#endif /* RTRS_SRV_H */148#endif /* RTRS_SRV_H */
drivers/infiniband/ulp/rtrs/rtrs.c+49−49
@@ -69,16 +69,16 @@ EXPORT_SYMBOL_GPL(rtrs_iu_free); · 68 unmodified lines
69 69
70int rtrs_iu_post_recv(struct rtrs_con *con, struct rtrs_iu *iu)70int rtrs_iu_post_recv(struct rtrs_con *con, struct rtrs_iu *iu)
71{71{
72 struct rtrs_sess *sess = con->sess;72 struct rtrs_path *path = con->path;
73 struct ib_recv_wr wr;73 struct ib_recv_wr wr;
74 struct ib_sge list;74 struct ib_sge list;
75 75
76 list.addr = iu->dma_addr;76 list.addr = iu->dma_addr;
77 list.length = iu->size;77 list.length = iu->size;
78 list.lkey = sess->dev->ib_pd->local_dma_lkey;78 list.lkey = path->dev->ib_pd->local_dma_lkey;
79 79
80 if (list.length == 0) {80 if (list.length == 0) {
81 rtrs_wrn(con->sess,81 rtrs_wrn(con->path,
82 "Posting receive work request failed, sg list is empty\n");82 "Posting receive work request failed, sg list is empty\n");
83 return -EINVAL;83 return -EINVAL;
84 }84 }
@@ -126,7 +126,7 @@ static int rtrs_post_send(struct ib_qp *qp, struct ib_send_wr *head, · 41 unmodified lines
126int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size,126int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size,
127 struct ib_send_wr *head)127 struct ib_send_wr *head)
128{128{
129 struct rtrs_sess *sess = con->sess;129 struct rtrs_path *path = con->path;
130 struct ib_send_wr wr;130 struct ib_send_wr wr;
131 struct ib_sge list;131 struct ib_sge list;
132 132
@@ -135,7 +135,7 @@ int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size, · 2 unmodified lines
135 135
136 list.addr = iu->dma_addr;136 list.addr = iu->dma_addr;
137 list.length = size;137 list.length = size;
138 list.lkey = sess->dev->ib_pd->local_dma_lkey;138 list.lkey = path->dev->ib_pd->local_dma_lkey;
139 139
140 wr = (struct ib_send_wr) {140 wr = (struct ib_send_wr) {
141 .wr_cqe = &iu->cqe,141 .wr_cqe = &iu->cqe,
@@ -188,11 +188,11 @@ static int rtrs_post_rdma_write_imm_empty(struct rtrs_con *con, · 46 unmodified lines
188 struct ib_send_wr *head)188 struct ib_send_wr *head)
189{189{
190 struct ib_rdma_wr wr;190 struct ib_rdma_wr wr;
191 struct rtrs_sess *sess = con->sess;191 struct rtrs_path *path = con->path;
192 enum ib_send_flags sflags;192 enum ib_send_flags sflags;
193 193
194 atomic_dec_if_positive(&con->sq_wr_avail);194 atomic_dec_if_positive(&con->sq_wr_avail);
195 sflags = (atomic_inc_return(&con->wr_cnt) % sess->signal_interval) ?195 sflags = (atomic_inc_return(&con->wr_cnt) % path->signal_interval) ?
196 0 : IB_SEND_SIGNALED;196 0 : IB_SEND_SIGNALED;
197 197
198 wr = (struct ib_rdma_wr) {198 wr = (struct ib_rdma_wr) {
@@ -211,12 +211,12 @@ static void qp_event_handler(struct ib_event *ev, void *ctx) · 12 unmodified lines
211 211
212 switch (ev->event) {212 switch (ev->event) {
213 case IB_EVENT_COMM_EST:213 case IB_EVENT_COMM_EST:
214 rtrs_info(con->sess, "QP event %s (%d) received\n",214 rtrs_info(con->path, "QP event %s (%d) received\n",
215 ib_event_msg(ev->event), ev->event);215 ib_event_msg(ev->event), ev->event);
216 rdma_notify(con->cm_id, IB_EVENT_COMM_EST);216 rdma_notify(con->cm_id, IB_EVENT_COMM_EST);
217 break;217 break;
218 default:218 default:
219 rtrs_info(con->sess, "Unhandled QP event %s (%d) received\n",219 rtrs_info(con->path, "Unhandled QP event %s (%d) received\n",
220 ib_event_msg(ev->event), ev->event);220 ib_event_msg(ev->event), ev->event);
221 break;221 break;
222 }222 }
@@ -224,7 +224,7 @@ static void qp_event_handler(struct ib_event *ev, void *ctx) · 1 unmodified line
224 224
225static bool is_pollqueue(struct rtrs_con *con)225static bool is_pollqueue(struct rtrs_con *con)
226{226{
227 return con->cid >= con->sess->irq_con_num;227 return con->cid >= con->path->irq_con_num;
228}228}
229 229
230static int create_cq(struct rtrs_con *con, int cq_vector, int nr_cqe,230static int create_cq(struct rtrs_con *con, int cq_vector, int nr_cqe,
@@ -240,7 +240,7 @@ static int create_cq(struct rtrs_con *con, int cq_vector, int nr_cqe, · 9 unmodified lines
240 cq = ib_cq_pool_get(cm_id->device, nr_cqe, cq_vector, poll_ctx);240 cq = ib_cq_pool_get(cm_id->device, nr_cqe, cq_vector, poll_ctx);
241 241
242 if (IS_ERR(cq)) {242 if (IS_ERR(cq)) {
243 rtrs_err(con->sess, "Creating completion queue failed, errno: %ld\n",243 rtrs_err(con->path, "Creating completion queue failed, errno: %ld\n",
244 PTR_ERR(cq));244 PTR_ERR(cq));
245 return PTR_ERR(cq);245 return PTR_ERR(cq);
246 }246 }
@@ -271,7 +271,7 @@ static int create_qp(struct rtrs_con *con, struct ib_pd *pd, · 24 unmodified lines
271 271
272 ret = rdma_create_qp(cm_id, pd, &init_attr);272 ret = rdma_create_qp(cm_id, pd, &init_attr);
273 if (ret) {273 if (ret) {
274 rtrs_err(con->sess, "Creating QP failed, err: %d\n", ret);274 rtrs_err(con->path, "Creating QP failed, err: %d\n", ret);
275 return ret;275 return ret;
276 }276 }
277 con->qp = cm_id->qp;277 con->qp = cm_id->qp;
@@ -290,7 +290,7 @@ static void destroy_cq(struct rtrs_con *con) · 12 unmodified lines
290 con->cq = NULL;290 con->cq = NULL;
291}291}
292 292
293int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con,293int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con,
294 u32 max_send_sge, int cq_vector, int nr_cqe,294 u32 max_send_sge, int cq_vector, int nr_cqe,
295 u32 max_send_wr, u32 max_recv_wr,295 u32 max_send_wr, u32 max_recv_wr,
296 enum ib_poll_context poll_ctx)296 enum ib_poll_context poll_ctx)
@@ -301,13 +301,13 @@ int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con, · 4 unmodified lines
301 if (err)301 if (err)
302 return err;302 return err;
303 303
304 err = create_qp(con, sess->dev->ib_pd, max_send_wr, max_recv_wr,304 err = create_qp(con, path->dev->ib_pd, max_send_wr, max_recv_wr,
305 max_send_sge);305 max_send_sge);
306 if (err) {306 if (err) {
307 destroy_cq(con);307 destroy_cq(con);
308 return err;308 return err;
309 }309 }
310 con->sess = sess;310 con->path = path;
311 311
312 return 0;312 return 0;
313}313}
@@ -323,24 +323,24 @@ void rtrs_cq_qp_destroy(struct rtrs_con *con) · 9 unmodified lines
323}323}
324EXPORT_SYMBOL_GPL(rtrs_cq_qp_destroy);324EXPORT_SYMBOL_GPL(rtrs_cq_qp_destroy);
325 325
326static void schedule_hb(struct rtrs_sess *sess)326static void schedule_hb(struct rtrs_path *path)
327{327{
328 queue_delayed_work(sess->hb_wq, &sess->hb_dwork,328 queue_delayed_work(path->hb_wq, &path->hb_dwork,
329 msecs_to_jiffies(sess->hb_interval_ms));329 msecs_to_jiffies(path->hb_interval_ms));
330}330}
331 331
332void rtrs_send_hb_ack(struct rtrs_sess *sess)332void rtrs_send_hb_ack(struct rtrs_path *path)
333{333{
334 struct rtrs_con *usr_con = sess->con[0];334 struct rtrs_con *usr_con = path->con[0];
335 u32 imm;335 u32 imm;
336 int err;336 int err;
337 337
338 imm = rtrs_to_imm(RTRS_HB_ACK_IMM, 0);338 imm = rtrs_to_imm(RTRS_HB_ACK_IMM, 0);
339 err = rtrs_post_rdma_write_imm_empty(usr_con, sess->hb_cqe, imm,339 err = rtrs_post_rdma_write_imm_empty(usr_con, path->hb_cqe, imm,
340 NULL);340 NULL);
341 if (err) {341 if (err) {
342 rtrs_err(sess, "send HB ACK failed, errno: %d\n", err);342 rtrs_err(path, "send HB ACK failed, errno: %d\n", err);
343 sess->hb_err_handler(usr_con);343 path->hb_err_handler(usr_con);
344 return;344 return;
345 }345 }
346}346}
@@ -349,63 +349,63 @@ EXPORT_SYMBOL_GPL(rtrs_send_hb_ack); · 2 unmodified lines
349static void hb_work(struct work_struct *work)349static void hb_work(struct work_struct *work)
350{350{
351 struct rtrs_con *usr_con;351 struct rtrs_con *usr_con;
352 struct rtrs_sess *sess;352 struct rtrs_path *path;
353 u32 imm;353 u32 imm;
354 int err;354 int err;
355 355
356 sess = container_of(to_delayed_work(work), typeof(*sess), hb_dwork);356 path = container_of(to_delayed_work(work), typeof(*path), hb_dwork);
357 usr_con = sess->con[0];357 usr_con = path->con[0];
358 358
359 if (sess->hb_missed_cnt > sess->hb_missed_max) {359 if (path->hb_missed_cnt > path->hb_missed_max) {
360 rtrs_err(sess, "HB missed max reached.\n");360 rtrs_err(path, "HB missed max reached.\n");
361 sess->hb_err_handler(usr_con);361 path->hb_err_handler(usr_con);
362 return;362 return;
363 }363 }
364 if (sess->hb_missed_cnt++) {364 if (path->hb_missed_cnt++) {
365 /* Reschedule work without sending hb */365 /* Reschedule work without sending hb */
366 schedule_hb(sess);366 schedule_hb(path);
367 return;367 return;
368 }368 }
369 369
370 sess->hb_last_sent = ktime_get();370 path->hb_last_sent = ktime_get();
371 371
372 imm = rtrs_to_imm(RTRS_HB_MSG_IMM, 0);372 imm = rtrs_to_imm(RTRS_HB_MSG_IMM, 0);
373 err = rtrs_post_rdma_write_imm_empty(usr_con, sess->hb_cqe, imm,373 err = rtrs_post_rdma_write_imm_empty(usr_con, path->hb_cqe, imm,
374 NULL);374 NULL);
375 if (err) {375 if (err) {
376 rtrs_err(sess, "HB send failed, errno: %d\n", err);376 rtrs_err(path, "HB send failed, errno: %d\n", err);
377 sess->hb_err_handler(usr_con);377 path->hb_err_handler(usr_con);
378 return;378 return;
379 }379 }
380 380
381 schedule_hb(sess);381 schedule_hb(path);
382}382}
383 383
384void rtrs_init_hb(struct rtrs_sess *sess, struct ib_cqe *cqe,384void rtrs_init_hb(struct rtrs_path *path, struct ib_cqe *cqe,
385 unsigned int interval_ms, unsigned int missed_max,385 unsigned int interval_ms, unsigned int missed_max,
386 void (*err_handler)(struct rtrs_con *con),386 void (*err_handler)(struct rtrs_con *con),
387 struct workqueue_struct *wq)387 struct workqueue_struct *wq)
388{388{
389 sess->hb_cqe = cqe;389 path->hb_cqe = cqe;
390 sess->hb_interval_ms = interval_ms;390 path->hb_interval_ms = interval_ms;
391 sess->hb_err_handler = err_handler;391 path->hb_err_handler = err_handler;
392 sess->hb_wq = wq;392 path->hb_wq = wq;
393 sess->hb_missed_max = missed_max;393 path->hb_missed_max = missed_max;
394 sess->hb_missed_cnt = 0;394 path->hb_missed_cnt = 0;
395 INIT_DELAYED_WORK(&sess->hb_dwork, hb_work);395 INIT_DELAYED_WORK(&path->hb_dwork, hb_work);
396}396}
397EXPORT_SYMBOL_GPL(rtrs_init_hb);397EXPORT_SYMBOL_GPL(rtrs_init_hb);
398 398
399void rtrs_start_hb(struct rtrs_sess *sess)399void rtrs_start_hb(struct rtrs_path *path)
400{400{
401 schedule_hb(sess);401 schedule_hb(path);
402}402}
403EXPORT_SYMBOL_GPL(rtrs_start_hb);403EXPORT_SYMBOL_GPL(rtrs_start_hb);
404 404
405void rtrs_stop_hb(struct rtrs_sess *sess)405void rtrs_stop_hb(struct rtrs_path *path)
406{406{
407 cancel_delayed_work_sync(&sess->hb_dwork);407 cancel_delayed_work_sync(&path->hb_dwork);
408 sess->hb_missed_cnt = 0;408 path->hb_missed_cnt = 0;
409}409}
410EXPORT_SYMBOL_GPL(rtrs_stop_hb);410EXPORT_SYMBOL_GPL(rtrs_stop_hb);
411 411
drivers/infiniband/ulp/rtrs/rtrs.h+18−16
@@ -13,9 +13,9 @@ · 12 unmodified lines
13#include <linux/scatterlist.h>13#include <linux/scatterlist.h>
14 14
15struct rtrs_permit;15struct rtrs_permit;
16struct rtrs_clt;16struct rtrs_clt_sess;
17struct rtrs_srv_ctx;17struct rtrs_srv_ctx;
18struct rtrs_srv;18struct rtrs_srv_sess;
19struct rtrs_srv_op;19struct rtrs_srv_op;
20 20
21/*21/*
@@ -52,14 +52,14 @@ struct rtrs_clt_ops { · 30 unmodified lines
52 void (*link_ev)(void *priv, enum rtrs_clt_link_ev ev);52 void (*link_ev)(void *priv, enum rtrs_clt_link_ev ev);
53};53};
54 54
55struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,55struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops,
56 const char *sessname,56 const char *pathname,
57 const struct rtrs_addr *paths,57 const struct rtrs_addr *paths,
58 size_t path_cnt, u16 port,58 size_t path_cnt, u16 port,
59 size_t pdu_sz, u8 reconnect_delay_sec,59 size_t pdu_sz, u8 reconnect_delay_sec,
60 s16 max_reconnect_attempts, u32 nr_poll_queues);60 s16 max_reconnect_attempts, u32 nr_poll_queues);
61 61
62void rtrs_clt_close(struct rtrs_clt *sess);62void rtrs_clt_close(struct rtrs_clt_sess *clt);
63 63
64enum wait_type {64enum wait_type {
65 RTRS_PERMIT_NOWAIT = 0,65 RTRS_PERMIT_NOWAIT = 0,
@@ -77,11 +77,12 @@ enum rtrs_clt_con_type { · 11 unmodified lines
77 RTRS_IO_CON77 RTRS_IO_CON
78};78};
79 79
80struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt *sess,80struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt_sess *sess,
81 enum rtrs_clt_con_type con_type,81 enum rtrs_clt_con_type con_type,
82 enum wait_type wait);82 enum wait_type wait);
83 83
84void rtrs_clt_put_permit(struct rtrs_clt *sess, struct rtrs_permit *permit);84void rtrs_clt_put_permit(struct rtrs_clt_sess *sess,
85 struct rtrs_permit *permit);
85 86
86/**87/**
87 * rtrs_clt_req_ops - it holds the request confirmation callback88 * rtrs_clt_req_ops - it holds the request confirmation callback
@@ -98,10 +99,10 @@ struct rtrs_clt_req_ops { · 10 unmodified lines
98};99};
99 100
100int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,101int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,
101 struct rtrs_clt *sess, struct rtrs_permit *permit,102 struct rtrs_clt_sess *sess, struct rtrs_permit *permit,
102 const struct kvec *vec, size_t nr, size_t len,103 const struct kvec *vec, size_t nr, size_t len,
103 struct scatterlist *sg, unsigned int sg_cnt);104 struct scatterlist *sg, unsigned int sg_cnt);
104int rtrs_clt_rdma_cq_direct(struct rtrs_clt *clt, unsigned int index);105int rtrs_clt_rdma_cq_direct(struct rtrs_clt_sess *clt, unsigned int index);
105 106
106/**107/**
107 * rtrs_attrs - RTRS session attributes108 * rtrs_attrs - RTRS session attributes
@@ -112,7 +113,7 @@ struct rtrs_attrs { · 4 unmodified lines
112 u32 max_segments;113 u32 max_segments;
113};114};
114 115
115int rtrs_clt_query(struct rtrs_clt *sess, struct rtrs_attrs *attr);116int rtrs_clt_query(struct rtrs_clt_sess *sess, struct rtrs_attrs *attr);
116 117
117/*118/*
118 * Here goes RTRS server API119 * Here goes RTRS server API
@@ -163,7 +164,7 @@ struct rtrs_srv_ops { · 44 unmodified lines
163 * @priv: Private data from user if previously set with164 * @priv: Private data from user if previously set with
164 * rtrs_srv_set_sess_priv()165 * rtrs_srv_set_sess_priv()
165 */166 */
166 int (*link_ev)(struct rtrs_srv *sess, enum rtrs_srv_link_ev ev,167 int (*link_ev)(struct rtrs_srv_sess *sess, enum rtrs_srv_link_ev ev,
167 void *priv);168 void *priv);
168};169};
169 170
@@ -173,11 +174,12 @@ void rtrs_srv_close(struct rtrs_srv_ctx *ctx); · 3 unmodified lines
173 174
174bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int errno);175bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int errno);
175 176
176void rtrs_srv_set_sess_priv(struct rtrs_srv *sess, void *priv);177void rtrs_srv_set_sess_priv(struct rtrs_srv_sess *sess, void *priv);
177 178
178int rtrs_srv_get_sess_name(struct rtrs_srv *sess, char *sessname, size_t len);179int rtrs_srv_get_path_name(struct rtrs_srv_sess *sess, char *pathname,
180 size_t len);
179 181
180int rtrs_srv_get_queue_depth(struct rtrs_srv *sess);182int rtrs_srv_get_queue_depth(struct rtrs_srv_sess *sess);
181 183
182int rtrs_addr_to_sockaddr(const char *str, size_t len, u16 port,184int rtrs_addr_to_sockaddr(const char *str, size_t len, u16 port,
183 struct rtrs_addr *addr);185 struct rtrs_addr *addr);
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h+1−1
@@ -497,7 +497,7 @@ struct cpl_t5_pass_accept_rpl { · 496 unmodified lines
497 __be32 opt2;497 __be32 opt2;
498 __be64 opt0;498 __be64 opt0;
499 __be32 iss;499 __be32 iss;
500 __be32 rsvd;500 __be32 rsvd[3];
501};501};
502 502
503struct cpl_act_open_req {503struct cpl_act_open_req {
include/rdma/ib_mad.h+1−0
@@ -276,6 +276,7 @@ enum ib_port_capability_mask2_bits { · 275 unmodified lines
276 IB_PORT_SWITCH_PORT_STATE_TABLE_SUP = 1 << 3,276 IB_PORT_SWITCH_PORT_STATE_TABLE_SUP = 1 << 3,
277 IB_PORT_LINK_WIDTH_2X_SUP = 1 << 4,277 IB_PORT_LINK_WIDTH_2X_SUP = 1 << 4,
278 IB_PORT_LINK_SPEED_HDR_SUP = 1 << 5,278 IB_PORT_LINK_SPEED_HDR_SUP = 1 << 5,
279 IB_PORT_LINK_SPEED_NDR_SUP = 1 << 10,
279};280};
280 281
281#define OPA_CLASS_PORT_INFO_PR_SUPPORT BIT(26)282#define OPA_CLASS_PORT_INFO_PR_SUPPORT BIT(26)
include/rdma/ib_smi.h+11−1
@@ -144,5 +144,15 @@ ib_get_smp_direction(struct ib_smp *smp) · 143 unmodified lines
144#define IB_NOTICE_TRAP_DR_NOTICE 0x80144#define IB_NOTICE_TRAP_DR_NOTICE 0x80
145#define IB_NOTICE_TRAP_DR_TRUNC 0x40145#define IB_NOTICE_TRAP_DR_TRUNC 0x40
146 146
147 147/**
148 * ib_init_query_mad - Initialize query MAD.
149 * @mad: MAD to initialize.
150 */
151static inline void ib_init_query_mad(struct ib_smp *mad)
152{
153 mad->base_version = IB_MGMT_BASE_VERSION;
154 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
155 mad->class_version = 1;
156 mad->method = IB_MGMT_METHOD_GET;
157}
148#endif /* IB_SMI_H */158#endif /* IB_SMI_H */
include/rdma/ib_verbs.h+17−0
@@ -4749,6 +4749,23 @@ static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn) · 4748 unmodified lines
4749 return (u32)(v & IB_GRH_FLOWLABEL_MASK);4749 return (u32)(v & IB_GRH_FLOWLABEL_MASK);
4750}4750}
4751 4751
4752/**
4753 * rdma_get_udp_sport - Calculate and set UDP source port based on the flow
4754 * label. If flow label is not defined in GRH then
4755 * calculate it based on lqpn/rqpn.
4756 *
4757 * @fl: flow label from GRH
4758 * @lqpn: local qp number
4759 * @rqpn: remote qp number
4760 */
4761static inline u16 rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
4762{
4763 if (!fl)
4764 fl = rdma_calc_flow_label(lqpn, rqpn);
4765
4766 return rdma_flow_label_to_udp_sport(fl);
4767}
4768
4752const struct ib_port_immutable*4769const struct ib_port_immutable*
4753ib_port_immutable_read(struct ib_device *dev, unsigned int port);4770ib_port_immutable_read(struct ib_device *dev, unsigned int port);
4754#endif /* IB_VERBS_H */4771#endif /* IB_VERBS_H */
include/uapi/rdma/hns-abi.h+2−0
@@ -77,10 +77,12 @@ enum hns_roce_qp_cap_flags { · 76 unmodified lines
77 HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0,77 HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0,
78 HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1,78 HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1,
79 HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2,79 HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2,
80 HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5,
80};81};
81 82
82struct hns_roce_ib_create_qp_resp {83struct hns_roce_ib_create_qp_resp {
83 __aligned_u64 cap_flags;84 __aligned_u64 cap_flags;
85 __aligned_u64 dwqe_mmap_key;
84};86};
85 87
86struct hns_roce_ib_alloc_ucontext_resp {88struct hns_roce_ib_alloc_ucontext_resp {