History

RDMA/hns: Add SRQ support for hip08 kernel mode

This patch implements the SRQ(Share Receive Queue) verbs and update the poll cq verbs to deal with SRQ complentions.

Lijun Ou committed 7 years ago

Commit c7bcb131 parent 5c1f16710 files · +1098 −52Browse files

Unified · Split

10 files
  1. drivers/infiniband/hw/hns/hns_roce_cmd.h +4 −0
  2. drivers/infiniband/hw/hns/hns_roce_device.h +57 −1
  3. drivers/infiniband/hw/hns/hns_roce_hem.c +31 −1
  4. drivers/infiniband/hw/hns/hns_roce_hw_v2.c +360 −24
  5. drivers/infiniband/hw/hns/hns_roce_hw_v2.h +86 −0
  6. drivers/infiniband/hw/hns/hns_roce_main.c +15 −0
  7. drivers/infiniband/hw/hns/hns_roce_mr.c +122 −15
  8. drivers/infiniband/hw/hns/hns_roce_qp.c +10 −11
  9. drivers/infiniband/hw/hns/hns_roce_srq.c +407 −0
  10. include/uapi/rdma/hns-abi.h +6 −0
drivers/infiniband/hw/hns/hns_roce_cmd.h+4−0
@@ -120,6 +120,10 @@ enum { · 119 unmodified lines
120 HNS_ROCE_CMD_SQD2RTS_QP = 0x20,120 HNS_ROCE_CMD_SQD2RTS_QP = 0x20,
121 HNS_ROCE_CMD_2RST_QP = 0x21,121 HNS_ROCE_CMD_2RST_QP = 0x21,
122 HNS_ROCE_CMD_QUERY_QP = 0x22,122 HNS_ROCE_CMD_QUERY_QP = 0x22,
123 HNS_ROCE_CMD_SW2HW_SRQ = 0x70,
124 HNS_ROCE_CMD_MODIFY_SRQC = 0x72,
125 HNS_ROCE_CMD_QUERY_SRQC = 0x73,
126 HNS_ROCE_CMD_HW2SW_SRQ = 0x74,
123};127};
124 128
125int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,129int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
drivers/infiniband/hw/hns/hns_roce_device.h+57−1
@@ -111,6 +111,9 @@ · 110 unmodified lines
111#define PAGES_SHIFT_24 24111#define PAGES_SHIFT_24 24
112#define PAGES_SHIFT_32 32112#define PAGES_SHIFT_32 32
113 113
114#define HNS_ROCE_IDX_QUE_ENTRY_SZ 4
115#define SRQ_DB_REG 0x230
116
114enum {117enum {
115 HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0,118 HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0,
116 HNS_ROCE_SUPPORT_SQ_RECORD_DB = 1 << 1,119 HNS_ROCE_SUPPORT_SQ_RECORD_DB = 1 << 1,
@@ -436,9 +439,37 @@ struct hns_roce_cq { · 319 unmodified lines
436 struct completion free;439 struct completion free;
437};440};
438 441
442struct hns_roce_idx_que {
443 struct hns_roce_buf idx_buf;
444 int entry_sz;
445 u32 buf_size;
446 struct ib_umem *umem;
447 struct hns_roce_mtt mtt;
448 u64 *bitmap;
449};
450
439struct hns_roce_srq {451struct hns_roce_srq {
440 struct ib_srq ibsrq;452 struct ib_srq ibsrq;
441 int srqn;453 void (*event)(struct hns_roce_srq *srq, enum hns_roce_event event);
454 unsigned long srqn;
455 int max;
456 int max_gs;
457 int wqe_shift;
458 void __iomem *db_reg_l;
459
460 atomic_t refcount;
461 struct completion free;
462
463 struct hns_roce_buf buf;
464 u64 *wrid;
465 struct ib_umem *umem;
466 struct hns_roce_mtt mtt;
467 struct hns_roce_idx_que idx_que;
468 spinlock_t lock;
469 int head;
470 int tail;
471 u16 wqe_ctr;
472 struct mutex mutex;
442};473};
443 474
444struct hns_roce_uar_table {475struct hns_roce_uar_table {
@@ -761,6 +792,12 @@ struct hns_roce_caps { · 316 unmodified lines
761 u32 cqe_ba_pg_sz;792 u32 cqe_ba_pg_sz;
762 u32 cqe_buf_pg_sz;793 u32 cqe_buf_pg_sz;
763 u32 cqe_hop_num;794 u32 cqe_hop_num;
795 u32 srqwqe_ba_pg_sz;
796 u32 srqwqe_buf_pg_sz;
797 u32 srqwqe_hop_num;
798 u32 idx_ba_pg_sz;
799 u32 idx_buf_pg_sz;
800 u32 idx_hop_num;
764 u32 eqe_ba_pg_sz;801 u32 eqe_ba_pg_sz;
765 u32 eqe_buf_pg_sz;802 u32 eqe_buf_pg_sz;
766 u32 eqe_hop_num;803 u32 eqe_hop_num;
@@ -829,6 +866,17 @@ struct hns_roce_hw { · 62 unmodified lines
829 int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);866 int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
830 int (*init_eq)(struct hns_roce_dev *hr_dev);867 int (*init_eq)(struct hns_roce_dev *hr_dev);
831 void (*cleanup_eq)(struct hns_roce_dev *hr_dev);868 void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
869 void (*write_srqc)(struct hns_roce_dev *hr_dev,
870 struct hns_roce_srq *srq, u32 pdn, u16 xrcd, u32 cqn,
871 void *mb_buf, u64 *mtts_wqe, u64 *mtts_idx,
872 dma_addr_t dma_handle_wqe,
873 dma_addr_t dma_handle_idx);
874 int (*modify_srq)(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr,
875 enum ib_srq_attr_mask srq_attr_mask,
876 struct ib_udata *udata);
877 int (*query_srq)(struct ib_srq *ibsrq, struct ib_srq_attr *attr);
878 int (*post_srq_recv)(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
879 const struct ib_recv_wr **bad_wr);
832};880};
833 881
834struct hns_roce_dev {882struct hns_roce_dev {
@@ -1038,6 +1086,14 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, · 203 unmodified lines
1038int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,1086int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
1039 struct hns_roce_mtt *mtt, struct ib_umem *umem);1087 struct hns_roce_mtt *mtt, struct ib_umem *umem);
1040 1088
1089struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
1090 struct ib_srq_init_attr *srq_init_attr,
1091 struct ib_udata *udata);
1092int hns_roce_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr,
1093 enum ib_srq_attr_mask srq_attr_mask,
1094 struct ib_udata *udata);
1095int hns_roce_destroy_srq(struct ib_srq *ibsrq);
1096
1041struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd,1097struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd,
1042 struct ib_qp_init_attr *init_attr,1098 struct ib_qp_init_attr *init_attr,
1043 struct ib_udata *udata);1099 struct ib_udata *udata);
drivers/infiniband/hw/hns/hns_roce_hem.c+31−1
@@ -46,7 +46,9 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) · 45 unmodified lines
46 (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||46 (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
47 (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||47 (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
48 (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||48 (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
49 (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT))49 (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT) ||
50 (hr_dev->caps.srqwqe_hop_num && type == HEM_TYPE_SRQWQE) ||
51 (hr_dev->caps.idx_hop_num && type == HEM_TYPE_IDX))
50 return true;52 return true;
51 53
52 return false;54 return false;
@@ -147,6 +149,22 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, · 94 unmodified lines
147 mhop->ba_l0_num = mhop->bt_chunk_size / 8;149 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
148 mhop->hop_num = hr_dev->caps.cqe_hop_num;150 mhop->hop_num = hr_dev->caps.cqe_hop_num;
149 break;151 break;
152 case HEM_TYPE_SRQWQE:
153 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz
154 + PAGE_SHIFT);
155 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
156 + PAGE_SHIFT);
157 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
158 mhop->hop_num = hr_dev->caps.srqwqe_hop_num;
159 break;
160 case HEM_TYPE_IDX:
161 mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz
162 + PAGE_SHIFT);
163 mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
164 + PAGE_SHIFT);
165 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
166 mhop->hop_num = hr_dev->caps.idx_hop_num;
167 break;
150 default:168 default:
151 dev_err(dev, "Table %d not support multi-hop addressing!\n",169 dev_err(dev, "Table %d not support multi-hop addressing!\n",
152 table->type);170 table->type);
@@ -906,6 +924,18 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, · 753 unmodified lines
906 bt_chunk_size = buf_chunk_size;924 bt_chunk_size = buf_chunk_size;
907 hop_num = hr_dev->caps.cqe_hop_num;925 hop_num = hr_dev->caps.cqe_hop_num;
908 break;926 break;
927 case HEM_TYPE_SRQWQE:
928 buf_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
929 + PAGE_SHIFT);
930 bt_chunk_size = buf_chunk_size;
931 hop_num = hr_dev->caps.srqwqe_hop_num;
932 break;
933 case HEM_TYPE_IDX:
934 buf_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
935 + PAGE_SHIFT);
936 bt_chunk_size = buf_chunk_size;
937 hop_num = hr_dev->caps.idx_hop_num;
938 break;
909 default:939 default:
910 dev_err(dev,940 dev_err(dev,
911 "Table %d not support to init hem table here!\n",941 "Table %d not support to init hem table here!\n",
drivers/infiniband/hw/hns/hns_roce_hw_v2.c+360−24
@@ -1339,6 +1339,12 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) · 1338 unmodified lines
1339 caps->cqe_ba_pg_sz = 0;1339 caps->cqe_ba_pg_sz = 0;
1340 caps->cqe_buf_pg_sz = 0;1340 caps->cqe_buf_pg_sz = 0;
1341 caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;1341 caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
1342 caps->srqwqe_ba_pg_sz = 0;
1343 caps->srqwqe_buf_pg_sz = 0;
1344 caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM;
1345 caps->idx_ba_pg_sz = 0;
1346 caps->idx_buf_pg_sz = 0;
1347 caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM;
1342 caps->eqe_ba_pg_sz = 0;1348 caps->eqe_ba_pg_sz = 0;
1343 caps->eqe_buf_pg_sz = 0;1349 caps->eqe_buf_pg_sz = 0;
1344 caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;1350 caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
@@ -2028,6 +2034,27 @@ static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq) · 683 unmodified lines
2028 return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);2034 return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2029}2035}
2030 2036
2037static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
2038{
2039 return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift);
2040}
2041
2042static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
2043{
2044 u32 bitmap_num;
2045 int bit_num;
2046
2047 /* always called with interrupts disabled. */
2048 spin_lock(&srq->lock);
2049
2050 bitmap_num = wqe_index / (sizeof(u64) * 8);
2051 bit_num = wqe_index % (sizeof(u64) * 8);
2052 srq->idx_que.bitmap[bitmap_num] |= (1ULL << bit_num);
2053 srq->tail++;
2054
2055 spin_unlock(&srq->lock);
2056}
2057
2031static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)2058static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2032{2059{
2033 *hr_cq->set_ci_db = cons_index & 0xffffff;2060 *hr_cq->set_ci_db = cons_index & 0xffffff;
@@ -2039,6 +2066,7 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, · 5 unmodified lines
2039 struct hns_roce_v2_cqe *cqe, *dest;2066 struct hns_roce_v2_cqe *cqe, *dest;
2040 u32 prod_index;2067 u32 prod_index;
2041 int nfreed = 0;2068 int nfreed = 0;
2069 int wqe_index;
2042 u8 owner_bit;2070 u8 owner_bit;
2043 2071
2044 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);2072 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
@@ -2056,7 +2084,13 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, · 11 unmodified lines
2056 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,2084 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2057 V2_CQE_BYTE_16_LCL_QPN_S) &2085 V2_CQE_BYTE_16_LCL_QPN_S) &
2058 HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {2086 HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
2059 /* In v1 engine, not support SRQ */2087 if (srq &&
2088 roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
2089 wqe_index = roce_get_field(cqe->byte_4,
2090 V2_CQE_BYTE_4_WQE_INDX_M,
2091 V2_CQE_BYTE_4_WQE_INDX_S);
2092 hns_roce_free_srq_wqe(srq, wqe_index);
2093 }
2060 ++nfreed;2094 ++nfreed;
2061 } else if (nfreed) {2095 } else if (nfreed) {
2062 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &2096 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
@@ -2233,6 +2267,7 @@ static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, · 170 unmodified lines
2233static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,2267static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2234 struct hns_roce_qp **cur_qp, struct ib_wc *wc)2268 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2235{2269{
2270 struct hns_roce_srq *srq = NULL;
2236 struct hns_roce_dev *hr_dev;2271 struct hns_roce_dev *hr_dev;
2237 struct hns_roce_v2_cqe *cqe;2272 struct hns_roce_v2_cqe *cqe;
2238 struct hns_roce_qp *hr_qp;2273 struct hns_roce_qp *hr_qp;
@@ -2275,6 +2310,37 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 36 unmodified lines
2275 wc->qp = &(*cur_qp)->ibqp;2310 wc->qp = &(*cur_qp)->ibqp;
2276 wc->vendor_err = 0;2311 wc->vendor_err = 0;
2277 2312
2313 if (is_send) {
2314 wq = &(*cur_qp)->sq;
2315 if ((*cur_qp)->sq_signal_bits) {
2316 /*
2317 * If sg_signal_bit is 1,
2318 * firstly tail pointer updated to wqe
2319 * which current cqe correspond to
2320 */
2321 wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2322 V2_CQE_BYTE_4_WQE_INDX_M,
2323 V2_CQE_BYTE_4_WQE_INDX_S);
2324 wq->tail += (wqe_ctr - (u16)wq->tail) &
2325 (wq->wqe_cnt - 1);
2326 }
2327
2328 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2329 ++wq->tail;
2330 } else if ((*cur_qp)->ibqp.srq) {
2331 srq = to_hr_srq((*cur_qp)->ibqp.srq);
2332 wqe_ctr = le16_to_cpu(roce_get_field(cqe->byte_4,
2333 V2_CQE_BYTE_4_WQE_INDX_M,
2334 V2_CQE_BYTE_4_WQE_INDX_S));
2335 wc->wr_id = srq->wrid[wqe_ctr];
2336 hns_roce_free_srq_wqe(srq, wqe_ctr);
2337 } else {
2338 /* Update tail pointer, record wr_id */
2339 wq = &(*cur_qp)->rq;
2340 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2341 ++wq->tail;
2342 }
2343
2278 status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,2344 status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2279 V2_CQE_BYTE_4_STATUS_S);2345 V2_CQE_BYTE_4_STATUS_S);
2280 switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {2346 switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
@@ -2394,23 +2460,6 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 113 unmodified lines
2394 wc->status = IB_WC_GENERAL_ERR;2460 wc->status = IB_WC_GENERAL_ERR;
2395 break;2461 break;
2396 }2462 }
2397
2398 wq = &(*cur_qp)->sq;
2399 if ((*cur_qp)->sq_signal_bits) {
2400 /*
2401 * If sg_signal_bit is 1,
2402 * firstly tail pointer updated to wqe
2403 * which current cqe correspond to
2404 */
2405 wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2406 V2_CQE_BYTE_4_WQE_INDX_M,
2407 V2_CQE_BYTE_4_WQE_INDX_S);
2408 wq->tail += (wqe_ctr - (u16)wq->tail) &
2409 (wq->wqe_cnt - 1);
2410 }
2411
2412 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2413 ++wq->tail;
2414 } else {2463 } else {
2415 /* RQ correspond to CQE */2464 /* RQ correspond to CQE */
2416 wc->byte_len = le32_to_cpu(cqe->byte_cnt);2465 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
@@ -2455,11 +2504,6 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 38 unmodified lines
2455 return -EAGAIN;2504 return -EAGAIN;
2456 }2505 }
2457 2506
2458 /* Update tail pointer, record wr_id */
2459 wq = &(*cur_qp)->rq;
2460 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2461 ++wq->tail;
2462
2463 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,2507 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2464 V2_CQE_BYTE_32_SL_S);2508 V2_CQE_BYTE_32_SL_S);
2465 wc->src_qp = (u8)roce_get_field(cqe->byte_32,2509 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
@@ -2768,6 +2812,8 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, · 302 unmodified lines
2768 2812
2769 roce_set_field(context->byte_20_smac_sgid_idx,2813 roce_set_field(context->byte_20_smac_sgid_idx,
2770 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,2814 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2815 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
2816 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || ibqp->srq) ? 0 :
2771 ilog2((unsigned int)hr_qp->rq.wqe_cnt));2817 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2772 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,2818 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2773 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);2819 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
@@ -3109,6 +3155,8 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, · 335 unmodified lines
3109 3155
3110 roce_set_field(context->byte_20_smac_sgid_idx,3156 roce_set_field(context->byte_20_smac_sgid_idx,
3111 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,3157 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
3158 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3159 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || ibqp->srq) ? 0 :
3112 ilog2((unsigned int)hr_qp->rq.wqe_cnt));3160 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3113 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,3161 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3114 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);3162 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
@@ -3810,6 +3858,11 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, · 695 unmodified lines
3810 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))3858 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
3811 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);3859 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
3812 3860
3861 roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
3862 ibqp->srq ? 1 : 0);
3863 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3864 V2_QPC_BYTE_108_INV_CREDIT_S, 0);
3865
3813 /* Every status migrate must change state */3866 /* Every status migrate must change state */
3814 roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,3867 roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
3815 V2_QPC_BYTE_60_QP_ST_S, new_state);3868 V2_QPC_BYTE_60_QP_ST_S, new_state);
@@ -4095,7 +4148,8 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, · 279 unmodified lines
4095 hns_roce_free_db(hr_dev, &hr_qp->rdb);4148 hns_roce_free_db(hr_dev, &hr_qp->rdb);
4096 }4149 }
4097 4150
4098 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {4151 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
4152 hr_qp->rq.wqe_cnt) {
4099 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);4153 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
4100 kfree(hr_qp->rq_inl_buf.wqe_list);4154 kfree(hr_qp->rq_inl_buf.wqe_list);
4101 }4155 }
@@ -5350,6 +5404,284 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) · 1248 unmodified lines
5350 destroy_workqueue(hr_dev->irq_workq);5404 destroy_workqueue(hr_dev->irq_workq);
5351}5405}
5352 5406
5407static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
5408 struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
5409 u32 cqn, void *mb_buf, u64 *mtts_wqe,
5410 u64 *mtts_idx, dma_addr_t dma_handle_wqe,
5411 dma_addr_t dma_handle_idx)
5412{
5413 struct hns_roce_srq_context *srq_context;
5414
5415 srq_context = mb_buf;
5416 memset(srq_context, 0, sizeof(*srq_context));
5417
5418 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
5419 SRQC_BYTE_4_SRQ_ST_S, 1);
5420
5421 roce_set_field(srq_context->byte_4_srqn_srqst,
5422 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
5423 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
5424 (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
5425 hr_dev->caps.srqwqe_hop_num));
5426 roce_set_field(srq_context->byte_4_srqn_srqst,
5427 SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
5428 ilog2(srq->max));
5429
5430 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
5431 SRQC_BYTE_4_SRQN_S, srq->srqn);
5432
5433 roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5434 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
5435
5436 roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
5437 SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
5438
5439 srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
5440
5441 roce_set_field(srq_context->byte_24_wqe_bt_ba,
5442 SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
5443 SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
5444 cpu_to_le32(dma_handle_wqe >> 35));
5445
5446 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
5447 SRQC_BYTE_28_PD_S, pdn);
5448 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
5449 SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
5450 fls(srq->max_gs - 1));
5451
5452 srq_context->idx_bt_ba = (u32)(dma_handle_idx >> 3);
5453 srq_context->idx_bt_ba = cpu_to_le32(srq_context->idx_bt_ba);
5454 roce_set_field(srq_context->rsv_idx_bt_ba,
5455 SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
5456 SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
5457 cpu_to_le32(dma_handle_idx >> 35));
5458
5459 srq_context->idx_cur_blk_addr = (u32)(mtts_idx[0] >> PAGE_ADDR_SHIFT);
5460 srq_context->idx_cur_blk_addr =
5461 cpu_to_le32(srq_context->idx_cur_blk_addr);
5462 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5463 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
5464 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
5465 cpu_to_le32((mtts_idx[0]) >> (32 + PAGE_ADDR_SHIFT)));
5466 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5467 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
5468 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
5469 hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
5470 hr_dev->caps.idx_hop_num);
5471
5472 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5473 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
5474 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
5475 hr_dev->caps.idx_ba_pg_sz);
5476 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5477 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
5478 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
5479 hr_dev->caps.idx_buf_pg_sz);
5480
5481 srq_context->idx_nxt_blk_addr = (u32)(mtts_idx[1] >> PAGE_ADDR_SHIFT);
5482 srq_context->idx_nxt_blk_addr =
5483 cpu_to_le32(srq_context->idx_nxt_blk_addr);
5484 roce_set_field(srq_context->rsv_idxnxtblkaddr,
5485 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
5486 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
5487 cpu_to_le32((mtts_idx[1]) >> (32 + PAGE_ADDR_SHIFT)));
5488 roce_set_field(srq_context->byte_56_xrc_cqn,
5489 SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
5490 cqn);
5491 roce_set_field(srq_context->byte_56_xrc_cqn,
5492 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
5493 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
5494 hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);
5495 roce_set_field(srq_context->byte_56_xrc_cqn,
5496 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
5497 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
5498 hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);
5499
5500 roce_set_bit(srq_context->db_record_addr_record_en,
5501 SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
5502}
5503
5504static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
5505 struct ib_srq_attr *srq_attr,
5506 enum ib_srq_attr_mask srq_attr_mask,
5507 struct ib_udata *udata)
5508{
5509 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5510 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5511 struct hns_roce_srq_context *srq_context;
5512 struct hns_roce_srq_context *srqc_mask;
5513 struct hns_roce_cmd_mailbox *mailbox;
5514 int ret;
5515
5516 if (srq_attr_mask & IB_SRQ_LIMIT) {
5517 if (srq_attr->srq_limit >= srq->max)
5518 return -EINVAL;
5519
5520 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5521 if (IS_ERR(mailbox))
5522 return PTR_ERR(mailbox);
5523
5524 srq_context = mailbox->buf;
5525 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
5526
5527 memset(srqc_mask, 0xff, sizeof(*srqc_mask));
5528
5529 roce_set_field(srq_context->byte_8_limit_wl,
5530 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5531 SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
5532 roce_set_field(srqc_mask->byte_8_limit_wl,
5533 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5534 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
5535
5536 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
5537 HNS_ROCE_CMD_MODIFY_SRQC,
5538 HNS_ROCE_CMD_TIMEOUT_MSECS);
5539 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5540 if (ret) {
5541 dev_err(hr_dev->dev,
5542 "MODIFY SRQ Failed to cmd mailbox.\n");
5543 return ret;
5544 }
5545 }
5546
5547 return 0;
5548}
5549
5550int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
5551{
5552 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5553 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5554 struct hns_roce_srq_context *srq_context;
5555 struct hns_roce_cmd_mailbox *mailbox;
5556 int limit_wl;
5557 int ret;
5558
5559 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5560 if (IS_ERR(mailbox))
5561 return PTR_ERR(mailbox);
5562
5563 srq_context = mailbox->buf;
5564 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
5565 HNS_ROCE_CMD_QUERY_SRQC,
5566 HNS_ROCE_CMD_TIMEOUT_MSECS);
5567 if (ret) {
5568 dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n");
5569 goto out;
5570 }
5571
5572 limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
5573 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5574 SRQC_BYTE_8_SRQ_LIMIT_WL_S);
5575
5576 attr->srq_limit = limit_wl;
5577 attr->max_wr = srq->max - 1;
5578 attr->max_sge = srq->max_gs;
5579
5580 memcpy(srq_context, mailbox->buf, sizeof(*srq_context));
5581
5582out:
5583 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5584 return ret;
5585}
5586
5587static int find_empty_entry(struct hns_roce_idx_que *idx_que)
5588{
5589 int bit_num;
5590 int i;
5591
5592 /* bitmap[i] is set zero if all bits are allocated */
5593 for (i = 0; idx_que->bitmap[i] == 0; ++i)
5594 ;
5595 bit_num = ffs(idx_que->bitmap[i]);
5596 idx_que->bitmap[i] &= ~(1ULL << (bit_num - 1));
5597
5598 return i * sizeof(u64) * 8 + (bit_num - 1);
5599}
5600
5601static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
5602 int cur_idx, int wqe_idx)
5603{
5604 unsigned int *addr;
5605
5606 addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf,
5607 cur_idx * idx_que->entry_sz);
5608 *addr = wqe_idx;
5609}
5610
5611static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
5612 const struct ib_recv_wr *wr,
5613 const struct ib_recv_wr **bad_wr)
5614{
5615 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5616 struct hns_roce_v2_wqe_data_seg *dseg;
5617 struct hns_roce_v2_db srq_db;
5618 unsigned long flags;
5619 int ret = 0;
5620 int wqe_idx;
5621 void *wqe;
5622 int nreq;
5623 int ind;
5624 int i;
5625
5626 spin_lock_irqsave(&srq->lock, flags);
5627
5628 ind = srq->head & (srq->max - 1);
5629
5630 for (nreq = 0; wr; ++nreq, wr = wr->next) {
5631 if (unlikely(wr->num_sge > srq->max_gs)) {
5632 ret = -EINVAL;
5633 *bad_wr = wr;
5634 break;
5635 }
5636
5637 if (unlikely(srq->head == srq->tail)) {
5638 ret = -ENOMEM;
5639 *bad_wr = wr;
5640 break;
5641 }
5642
5643 wqe_idx = find_empty_entry(&srq->idx_que);
5644 fill_idx_queue(&srq->idx_que, ind, wqe_idx);
5645 wqe = get_srq_wqe(srq, wqe_idx);
5646 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
5647
5648 for (i = 0; i < wr->num_sge; ++i) {
5649 dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
5650 dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
5651 dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
5652 }
5653
5654 if (i < srq->max_gs) {
5655 dseg->len = 0;
5656 dseg->lkey = cpu_to_le32(0x100);
5657 dseg->addr = 0;
5658 }
5659
5660 srq->wrid[wqe_idx] = wr->wr_id;
5661 ind = (ind + 1) & (srq->max - 1);
5662 }
5663
5664 if (likely(nreq)) {
5665 srq->head += nreq;
5666
5667 /*
5668 * Make sure that descriptors are written before
5669 * doorbell record.
5670 */
5671 wmb();
5672
5673 srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << 24 | srq->srqn;
5674 srq_db.parameter = srq->head;
5675
5676 hns_roce_write64_k((__le32 *)&srq_db, srq->db_reg_l);
5677
5678 }
5679
5680 spin_unlock_irqrestore(&srq->lock, flags);
5681
5682 return ret;
5683}
5684
5353static const struct hns_roce_hw hns_roce_hw_v2 = {5685static const struct hns_roce_hw hns_roce_hw_v2 = {
5354 .cmq_init = hns_roce_v2_cmq_init,5686 .cmq_init = hns_roce_v2_cmq_init,
5355 .cmq_exit = hns_roce_v2_cmq_exit,5687 .cmq_exit = hns_roce_v2_cmq_exit,
@@ -5377,6 +5709,10 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { · 21 unmodified lines
5377 .poll_cq = hns_roce_v2_poll_cq,5709 .poll_cq = hns_roce_v2_poll_cq,
5378 .init_eq = hns_roce_v2_init_eq_table,5710 .init_eq = hns_roce_v2_init_eq_table,
5379 .cleanup_eq = hns_roce_v2_cleanup_eq_table,5711 .cleanup_eq = hns_roce_v2_cleanup_eq_table,
5712 .write_srqc = hns_roce_v2_write_srqc,
5713 .modify_srq = hns_roce_v2_modify_srq,
5714 .query_srq = hns_roce_v2_query_srq,
5715 .post_srq_recv = hns_roce_v2_post_srq_recv,
5380};5716};
5381 5717
5382static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {5718static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
drivers/infiniband/hw/hns/hns_roce_hw_v2.h+86−0
@@ -93,8 +93,10 @@ · 92 unmodified lines
93#define HNS_ROCE_CONTEXT_HOP_NUM 193#define HNS_ROCE_CONTEXT_HOP_NUM 1
94#define HNS_ROCE_MTT_HOP_NUM 194#define HNS_ROCE_MTT_HOP_NUM 1
95#define HNS_ROCE_CQE_HOP_NUM 195#define HNS_ROCE_CQE_HOP_NUM 1
96#define HNS_ROCE_SRQWQE_HOP_NUM 1
96#define HNS_ROCE_PBL_HOP_NUM 297#define HNS_ROCE_PBL_HOP_NUM 2
97#define HNS_ROCE_EQE_HOP_NUM 298#define HNS_ROCE_EQE_HOP_NUM 2
99#define HNS_ROCE_IDX_HOP_NUM 1
98 100
99#define HNS_ROCE_V2_GID_INDEX_NUM 256101#define HNS_ROCE_V2_GID_INDEX_NUM 256
100 102
@@ -334,6 +336,90 @@ struct hns_roce_v2_cq_context { · 233 unmodified lines
334#define V2_CQC_BYTE_64_SE_CQE_IDX_S 0336#define V2_CQC_BYTE_64_SE_CQE_IDX_S 0
335#define V2_CQC_BYTE_64_SE_CQE_IDX_M GENMASK(23, 0)337#define V2_CQC_BYTE_64_SE_CQE_IDX_M GENMASK(23, 0)
336 338
339struct hns_roce_srq_context {
340 __le32 byte_4_srqn_srqst;
341 __le32 byte_8_limit_wl;
342 __le32 byte_12_xrcd;
343 __le32 byte_16_pi_ci;
344 __le32 wqe_bt_ba;
345 __le32 byte_24_wqe_bt_ba;
346 __le32 byte_28_rqws_pd;
347 __le32 idx_bt_ba;
348 __le32 rsv_idx_bt_ba;
349 __le32 idx_cur_blk_addr;
350 __le32 byte_44_idxbufpgsz_addr;
351 __le32 idx_nxt_blk_addr;
352 __le32 rsv_idxnxtblkaddr;
353 __le32 byte_56_xrc_cqn;
354 __le32 db_record_addr_record_en;
355 __le32 db_record_addr;
356};
357
358#define SRQC_BYTE_4_SRQ_ST_S 0
359#define SRQC_BYTE_4_SRQ_ST_M GENMASK(1, 0)
360
361#define SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S 2
362#define SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M GENMASK(3, 2)
363
364#define SRQC_BYTE_4_SRQ_SHIFT_S 4
365#define SRQC_BYTE_4_SRQ_SHIFT_M GENMASK(7, 4)
366
367#define SRQC_BYTE_4_SRQN_S 8
368#define SRQC_BYTE_4_SRQN_M GENMASK(31, 8)
369
370#define SRQC_BYTE_8_SRQ_LIMIT_WL_S 0
371#define SRQC_BYTE_8_SRQ_LIMIT_WL_M GENMASK(15, 0)
372
373#define SRQC_BYTE_12_SRQ_XRCD_S 0
374#define SRQC_BYTE_12_SRQ_XRCD_M GENMASK(23, 0)
375
376#define SRQC_BYTE_16_SRQ_PRODUCER_IDX_S 0
377#define SRQC_BYTE_16_SRQ_PRODUCER_IDX_M GENMASK(15, 0)
378
379#define SRQC_BYTE_16_SRQ_CONSUMER_IDX_S 0
380#define SRQC_BYTE_16_SRQ_CONSUMER_IDX_M GENMASK(31, 16)
381
382#define SRQC_BYTE_24_SRQ_WQE_BT_BA_S 0
383#define SRQC_BYTE_24_SRQ_WQE_BT_BA_M GENMASK(28, 0)
384
385#define SRQC_BYTE_28_PD_S 0
386#define SRQC_BYTE_28_PD_M GENMASK(23, 0)
387
388#define SRQC_BYTE_28_RQWS_S 24
389#define SRQC_BYTE_28_RQWS_M GENMASK(27, 24)
390
391#define SRQC_BYTE_36_SRQ_IDX_BT_BA_S 0
392#define SRQC_BYTE_36_SRQ_IDX_BT_BA_M GENMASK(28, 0)
393
394#define SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S 0
395#define SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M GENMASK(19, 0)
396
397#define SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S 22
398#define SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M GENMASK(23, 22)
399
400#define SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S 24
401#define SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M GENMASK(27, 24)
402
403#define SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S 28
404#define SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M GENMASK(31, 28)
405
406#define SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S 0
407#define SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M GENMASK(19, 0)
408
409#define SRQC_BYTE_56_SRQ_XRC_CQN_S 0
410#define SRQC_BYTE_56_SRQ_XRC_CQN_M GENMASK(23, 0)
411
412#define SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S 24
413#define SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M GENMASK(27, 24)
414
415#define SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S 28
416#define SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M GENMASK(31, 28)
417
418#define SRQC_BYTE_60_SRQ_RECORD_EN_S 0
419
420#define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_S 1
421#define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_M GENMASK(31, 1)
422
337enum{423enum{
338 V2_MPT_ST_VALID = 0x1,424 V2_MPT_ST_VALID = 0x1,
339 V2_MPT_ST_FREE = 0x2,425 V2_MPT_ST_FREE = 0x2,
drivers/infiniband/hw/hns/hns_roce_main.c+15−0
@@ -546,6 +546,21 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) · 545 unmodified lines
546 ib_dev->map_mr_sg = hns_roce_map_mr_sg;546 ib_dev->map_mr_sg = hns_roce_map_mr_sg;
547 }547 }
548 548
549 /* SRQ */
550 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
551 ib_dev->create_srq = hns_roce_create_srq;
552 ib_dev->modify_srq = hr_dev->hw->modify_srq;
553 ib_dev->query_srq = hr_dev->hw->query_srq;
554 ib_dev->destroy_srq = hns_roce_destroy_srq;
555 ib_dev->post_srq_recv = hr_dev->hw->post_srq_recv;
556 ib_dev->uverbs_cmd_mask |=
557 (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) |
558 (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) |
559 (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) |
560 (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) |
561 (1ULL << IB_USER_VERBS_CMD_POST_SRQ_RECV);
562 }
563
549 /* OTHERS */564 /* OTHERS */
550 ib_dev->get_port_immutable = hns_roce_port_immutable;565 ib_dev->get_port_immutable = hns_roce_port_immutable;
551 ib_dev->disassociate_ucontext = hns_roce_disassociate_ucontext;566 ib_dev->disassociate_ucontext = hns_roce_disassociate_ucontext;
drivers/infiniband/hw/hns/hns_roce_mr.c+122−15
@@ -184,12 +184,27 @@ static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order, · 183 unmodified lines
184 struct hns_roce_buddy *buddy;184 struct hns_roce_buddy *buddy;
185 int ret;185 int ret;
186 186
187 if (mtt_type == MTT_TYPE_WQE) {187 switch (mtt_type) {
188 case MTT_TYPE_WQE:
188 buddy = &mr_table->mtt_buddy;189 buddy = &mr_table->mtt_buddy;
189 table = &mr_table->mtt_table;190 table = &mr_table->mtt_table;
190 } else {191 break;
192 case MTT_TYPE_CQE:
191 buddy = &mr_table->mtt_cqe_buddy;193 buddy = &mr_table->mtt_cqe_buddy;
192 table = &mr_table->mtt_cqe_table;194 table = &mr_table->mtt_cqe_table;
195 break;
196 case MTT_TYPE_SRQWQE:
197 buddy = &mr_table->mtt_srqwqe_buddy;
198 table = &mr_table->mtt_srqwqe_table;
199 break;
200 case MTT_TYPE_IDX:
201 buddy = &mr_table->mtt_idx_buddy;
202 table = &mr_table->mtt_idx_table;
203 break;
204 default:
205 dev_err(hr_dev->dev, "Unsupport MTT table type: %d\n",
206 mtt_type);
207 return -EINVAL;
193 }208 }
194 209
195 ret = hns_roce_buddy_alloc(buddy, order, seg);210 ret = hns_roce_buddy_alloc(buddy, order, seg);
@@ -242,18 +257,40 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt) · 46 unmodified lines
242 if (mtt->order < 0)257 if (mtt->order < 0)
243 return;258 return;
244 259
245 if (mtt->mtt_type == MTT_TYPE_WQE) {260 switch (mtt->mtt_type) {
261 case MTT_TYPE_WQE:
246 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg,262 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg,
247 mtt->order);263 mtt->order);
248 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table,264 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table,
249 mtt->first_seg,265 mtt->first_seg,
250 mtt->first_seg + (1 << mtt->order) - 1);266 mtt->first_seg + (1 << mtt->order) - 1);
251 } else {267 break;
268 case MTT_TYPE_CQE:
252 hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg,269 hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg,
253 mtt->order);270 mtt->order);
254 hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table,271 hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table,
255 mtt->first_seg,272 mtt->first_seg,
256 mtt->first_seg + (1 << mtt->order) - 1);273 mtt->first_seg + (1 << mtt->order) - 1);
274 break;
275 case MTT_TYPE_SRQWQE:
276 hns_roce_buddy_free(&mr_table->mtt_srqwqe_buddy, mtt->first_seg,
277 mtt->order);
278 hns_roce_table_put_range(hr_dev, &mr_table->mtt_srqwqe_table,
279 mtt->first_seg,
280 mtt->first_seg + (1 << mtt->order) - 1);
281 break;
282 case MTT_TYPE_IDX:
283 hns_roce_buddy_free(&mr_table->mtt_idx_buddy, mtt->first_seg,
284 mtt->order);
285 hns_roce_table_put_range(hr_dev, &mr_table->mtt_idx_table,
286 mtt->first_seg,
287 mtt->first_seg + (1 << mtt->order) - 1);
288 break;
289 default:
290 dev_err(hr_dev->dev,
291 "Unsupport mtt type %d, clean mtt failed\n",
292 mtt->mtt_type);
293 break;
257 }294 }
258}295}
259EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup);296EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup);
@@ -713,10 +750,26 @@ static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, · 453 unmodified lines
713 u32 bt_page_size;750 u32 bt_page_size;
714 u32 i;751 u32 i;
715 752
716 if (mtt->mtt_type == MTT_TYPE_WQE)753 switch (mtt->mtt_type) {
754 case MTT_TYPE_WQE:
755 table = &hr_dev->mr_table.mtt_table;
717 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);756 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
718 else757 break;
758 case MTT_TYPE_CQE:
759 table = &hr_dev->mr_table.mtt_cqe_table;
719 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);760 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
761 break;
762 case MTT_TYPE_SRQWQE:
763 table = &hr_dev->mr_table.mtt_srqwqe_table;
764 bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
765 break;
766 case MTT_TYPE_IDX:
767 table = &hr_dev->mr_table.mtt_idx_table;
768 bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
769 break;
770 default:
771 return -EINVAL;
772 }
720 773
721 /* All MTTs must fit in the same page */774 /* All MTTs must fit in the same page */
722 if (start_index / (bt_page_size / sizeof(u64)) !=775 if (start_index / (bt_page_size / sizeof(u64)) !=
@@ -726,11 +779,6 @@ static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, · 3 unmodified lines
726 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))779 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))
727 return -EINVAL;780 return -EINVAL;
728 781
729 if (mtt->mtt_type == MTT_TYPE_WQE)
730 table = &hr_dev->mr_table.mtt_table;
731 else
732 table = &hr_dev->mr_table.mtt_cqe_table;
733
734 mtts = hns_roce_table_find(hr_dev, table,782 mtts = hns_roce_table_find(hr_dev, table,
735 mtt->first_seg + s / hr_dev->caps.mtt_entry_sz,783 mtt->first_seg + s / hr_dev->caps.mtt_entry_sz,
736 &dma_handle);784 &dma_handle);
@@ -759,10 +807,25 @@ static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev, · 22 unmodified lines
759 if (mtt->order < 0)807 if (mtt->order < 0)
760 return -EINVAL;808 return -EINVAL;
761 809
762 if (mtt->mtt_type == MTT_TYPE_WQE)810 switch (mtt->mtt_type) {
811 case MTT_TYPE_WQE:
763 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);812 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
764 else813 break;
814 case MTT_TYPE_CQE:
765 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);815 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
816 break;
817 case MTT_TYPE_SRQWQE:
818 bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
819 break;
820 case MTT_TYPE_IDX:
821 bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
822 break;
823 default:
824 dev_err(hr_dev->dev,
825 "Unsupport mtt type %d, write mtt failed\n",
826 mtt->mtt_type);
827 return -EINVAL;
828 }
766 829
767 while (npages > 0) {830 while (npages > 0) {
768 chunk = min_t(int, bt_page_size / sizeof(u64), npages);831 chunk = min_t(int, bt_page_size / sizeof(u64), npages);
@@ -828,8 +891,31 @@ int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) · 59 unmodified lines
828 if (ret)891 if (ret)
829 goto err_buddy_cqe;892 goto err_buddy_cqe;
830 }893 }
894
895 if (hr_dev->caps.num_srqwqe_segs) {
896 ret = hns_roce_buddy_init(&mr_table->mtt_srqwqe_buddy,
897 ilog2(hr_dev->caps.num_srqwqe_segs));
898 if (ret)
899 goto err_buddy_srqwqe;
900 }
901
902 if (hr_dev->caps.num_idx_segs) {
903 ret = hns_roce_buddy_init(&mr_table->mtt_idx_buddy,
904 ilog2(hr_dev->caps.num_idx_segs));
905 if (ret)
906 goto err_buddy_idx;
907 }
908
831 return 0;909 return 0;
832 910
911err_buddy_idx:
912 if (hr_dev->caps.num_srqwqe_segs)
913 hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
914
915err_buddy_srqwqe:
916 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
917 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
918
833err_buddy_cqe:919err_buddy_cqe:
834 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);920 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
835 921
@@ -842,6 +928,10 @@ void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev) · 6 unmodified lines
842{928{
843 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;929 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
844 930
931 if (hr_dev->caps.num_idx_segs)
932 hns_roce_buddy_cleanup(&mr_table->mtt_idx_buddy);
933 if (hr_dev->caps.num_srqwqe_segs)
934 hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
845 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);935 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
846 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))936 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
847 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);937 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
@@ -897,8 +987,25 @@ int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, · 49 unmodified lines
897 u32 bt_page_size;987 u32 bt_page_size;
898 u32 n;988 u32 n;
899 989
900 order = mtt->mtt_type == MTT_TYPE_WQE ? hr_dev->caps.mtt_ba_pg_sz :990 switch (mtt->mtt_type) {
901 hr_dev->caps.cqe_ba_pg_sz;991 case MTT_TYPE_WQE:
992 order = hr_dev->caps.mtt_ba_pg_sz;
993 break;
994 case MTT_TYPE_CQE:
995 order = hr_dev->caps.cqe_ba_pg_sz;
996 break;
997 case MTT_TYPE_SRQWQE:
998 order = hr_dev->caps.srqwqe_ba_pg_sz;
999 break;
1000 case MTT_TYPE_IDX:
1001 order = hr_dev->caps.idx_ba_pg_sz;
1002 break;
1003 default:
1004 dev_err(dev, "Unsupport mtt type %d, write mtt failed\n",
1005 mtt->mtt_type);
1006 return -EINVAL;
1007 }
1008
902 bt_page_size = 1 << (order + PAGE_SHIFT);1009 bt_page_size = 1 << (order + PAGE_SHIFT);
903 1010
904 pages = (u64 *) __get_free_pages(GFP_KERNEL, order);1011 pages = (u64 *) __get_free_pages(GFP_KERNEL, order);
drivers/infiniband/hw/hns/hns_roce_qp.c+10−11
@@ -280,7 +280,7 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, · 279 unmodified lines
280EXPORT_SYMBOL_GPL(hns_roce_release_range_qp);280EXPORT_SYMBOL_GPL(hns_roce_release_range_qp);
281 281
282static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,282static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,
283 struct ib_qp_cap *cap, int is_user, int has_srq,283 struct ib_qp_cap *cap, int is_user, int has_rq,
284 struct hns_roce_qp *hr_qp)284 struct hns_roce_qp *hr_qp)
285{285{
286 struct device *dev = hr_dev->dev;286 struct device *dev = hr_dev->dev;
@@ -294,14 +294,12 @@ static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, · 7 unmodified lines
294 return -EINVAL;294 return -EINVAL;
295 }295 }
296 296
297 /* If srq exit, set zero for relative number of rq */297 /* If srq exist, set zero for relative number of rq */
298 if (has_srq) {298 if (!has_rq) {
299 if (cap->max_recv_wr) {299 hr_qp->rq.wqe_cnt = 0;
300 dev_dbg(dev, "srq no need config max_recv_wr\n");300 hr_qp->rq.max_gs = 0;
301 return -EINVAL;301 cap->max_recv_wr = 0;
302 }302 cap->max_recv_sge = 0;
303
304 hr_qp->rq.wqe_cnt = hr_qp->rq.max_gs = 0;
305 } else {303 } else {
306 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) {304 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) {
307 dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n");305 dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n");
@@ -563,13 +561,14 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, · 255 unmodified lines
563 hr_qp->sq_signal_bits = cpu_to_le32(IB_SIGNAL_REQ_WR);561 hr_qp->sq_signal_bits = cpu_to_le32(IB_SIGNAL_REQ_WR);
564 562
565 ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, !!ib_pd->uobject,563 ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, !!ib_pd->uobject,
566 !!init_attr->srq, hr_qp);564 hns_roce_qp_has_rq(init_attr), hr_qp);
567 if (ret) {565 if (ret) {
568 dev_err(dev, "hns_roce_set_rq_size failed\n");566 dev_err(dev, "hns_roce_set_rq_size failed\n");
569 goto err_out;567 goto err_out;
570 }568 }
571 569
572 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {570 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
571 hns_roce_qp_has_rq(init_attr)) {
573 /* allocate recv inline buf */572 /* allocate recv inline buf */
574 hr_qp->rq_inl_buf.wqe_list = kcalloc(hr_qp->rq.wqe_cnt,573 hr_qp->rq_inl_buf.wqe_list = kcalloc(hr_qp->rq.wqe_cnt,
575 sizeof(struct hns_roce_rinl_wqe),574 sizeof(struct hns_roce_rinl_wqe),
drivers/infiniband/hw/hns/hns_roce_srq.c+407−0
@@ -9,6 +9,413 @@ · 8 unmodified lines
9#include "hns_roce_cmd.h"9#include "hns_roce_cmd.h"
10#include "hns_roce_hem.h"10#include "hns_roce_hem.h"
11 11
12static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,
13 enum hns_roce_event event_type)
14{
15 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
16 struct ib_srq *ibsrq = &srq->ibsrq;
17 struct ib_event event;
18
19 if (ibsrq->event_handler) {
20 event.device = ibsrq->device;
21 event.element.srq = ibsrq;
22 switch (event_type) {
23 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
24 event.event = IB_EVENT_SRQ_LIMIT_REACHED;
25 break;
26 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
27 event.event = IB_EVENT_SRQ_ERR;
28 break;
29 default:
30 dev_err(hr_dev->dev,
31 "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n",
32 event_type, srq->srqn);
33 return;
34 }
35
36 ibsrq->event_handler(&event, ibsrq->srq_context);
37 }
38}
39
40static int hns_roce_sw2hw_srq(struct hns_roce_dev *dev,
41 struct hns_roce_cmd_mailbox *mailbox,
42 unsigned long srq_num)
43{
44 return hns_roce_cmd_mbox(dev, mailbox->dma, 0, srq_num, 0,
45 HNS_ROCE_CMD_SW2HW_SRQ,
46 HNS_ROCE_CMD_TIMEOUT_MSECS);
47}
48
49static int hns_roce_hw2sw_srq(struct hns_roce_dev *dev,
50 struct hns_roce_cmd_mailbox *mailbox,
51 unsigned long srq_num)
52{
53 return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, srq_num,
54 mailbox ? 0 : 1, HNS_ROCE_CMD_HW2SW_SRQ,
55 HNS_ROCE_CMD_TIMEOUT_MSECS);
56}
57
58int hns_roce_srq_alloc(struct hns_roce_dev *hr_dev, u32 pdn, u32 cqn, u16 xrcd,
59 struct hns_roce_mtt *hr_mtt, u64 db_rec_addr,
60 struct hns_roce_srq *srq)
61{
62 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
63 struct hns_roce_cmd_mailbox *mailbox;
64 dma_addr_t dma_handle_wqe;
65 dma_addr_t dma_handle_idx;
66 u64 *mtts_wqe;
67 u64 *mtts_idx;
68 int ret;
69
70 /* Get the physical address of srq buf */
71 mtts_wqe = hns_roce_table_find(hr_dev,
72 &hr_dev->mr_table.mtt_srqwqe_table,
73 srq->mtt.first_seg,
74 &dma_handle_wqe);
75 if (!mtts_wqe) {
76 dev_err(hr_dev->dev,
77 "SRQ alloc.Failed to find srq buf addr.\n");
78 return -EINVAL;
79 }
80
81 /* Get physical address of idx que buf */
82 mtts_idx = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_idx_table,
83 srq->idx_que.mtt.first_seg,
84 &dma_handle_idx);
85 if (!mtts_idx) {
86 dev_err(hr_dev->dev,
87 "SRQ alloc.Failed to find idx que buf addr.\n");
88 return -EINVAL;
89 }
90
91 ret = hns_roce_bitmap_alloc(&srq_table->bitmap, &srq->srqn);
92 if (ret == -1) {
93 dev_err(hr_dev->dev, "SRQ alloc.Failed to alloc index.\n");
94 return -ENOMEM;
95 }
96
97 ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);
98 if (ret)
99 goto err_out;
100
101 ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));
102 if (ret)
103 goto err_put;
104
105 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
106 if (IS_ERR(mailbox)) {
107 ret = PTR_ERR(mailbox);
108 goto err_xa;
109 }
110
111 hr_dev->hw->write_srqc(hr_dev, srq, pdn, xrcd, cqn, mailbox->buf,
112 mtts_wqe, mtts_idx, dma_handle_wqe,
113 dma_handle_idx);
114
115 ret = hns_roce_sw2hw_srq(hr_dev, mailbox, srq->srqn);
116 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
117 if (ret)
118 goto err_xa;
119
120 atomic_set(&srq->refcount, 1);
121 init_completion(&srq->free);
122 return ret;
123
124err_xa:
125 xa_erase(&srq_table->xa, srq->srqn);
126
127err_put:
128 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
129
130err_out:
131 hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR);
132 return ret;
133}
134
135void hns_roce_srq_free(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
136{
137 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
138 int ret;
139
140 ret = hns_roce_hw2sw_srq(hr_dev, NULL, srq->srqn);
141 if (ret)
142 dev_err(hr_dev->dev, "HW2SW_SRQ failed (%d) for CQN %06lx\n",
143 ret, srq->srqn);
144
145 xa_erase(&srq_table->xa, srq->srqn);
146
147 if (atomic_dec_and_test(&srq->refcount))
148 complete(&srq->free);
149 wait_for_completion(&srq->free);
150
151 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
152 hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR);
153}
154
155static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq,
156 u32 page_shift)
157{
158 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
159 struct hns_roce_idx_que *idx_que = &srq->idx_que;
160 u32 bitmap_num;
161 int i;
162
163 bitmap_num = HNS_ROCE_ALOGN_UP(srq->max, 8 * sizeof(u64));
164
165 idx_que->bitmap = kcalloc(1, bitmap_num / 8, GFP_KERNEL);
166 if (!idx_que->bitmap)
167 return -ENOMEM;
168
169 bitmap_num = bitmap_num / (8 * sizeof(u64));
170
171 idx_que->buf_size = srq->idx_que.buf_size;
172
173 if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2,
174 &idx_que->idx_buf, page_shift)) {
175 kfree(idx_que->bitmap);
176 return -ENOMEM;
177 }
178
179 for (i = 0; i < bitmap_num; i++)
180 idx_que->bitmap[i] = ~(0UL);
181
182 return 0;
183}
184
185struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
186 struct ib_srq_init_attr *srq_init_attr,
187 struct ib_udata *udata)
188{
189 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
190 struct hns_roce_srq *srq;
191 int srq_desc_size;
192 int srq_buf_size;
193 u32 page_shift;
194 int ret = 0;
195 u32 npages;
196 u32 cqn;
197
198 /* Check the actual SRQ wqe and SRQ sge num */
199 if (srq_init_attr->attr.max_wr >= hr_dev->caps.max_srq_wrs ||
200 srq_init_attr->attr.max_sge > hr_dev->caps.max_srq_sges)
201 return ERR_PTR(-EINVAL);
202
203 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
204 if (!srq)
205 return ERR_PTR(-ENOMEM);
206
207 mutex_init(&srq->mutex);
208 spin_lock_init(&srq->lock);
209
210 srq->max = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
211 srq->max_gs = srq_init_attr->attr.max_sge;
212
213 srq_desc_size = max(16, 16 * srq->max_gs);
214
215 srq->wqe_shift = ilog2(srq_desc_size);
216
217 srq_buf_size = srq->max * srq_desc_size;
218
219 srq->idx_que.entry_sz = HNS_ROCE_IDX_QUE_ENTRY_SZ;
220 srq->idx_que.buf_size = srq->max * srq->idx_que.entry_sz;
221 srq->mtt.mtt_type = MTT_TYPE_SRQWQE;
222 srq->idx_que.mtt.mtt_type = MTT_TYPE_IDX;
223
224 if (udata) {
225 struct hns_roce_ib_create_srq ucmd;
226
227 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
228 ret = -EFAULT;
229 goto err_srq;
230 }
231
232 srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
233 srq_buf_size, 0, 0);
234 if (IS_ERR(srq->umem)) {
235 ret = PTR_ERR(srq->umem);
236 goto err_srq;
237 }
238
239 if (hr_dev->caps.srqwqe_buf_pg_sz) {
240 npages = (ib_umem_page_count(srq->umem) +
241 (1 << hr_dev->caps.srqwqe_buf_pg_sz) - 1) /
242 (1 << hr_dev->caps.srqwqe_buf_pg_sz);
243 page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
244 ret = hns_roce_mtt_init(hr_dev, npages,
245 page_shift,
246 &srq->mtt);
247 } else
248 ret = hns_roce_mtt_init(hr_dev,
249 ib_umem_page_count(srq->umem),
250 srq->umem->page_shift,
251 &srq->mtt);
252 if (ret)
253 goto err_buf;
254
255 ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->mtt, srq->umem);
256 if (ret)
257 goto err_srq_mtt;
258
259 /* config index queue BA */
260 srq->idx_que.umem = ib_umem_get(pd->uobject->context,
261 ucmd.que_addr,
262 srq->idx_que.buf_size, 0, 0);
263 if (IS_ERR(srq->idx_que.umem)) {
264 dev_err(hr_dev->dev,
265 "ib_umem_get error for index queue\n");
266 goto err_srq_mtt;
267 }
268
269 if (hr_dev->caps.idx_buf_pg_sz) {
270 npages = (ib_umem_page_count(srq->idx_que.umem) +
271 (1 << hr_dev->caps.idx_buf_pg_sz) - 1) /
272 (1 << hr_dev->caps.idx_buf_pg_sz);
273 page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
274 ret = hns_roce_mtt_init(hr_dev, npages,
275 page_shift, &srq->idx_que.mtt);
276 } else {
277 ret = hns_roce_mtt_init(hr_dev,
278 ib_umem_page_count(srq->idx_que.umem),
279 srq->idx_que.umem->page_shift,
280 &srq->idx_que.mtt);
281 }
282
283 if (ret) {
284 dev_err(hr_dev->dev,
285 "hns_roce_mtt_init error for idx que\n");
286 goto err_idx_mtt;
287 }
288
289 ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->idx_que.mtt,
290 srq->idx_que.umem);
291 if (ret) {
292 dev_err(hr_dev->dev,
293 "hns_roce_ib_umem_write_mtt error for idx que\n");
294 goto err_idx_buf;
295 }
296 } else {
297 page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
298 if (hns_roce_buf_alloc(hr_dev, srq_buf_size,
299 (1 << page_shift) * 2,
300 &srq->buf, page_shift)) {
301 ret = -ENOMEM;
302 goto err_srq;
303 }
304
305 srq->head = 0;
306 srq->tail = srq->max - 1;
307
308 ret = hns_roce_mtt_init(hr_dev, srq->buf.npages,
309 srq->buf.page_shift, &srq->mtt);
310 if (ret)
311 goto err_buf;
312
313 ret = hns_roce_buf_write_mtt(hr_dev, &srq->mtt, &srq->buf);
314 if (ret)
315 goto err_srq_mtt;
316
317 page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
318 ret = hns_roce_create_idx_que(pd, srq, page_shift);
319 if (ret) {
320 dev_err(hr_dev->dev, "Create idx queue fail(%d)!\n",
321 ret);
322 goto err_srq_mtt;
323 }
324
325 /* Init mtt table for idx_que */
326 ret = hns_roce_mtt_init(hr_dev, srq->idx_que.idx_buf.npages,
327 srq->idx_que.idx_buf.page_shift,
328 &srq->idx_que.mtt);
329 if (ret)
330 goto err_create_idx;
331
332 /* Write buffer address into the mtt table */
333 ret = hns_roce_buf_write_mtt(hr_dev, &srq->idx_que.mtt,
334 &srq->idx_que.idx_buf);
335 if (ret)
336 goto err_idx_buf;
337
338 srq->wrid = kvmalloc_array(srq->max, sizeof(u64), GFP_KERNEL);
339 if (!srq->wrid) {
340 ret = -ENOMEM;
341 goto err_idx_buf;
342 }
343 }
344
345 cqn = ib_srq_has_cq(srq_init_attr->srq_type) ?
346 to_hr_cq(srq_init_attr->ext.cq)->cqn : 0;
347
348 srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG;
349
350 ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(pd)->pdn, cqn, 0,
351 &srq->mtt, 0, srq);
352 if (ret)
353 goto err_wrid;
354
355 srq->event = hns_roce_ib_srq_event;
356 srq->ibsrq.ext.xrc.srq_num = srq->srqn;
357
358 if (pd->uobject) {
359 if (ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) {
360 ret = -EFAULT;
361 goto err_wrid;
362 }
363 }
364
365 return &srq->ibsrq;
366
367err_wrid:
368 kvfree(srq->wrid);
369
370err_idx_buf:
371 hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
372
373err_idx_mtt:
374 if (udata)
375 ib_umem_release(srq->idx_que.umem);
376
377err_create_idx:
378 hns_roce_buf_free(hr_dev, srq->idx_que.buf_size,
379 &srq->idx_que.idx_buf);
380 kfree(srq->idx_que.bitmap);
381
382err_srq_mtt:
383 hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
384
385err_buf:
386 if (udata)
387 ib_umem_release(srq->umem);
388 else
389 hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf);
390
391err_srq:
392 kfree(srq);
393 return ERR_PTR(ret);
394}
395
396int hns_roce_destroy_srq(struct ib_srq *ibsrq)
397{
398 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
399 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
400
401 hns_roce_srq_free(hr_dev, srq);
402 hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
403
404 if (ibsrq->uobject) {
405 hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
406 ib_umem_release(srq->idx_que.umem);
407 ib_umem_release(srq->umem);
408 } else {
409 kvfree(srq->wrid);
410 hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift,
411 &srq->buf);
412 }
413
414 kfree(srq);
415
416 return 0;
417}
418
12int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)419int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)
13{420{
14 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;421 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
include/uapi/rdma/hns-abi.h+6−0
@@ -46,6 +46,12 @@ struct hns_roce_ib_create_cq_resp { · 45 unmodified lines
46 __aligned_u64 cap_flags;46 __aligned_u64 cap_flags;
47};47};
48 48
49struct hns_roce_ib_create_srq {
50 __aligned_u64 buf_addr;
51 __aligned_u64 db_addr;
52 __aligned_u64 que_addr;
53};
54
49struct hns_roce_ib_create_qp {55struct hns_roce_ib_create_qp {
50 __aligned_u64 buf_addr;56 __aligned_u64 buf_addr;
51 __aligned_u64 db_addr;57 __aligned_u64 db_addr;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t