RDMA/hns: Add SRQ support for hip08 kernel mode

This patch implements the SRQ(Share Receive Queue) verbs and update the poll cq verbs to deal with SRQ complentions.

Lijun Ou committed 7 years ago

Commit c7bcb131 parent 5c1f16710 files · +1098 −52Browse files

Unified · Split

10 files

drivers/infiniband/hw/hns/hns_roce_cmd.h +4 −0
drivers/infiniband/hw/hns/hns_roce_device.h +57 −1
drivers/infiniband/hw/hns/hns_roce_hem.c +31 −1
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +360 −24
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +86 −0
drivers/infiniband/hw/hns/hns_roce_main.c +15 −0
drivers/infiniband/hw/hns/hns_roce_mr.c +122 −15
drivers/infiniband/hw/hns/hns_roce_qp.c +10 −11
drivers/infiniband/hw/hns/hns_roce_srq.c +407 −0
include/uapi/rdma/hns-abi.h +6 −0

drivers/infiniband/hw/hns/hns_roce_cmd.h+4−0

@@ -120,6 +120,10 @@ enum { · 119 unmodified lines
120	120	HNS_ROCE_CMD_SQD2RTS_QP = 0x20,
121	121	HNS_ROCE_CMD_2RST_QP = 0x21,
122	122	HNS_ROCE_CMD_QUERY_QP = 0x22,
	123	+ HNS_ROCE_CMD_SW2HW_SRQ = 0x70,
	124	+ HNS_ROCE_CMD_MODIFY_SRQC = 0x72,
	125	+ HNS_ROCE_CMD_QUERY_SRQC = 0x73,
	126	+ HNS_ROCE_CMD_HW2SW_SRQ = 0x74,
123	127	};
124	128
125	129	int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,

drivers/infiniband/hw/hns/hns_roce_device.h+57−1

@@ -111,6 +111,9 @@ · 110 unmodified lines
111	111	#define PAGES_SHIFT_24 24
112	112	#define PAGES_SHIFT_32 32
113	113
	114	+#define HNS_ROCE_IDX_QUE_ENTRY_SZ 4
	115	+#define SRQ_DB_REG 0x230
	116	+
114	117	enum {
115	118	HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0,
116	119	HNS_ROCE_SUPPORT_SQ_RECORD_DB = 1 << 1,
@@ -436,9 +439,37 @@ struct hns_roce_cq { · 319 unmodified lines
436	439	struct completion free;
437	440	};
438	441
	442	+struct hns_roce_idx_que {
	443	+ struct hns_roce_buf idx_buf;
	444	+ int entry_sz;
	445	+ u32 buf_size;
	446	+ struct ib_umem *umem;
	447	+ struct hns_roce_mtt mtt;
	448	+ u64 *bitmap;
	449	+};
	450	+
439	451	struct hns_roce_srq {
440	452	struct ib_srq ibsrq;
441		− int srqn;
	453	+ void (event)(struct hns_roce_srq srq, enum hns_roce_event event);
	454	+ unsigned long srqn;
	455	+ int max;
	456	+ int max_gs;
	457	+ int wqe_shift;
	458	+ void __iomem *db_reg_l;
	459	+
	460	+ atomic_t refcount;
	461	+ struct completion free;
	462	+
	463	+ struct hns_roce_buf buf;
	464	+ u64 *wrid;
	465	+ struct ib_umem *umem;
	466	+ struct hns_roce_mtt mtt;
	467	+ struct hns_roce_idx_que idx_que;
	468	+ spinlock_t lock;
	469	+ int head;
	470	+ int tail;
	471	+ u16 wqe_ctr;
	472	+ struct mutex mutex;
442	473	};
443	474
444	475	struct hns_roce_uar_table {
@@ -761,6 +792,12 @@ struct hns_roce_caps { · 316 unmodified lines
761	792	u32 cqe_ba_pg_sz;
762	793	u32 cqe_buf_pg_sz;
763	794	u32 cqe_hop_num;
	795	+ u32 srqwqe_ba_pg_sz;
	796	+ u32 srqwqe_buf_pg_sz;
	797	+ u32 srqwqe_hop_num;
	798	+ u32 idx_ba_pg_sz;
	799	+ u32 idx_buf_pg_sz;
	800	+ u32 idx_hop_num;
764	801	u32 eqe_ba_pg_sz;
765	802	u32 eqe_buf_pg_sz;
766	803	u32 eqe_hop_num;
@@ -829,6 +866,17 @@ struct hns_roce_hw { · 62 unmodified lines
829	866	int (modify_cq)(struct ib_cq cq, u16 cq_count, u16 cq_period);
830	867	int (init_eq)(struct hns_roce_dev hr_dev);
831	868	void (cleanup_eq)(struct hns_roce_dev hr_dev);
	869	+ void (write_srqc)(struct hns_roce_dev hr_dev,
	870	+ struct hns_roce_srq *srq, u32 pdn, u16 xrcd, u32 cqn,
	871	+ void mb_buf, u64 mtts_wqe, u64 *mtts_idx,
	872	+ dma_addr_t dma_handle_wqe,
	873	+ dma_addr_t dma_handle_idx);
	874	+ int (modify_srq)(struct ib_srq ibsrq, struct ib_srq_attr *srq_attr,
	875	+ enum ib_srq_attr_mask srq_attr_mask,
	876	+ struct ib_udata *udata);
	877	+ int (query_srq)(struct ib_srq ibsrq, struct ib_srq_attr *attr);
	878	+ int (post_srq_recv)(struct ib_srq ibsrq, const struct ib_recv_wr *wr,
	879	+ const struct ib_recv_wr **bad_wr);
832	880	};
833	881
834	882	struct hns_roce_dev {
@@ -1038,6 +1086,14 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, · 203 unmodified lines
1038	1086	int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
1039	1087	struct hns_roce_mtt mtt, struct ib_umem umem);
1040	1088
	1089	+struct ib_srq hns_roce_create_srq(struct ib_pd pd,
	1090	+ struct ib_srq_init_attr *srq_init_attr,
	1091	+ struct ib_udata *udata);
	1092	+int hns_roce_modify_srq(struct ib_srq ibsrq, struct ib_srq_attr srq_attr,
	1093	+ enum ib_srq_attr_mask srq_attr_mask,
	1094	+ struct ib_udata *udata);
	1095	+int hns_roce_destroy_srq(struct ib_srq *ibsrq);
	1096	+
1041	1097	struct ib_qp hns_roce_create_qp(struct ib_pd ib_pd,
1042	1098	struct ib_qp_init_attr *init_attr,
1043	1099	struct ib_udata *udata);

drivers/infiniband/hw/hns/hns_roce_hem.c+31−1

@@ -46,7 +46,9 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) · 45 unmodified lines
46	46	(hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) \|\|
47	47	(hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) \|\|
48	48	(hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) \|\|
49		− (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT))
	49	+ (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT) \|\|
	50	+ (hr_dev->caps.srqwqe_hop_num && type == HEM_TYPE_SRQWQE) \|\|
	51	+ (hr_dev->caps.idx_hop_num && type == HEM_TYPE_IDX))
50	52	return true;
51	53
52	54	return false;
@@ -147,6 +149,22 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, · 94 unmodified lines
147	149	mhop->ba_l0_num = mhop->bt_chunk_size / 8;
148	150	mhop->hop_num = hr_dev->caps.cqe_hop_num;
149	151	break;
	152	+ case HEM_TYPE_SRQWQE:
	153	+ mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz
	154	+ + PAGE_SHIFT);
	155	+ mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
	156	+ + PAGE_SHIFT);
	157	+ mhop->ba_l0_num = mhop->bt_chunk_size / 8;
	158	+ mhop->hop_num = hr_dev->caps.srqwqe_hop_num;
	159	+ break;
	160	+ case HEM_TYPE_IDX:
	161	+ mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz
	162	+ + PAGE_SHIFT);
	163	+ mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
	164	+ + PAGE_SHIFT);
	165	+ mhop->ba_l0_num = mhop->bt_chunk_size / 8;
	166	+ mhop->hop_num = hr_dev->caps.idx_hop_num;
	167	+ break;
150	168	default:
151	169	dev_err(dev, "Table %d not support multi-hop addressing!\n",
152	170	table->type);
@@ -906,6 +924,18 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, · 753 unmodified lines
906	924	bt_chunk_size = buf_chunk_size;
907	925	hop_num = hr_dev->caps.cqe_hop_num;
908	926	break;
	927	+ case HEM_TYPE_SRQWQE:
	928	+ buf_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
	929	+ + PAGE_SHIFT);
	930	+ bt_chunk_size = buf_chunk_size;
	931	+ hop_num = hr_dev->caps.srqwqe_hop_num;
	932	+ break;
	933	+ case HEM_TYPE_IDX:
	934	+ buf_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
	935	+ + PAGE_SHIFT);
	936	+ bt_chunk_size = buf_chunk_size;
	937	+ hop_num = hr_dev->caps.idx_hop_num;
	938	+ break;
909	939	default:
910	940	dev_err(dev,
911	941	"Table %d not support to init hem table here!\n",

drivers/infiniband/hw/hns/hns_roce_hw_v2.c+360−24

@@ -1339,6 +1339,12 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) · 1338 unmodified lines
1339	1339	caps->cqe_ba_pg_sz = 0;
1340	1340	caps->cqe_buf_pg_sz = 0;
1341	1341	caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
	1342	+ caps->srqwqe_ba_pg_sz = 0;
	1343	+ caps->srqwqe_buf_pg_sz = 0;
	1344	+ caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM;
	1345	+ caps->idx_ba_pg_sz = 0;
	1346	+ caps->idx_buf_pg_sz = 0;
	1347	+ caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM;
1342	1348	caps->eqe_ba_pg_sz = 0;
1343	1349	caps->eqe_buf_pg_sz = 0;
1344	1350	caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
@@ -2028,6 +2034,27 @@ static struct hns_roce_v2_cqe next_cqe_sw_v2(struct hns_roce_cq hr_cq) · 683 unmodified lines
2028	2034	return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2029	2035	}
2030	2036
	2037	+static void get_srq_wqe(struct hns_roce_srq srq, int n)
	2038	+{
	2039	+ return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift);
	2040	+}
	2041	+
	2042	+static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
	2043	+{
	2044	+ u32 bitmap_num;
	2045	+ int bit_num;
	2046	+
	2047	+ /* always called with interrupts disabled. */
	2048	+ spin_lock(&srq->lock);
	2049	+
	2050	+ bitmap_num = wqe_index / (sizeof(u64) * 8);
	2051	+ bit_num = wqe_index % (sizeof(u64) * 8);
	2052	+ srq->idx_que.bitmap[bitmap_num] \|= (1ULL << bit_num);
	2053	+ srq->tail++;
	2054	+
	2055	+ spin_unlock(&srq->lock);
	2056	+}
	2057	+
2031	2058	static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2032	2059	{
2033	2060	*hr_cq->set_ci_db = cons_index & 0xffffff;
@@ -2039,6 +2066,7 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, · 5 unmodified lines
2039	2066	struct hns_roce_v2_cqe cqe, dest;
2040	2067	u32 prod_index;
2041	2068	int nfreed = 0;
	2069	+ int wqe_index;
2042	2070	u8 owner_bit;
2043	2071
2044	2072	for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
@@ -2056,7 +2084,13 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, · 11 unmodified lines
2056	2084	if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2057	2085	V2_CQE_BYTE_16_LCL_QPN_S) &
2058	2086	HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
2059		− /* In v1 engine, not support SRQ */
	2087	+ if (srq &&
	2088	+ roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
	2089	+ wqe_index = roce_get_field(cqe->byte_4,
	2090	+ V2_CQE_BYTE_4_WQE_INDX_M,
	2091	+ V2_CQE_BYTE_4_WQE_INDX_S);
	2092	+ hns_roce_free_srq_wqe(srq, wqe_index);
	2093	+ }
2060	2094	++nfreed;
2061	2095	} else if (nfreed) {
2062	2096	dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
@@ -2233,6 +2267,7 @@ static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, · 170 unmodified lines
2233	2267	static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2234	2268	struct hns_roce_qp *cur_qp, struct ib_wc wc)
2235	2269	{
	2270	+ struct hns_roce_srq *srq = NULL;
2236	2271	struct hns_roce_dev *hr_dev;
2237	2272	struct hns_roce_v2_cqe *cqe;
2238	2273	struct hns_roce_qp *hr_qp;
@@ -2275,6 +2310,37 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 36 unmodified lines
2275	2310	wc->qp = &(*cur_qp)->ibqp;
2276	2311	wc->vendor_err = 0;
2277	2312
	2313	+ if (is_send) {
	2314	+ wq = &(*cur_qp)->sq;
	2315	+ if ((*cur_qp)->sq_signal_bits) {
	2316	+ /*
	2317	+ * If sg_signal_bit is 1,
	2318	+ * firstly tail pointer updated to wqe
	2319	+ * which current cqe correspond to
	2320	+ */
	2321	+ wqe_ctr = (u16)roce_get_field(cqe->byte_4,
	2322	+ V2_CQE_BYTE_4_WQE_INDX_M,
	2323	+ V2_CQE_BYTE_4_WQE_INDX_S);
	2324	+ wq->tail += (wqe_ctr - (u16)wq->tail) &
	2325	+ (wq->wqe_cnt - 1);
	2326	+ }
	2327	+
	2328	+ wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
	2329	+ ++wq->tail;
	2330	+ } else if ((*cur_qp)->ibqp.srq) {
	2331	+ srq = to_hr_srq((*cur_qp)->ibqp.srq);
	2332	+ wqe_ctr = le16_to_cpu(roce_get_field(cqe->byte_4,
	2333	+ V2_CQE_BYTE_4_WQE_INDX_M,
	2334	+ V2_CQE_BYTE_4_WQE_INDX_S));
	2335	+ wc->wr_id = srq->wrid[wqe_ctr];
	2336	+ hns_roce_free_srq_wqe(srq, wqe_ctr);
	2337	+ } else {
	2338	+ /* Update tail pointer, record wr_id */
	2339	+ wq = &(*cur_qp)->rq;
	2340	+ wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
	2341	+ ++wq->tail;
	2342	+ }
	2343	+
2278	2344	status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2279	2345	V2_CQE_BYTE_4_STATUS_S);
2280	2346	switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
@@ -2394,23 +2460,6 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 113 unmodified lines
2394	2460	wc->status = IB_WC_GENERAL_ERR;
2395	2461	break;
2396	2462	}
2397		−
2398		− wq = &(*cur_qp)->sq;
2399		− if ((*cur_qp)->sq_signal_bits) {
2400		− /*
2401		− * If sg_signal_bit is 1,
2402		− * firstly tail pointer updated to wqe
2403		− * which current cqe correspond to
2404		− */
2405		− wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2406		− V2_CQE_BYTE_4_WQE_INDX_M,
2407		− V2_CQE_BYTE_4_WQE_INDX_S);
2408		− wq->tail += (wqe_ctr - (u16)wq->tail) &
2409		− (wq->wqe_cnt - 1);
2410		− }
2411		−
2412		− wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2413		− ++wq->tail;
2414	2463	} else {
2415	2464	/* RQ correspond to CQE */
2416	2465	wc->byte_len = le32_to_cpu(cqe->byte_cnt);
@@ -2455,11 +2504,6 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 38 unmodified lines
2455	2504	return -EAGAIN;
2456	2505	}
2457	2506
2458		− /* Update tail pointer, record wr_id */
2459		− wq = &(*cur_qp)->rq;
2460		− wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2461		− ++wq->tail;
2462		−
2463	2507	wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2464	2508	V2_CQE_BYTE_32_SL_S);
2465	2509	wc->src_qp = (u8)roce_get_field(cqe->byte_32,
@@ -2768,6 +2812,8 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, · 302 unmodified lines
2768	2812
2769	2813	roce_set_field(context->byte_20_smac_sgid_idx,
2770	2814	V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
	2815	+ (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI \|\|
	2816	+ hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT \|\| ibqp->srq) ? 0 :
2771	2817	ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2772	2818	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2773	2819	V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
@@ -3109,6 +3155,8 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, · 335 unmodified lines
3109	3155
3110	3156	roce_set_field(context->byte_20_smac_sgid_idx,
3111	3157	V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
	3158	+ (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI \|\|
	3159	+ hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT \|\| ibqp->srq) ? 0 :
3112	3160	ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3113	3161	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3114	3162	V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
@@ -3810,6 +3858,11 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, · 695 unmodified lines
3810	3858	if (attr_mask & (IB_QP_ACCESS_FLAGS \| IB_QP_MAX_DEST_RD_ATOMIC))
3811	3859	set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
3812	3860
	3861	+ roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
	3862	+ ibqp->srq ? 1 : 0);
	3863	+ roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
	3864	+ V2_QPC_BYTE_108_INV_CREDIT_S, 0);
	3865	+
3813	3866	/* Every status migrate must change state */
3814	3867	roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
3815	3868	V2_QPC_BYTE_60_QP_ST_S, new_state);
@@ -4095,7 +4148,8 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, · 279 unmodified lines
4095	4148	hns_roce_free_db(hr_dev, &hr_qp->rdb);
4096	4149	}
4097	4150
4098		− if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
	4151	+ if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
	4152	+ hr_qp->rq.wqe_cnt) {
4099	4153	kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
4100	4154	kfree(hr_qp->rq_inl_buf.wqe_list);
4101	4155	}
@@ -5350,6 +5404,284 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) · 1248 unmodified lines
5350	5404	destroy_workqueue(hr_dev->irq_workq);
5351	5405	}
5352	5406
	5407	+static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
	5408	+ struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
	5409	+ u32 cqn, void mb_buf, u64 mtts_wqe,
	5410	+ u64 *mtts_idx, dma_addr_t dma_handle_wqe,
	5411	+ dma_addr_t dma_handle_idx)
	5412	+{
	5413	+ struct hns_roce_srq_context *srq_context;
	5414	+
	5415	+ srq_context = mb_buf;
	5416	+ memset(srq_context, 0, sizeof(*srq_context));
	5417	+
	5418	+ roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
	5419	+ SRQC_BYTE_4_SRQ_ST_S, 1);
	5420	+
	5421	+ roce_set_field(srq_context->byte_4_srqn_srqst,
	5422	+ SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
	5423	+ SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
	5424	+ (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
	5425	+ hr_dev->caps.srqwqe_hop_num));
	5426	+ roce_set_field(srq_context->byte_4_srqn_srqst,
	5427	+ SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
	5428	+ ilog2(srq->max));
	5429	+
	5430	+ roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
	5431	+ SRQC_BYTE_4_SRQN_S, srq->srqn);
	5432	+
	5433	+ roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5434	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
	5435	+
	5436	+ roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
	5437	+ SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
	5438	+
	5439	+ srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
	5440	+
	5441	+ roce_set_field(srq_context->byte_24_wqe_bt_ba,
	5442	+ SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
	5443	+ SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
	5444	+ cpu_to_le32(dma_handle_wqe >> 35));
	5445	+
	5446	+ roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
	5447	+ SRQC_BYTE_28_PD_S, pdn);
	5448	+ roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
	5449	+ SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
	5450	+ fls(srq->max_gs - 1));
	5451	+
	5452	+ srq_context->idx_bt_ba = (u32)(dma_handle_idx >> 3);
	5453	+ srq_context->idx_bt_ba = cpu_to_le32(srq_context->idx_bt_ba);
	5454	+ roce_set_field(srq_context->rsv_idx_bt_ba,
	5455	+ SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
	5456	+ SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
	5457	+ cpu_to_le32(dma_handle_idx >> 35));
	5458	+
	5459	+ srq_context->idx_cur_blk_addr = (u32)(mtts_idx[0] >> PAGE_ADDR_SHIFT);
	5460	+ srq_context->idx_cur_blk_addr =
	5461	+ cpu_to_le32(srq_context->idx_cur_blk_addr);
	5462	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5463	+ SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
	5464	+ SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
	5465	+ cpu_to_le32((mtts_idx[0]) >> (32 + PAGE_ADDR_SHIFT)));
	5466	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5467	+ SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
	5468	+ SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
	5469	+ hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
	5470	+ hr_dev->caps.idx_hop_num);
	5471	+
	5472	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5473	+ SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
	5474	+ SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
	5475	+ hr_dev->caps.idx_ba_pg_sz);
	5476	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5477	+ SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
	5478	+ SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
	5479	+ hr_dev->caps.idx_buf_pg_sz);
	5480	+
	5481	+ srq_context->idx_nxt_blk_addr = (u32)(mtts_idx[1] >> PAGE_ADDR_SHIFT);
	5482	+ srq_context->idx_nxt_blk_addr =
	5483	+ cpu_to_le32(srq_context->idx_nxt_blk_addr);
	5484	+ roce_set_field(srq_context->rsv_idxnxtblkaddr,
	5485	+ SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
	5486	+ SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
	5487	+ cpu_to_le32((mtts_idx[1]) >> (32 + PAGE_ADDR_SHIFT)));
	5488	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5489	+ SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
	5490	+ cqn);
	5491	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5492	+ SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
	5493	+ SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
	5494	+ hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);
	5495	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5496	+ SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
	5497	+ SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
	5498	+ hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);
	5499	+
	5500	+ roce_set_bit(srq_context->db_record_addr_record_en,
	5501	+ SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
	5502	+}
	5503	+
	5504	+static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
	5505	+ struct ib_srq_attr *srq_attr,
	5506	+ enum ib_srq_attr_mask srq_attr_mask,
	5507	+ struct ib_udata *udata)
	5508	+{
	5509	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	5510	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	5511	+ struct hns_roce_srq_context *srq_context;
	5512	+ struct hns_roce_srq_context *srqc_mask;
	5513	+ struct hns_roce_cmd_mailbox *mailbox;
	5514	+ int ret;
	5515	+
	5516	+ if (srq_attr_mask & IB_SRQ_LIMIT) {
	5517	+ if (srq_attr->srq_limit >= srq->max)
	5518	+ return -EINVAL;
	5519	+
	5520	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	5521	+ if (IS_ERR(mailbox))
	5522	+ return PTR_ERR(mailbox);
	5523	+
	5524	+ srq_context = mailbox->buf;
	5525	+ srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
	5526	+
	5527	+ memset(srqc_mask, 0xff, sizeof(*srqc_mask));
	5528	+
	5529	+ roce_set_field(srq_context->byte_8_limit_wl,
	5530	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5531	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
	5532	+ roce_set_field(srqc_mask->byte_8_limit_wl,
	5533	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5534	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
	5535	+
	5536	+ ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
	5537	+ HNS_ROCE_CMD_MODIFY_SRQC,
	5538	+ HNS_ROCE_CMD_TIMEOUT_MSECS);
	5539	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	5540	+ if (ret) {
	5541	+ dev_err(hr_dev->dev,
	5542	+ "MODIFY SRQ Failed to cmd mailbox.\n");
	5543	+ return ret;
	5544	+ }
	5545	+ }
	5546	+
	5547	+ return 0;
	5548	+}
	5549	+
	5550	+int hns_roce_v2_query_srq(struct ib_srq ibsrq, struct ib_srq_attr attr)
	5551	+{
	5552	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	5553	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	5554	+ struct hns_roce_srq_context *srq_context;
	5555	+ struct hns_roce_cmd_mailbox *mailbox;
	5556	+ int limit_wl;
	5557	+ int ret;
	5558	+
	5559	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	5560	+ if (IS_ERR(mailbox))
	5561	+ return PTR_ERR(mailbox);
	5562	+
	5563	+ srq_context = mailbox->buf;
	5564	+ ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
	5565	+ HNS_ROCE_CMD_QUERY_SRQC,
	5566	+ HNS_ROCE_CMD_TIMEOUT_MSECS);
	5567	+ if (ret) {
	5568	+ dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n");
	5569	+ goto out;
	5570	+ }
	5571	+
	5572	+ limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
	5573	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5574	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S);
	5575	+
	5576	+ attr->srq_limit = limit_wl;
	5577	+ attr->max_wr = srq->max - 1;
	5578	+ attr->max_sge = srq->max_gs;
	5579	+
	5580	+ memcpy(srq_context, mailbox->buf, sizeof(*srq_context));
	5581	+
	5582	+out:
	5583	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	5584	+ return ret;
	5585	+}
	5586	+
	5587	+static int find_empty_entry(struct hns_roce_idx_que *idx_que)
	5588	+{
	5589	+ int bit_num;
	5590	+ int i;
	5591	+
	5592	+ /* bitmap[i] is set zero if all bits are allocated */
	5593	+ for (i = 0; idx_que->bitmap[i] == 0; ++i)
	5594	+ ;
	5595	+ bit_num = ffs(idx_que->bitmap[i]);
	5596	+ idx_que->bitmap[i] &= ~(1ULL << (bit_num - 1));
	5597	+
	5598	+ return i * sizeof(u64) * 8 + (bit_num - 1);
	5599	+}
	5600	+
	5601	+static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
	5602	+ int cur_idx, int wqe_idx)
	5603	+{
	5604	+ unsigned int *addr;
	5605	+
	5606	+ addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf,
	5607	+ cur_idx * idx_que->entry_sz);
	5608	+ *addr = wqe_idx;
	5609	+}
	5610	+
	5611	+static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
	5612	+ const struct ib_recv_wr *wr,
	5613	+ const struct ib_recv_wr **bad_wr)
	5614	+{
	5615	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	5616	+ struct hns_roce_v2_wqe_data_seg *dseg;
	5617	+ struct hns_roce_v2_db srq_db;
	5618	+ unsigned long flags;
	5619	+ int ret = 0;
	5620	+ int wqe_idx;
	5621	+ void *wqe;
	5622	+ int nreq;
	5623	+ int ind;
	5624	+ int i;
	5625	+
	5626	+ spin_lock_irqsave(&srq->lock, flags);
	5627	+
	5628	+ ind = srq->head & (srq->max - 1);
	5629	+
	5630	+ for (nreq = 0; wr; ++nreq, wr = wr->next) {
	5631	+ if (unlikely(wr->num_sge > srq->max_gs)) {
	5632	+ ret = -EINVAL;
	5633	+ *bad_wr = wr;
	5634	+ break;
	5635	+ }
	5636	+
	5637	+ if (unlikely(srq->head == srq->tail)) {
	5638	+ ret = -ENOMEM;
	5639	+ *bad_wr = wr;
	5640	+ break;
	5641	+ }
	5642	+
	5643	+ wqe_idx = find_empty_entry(&srq->idx_que);
	5644	+ fill_idx_queue(&srq->idx_que, ind, wqe_idx);
	5645	+ wqe = get_srq_wqe(srq, wqe_idx);
	5646	+ dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
	5647	+
	5648	+ for (i = 0; i < wr->num_sge; ++i) {
	5649	+ dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
	5650	+ dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
	5651	+ dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
	5652	+ }
	5653	+
	5654	+ if (i < srq->max_gs) {
	5655	+ dseg->len = 0;
	5656	+ dseg->lkey = cpu_to_le32(0x100);
	5657	+ dseg->addr = 0;
	5658	+ }
	5659	+
	5660	+ srq->wrid[wqe_idx] = wr->wr_id;
	5661	+ ind = (ind + 1) & (srq->max - 1);
	5662	+ }
	5663	+
	5664	+ if (likely(nreq)) {
	5665	+ srq->head += nreq;
	5666	+
	5667	+ /*
	5668	+ * Make sure that descriptors are written before
	5669	+ * doorbell record.
	5670	+ */
	5671	+ wmb();
	5672	+
	5673	+ srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << 24 \| srq->srqn;
	5674	+ srq_db.parameter = srq->head;
	5675	+
	5676	+ hns_roce_write64_k((__le32 *)&srq_db, srq->db_reg_l);
	5677	+
	5678	+ }
	5679	+
	5680	+ spin_unlock_irqrestore(&srq->lock, flags);
	5681	+
	5682	+ return ret;
	5683	+}
	5684	+
5353	5685	static const struct hns_roce_hw hns_roce_hw_v2 = {
5354	5686	.cmq_init = hns_roce_v2_cmq_init,
5355	5687	.cmq_exit = hns_roce_v2_cmq_exit,
@@ -5377,6 +5709,10 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { · 21 unmodified lines
5377	5709	.poll_cq = hns_roce_v2_poll_cq,
5378	5710	.init_eq = hns_roce_v2_init_eq_table,
5379	5711	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
	5712	+ .write_srqc = hns_roce_v2_write_srqc,
	5713	+ .modify_srq = hns_roce_v2_modify_srq,
	5714	+ .query_srq = hns_roce_v2_query_srq,
	5715	+ .post_srq_recv = hns_roce_v2_post_srq_recv,
5380	5716	};
5381	5717
5382	5718	static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {

drivers/infiniband/hw/hns/hns_roce_hw_v2.h+86−0

@@ -93,8 +93,10 @@ · 92 unmodified lines
93	93	#define HNS_ROCE_CONTEXT_HOP_NUM 1
94	94	#define HNS_ROCE_MTT_HOP_NUM 1
95	95	#define HNS_ROCE_CQE_HOP_NUM 1
	96	+#define HNS_ROCE_SRQWQE_HOP_NUM 1
96	97	#define HNS_ROCE_PBL_HOP_NUM 2
97	98	#define HNS_ROCE_EQE_HOP_NUM 2
	99	+#define HNS_ROCE_IDX_HOP_NUM 1
98	100
99	101	#define HNS_ROCE_V2_GID_INDEX_NUM 256
100	102
@@ -334,6 +336,90 @@ struct hns_roce_v2_cq_context { · 233 unmodified lines
334	336	#define V2_CQC_BYTE_64_SE_CQE_IDX_S 0
335	337	#define V2_CQC_BYTE_64_SE_CQE_IDX_M GENMASK(23, 0)
336	338
	339	+struct hns_roce_srq_context {
	340	+ __le32 byte_4_srqn_srqst;
	341	+ __le32 byte_8_limit_wl;
	342	+ __le32 byte_12_xrcd;
	343	+ __le32 byte_16_pi_ci;
	344	+ __le32 wqe_bt_ba;
	345	+ __le32 byte_24_wqe_bt_ba;
	346	+ __le32 byte_28_rqws_pd;
	347	+ __le32 idx_bt_ba;
	348	+ __le32 rsv_idx_bt_ba;
	349	+ __le32 idx_cur_blk_addr;
	350	+ __le32 byte_44_idxbufpgsz_addr;
	351	+ __le32 idx_nxt_blk_addr;
	352	+ __le32 rsv_idxnxtblkaddr;
	353	+ __le32 byte_56_xrc_cqn;
	354	+ __le32 db_record_addr_record_en;
	355	+ __le32 db_record_addr;
	356	+};
	357	+
	358	+#define SRQC_BYTE_4_SRQ_ST_S 0
	359	+#define SRQC_BYTE_4_SRQ_ST_M GENMASK(1, 0)
	360	+
	361	+#define SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S 2
	362	+#define SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M GENMASK(3, 2)
	363	+
	364	+#define SRQC_BYTE_4_SRQ_SHIFT_S 4
	365	+#define SRQC_BYTE_4_SRQ_SHIFT_M GENMASK(7, 4)
	366	+
	367	+#define SRQC_BYTE_4_SRQN_S 8
	368	+#define SRQC_BYTE_4_SRQN_M GENMASK(31, 8)
	369	+
	370	+#define SRQC_BYTE_8_SRQ_LIMIT_WL_S 0
	371	+#define SRQC_BYTE_8_SRQ_LIMIT_WL_M GENMASK(15, 0)
	372	+
	373	+#define SRQC_BYTE_12_SRQ_XRCD_S 0
	374	+#define SRQC_BYTE_12_SRQ_XRCD_M GENMASK(23, 0)
	375	+
	376	+#define SRQC_BYTE_16_SRQ_PRODUCER_IDX_S 0
	377	+#define SRQC_BYTE_16_SRQ_PRODUCER_IDX_M GENMASK(15, 0)
	378	+
	379	+#define SRQC_BYTE_16_SRQ_CONSUMER_IDX_S 0
	380	+#define SRQC_BYTE_16_SRQ_CONSUMER_IDX_M GENMASK(31, 16)
	381	+
	382	+#define SRQC_BYTE_24_SRQ_WQE_BT_BA_S 0
	383	+#define SRQC_BYTE_24_SRQ_WQE_BT_BA_M GENMASK(28, 0)
	384	+
	385	+#define SRQC_BYTE_28_PD_S 0
	386	+#define SRQC_BYTE_28_PD_M GENMASK(23, 0)
	387	+
	388	+#define SRQC_BYTE_28_RQWS_S 24
	389	+#define SRQC_BYTE_28_RQWS_M GENMASK(27, 24)
	390	+
	391	+#define SRQC_BYTE_36_SRQ_IDX_BT_BA_S 0
	392	+#define SRQC_BYTE_36_SRQ_IDX_BT_BA_M GENMASK(28, 0)
	393	+
	394	+#define SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S 0
	395	+#define SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M GENMASK(19, 0)
	396	+
	397	+#define SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S 22
	398	+#define SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M GENMASK(23, 22)
	399	+
	400	+#define SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S 24
	401	+#define SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M GENMASK(27, 24)
	402	+
	403	+#define SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S 28
	404	+#define SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M GENMASK(31, 28)
	405	+
	406	+#define SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S 0
	407	+#define SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M GENMASK(19, 0)
	408	+
	409	+#define SRQC_BYTE_56_SRQ_XRC_CQN_S 0
	410	+#define SRQC_BYTE_56_SRQ_XRC_CQN_M GENMASK(23, 0)
	411	+
	412	+#define SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S 24
	413	+#define SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M GENMASK(27, 24)
	414	+
	415	+#define SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S 28
	416	+#define SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M GENMASK(31, 28)
	417	+
	418	+#define SRQC_BYTE_60_SRQ_RECORD_EN_S 0
	419	+
	420	+#define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_S 1
	421	+#define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_M GENMASK(31, 1)
	422	+
337	423	enum{
338	424	V2_MPT_ST_VALID = 0x1,
339	425	V2_MPT_ST_FREE = 0x2,

drivers/infiniband/hw/hns/hns_roce_main.c+15−0

@@ -546,6 +546,21 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) · 545 unmodified lines
546	546	ib_dev->map_mr_sg = hns_roce_map_mr_sg;
547	547	}
548	548
	549	+ /* SRQ */
	550	+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
	551	+ ib_dev->create_srq = hns_roce_create_srq;
	552	+ ib_dev->modify_srq = hr_dev->hw->modify_srq;
	553	+ ib_dev->query_srq = hr_dev->hw->query_srq;
	554	+ ib_dev->destroy_srq = hns_roce_destroy_srq;
	555	+ ib_dev->post_srq_recv = hr_dev->hw->post_srq_recv;
	556	+ ib_dev->uverbs_cmd_mask \|=
	557	+ (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) \|
	558	+ (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) \|
	559	+ (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) \|
	560	+ (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) \|
	561	+ (1ULL << IB_USER_VERBS_CMD_POST_SRQ_RECV);
	562	+ }
	563	+
549	564	/* OTHERS */
550	565	ib_dev->get_port_immutable = hns_roce_port_immutable;
551	566	ib_dev->disassociate_ucontext = hns_roce_disassociate_ucontext;

drivers/infiniband/hw/hns/hns_roce_mr.c+122−15

@@ -184,12 +184,27 @@ static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order, · 183 unmodified lines
184	184	struct hns_roce_buddy *buddy;
185	185	int ret;
186	186
187		− if (mtt_type == MTT_TYPE_WQE) {
	187	+ switch (mtt_type) {
	188	+ case MTT_TYPE_WQE:
188	189	buddy = &mr_table->mtt_buddy;
189	190	table = &mr_table->mtt_table;
190		− } else {
	191	+ break;
	192	+ case MTT_TYPE_CQE:
191	193	buddy = &mr_table->mtt_cqe_buddy;
192	194	table = &mr_table->mtt_cqe_table;
	195	+ break;
	196	+ case MTT_TYPE_SRQWQE:
	197	+ buddy = &mr_table->mtt_srqwqe_buddy;
	198	+ table = &mr_table->mtt_srqwqe_table;
	199	+ break;
	200	+ case MTT_TYPE_IDX:
	201	+ buddy = &mr_table->mtt_idx_buddy;
	202	+ table = &mr_table->mtt_idx_table;
	203	+ break;
	204	+ default:
	205	+ dev_err(hr_dev->dev, "Unsupport MTT table type: %d\n",
	206	+ mtt_type);
	207	+ return -EINVAL;
193	208	}
194	209
195	210	ret = hns_roce_buddy_alloc(buddy, order, seg);
@@ -242,18 +257,40 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev hr_dev, struct hns_roce_mtt mtt) · 46 unmodified lines
242	257	if (mtt->order < 0)
243	258	return;
244	259
245		− if (mtt->mtt_type == MTT_TYPE_WQE) {
	260	+ switch (mtt->mtt_type) {
	261	+ case MTT_TYPE_WQE:
246	262	hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg,
247	263	mtt->order);
248	264	hns_roce_table_put_range(hr_dev, &mr_table->mtt_table,
249	265	mtt->first_seg,
250	266	mtt->first_seg + (1 << mtt->order) - 1);
251		− } else {
	267	+ break;
	268	+ case MTT_TYPE_CQE:
252	269	hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg,
253	270	mtt->order);
254	271	hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table,
255	272	mtt->first_seg,
256	273	mtt->first_seg + (1 << mtt->order) - 1);
	274	+ break;
	275	+ case MTT_TYPE_SRQWQE:
	276	+ hns_roce_buddy_free(&mr_table->mtt_srqwqe_buddy, mtt->first_seg,
	277	+ mtt->order);
	278	+ hns_roce_table_put_range(hr_dev, &mr_table->mtt_srqwqe_table,
	279	+ mtt->first_seg,
	280	+ mtt->first_seg + (1 << mtt->order) - 1);
	281	+ break;
	282	+ case MTT_TYPE_IDX:
	283	+ hns_roce_buddy_free(&mr_table->mtt_idx_buddy, mtt->first_seg,
	284	+ mtt->order);
	285	+ hns_roce_table_put_range(hr_dev, &mr_table->mtt_idx_table,
	286	+ mtt->first_seg,
	287	+ mtt->first_seg + (1 << mtt->order) - 1);
	288	+ break;
	289	+ default:
	290	+ dev_err(hr_dev->dev,
	291	+ "Unsupport mtt type %d, clean mtt failed\n",
	292	+ mtt->mtt_type);
	293	+ break;
257	294	}
258	295	}
259	296	EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup);
@@ -713,10 +750,26 @@ static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, · 453 unmodified lines
713	750	u32 bt_page_size;
714	751	u32 i;
715	752
716		− if (mtt->mtt_type == MTT_TYPE_WQE)
	753	+ switch (mtt->mtt_type) {
	754	+ case MTT_TYPE_WQE:
	755	+ table = &hr_dev->mr_table.mtt_table;
717	756	bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
718		− else
	757	+ break;
	758	+ case MTT_TYPE_CQE:
	759	+ table = &hr_dev->mr_table.mtt_cqe_table;
719	760	bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
	761	+ break;
	762	+ case MTT_TYPE_SRQWQE:
	763	+ table = &hr_dev->mr_table.mtt_srqwqe_table;
	764	+ bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
	765	+ break;
	766	+ case MTT_TYPE_IDX:
	767	+ table = &hr_dev->mr_table.mtt_idx_table;
	768	+ bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
	769	+ break;
	770	+ default:
	771	+ return -EINVAL;
	772	+ }
720	773
721	774	/* All MTTs must fit in the same page */
722	775	if (start_index / (bt_page_size / sizeof(u64)) !=
@@ -726,11 +779,6 @@ static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, · 3 unmodified lines
726	779	if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))
727	780	return -EINVAL;
728	781
729		− if (mtt->mtt_type == MTT_TYPE_WQE)
730		− table = &hr_dev->mr_table.mtt_table;
731		− else
732		− table = &hr_dev->mr_table.mtt_cqe_table;
733		−
734	782	mtts = hns_roce_table_find(hr_dev, table,
735	783	mtt->first_seg + s / hr_dev->caps.mtt_entry_sz,
736	784	&dma_handle);
@@ -759,10 +807,25 @@ static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev, · 22 unmodified lines
759	807	if (mtt->order < 0)
760	808	return -EINVAL;
761	809
762		− if (mtt->mtt_type == MTT_TYPE_WQE)
	810	+ switch (mtt->mtt_type) {
	811	+ case MTT_TYPE_WQE:
763	812	bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
764		− else
	813	+ break;
	814	+ case MTT_TYPE_CQE:
765	815	bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
	816	+ break;
	817	+ case MTT_TYPE_SRQWQE:
	818	+ bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
	819	+ break;
	820	+ case MTT_TYPE_IDX:
	821	+ bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
	822	+ break;
	823	+ default:
	824	+ dev_err(hr_dev->dev,
	825	+ "Unsupport mtt type %d, write mtt failed\n",
	826	+ mtt->mtt_type);
	827	+ return -EINVAL;
	828	+ }
766	829
767	830	while (npages > 0) {
768	831	chunk = min_t(int, bt_page_size / sizeof(u64), npages);
@@ -828,8 +891,31 @@ int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) · 59 unmodified lines
828	891	if (ret)
829	892	goto err_buddy_cqe;
830	893	}
	894	+
	895	+ if (hr_dev->caps.num_srqwqe_segs) {
	896	+ ret = hns_roce_buddy_init(&mr_table->mtt_srqwqe_buddy,
	897	+ ilog2(hr_dev->caps.num_srqwqe_segs));
	898	+ if (ret)
	899	+ goto err_buddy_srqwqe;
	900	+ }
	901	+
	902	+ if (hr_dev->caps.num_idx_segs) {
	903	+ ret = hns_roce_buddy_init(&mr_table->mtt_idx_buddy,
	904	+ ilog2(hr_dev->caps.num_idx_segs));
	905	+ if (ret)
	906	+ goto err_buddy_idx;
	907	+ }
	908	+
831	909	return 0;
832	910
	911	+err_buddy_idx:
	912	+ if (hr_dev->caps.num_srqwqe_segs)
	913	+ hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
	914	+
	915	+err_buddy_srqwqe:
	916	+ if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
	917	+ hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
	918	+
833	919	err_buddy_cqe:
834	920	hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
835	921
@@ -842,6 +928,10 @@ void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev) · 6 unmodified lines
842	928	{
843	929	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
844	930
	931	+ if (hr_dev->caps.num_idx_segs)
	932	+ hns_roce_buddy_cleanup(&mr_table->mtt_idx_buddy);
	933	+ if (hr_dev->caps.num_srqwqe_segs)
	934	+ hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
845	935	hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
846	936	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
847	937	hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
@@ -897,8 +987,25 @@ int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, · 49 unmodified lines
897	987	u32 bt_page_size;
898	988	u32 n;
899	989
900		− order = mtt->mtt_type == MTT_TYPE_WQE ? hr_dev->caps.mtt_ba_pg_sz :
901		− hr_dev->caps.cqe_ba_pg_sz;
	990	+ switch (mtt->mtt_type) {
	991	+ case MTT_TYPE_WQE:
	992	+ order = hr_dev->caps.mtt_ba_pg_sz;
	993	+ break;
	994	+ case MTT_TYPE_CQE:
	995	+ order = hr_dev->caps.cqe_ba_pg_sz;
	996	+ break;
	997	+ case MTT_TYPE_SRQWQE:
	998	+ order = hr_dev->caps.srqwqe_ba_pg_sz;
	999	+ break;
	1000	+ case MTT_TYPE_IDX:
	1001	+ order = hr_dev->caps.idx_ba_pg_sz;
	1002	+ break;
	1003	+ default:
	1004	+ dev_err(dev, "Unsupport mtt type %d, write mtt failed\n",
	1005	+ mtt->mtt_type);
	1006	+ return -EINVAL;
	1007	+ }
	1008	+
902	1009	bt_page_size = 1 << (order + PAGE_SHIFT);
903	1010
904	1011	pages = (u64 *) __get_free_pages(GFP_KERNEL, order);

drivers/infiniband/hw/hns/hns_roce_qp.c+10−11

@@ -280,7 +280,7 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, · 279 unmodified lines
280	280	EXPORT_SYMBOL_GPL(hns_roce_release_range_qp);
281	281
282	282	static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,
283		− struct ib_qp_cap *cap, int is_user, int has_srq,
	283	+ struct ib_qp_cap *cap, int is_user, int has_rq,
284	284	struct hns_roce_qp *hr_qp)
285	285	{
286	286	struct device *dev = hr_dev->dev;
@@ -294,14 +294,12 @@ static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, · 7 unmodified lines
294	294	return -EINVAL;
295	295	}
296	296
297		− /* If srq exit, set zero for relative number of rq */
298		− if (has_srq) {
299		− if (cap->max_recv_wr) {
300		− dev_dbg(dev, "srq no need config max_recv_wr\n");
301		− return -EINVAL;
302		− }
303		−
304		− hr_qp->rq.wqe_cnt = hr_qp->rq.max_gs = 0;
	297	+ /* If srq exist, set zero for relative number of rq */
	298	+ if (!has_rq) {
	299	+ hr_qp->rq.wqe_cnt = 0;
	300	+ hr_qp->rq.max_gs = 0;
	301	+ cap->max_recv_wr = 0;
	302	+ cap->max_recv_sge = 0;
305	303	} else {
306	304	if (is_user && (!cap->max_recv_wr \|\| !cap->max_recv_sge)) {
307	305	dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n");
@@ -563,13 +561,14 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, · 255 unmodified lines
563	561	hr_qp->sq_signal_bits = cpu_to_le32(IB_SIGNAL_REQ_WR);
564	562
565	563	ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, !!ib_pd->uobject,
566		− !!init_attr->srq, hr_qp);
	564	+ hns_roce_qp_has_rq(init_attr), hr_qp);
567	565	if (ret) {
568	566	dev_err(dev, "hns_roce_set_rq_size failed\n");
569	567	goto err_out;
570	568	}
571	569
572		− if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
	570	+ if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
	571	+ hns_roce_qp_has_rq(init_attr)) {
573	572	/* allocate recv inline buf */
574	573	hr_qp->rq_inl_buf.wqe_list = kcalloc(hr_qp->rq.wqe_cnt,
575	574	sizeof(struct hns_roce_rinl_wqe),

drivers/infiniband/hw/hns/hns_roce_srq.c+407−0

@@ -9,6 +9,413 @@ · 8 unmodified lines
9	9	#include "hns_roce_cmd.h"
10	10	#include "hns_roce_hem.h"
11	11
	12	+static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,
	13	+ enum hns_roce_event event_type)
	14	+{
	15	+ struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
	16	+ struct ib_srq *ibsrq = &srq->ibsrq;
	17	+ struct ib_event event;
	18	+
	19	+ if (ibsrq->event_handler) {
	20	+ event.device = ibsrq->device;
	21	+ event.element.srq = ibsrq;
	22	+ switch (event_type) {
	23	+ case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
	24	+ event.event = IB_EVENT_SRQ_LIMIT_REACHED;
	25	+ break;
	26	+ case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
	27	+ event.event = IB_EVENT_SRQ_ERR;
	28	+ break;
	29	+ default:
	30	+ dev_err(hr_dev->dev,
	31	+ "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n",
	32	+ event_type, srq->srqn);
	33	+ return;
	34	+ }
	35	+
	36	+ ibsrq->event_handler(&event, ibsrq->srq_context);
	37	+ }
	38	+}
	39	+
	40	+static int hns_roce_sw2hw_srq(struct hns_roce_dev *dev,
	41	+ struct hns_roce_cmd_mailbox *mailbox,
	42	+ unsigned long srq_num)
	43	+{
	44	+ return hns_roce_cmd_mbox(dev, mailbox->dma, 0, srq_num, 0,
	45	+ HNS_ROCE_CMD_SW2HW_SRQ,
	46	+ HNS_ROCE_CMD_TIMEOUT_MSECS);
	47	+}
	48	+
	49	+static int hns_roce_hw2sw_srq(struct hns_roce_dev *dev,
	50	+ struct hns_roce_cmd_mailbox *mailbox,
	51	+ unsigned long srq_num)
	52	+{
	53	+ return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, srq_num,
	54	+ mailbox ? 0 : 1, HNS_ROCE_CMD_HW2SW_SRQ,
	55	+ HNS_ROCE_CMD_TIMEOUT_MSECS);
	56	+}
	57	+
	58	+int hns_roce_srq_alloc(struct hns_roce_dev *hr_dev, u32 pdn, u32 cqn, u16 xrcd,
	59	+ struct hns_roce_mtt *hr_mtt, u64 db_rec_addr,
	60	+ struct hns_roce_srq *srq)
	61	+{
	62	+ struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
	63	+ struct hns_roce_cmd_mailbox *mailbox;
	64	+ dma_addr_t dma_handle_wqe;
	65	+ dma_addr_t dma_handle_idx;
	66	+ u64 *mtts_wqe;
	67	+ u64 *mtts_idx;
	68	+ int ret;
	69	+
	70	+ /* Get the physical address of srq buf */
	71	+ mtts_wqe = hns_roce_table_find(hr_dev,
	72	+ &hr_dev->mr_table.mtt_srqwqe_table,
	73	+ srq->mtt.first_seg,
	74	+ &dma_handle_wqe);
	75	+ if (!mtts_wqe) {
	76	+ dev_err(hr_dev->dev,
	77	+ "SRQ alloc.Failed to find srq buf addr.\n");
	78	+ return -EINVAL;
	79	+ }
	80	+
	81	+ /* Get physical address of idx que buf */
	82	+ mtts_idx = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_idx_table,
	83	+ srq->idx_que.mtt.first_seg,
	84	+ &dma_handle_idx);
	85	+ if (!mtts_idx) {
	86	+ dev_err(hr_dev->dev,
	87	+ "SRQ alloc.Failed to find idx que buf addr.\n");
	88	+ return -EINVAL;
	89	+ }
	90	+
	91	+ ret = hns_roce_bitmap_alloc(&srq_table->bitmap, &srq->srqn);
	92	+ if (ret == -1) {
	93	+ dev_err(hr_dev->dev, "SRQ alloc.Failed to alloc index.\n");
	94	+ return -ENOMEM;
	95	+ }
	96	+
	97	+ ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);
	98	+ if (ret)
	99	+ goto err_out;
	100	+
	101	+ ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));
	102	+ if (ret)
	103	+ goto err_put;
	104	+
	105	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	106	+ if (IS_ERR(mailbox)) {
	107	+ ret = PTR_ERR(mailbox);
	108	+ goto err_xa;
	109	+ }
	110	+
	111	+ hr_dev->hw->write_srqc(hr_dev, srq, pdn, xrcd, cqn, mailbox->buf,
	112	+ mtts_wqe, mtts_idx, dma_handle_wqe,
	113	+ dma_handle_idx);
	114	+
	115	+ ret = hns_roce_sw2hw_srq(hr_dev, mailbox, srq->srqn);
	116	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	117	+ if (ret)
	118	+ goto err_xa;
	119	+
	120	+ atomic_set(&srq->refcount, 1);
	121	+ init_completion(&srq->free);
	122	+ return ret;
	123	+
	124	+err_xa:
	125	+ xa_erase(&srq_table->xa, srq->srqn);
	126	+
	127	+err_put:
	128	+ hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
	129	+
	130	+err_out:
	131	+ hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR);
	132	+ return ret;
	133	+}
	134	+
	135	+void hns_roce_srq_free(struct hns_roce_dev hr_dev, struct hns_roce_srq srq)
	136	+{
	137	+ struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
	138	+ int ret;
	139	+
	140	+ ret = hns_roce_hw2sw_srq(hr_dev, NULL, srq->srqn);
	141	+ if (ret)
	142	+ dev_err(hr_dev->dev, "HW2SW_SRQ failed (%d) for CQN %06lx\n",
	143	+ ret, srq->srqn);
	144	+
	145	+ xa_erase(&srq_table->xa, srq->srqn);
	146	+
	147	+ if (atomic_dec_and_test(&srq->refcount))
	148	+ complete(&srq->free);
	149	+ wait_for_completion(&srq->free);
	150	+
	151	+ hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
	152	+ hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR);
	153	+}
	154	+
	155	+static int hns_roce_create_idx_que(struct ib_pd pd, struct hns_roce_srq srq,
	156	+ u32 page_shift)
	157	+{
	158	+ struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
	159	+ struct hns_roce_idx_que *idx_que = &srq->idx_que;
	160	+ u32 bitmap_num;
	161	+ int i;
	162	+
	163	+ bitmap_num = HNS_ROCE_ALOGN_UP(srq->max, 8 * sizeof(u64));
	164	+
	165	+ idx_que->bitmap = kcalloc(1, bitmap_num / 8, GFP_KERNEL);
	166	+ if (!idx_que->bitmap)
	167	+ return -ENOMEM;
	168	+
	169	+ bitmap_num = bitmap_num / (8 * sizeof(u64));
	170	+
	171	+ idx_que->buf_size = srq->idx_que.buf_size;
	172	+
	173	+ if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2,
	174	+ &idx_que->idx_buf, page_shift)) {
	175	+ kfree(idx_que->bitmap);
	176	+ return -ENOMEM;
	177	+ }
	178	+
	179	+ for (i = 0; i < bitmap_num; i++)
	180	+ idx_que->bitmap[i] = ~(0UL);
	181	+
	182	+ return 0;
	183	+}
	184	+
	185	+struct ib_srq hns_roce_create_srq(struct ib_pd pd,
	186	+ struct ib_srq_init_attr *srq_init_attr,
	187	+ struct ib_udata *udata)
	188	+{
	189	+ struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
	190	+ struct hns_roce_srq *srq;
	191	+ int srq_desc_size;
	192	+ int srq_buf_size;
	193	+ u32 page_shift;
	194	+ int ret = 0;
	195	+ u32 npages;
	196	+ u32 cqn;
	197	+
	198	+ /* Check the actual SRQ wqe and SRQ sge num */
	199	+ if (srq_init_attr->attr.max_wr >= hr_dev->caps.max_srq_wrs \|\|
	200	+ srq_init_attr->attr.max_sge > hr_dev->caps.max_srq_sges)
	201	+ return ERR_PTR(-EINVAL);
	202	+
	203	+ srq = kzalloc(sizeof(*srq), GFP_KERNEL);
	204	+ if (!srq)
	205	+ return ERR_PTR(-ENOMEM);
	206	+
	207	+ mutex_init(&srq->mutex);
	208	+ spin_lock_init(&srq->lock);
	209	+
	210	+ srq->max = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
	211	+ srq->max_gs = srq_init_attr->attr.max_sge;
	212	+
	213	+ srq_desc_size = max(16, 16 * srq->max_gs);
	214	+
	215	+ srq->wqe_shift = ilog2(srq_desc_size);
	216	+
	217	+ srq_buf_size = srq->max * srq_desc_size;
	218	+
	219	+ srq->idx_que.entry_sz = HNS_ROCE_IDX_QUE_ENTRY_SZ;
	220	+ srq->idx_que.buf_size = srq->max * srq->idx_que.entry_sz;
	221	+ srq->mtt.mtt_type = MTT_TYPE_SRQWQE;
	222	+ srq->idx_que.mtt.mtt_type = MTT_TYPE_IDX;
	223	+
	224	+ if (udata) {
	225	+ struct hns_roce_ib_create_srq ucmd;
	226	+
	227	+ if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
	228	+ ret = -EFAULT;
	229	+ goto err_srq;
	230	+ }
	231	+
	232	+ srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
	233	+ srq_buf_size, 0, 0);
	234	+ if (IS_ERR(srq->umem)) {
	235	+ ret = PTR_ERR(srq->umem);
	236	+ goto err_srq;
	237	+ }
	238	+
	239	+ if (hr_dev->caps.srqwqe_buf_pg_sz) {
	240	+ npages = (ib_umem_page_count(srq->umem) +
	241	+ (1 << hr_dev->caps.srqwqe_buf_pg_sz) - 1) /
	242	+ (1 << hr_dev->caps.srqwqe_buf_pg_sz);
	243	+ page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
	244	+ ret = hns_roce_mtt_init(hr_dev, npages,
	245	+ page_shift,
	246	+ &srq->mtt);
	247	+ } else
	248	+ ret = hns_roce_mtt_init(hr_dev,
	249	+ ib_umem_page_count(srq->umem),
	250	+ srq->umem->page_shift,
	251	+ &srq->mtt);
	252	+ if (ret)
	253	+ goto err_buf;
	254	+
	255	+ ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->mtt, srq->umem);
	256	+ if (ret)
	257	+ goto err_srq_mtt;
	258	+
	259	+ /* config index queue BA */
	260	+ srq->idx_que.umem = ib_umem_get(pd->uobject->context,
	261	+ ucmd.que_addr,
	262	+ srq->idx_que.buf_size, 0, 0);
	263	+ if (IS_ERR(srq->idx_que.umem)) {
	264	+ dev_err(hr_dev->dev,
	265	+ "ib_umem_get error for index queue\n");
	266	+ goto err_srq_mtt;
	267	+ }
	268	+
	269	+ if (hr_dev->caps.idx_buf_pg_sz) {
	270	+ npages = (ib_umem_page_count(srq->idx_que.umem) +
	271	+ (1 << hr_dev->caps.idx_buf_pg_sz) - 1) /
	272	+ (1 << hr_dev->caps.idx_buf_pg_sz);
	273	+ page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
	274	+ ret = hns_roce_mtt_init(hr_dev, npages,
	275	+ page_shift, &srq->idx_que.mtt);
	276	+ } else {
	277	+ ret = hns_roce_mtt_init(hr_dev,
	278	+ ib_umem_page_count(srq->idx_que.umem),
	279	+ srq->idx_que.umem->page_shift,
	280	+ &srq->idx_que.mtt);
	281	+ }
	282	+
	283	+ if (ret) {
	284	+ dev_err(hr_dev->dev,
	285	+ "hns_roce_mtt_init error for idx que\n");
	286	+ goto err_idx_mtt;
	287	+ }
	288	+
	289	+ ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->idx_que.mtt,
	290	+ srq->idx_que.umem);
	291	+ if (ret) {
	292	+ dev_err(hr_dev->dev,
	293	+ "hns_roce_ib_umem_write_mtt error for idx que\n");
	294	+ goto err_idx_buf;
	295	+ }
	296	+ } else {
	297	+ page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
	298	+ if (hns_roce_buf_alloc(hr_dev, srq_buf_size,
	299	+ (1 << page_shift) * 2,
	300	+ &srq->buf, page_shift)) {
	301	+ ret = -ENOMEM;
	302	+ goto err_srq;
	303	+ }
	304	+
	305	+ srq->head = 0;
	306	+ srq->tail = srq->max - 1;
	307	+
	308	+ ret = hns_roce_mtt_init(hr_dev, srq->buf.npages,
	309	+ srq->buf.page_shift, &srq->mtt);
	310	+ if (ret)
	311	+ goto err_buf;
	312	+
	313	+ ret = hns_roce_buf_write_mtt(hr_dev, &srq->mtt, &srq->buf);
	314	+ if (ret)
	315	+ goto err_srq_mtt;
	316	+
	317	+ page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
	318	+ ret = hns_roce_create_idx_que(pd, srq, page_shift);
	319	+ if (ret) {
	320	+ dev_err(hr_dev->dev, "Create idx queue fail(%d)!\n",
	321	+ ret);
	322	+ goto err_srq_mtt;
	323	+ }
	324	+
	325	+ /* Init mtt table for idx_que */
	326	+ ret = hns_roce_mtt_init(hr_dev, srq->idx_que.idx_buf.npages,
	327	+ srq->idx_que.idx_buf.page_shift,
	328	+ &srq->idx_que.mtt);
	329	+ if (ret)
	330	+ goto err_create_idx;
	331	+
	332	+ /* Write buffer address into the mtt table */
	333	+ ret = hns_roce_buf_write_mtt(hr_dev, &srq->idx_que.mtt,
	334	+ &srq->idx_que.idx_buf);
	335	+ if (ret)
	336	+ goto err_idx_buf;
	337	+
	338	+ srq->wrid = kvmalloc_array(srq->max, sizeof(u64), GFP_KERNEL);
	339	+ if (!srq->wrid) {
	340	+ ret = -ENOMEM;
	341	+ goto err_idx_buf;
	342	+ }
	343	+ }
	344	+
	345	+ cqn = ib_srq_has_cq(srq_init_attr->srq_type) ?
	346	+ to_hr_cq(srq_init_attr->ext.cq)->cqn : 0;
	347	+
	348	+ srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG;
	349	+
	350	+ ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(pd)->pdn, cqn, 0,
	351	+ &srq->mtt, 0, srq);
	352	+ if (ret)
	353	+ goto err_wrid;
	354	+
	355	+ srq->event = hns_roce_ib_srq_event;
	356	+ srq->ibsrq.ext.xrc.srq_num = srq->srqn;
	357	+
	358	+ if (pd->uobject) {
	359	+ if (ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) {
	360	+ ret = -EFAULT;
	361	+ goto err_wrid;
	362	+ }
	363	+ }
	364	+
	365	+ return &srq->ibsrq;
	366	+
	367	+err_wrid:
	368	+ kvfree(srq->wrid);
	369	+
	370	+err_idx_buf:
	371	+ hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
	372	+
	373	+err_idx_mtt:
	374	+ if (udata)
	375	+ ib_umem_release(srq->idx_que.umem);
	376	+
	377	+err_create_idx:
	378	+ hns_roce_buf_free(hr_dev, srq->idx_que.buf_size,
	379	+ &srq->idx_que.idx_buf);
	380	+ kfree(srq->idx_que.bitmap);
	381	+
	382	+err_srq_mtt:
	383	+ hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
	384	+
	385	+err_buf:
	386	+ if (udata)
	387	+ ib_umem_release(srq->umem);
	388	+ else
	389	+ hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf);
	390	+
	391	+err_srq:
	392	+ kfree(srq);
	393	+ return ERR_PTR(ret);
	394	+}
	395	+
	396	+int hns_roce_destroy_srq(struct ib_srq *ibsrq)
	397	+{
	398	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	399	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	400	+
	401	+ hns_roce_srq_free(hr_dev, srq);
	402	+ hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
	403	+
	404	+ if (ibsrq->uobject) {
	405	+ hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
	406	+ ib_umem_release(srq->idx_que.umem);
	407	+ ib_umem_release(srq->umem);
	408	+ } else {
	409	+ kvfree(srq->wrid);
	410	+ hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift,
	411	+ &srq->buf);
	412	+ }
	413	+
	414	+ kfree(srq);
	415	+
	416	+ return 0;
	417	+}
	418	+
12	419	int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)
13	420	{
14	421	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;

include/uapi/rdma/hns-abi.h+6−0

@@ -46,6 +46,12 @@ struct hns_roce_ib_create_cq_resp { · 45 unmodified lines
46	46	__aligned_u64 cap_flags;
47	47	};
48	48
	49	+struct hns_roce_ib_create_srq {
	50	+ __aligned_u64 buf_addr;
	51	+ __aligned_u64 db_addr;
	52	+ __aligned_u64 que_addr;
	53	+};
	54	+
49	55	struct hns_roce_ib_create_qp {
50	56	__aligned_u64 buf_addr;
51	57	__aligned_u64 db_addr;

RDMA/hns: Add SRQ support for hip08 kernel mode

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code