History

RDMA/mlx5: Lower setting the umem's PAS for SRQ

Some of the SRQ types are created using a WQ, and the WQ requires a different parameter set to mlx5_umem_find_best_quantized_pgoff() as it has a 5 bit page_offset. Add the umem to the mlx5_srq_attr and defer computing the PAS data until the code has figured out what kind of mailbox to use. Compute the PAS directly from the umem for each of the four unique mailbox types. This also avoids allocating memory to store the user PAS, instead it is written directly to the mailbox as in most other cases.

Jason Gunthorpe committed 6 years ago

Commit 8a7904a1 parent 878f7b33 files · +79 −29Browse files

Unified · Split

3 files
  1. drivers/infiniband/hw/mlx5/srq.c +2 −25
  2. drivers/infiniband/hw/mlx5/srq.h +1 −0
  3. drivers/infiniband/hw/mlx5/srq_cmd.c +76 −4
drivers/infiniband/hw/mlx5/srq.c+2−25
@@ -51,8 +51,6 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, · 50 unmodified lines
51 udata, struct mlx5_ib_ucontext, ibucontext);51 udata, struct mlx5_ib_ucontext, ibucontext);
52 size_t ucmdlen;52 size_t ucmdlen;
53 int err;53 int err;
54 unsigned int page_offset_quantized;
55 unsigned int page_size;
56 u32 uidx = MLX5_IB_DEFAULT_UIDX;54 u32 uidx = MLX5_IB_DEFAULT_UIDX;
57 55
58 ucmdlen = min(udata->inlen, sizeof(ucmd));56 ucmdlen = min(udata->inlen, sizeof(ucmd));
@@ -84,32 +82,14 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, · 25 unmodified lines
84 err = PTR_ERR(srq->umem);82 err = PTR_ERR(srq->umem);
85 return err;83 return err;
86 }84 }
87 85 in->umem = srq->umem;
88 page_size = mlx5_umem_find_best_quantized_pgoff(
89 srq->umem, srqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,
90 page_offset, 64, &page_offset_quantized);
91 if (!page_size) {
92 mlx5_ib_warn(dev, "bad offset\n");
93 goto err_umem;
94 }
95
96 in->pas = kvcalloc(ib_umem_num_dma_blocks(srq->umem, page_size),
97 sizeof(*in->pas), GFP_KERNEL);
98 if (!in->pas) {
99 err = -ENOMEM;
100 goto err_umem;
101 }
102
103 mlx5_ib_populate_pas(srq->umem, page_size, in->pas, 0);
104 86
105 err = mlx5_ib_db_map_user(ucontext, udata, ucmd.db_addr, &srq->db);87 err = mlx5_ib_db_map_user(ucontext, udata, ucmd.db_addr, &srq->db);
106 if (err) {88 if (err) {
107 mlx5_ib_dbg(dev, "map doorbell failed\n");89 mlx5_ib_dbg(dev, "map doorbell failed\n");
108 goto err_in;90 goto err_umem;
109 }91 }
110 92
111 in->log_page_size = order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT;
112 in->page_offset = page_offset_quantized;
113 in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0;93 in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0;
114 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&94 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
115 in->type != IB_SRQT_BASIC)95 in->type != IB_SRQT_BASIC)
@@ -117,9 +97,6 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, · 1 unmodified line
117 97
118 return 0;98 return 0;
119 99
120err_in:
121 kvfree(in->pas);
122
123err_umem:100err_umem:
124 ib_umem_release(srq->umem);101 ib_umem_release(srq->umem);
125 102
drivers/infiniband/hw/mlx5/srq.h+1−0
@@ -28,6 +28,7 @@ struct mlx5_srq_attr { · 27 unmodified lines
28 u32 user_index;28 u32 user_index;
29 u64 db_record;29 u64 db_record;
30 __be64 *pas;30 __be64 *pas;
31 struct ib_umem *umem;
31 u32 tm_log_list_size;32 u32 tm_log_list_size;
32 u32 tm_next_tag;33 u32 tm_next_tag;
33 u32 tm_hw_phase_cnt;34 u32 tm_hw_phase_cnt;
drivers/infiniband/hw/mlx5/srq_cmd.c+76−4
@@ -92,6 +92,25 @@ struct mlx5_core_srq *mlx5_cmd_get_srq(struct mlx5_ib_dev *dev, u32 srqn) · 91 unmodified lines
92 return srq;92 return srq;
93}93}
94 94
95static int __set_srq_page_size(struct mlx5_srq_attr *in,
96 unsigned long page_size)
97{
98 if (!page_size)
99 return -EINVAL;
100 in->log_page_size = order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT;
101
102 if (WARN_ON(get_pas_size(in) !=
103 ib_umem_num_dma_blocks(in->umem, page_size) * sizeof(u64)))
104 return -EINVAL;
105 return 0;
106}
107
108#define set_srq_page_size(in, typ, log_pgsz_fld) \
109 __set_srq_page_size(in, mlx5_umem_find_best_quantized_pgoff( \
110 (in)->umem, typ, log_pgsz_fld, \
111 MLX5_ADAPTER_PAGE_SHIFT, page_offset, \
112 64, &(in)->page_offset))
113
95static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,114static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
96 struct mlx5_srq_attr *in)115 struct mlx5_srq_attr *in)
97{116{
@@ -103,6 +122,12 @@ static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 5 unmodified lines
103 int inlen;122 int inlen;
104 int err;123 int err;
105 124
125 if (in->umem) {
126 err = set_srq_page_size(in, srqc, log_page_size);
127 if (err)
128 return err;
129 }
130
106 pas_size = get_pas_size(in);131 pas_size = get_pas_size(in);
107 inlen = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;132 inlen = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;
108 create_in = kvzalloc(inlen, GFP_KERNEL);133 create_in = kvzalloc(inlen, GFP_KERNEL);
@@ -114,7 +139,13 @@ static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 5 unmodified lines
114 pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);139 pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);
115 140
116 set_srqc(srqc, in);141 set_srqc(srqc, in);
117 memcpy(pas, in->pas, pas_size);142 if (in->umem)
143 mlx5_ib_populate_pas(
144 in->umem,
145 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
146 pas, 0);
147 else
148 memcpy(pas, in->pas, pas_size);
118 149
119 MLX5_SET(create_srq_in, create_in, opcode,150 MLX5_SET(create_srq_in, create_in, opcode,
120 MLX5_CMD_OP_CREATE_SRQ);151 MLX5_CMD_OP_CREATE_SRQ);
@@ -194,6 +225,12 @@ static int create_xrc_srq_cmd(struct mlx5_ib_dev *dev, · 73 unmodified lines
194 int inlen;225 int inlen;
195 int err;226 int err;
196 227
228 if (in->umem) {
229 err = set_srq_page_size(in, xrc_srqc, log_page_size);
230 if (err)
231 return err;
232 }
233
197 pas_size = get_pas_size(in);234 pas_size = get_pas_size(in);
198 inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;235 inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
199 create_in = kvzalloc(inlen, GFP_KERNEL);236 create_in = kvzalloc(inlen, GFP_KERNEL);
@@ -207,7 +244,13 @@ static int create_xrc_srq_cmd(struct mlx5_ib_dev *dev, · 7 unmodified lines
207 244
208 set_srqc(xrc_srqc, in);245 set_srqc(xrc_srqc, in);
209 MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);246 MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);
210 memcpy(pas, in->pas, pas_size);247 if (in->umem)
248 mlx5_ib_populate_pas(
249 in->umem,
250 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
251 pas, 0);
252 else
253 memcpy(pas, in->pas, pas_size);
211 MLX5_SET(create_xrc_srq_in, create_in, opcode,254 MLX5_SET(create_xrc_srq_in, create_in, opcode,
212 MLX5_CMD_OP_CREATE_XRC_SRQ);255 MLX5_CMD_OP_CREATE_XRC_SRQ);
213 256
@@ -289,11 +332,18 @@ static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 75 unmodified lines
289 void *create_in = NULL;332 void *create_in = NULL;
290 void *rmpc;333 void *rmpc;
291 void *wq;334 void *wq;
335 void *pas;
292 int pas_size;336 int pas_size;
293 int outlen;337 int outlen;
294 int inlen;338 int inlen;
295 int err;339 int err;
296 340
341 if (in->umem) {
342 err = set_srq_page_size(in, wq, log_wq_pg_sz);
343 if (err)
344 return err;
345 }
346
297 pas_size = get_pas_size(in);347 pas_size = get_pas_size(in);
298 inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;348 inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
299 outlen = MLX5_ST_SZ_BYTES(create_rmp_out);349 outlen = MLX5_ST_SZ_BYTES(create_rmp_out);
@@ -309,8 +359,16 @@ static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 9 unmodified lines
309 359
310 MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);360 MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
311 MLX5_SET(create_rmp_in, create_in, uid, in->uid);361 MLX5_SET(create_rmp_in, create_in, uid, in->uid);
362 pas = MLX5_ADDR_OF(rmpc, rmpc, wq.pas);
363
312 set_wq(wq, in);364 set_wq(wq, in);
313 memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);365 if (in->umem)
366 mlx5_ib_populate_pas(
367 in->umem,
368 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
369 pas, 0);
370 else
371 memcpy(pas, in->pas, pas_size);
314 372
315 MLX5_SET(create_rmp_in, create_in, opcode, MLX5_CMD_OP_CREATE_RMP);373 MLX5_SET(create_rmp_in, create_in, opcode, MLX5_CMD_OP_CREATE_RMP);
316 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen);374 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen);
@@ -421,10 +479,17 @@ static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 104 unmodified lines
421 void *create_in;479 void *create_in;
422 void *xrqc;480 void *xrqc;
423 void *wq;481 void *wq;
482 void *pas;
424 int pas_size;483 int pas_size;
425 int inlen;484 int inlen;
426 int err;485 int err;
427 486
487 if (in->umem) {
488 err = set_srq_page_size(in, wq, log_wq_pg_sz);
489 if (err)
490 return err;
491 }
492
428 pas_size = get_pas_size(in);493 pas_size = get_pas_size(in);
429 inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;494 inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;
430 create_in = kvzalloc(inlen, GFP_KERNEL);495 create_in = kvzalloc(inlen, GFP_KERNEL);
@@ -433,9 +498,16 @@ static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, · 2 unmodified lines
433 498
434 xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);499 xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);
435 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);500 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
501 pas = MLX5_ADDR_OF(xrqc, xrqc, wq.pas);
436 502
437 set_wq(wq, in);503 set_wq(wq, in);
438 memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size);504 if (in->umem)
505 mlx5_ib_populate_pas(
506 in->umem,
507 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
508 pas, 0);
509 else
510 memcpy(pas, in->pas, pas_size);
439 511
440 if (in->type == IB_SRQT_TM) {512 if (in->type == IB_SRQT_TM) {
441 MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);513 MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);