History

Merge branch 'mlx5_memic_ops' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux

Maor Gottlieb says: ==================== This series from Maor extends MEMIC to support atomic operations from the host in addition to already supported regular read/write. ==================== * 'memic_ops': RDMA/mlx5: Expose UAPI to query DM RDMA/mlx5: Add support in MEMIC operations RDMA/mlx5: Add support to MODIFY_MEMIC command RDMA/mlx5: Re-organize the DM code RDMA/mlx5: Move all DM logic to separate file RDMA/uverbs: Make UVERBS_OBJECT_METHODS to consider line number net/mlx5: Add MEMIC operations related bits

Jason Gunthorpe committed 5 years ago

Commit fe73f962 parents 780278c 187316411 files · +720 −369Browse files

Unified · Split

11 files
  1. drivers/infiniband/hw/mlx5/Makefile +1 −0
  2. drivers/infiniband/hw/mlx5/cmd.c +0 −101
  3. drivers/infiniband/hw/mlx5/cmd.h +0 −3
  4. drivers/infiniband/hw/mlx5/dm.c +584 −0
  5. drivers/infiniband/hw/mlx5/dm.h +68 −0
  6. drivers/infiniband/hw/mlx5/main.c +4 −239
  7. drivers/infiniband/hw/mlx5/mlx5_ib.h +1 −24
  8. drivers/infiniband/hw/mlx5/mr.c +1 −0
  9. include/linux/mlx5/mlx5_ifc.h +41 −1
  10. include/rdma/uverbs_named_ioctl.h +1 −1
  11. include/uapi/rdma/mlx5_user_ioctl_cmds.h +19 −0
drivers/infiniband/hw/mlx5/Makefile+1−0
@@ -6,6 +6,7 @@ mlx5_ib-y := ah.o \ · 5 unmodified lines
6 cong.o \6 cong.o \
7 counters.o \7 counters.o \
8 cq.o \8 cq.o \
9 dm.o \
9 doorbell.o \10 doorbell.o \
10 gsi.o \11 gsi.o \
11 ib_virt.o \12 ib_virt.o \
drivers/infiniband/hw/mlx5/cmd.c+0−101
@@ -47,107 +47,6 @@ int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, · 46 unmodified lines
47 return mlx5_cmd_exec_inout(dev, query_cong_params, in, out);47 return mlx5_cmd_exec_inout(dev, query_cong_params, in, out);
48}48}
49 49
50int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
51 u64 length, u32 alignment)
52{
53 struct mlx5_core_dev *dev = dm->dev;
54 u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)
55 >> PAGE_SHIFT;
56 u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
57 u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);
58 u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
59 u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};
60 u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};
61 u32 mlx5_alignment;
62 u64 page_idx = 0;
63 int ret = 0;
64
65 if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK))
66 return -EINVAL;
67
68 /* mlx5 device sets alignment as 64*2^driver_value
69 * so normalizing is needed.
70 */
71 mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 :
72 alignment - MLX5_MEMIC_BASE_ALIGN;
73 if (mlx5_alignment > max_alignment)
74 return -EINVAL;
75
76 MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);
77 MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);
78 MLX5_SET(alloc_memic_in, in, memic_size, length);
79 MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,
80 mlx5_alignment);
81
82 while (page_idx < num_memic_hw_pages) {
83 spin_lock(&dm->lock);
84 page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,
85 num_memic_hw_pages,
86 page_idx,
87 num_pages, 0);
88
89 if (page_idx < num_memic_hw_pages)
90 bitmap_set(dm->memic_alloc_pages,
91 page_idx, num_pages);
92
93 spin_unlock(&dm->lock);
94
95 if (page_idx >= num_memic_hw_pages)
96 break;
97
98 MLX5_SET64(alloc_memic_in, in, range_start_addr,
99 hw_start_addr + (page_idx * PAGE_SIZE));
100
101 ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);
102 if (ret) {
103 spin_lock(&dm->lock);
104 bitmap_clear(dm->memic_alloc_pages,
105 page_idx, num_pages);
106 spin_unlock(&dm->lock);
107
108 if (ret == -EAGAIN) {
109 page_idx++;
110 continue;
111 }
112
113 return ret;
114 }
115
116 *addr = dev->bar_addr +
117 MLX5_GET64(alloc_memic_out, out, memic_start_addr);
118
119 return 0;
120 }
121
122 return -ENOMEM;
123}
124
125void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)
126{
127 struct mlx5_core_dev *dev = dm->dev;
128 u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
129 u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
130 u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};
131 u64 start_page_idx;
132 int err;
133
134 addr -= dev->bar_addr;
135 start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT;
136
137 MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);
138 MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);
139 MLX5_SET(dealloc_memic_in, in, memic_size, length);
140
141 err = mlx5_cmd_exec_in(dev, dealloc_memic, in);
142 if (err)
143 return;
144
145 spin_lock(&dm->lock);
146 bitmap_clear(dm->memic_alloc_pages,
147 start_page_idx, num_pages);
148 spin_unlock(&dm->lock);
149}
150
151void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)50void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)
152{51{
153 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};52 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};
drivers/infiniband/hw/mlx5/cmd.h+0−3
@@ -41,9 +41,6 @@ int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey); · 40 unmodified lines
41int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey);41int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey);
42int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,42int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
43 void *out);43 void *out);
44int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
45 u64 length, u32 alignment);
46void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);
47int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);44int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);
48void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid);45void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid);
49void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid);46void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid);
drivers/infiniband/hw/mlx5/dm.c+584−0
@@ -0,0 +1,584 @@
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include <rdma/uverbs_std_types.h>
7#include "dm.h"
8
9#define UVERBS_MODULE_NAME mlx5_ib
10#include <rdma/uverbs_named_ioctl.h>
11
12static int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
13 u64 length, u32 alignment)
14{
15 struct mlx5_core_dev *dev = dm->dev;
16 u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)
17 >> PAGE_SHIFT;
18 u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
19 u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);
20 u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
21 u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};
22 u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};
23 u32 mlx5_alignment;
24 u64 page_idx = 0;
25 int ret = 0;
26
27 if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK))
28 return -EINVAL;
29
30 /* mlx5 device sets alignment as 64*2^driver_value
31 * so normalizing is needed.
32 */
33 mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 :
34 alignment - MLX5_MEMIC_BASE_ALIGN;
35 if (mlx5_alignment > max_alignment)
36 return -EINVAL;
37
38 MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);
39 MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);
40 MLX5_SET(alloc_memic_in, in, memic_size, length);
41 MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,
42 mlx5_alignment);
43
44 while (page_idx < num_memic_hw_pages) {
45 spin_lock(&dm->lock);
46 page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,
47 num_memic_hw_pages,
48 page_idx,
49 num_pages, 0);
50
51 if (page_idx < num_memic_hw_pages)
52 bitmap_set(dm->memic_alloc_pages,
53 page_idx, num_pages);
54
55 spin_unlock(&dm->lock);
56
57 if (page_idx >= num_memic_hw_pages)
58 break;
59
60 MLX5_SET64(alloc_memic_in, in, range_start_addr,
61 hw_start_addr + (page_idx * PAGE_SIZE));
62
63 ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);
64 if (ret) {
65 spin_lock(&dm->lock);
66 bitmap_clear(dm->memic_alloc_pages,
67 page_idx, num_pages);
68 spin_unlock(&dm->lock);
69
70 if (ret == -EAGAIN) {
71 page_idx++;
72 continue;
73 }
74
75 return ret;
76 }
77
78 *addr = dev->bar_addr +
79 MLX5_GET64(alloc_memic_out, out, memic_start_addr);
80
81 return 0;
82 }
83
84 return -ENOMEM;
85}
86
87void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,
88 u64 length)
89{
90 struct mlx5_core_dev *dev = dm->dev;
91 u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
92 u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
93 u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};
94 u64 start_page_idx;
95 int err;
96
97 addr -= dev->bar_addr;
98 start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT;
99
100 MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);
101 MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);
102 MLX5_SET(dealloc_memic_in, in, memic_size, length);
103
104 err = mlx5_cmd_exec_in(dev, dealloc_memic, in);
105 if (err)
106 return;
107
108 spin_lock(&dm->lock);
109 bitmap_clear(dm->memic_alloc_pages,
110 start_page_idx, num_pages);
111 spin_unlock(&dm->lock);
112}
113
114void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
115 u8 operation)
116{
117 u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};
118 struct mlx5_core_dev *dev = dm->dev;
119
120 MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);
121 MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC);
122 MLX5_SET(modify_memic_in, in, memic_operation_type, operation);
123 MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr);
124
125 mlx5_cmd_exec_in(dev, modify_memic, in);
126}
127
128static int mlx5_cmd_alloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
129 u8 operation, phys_addr_t *op_addr)
130{
131 u32 out[MLX5_ST_SZ_DW(modify_memic_out)] = {};
132 u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};
133 struct mlx5_core_dev *dev = dm->dev;
134 int err;
135
136 MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);
137 MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_ALLOC);
138 MLX5_SET(modify_memic_in, in, memic_operation_type, operation);
139 MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr);
140
141 err = mlx5_cmd_exec_inout(dev, modify_memic, in, out);
142 if (err)
143 return err;
144
145 *op_addr = dev->bar_addr +
146 MLX5_GET64(modify_memic_out, out, memic_operation_addr);
147 return 0;
148}
149
150static int add_dm_mmap_entry(struct ib_ucontext *context,
151 struct mlx5_user_mmap_entry *mentry, u8 mmap_flag,
152 size_t size, u64 address)
153{
154 mentry->mmap_flag = mmap_flag;
155 mentry->address = address;
156
157 return rdma_user_mmap_entry_insert_range(
158 context, &mentry->rdma_entry, size,
159 MLX5_IB_MMAP_DEVICE_MEM << 16,
160 (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1);
161}
162
163static void mlx5_ib_dm_memic_free(struct kref *kref)
164{
165 struct mlx5_ib_dm_memic *dm =
166 container_of(kref, struct mlx5_ib_dm_memic, ref);
167 struct mlx5_ib_dev *dev = to_mdev(dm->base.ibdm.device);
168
169 mlx5_cmd_dealloc_memic(&dev->dm, dm->base.dev_addr, dm->base.size);
170 kfree(dm);
171}
172
173static int copy_op_to_user(struct mlx5_ib_dm_op_entry *op_entry,
174 struct uverbs_attr_bundle *attrs)
175{
176 u64 start_offset;
177 u16 page_idx;
178 int err;
179
180 page_idx = op_entry->mentry.rdma_entry.start_pgoff & 0xFFFF;
181 start_offset = op_entry->op_addr & ~PAGE_MASK;
182 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
183 &page_idx, sizeof(page_idx));
184 if (err)
185 return err;
186
187 return uverbs_copy_to(attrs,
188 MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
189 &start_offset, sizeof(start_offset));
190}
191
192static int map_existing_op(struct mlx5_ib_dm_memic *dm, u8 op,
193 struct uverbs_attr_bundle *attrs)
194{
195 struct mlx5_ib_dm_op_entry *op_entry;
196
197 op_entry = xa_load(&dm->ops, op);
198 if (!op_entry)
199 return -ENOENT;
200
201 return copy_op_to_user(op_entry, attrs);
202}
203
204static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_MAP_OP_ADDR)(
205 struct uverbs_attr_bundle *attrs)
206{
207 struct ib_uobject *uobj = uverbs_attr_get_uobject(
208 attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE);
209 struct mlx5_ib_dev *dev = to_mdev(uobj->context->device);
210 struct ib_dm *ibdm = uobj->object;
211 struct mlx5_ib_dm_memic *dm = to_memic(ibdm);
212 struct mlx5_ib_dm_op_entry *op_entry;
213 int err;
214 u8 op;
215
216 err = uverbs_copy_from(&op, attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP);
217 if (err)
218 return err;
219
220 if (!(MLX5_CAP_DEV_MEM(dev->mdev, memic_operations) & BIT(op)))
221 return -EOPNOTSUPP;
222
223 mutex_lock(&dm->ops_xa_lock);
224 err = map_existing_op(dm, op, attrs);
225 if (!err || err != -ENOENT)
226 goto err_unlock;
227
228 op_entry = kzalloc(sizeof(*op_entry), GFP_KERNEL);
229 if (!op_entry)
230 goto err_unlock;
231
232 err = mlx5_cmd_alloc_memic_op(&dev->dm, dm->base.dev_addr, op,
233 &op_entry->op_addr);
234 if (err) {
235 kfree(op_entry);
236 goto err_unlock;
237 }
238 op_entry->op = op;
239 op_entry->dm = dm;
240
241 err = add_dm_mmap_entry(uobj->context, &op_entry->mentry,
242 MLX5_IB_MMAP_TYPE_MEMIC_OP, dm->base.size,
243 op_entry->op_addr & PAGE_MASK);
244 if (err) {
245 mlx5_cmd_dealloc_memic_op(&dev->dm, dm->base.dev_addr, op);
246 kfree(op_entry);
247 goto err_unlock;
248 }
249 /* From this point, entry will be freed by mmap_free */
250 kref_get(&dm->ref);
251
252 err = copy_op_to_user(op_entry, attrs);
253 if (err)
254 goto err_remove;
255
256 err = xa_insert(&dm->ops, op, op_entry, GFP_KERNEL);
257 if (err)
258 goto err_remove;
259 mutex_unlock(&dm->ops_xa_lock);
260
261 return 0;
262
263err_remove:
264 rdma_user_mmap_entry_remove(&op_entry->mentry.rdma_entry);
265err_unlock:
266 mutex_unlock(&dm->ops_xa_lock);
267
268 return err;
269}
270
271static struct ib_dm *handle_alloc_dm_memic(struct ib_ucontext *ctx,
272 struct ib_dm_alloc_attr *attr,
273 struct uverbs_attr_bundle *attrs)
274{
275 struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
276 struct mlx5_ib_dm_memic *dm;
277 u64 start_offset;
278 u16 page_idx;
279 int err;
280 u64 address;
281
282 if (!MLX5_CAP_DEV_MEM(dm_db->dev, memic))
283 return ERR_PTR(-EOPNOTSUPP);
284
285 dm = kzalloc(sizeof(*dm), GFP_KERNEL);
286 if (!dm)
287 return ERR_PTR(-ENOMEM);
288
289 dm->base.type = MLX5_IB_UAPI_DM_TYPE_MEMIC;
290 dm->base.size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
291 dm->base.ibdm.device = ctx->device;
292
293 kref_init(&dm->ref);
294 xa_init(&dm->ops);
295 mutex_init(&dm->ops_xa_lock);
296 dm->req_length = attr->length;
297
298 err = mlx5_cmd_alloc_memic(dm_db, &dm->base.dev_addr,
299 dm->base.size, attr->alignment);
300 if (err) {
301 kfree(dm);
302 return ERR_PTR(err);
303 }
304
305 address = dm->base.dev_addr & PAGE_MASK;
306 err = add_dm_mmap_entry(ctx, &dm->mentry, MLX5_IB_MMAP_TYPE_MEMIC,
307 dm->base.size, address);
308 if (err) {
309 mlx5_cmd_dealloc_memic(dm_db, dm->base.dev_addr, dm->base.size);
310 kfree(dm);
311 return ERR_PTR(err);
312 }
313
314 page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF;
315 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
316 &page_idx, sizeof(page_idx));
317 if (err)
318 goto err_copy;
319
320 start_offset = dm->base.dev_addr & ~PAGE_MASK;
321 err = uverbs_copy_to(attrs,
322 MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
323 &start_offset, sizeof(start_offset));
324 if (err)
325 goto err_copy;
326
327 return &dm->base.ibdm;
328
329err_copy:
330 rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
331 return ERR_PTR(err);
332}
333
334static struct ib_dm *handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,
335 struct ib_dm_alloc_attr *attr,
336 struct uverbs_attr_bundle *attrs,
337 int type)
338{
339 struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev;
340 struct mlx5_ib_dm_icm *dm;
341 u64 act_size;
342 int err;
343
344 dm = kzalloc(sizeof(*dm), GFP_KERNEL);
345 if (!dm)
346 return ERR_PTR(-ENOMEM);
347
348 dm->base.type = type;
349 dm->base.ibdm.device = ctx->device;
350
351 if (!capable(CAP_SYS_RAWIO) || !capable(CAP_NET_RAW)) {
352 err = -EPERM;
353 goto free;
354 }
355
356 if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner) ||
357 MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner) ||
358 MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) ||
359 MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2))) {
360 err = -EOPNOTSUPP;
361 goto free;
362 }
363
364 /* Allocation size must a multiple of the basic block size
365 * and a power of 2.
366 */
367 act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev));
368 act_size = roundup_pow_of_two(act_size);
369
370 dm->base.size = act_size;
371 err = mlx5_dm_sw_icm_alloc(dev, type, act_size, attr->alignment,
372 to_mucontext(ctx)->devx_uid,
373 &dm->base.dev_addr, &dm->obj_id);
374 if (err)
375 goto free;
376
377 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
378 &dm->base.dev_addr, sizeof(dm->base.dev_addr));
379 if (err) {
380 mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size,
381 to_mucontext(ctx)->devx_uid,
382 dm->base.dev_addr, dm->obj_id);
383 goto free;
384 }
385 return &dm->base.ibdm;
386free:
387 kfree(dm);
388 return ERR_PTR(err);
389}
390
391struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
392 struct ib_ucontext *context,
393 struct ib_dm_alloc_attr *attr,
394 struct uverbs_attr_bundle *attrs)
395{
396 enum mlx5_ib_uapi_dm_type type;
397 int err;
398
399 err = uverbs_get_const_default(&type, attrs,
400 MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
401 MLX5_IB_UAPI_DM_TYPE_MEMIC);
402 if (err)
403 return ERR_PTR(err);
404
405 mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",
406 type, attr->length, attr->alignment);
407
408 switch (type) {
409 case MLX5_IB_UAPI_DM_TYPE_MEMIC:
410 return handle_alloc_dm_memic(context, attr, attrs);
411 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
412 return handle_alloc_dm_sw_icm(context, attr, attrs,
413 MLX5_SW_ICM_TYPE_STEERING);
414 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
415 return handle_alloc_dm_sw_icm(context, attr, attrs,
416 MLX5_SW_ICM_TYPE_HEADER_MODIFY);
417 default:
418 return ERR_PTR(-EOPNOTSUPP);
419 }
420}
421
422static void dm_memic_remove_ops(struct mlx5_ib_dm_memic *dm)
423{
424 struct mlx5_ib_dm_op_entry *entry;
425 unsigned long idx;
426
427 mutex_lock(&dm->ops_xa_lock);
428 xa_for_each(&dm->ops, idx, entry) {
429 xa_erase(&dm->ops, idx);
430 rdma_user_mmap_entry_remove(&entry->mentry.rdma_entry);
431 }
432 mutex_unlock(&dm->ops_xa_lock);
433}
434
435static void mlx5_dm_memic_dealloc(struct mlx5_ib_dm_memic *dm)
436{
437 dm_memic_remove_ops(dm);
438 rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
439}
440
441static int mlx5_dm_icm_dealloc(struct mlx5_ib_ucontext *ctx,
442 struct mlx5_ib_dm_icm *dm)
443{
444 enum mlx5_sw_icm_type type =
445 dm->base.type == MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM ?
446 MLX5_SW_ICM_TYPE_STEERING :
447 MLX5_SW_ICM_TYPE_HEADER_MODIFY;
448 struct mlx5_core_dev *dev = to_mdev(dm->base.ibdm.device)->mdev;
449 int err;
450
451 err = mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size, ctx->devx_uid,
452 dm->base.dev_addr, dm->obj_id);
453 if (!err)
454 kfree(dm);
455 return 0;
456}
457
458static int mlx5_ib_dealloc_dm(struct ib_dm *ibdm,
459 struct uverbs_attr_bundle *attrs)
460{
461 struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(
462 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
463 struct mlx5_ib_dm *dm = to_mdm(ibdm);
464
465 switch (dm->type) {
466 case MLX5_IB_UAPI_DM_TYPE_MEMIC:
467 mlx5_dm_memic_dealloc(to_memic(ibdm));
468 return 0;
469 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
470 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
471 return mlx5_dm_icm_dealloc(ctx, to_icm(ibdm));
472 default:
473 return -EOPNOTSUPP;
474 }
475}
476
477static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_QUERY)(
478 struct uverbs_attr_bundle *attrs)
479{
480 struct ib_dm *ibdm =
481 uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE);
482 struct mlx5_ib_dm *dm = to_mdm(ibdm);
483 struct mlx5_ib_dm_memic *memic;
484 u64 start_offset;
485 u16 page_idx;
486 int err;
487
488 if (dm->type != MLX5_IB_UAPI_DM_TYPE_MEMIC)
489 return -EOPNOTSUPP;
490
491 memic = to_memic(ibdm);
492 page_idx = memic->mentry.rdma_entry.start_pgoff & 0xFFFF;
493 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
494 &page_idx, sizeof(page_idx));
495 if (err)
496 return err;
497
498 start_offset = memic->base.dev_addr & ~PAGE_MASK;
499 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
500 &start_offset, sizeof(start_offset));
501 if (err)
502 return err;
503
504 return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
505 &memic->req_length,
506 sizeof(memic->req_length));
507}
508
509void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev,
510 struct mlx5_user_mmap_entry *mentry)
511{
512 struct mlx5_ib_dm_op_entry *op_entry;
513 struct mlx5_ib_dm_memic *mdm;
514
515 switch (mentry->mmap_flag) {
516 case MLX5_IB_MMAP_TYPE_MEMIC:
517 mdm = container_of(mentry, struct mlx5_ib_dm_memic, mentry);
518 kref_put(&mdm->ref, mlx5_ib_dm_memic_free);
519 break;
520 case MLX5_IB_MMAP_TYPE_MEMIC_OP:
521 op_entry = container_of(mentry, struct mlx5_ib_dm_op_entry,
522 mentry);
523 mdm = op_entry->dm;
524 mlx5_cmd_dealloc_memic_op(&dev->dm, mdm->base.dev_addr,
525 op_entry->op);
526 kfree(op_entry);
527 kref_put(&mdm->ref, mlx5_ib_dm_memic_free);
528 break;
529 default:
530 WARN_ON(true);
531 }
532}
533
534DECLARE_UVERBS_NAMED_METHOD(
535 MLX5_IB_METHOD_DM_QUERY,
536 UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE, UVERBS_OBJECT_DM,
537 UVERBS_ACCESS_READ, UA_MANDATORY),
538 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
539 UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
540 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
541 UVERBS_ATTR_TYPE(u16), UA_MANDATORY),
542 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
543 UVERBS_ATTR_TYPE(u64), UA_MANDATORY));
544
545ADD_UVERBS_ATTRIBUTES_SIMPLE(
546 mlx5_ib_dm, UVERBS_OBJECT_DM, UVERBS_METHOD_DM_ALLOC,
547 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
548 UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
549 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
550 UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
551 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
552 enum mlx5_ib_uapi_dm_type, UA_OPTIONAL));
553
554DECLARE_UVERBS_NAMED_METHOD(
555 MLX5_IB_METHOD_DM_MAP_OP_ADDR,
556 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE,
557 UVERBS_OBJECT_DM,
558 UVERBS_ACCESS_READ,
559 UA_MANDATORY),
560 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP,
561 UVERBS_ATTR_TYPE(u8),
562 UA_MANDATORY),
563 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
564 UVERBS_ATTR_TYPE(u64),
565 UA_MANDATORY),
566 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
567 UVERBS_ATTR_TYPE(u16),
568 UA_OPTIONAL));
569
570DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DM,
571 &UVERBS_METHOD(MLX5_IB_METHOD_DM_MAP_OP_ADDR),
572 &UVERBS_METHOD(MLX5_IB_METHOD_DM_QUERY));
573
574const struct uapi_definition mlx5_ib_dm_defs[] = {
575 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
576 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DM),
577 {},
578};
579
580const struct ib_device_ops mlx5_ib_dev_dm_ops = {
581 .alloc_dm = mlx5_ib_alloc_dm,
582 .dealloc_dm = mlx5_ib_dealloc_dm,
583 .reg_dm_mr = mlx5_ib_reg_dm_mr,
584};
drivers/infiniband/hw/mlx5/dm.h+68−0
@@ -0,0 +1,68 @@
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2/*
3 * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.
4 */
5
6#ifndef _MLX5_IB_DM_H
7#define _MLX5_IB_DM_H
8
9#include "mlx5_ib.h"
10
11extern const struct ib_device_ops mlx5_ib_dev_dm_ops;
12extern const struct uapi_definition mlx5_ib_dm_defs[];
13
14struct mlx5_ib_dm {
15 struct ib_dm ibdm;
16 u32 type;
17 phys_addr_t dev_addr;
18 size_t size;
19};
20
21struct mlx5_ib_dm_op_entry {
22 struct mlx5_user_mmap_entry mentry;
23 phys_addr_t op_addr;
24 struct mlx5_ib_dm_memic *dm;
25 u8 op;
26};
27
28struct mlx5_ib_dm_memic {
29 struct mlx5_ib_dm base;
30 struct mlx5_user_mmap_entry mentry;
31 struct xarray ops;
32 struct mutex ops_xa_lock;
33 struct kref ref;
34 size_t req_length;
35};
36
37struct mlx5_ib_dm_icm {
38 struct mlx5_ib_dm base;
39 u32 obj_id;
40};
41
42static inline struct mlx5_ib_dm *to_mdm(struct ib_dm *ibdm)
43{
44 return container_of(ibdm, struct mlx5_ib_dm, ibdm);
45}
46
47static inline struct mlx5_ib_dm_memic *to_memic(struct ib_dm *ibdm)
48{
49 return container_of(ibdm, struct mlx5_ib_dm_memic, base.ibdm);
50}
51
52static inline struct mlx5_ib_dm_icm *to_icm(struct ib_dm *ibdm)
53{
54 return container_of(ibdm, struct mlx5_ib_dm_icm, base.ibdm);
55}
56
57struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
58 struct ib_ucontext *context,
59 struct ib_dm_alloc_attr *attr,
60 struct uverbs_attr_bundle *attrs);
61void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev,
62 struct mlx5_user_mmap_entry *mentry);
63void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,
64 u64 length);
65void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
66 u8 operation);
67
68#endif /* _MLX5_IB_DM_H */
drivers/infiniband/hw/mlx5/main.c+4−239
@@ -34,6 +34,7 @@ · 33 unmodified lines
34#include "ib_rep.h"34#include "ib_rep.h"
35#include "cmd.h"35#include "cmd.h"
36#include "devx.h"36#include "devx.h"
37#include "dm.h"
37#include "fs.h"38#include "fs.h"
38#include "srq.h"39#include "srq.h"
39#include "qp.h"40#include "qp.h"
@@ -2088,14 +2089,11 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry) · 2048 unmodified lines
2088 struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);2089 struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
2089 struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);2090 struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
2090 struct mlx5_var_table *var_table = &dev->var_table;2091 struct mlx5_var_table *var_table = &dev->var_table;
2091 struct mlx5_ib_dm *mdm;
2092 2092
2093 switch (mentry->mmap_flag) {2093 switch (mentry->mmap_flag) {
2094 case MLX5_IB_MMAP_TYPE_MEMIC:2094 case MLX5_IB_MMAP_TYPE_MEMIC:
2095 mdm = container_of(mentry, struct mlx5_ib_dm, mentry);2095 case MLX5_IB_MMAP_TYPE_MEMIC_OP:
2096 mlx5_cmd_dealloc_memic(&dev->dm, mdm->dev_addr,2096 mlx5_ib_dm_mmap_free(dev, mentry);
2097 mdm->size);
2098 kfree(mdm);
2099 break;2097 break;
2100 case MLX5_IB_MMAP_TYPE_VAR:2098 case MLX5_IB_MMAP_TYPE_VAR:
2101 mutex_lock(&var_table->bitmap_lock);2099 mutex_lock(&var_table->bitmap_lock);
@@ -2220,19 +2218,6 @@ free_bfreg: · 118 unmodified lines
2220 return err;2218 return err;
2221}2219}
2222 2220
2223static int add_dm_mmap_entry(struct ib_ucontext *context,
2224 struct mlx5_ib_dm *mdm,
2225 u64 address)
2226{
2227 mdm->mentry.mmap_flag = MLX5_IB_MMAP_TYPE_MEMIC;
2228 mdm->mentry.address = address;
2229 return rdma_user_mmap_entry_insert_range(
2230 context, &mdm->mentry.rdma_entry,
2231 mdm->size,
2232 MLX5_IB_MMAP_DEVICE_MEM << 16,
2233 (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1);
2234}
2235
2236static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)2221static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
2237{2222{
2238 unsigned long idx;2223 unsigned long idx;
@@ -2334,206 +2319,6 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm · 95 unmodified lines
2334 return 0;2319 return 0;
2335}2320}
2336 2321
2337static inline int check_dm_type_support(struct mlx5_ib_dev *dev,
2338 u32 type)
2339{
2340 switch (type) {
2341 case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2342 if (!MLX5_CAP_DEV_MEM(dev->mdev, memic))
2343 return -EOPNOTSUPP;
2344 break;
2345 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2346 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2347 if (!capable(CAP_SYS_RAWIO) ||
2348 !capable(CAP_NET_RAW))
2349 return -EPERM;
2350
2351 if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
2352 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner) ||
2353 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2) ||
2354 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner_v2)))
2355 return -EOPNOTSUPP;
2356 break;
2357 }
2358
2359 return 0;
2360}
2361
2362static int handle_alloc_dm_memic(struct ib_ucontext *ctx,
2363 struct mlx5_ib_dm *dm,
2364 struct ib_dm_alloc_attr *attr,
2365 struct uverbs_attr_bundle *attrs)
2366{
2367 struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
2368 u64 start_offset;
2369 u16 page_idx;
2370 int err;
2371 u64 address;
2372
2373 dm->size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
2374
2375 err = mlx5_cmd_alloc_memic(dm_db, &dm->dev_addr,
2376 dm->size, attr->alignment);
2377 if (err)
2378 return err;
2379
2380 address = dm->dev_addr & PAGE_MASK;
2381 err = add_dm_mmap_entry(ctx, dm, address);
2382 if (err)
2383 goto err_dealloc;
2384
2385 page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF;
2386 err = uverbs_copy_to(attrs,
2387 MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
2388 &page_idx,
2389 sizeof(page_idx));
2390 if (err)
2391 goto err_copy;
2392
2393 start_offset = dm->dev_addr & ~PAGE_MASK;
2394 err = uverbs_copy_to(attrs,
2395 MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
2396 &start_offset, sizeof(start_offset));
2397 if (err)
2398 goto err_copy;
2399
2400 return 0;
2401
2402err_copy:
2403 rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
2404err_dealloc:
2405 mlx5_cmd_dealloc_memic(dm_db, dm->dev_addr, dm->size);
2406
2407 return err;
2408}
2409
2410static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,
2411 struct mlx5_ib_dm *dm,
2412 struct ib_dm_alloc_attr *attr,
2413 struct uverbs_attr_bundle *attrs,
2414 int type)
2415{
2416 struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev;
2417 u64 act_size;
2418 int err;
2419
2420 /* Allocation size must a multiple of the basic block size
2421 * and a power of 2.
2422 */
2423 act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev));
2424 act_size = roundup_pow_of_two(act_size);
2425
2426 dm->size = act_size;
2427 err = mlx5_dm_sw_icm_alloc(dev, type, act_size, attr->alignment,
2428 to_mucontext(ctx)->devx_uid, &dm->dev_addr,
2429 &dm->icm_dm.obj_id);
2430 if (err)
2431 return err;
2432
2433 err = uverbs_copy_to(attrs,
2434 MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
2435 &dm->dev_addr, sizeof(dm->dev_addr));
2436 if (err)
2437 mlx5_dm_sw_icm_dealloc(dev, type, dm->size,
2438 to_mucontext(ctx)->devx_uid, dm->dev_addr,
2439 dm->icm_dm.obj_id);
2440
2441 return err;
2442}
2443
2444struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
2445 struct ib_ucontext *context,
2446 struct ib_dm_alloc_attr *attr,
2447 struct uverbs_attr_bundle *attrs)
2448{
2449 struct mlx5_ib_dm *dm;
2450 enum mlx5_ib_uapi_dm_type type;
2451 int err;
2452
2453 err = uverbs_get_const_default(&type, attrs,
2454 MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
2455 MLX5_IB_UAPI_DM_TYPE_MEMIC);
2456 if (err)
2457 return ERR_PTR(err);
2458
2459 mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",
2460 type, attr->length, attr->alignment);
2461
2462 err = check_dm_type_support(to_mdev(ibdev), type);
2463 if (err)
2464 return ERR_PTR(err);
2465
2466 dm = kzalloc(sizeof(*dm), GFP_KERNEL);
2467 if (!dm)
2468 return ERR_PTR(-ENOMEM);
2469
2470 dm->type = type;
2471
2472 switch (type) {
2473 case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2474 err = handle_alloc_dm_memic(context, dm,
2475 attr,
2476 attrs);
2477 break;
2478 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2479 err = handle_alloc_dm_sw_icm(context, dm,
2480 attr, attrs,
2481 MLX5_SW_ICM_TYPE_STEERING);
2482 break;
2483 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2484 err = handle_alloc_dm_sw_icm(context, dm,
2485 attr, attrs,
2486 MLX5_SW_ICM_TYPE_HEADER_MODIFY);
2487 break;
2488 default:
2489 err = -EOPNOTSUPP;
2490 }
2491
2492 if (err)
2493 goto err_free;
2494
2495 return &dm->ibdm;
2496
2497err_free:
2498 kfree(dm);
2499 return ERR_PTR(err);
2500}
2501
2502int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs)
2503{
2504 struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(
2505 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2506 struct mlx5_core_dev *dev = to_mdev(ibdm->device)->mdev;
2507 struct mlx5_ib_dm *dm = to_mdm(ibdm);
2508 int ret;
2509
2510 switch (dm->type) {
2511 case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2512 rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
2513 return 0;
2514 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2515 ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_STEERING,
2516 dm->size, ctx->devx_uid, dm->dev_addr,
2517 dm->icm_dm.obj_id);
2518 if (ret)
2519 return ret;
2520 break;
2521 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2522 ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_HEADER_MODIFY,
2523 dm->size, ctx->devx_uid, dm->dev_addr,
2524 dm->icm_dm.obj_id);
2525 if (ret)
2526 return ret;
2527 break;
2528 default:
2529 return -EOPNOTSUPP;
2530 }
2531
2532 kfree(dm);
2533
2534 return 0;
2535}
2536
2537static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)2322static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
2538{2323{
2539 struct mlx5_ib_pd *pd = to_mpd(ibpd);2324 struct mlx5_ib_pd *pd = to_mpd(ibpd);
@@ -3817,20 +3602,6 @@ DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR, · 1277 unmodified lines
3817 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),3602 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
3818 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));3603 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
3819 3604
3820ADD_UVERBS_ATTRIBUTES_SIMPLE(
3821 mlx5_ib_dm,
3822 UVERBS_OBJECT_DM,
3823 UVERBS_METHOD_DM_ALLOC,
3824 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
3825 UVERBS_ATTR_TYPE(u64),
3826 UA_MANDATORY),
3827 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
3828 UVERBS_ATTR_TYPE(u16),
3829 UA_OPTIONAL),
3830 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
3831 enum mlx5_ib_uapi_dm_type,
3832 UA_OPTIONAL));
3833
3834ADD_UVERBS_ATTRIBUTES_SIMPLE(3605ADD_UVERBS_ATTRIBUTES_SIMPLE(
3835 mlx5_ib_flow_action,3606 mlx5_ib_flow_action,
3836 UVERBS_OBJECT_FLOW_ACTION,3607 UVERBS_OBJECT_FLOW_ACTION,
@@ -3853,10 +3624,10 @@ static const struct uapi_definition mlx5_ib_defs[] = { · 16 unmodified lines
3853 UAPI_DEF_CHAIN(mlx5_ib_flow_defs),3624 UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
3854 UAPI_DEF_CHAIN(mlx5_ib_qos_defs),3625 UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
3855 UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),3626 UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
3627 UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
3856 3628
3857 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,3629 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
3858 &mlx5_ib_flow_action),3630 &mlx5_ib_flow_action),
3859 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
3860 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),3631 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
3861 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,3632 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
3862 UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),3633 UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
@@ -4032,12 +3803,6 @@ static const struct ib_device_ops mlx5_ib_dev_xrc_ops = { · 169 unmodified lines
4032 INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),3803 INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
4033};3804};
4034 3805
4035static const struct ib_device_ops mlx5_ib_dev_dm_ops = {
4036 .alloc_dm = mlx5_ib_alloc_dm,
4037 .dealloc_dm = mlx5_ib_dealloc_dm,
4038 .reg_dm_mr = mlx5_ib_reg_dm_mr,
4039};
4040
4041static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)3806static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
4042{3807{
4043 struct mlx5_core_dev *mdev = dev->mdev;3808 struct mlx5_core_dev *mdev = dev->mdev;
drivers/infiniband/hw/mlx5/mlx5_ib.h+1−24
@@ -166,6 +166,7 @@ enum mlx5_ib_mmap_type { · 165 unmodified lines
166 MLX5_IB_MMAP_TYPE_VAR = 2,166 MLX5_IB_MMAP_TYPE_VAR = 2,
167 MLX5_IB_MMAP_TYPE_UAR_WC = 3,167 MLX5_IB_MMAP_TYPE_UAR_WC = 3,
168 MLX5_IB_MMAP_TYPE_UAR_NC = 4,168 MLX5_IB_MMAP_TYPE_UAR_NC = 4,
169 MLX5_IB_MMAP_TYPE_MEMIC_OP = 5,
169};170};
170 171
171struct mlx5_bfreg_info {172struct mlx5_bfreg_info {
@@ -618,20 +619,6 @@ struct mlx5_user_mmap_entry { · 446 unmodified lines
618 u32 page_idx;619 u32 page_idx;
619};620};
620 621
621struct mlx5_ib_dm {
622 struct ib_dm ibdm;
623 phys_addr_t dev_addr;
624 u32 type;
625 size_t size;
626 union {
627 struct {
628 u32 obj_id;
629 } icm_dm;
630 /* other dm types specific params should be added here */
631 };
632 struct mlx5_user_mmap_entry mentry;
633};
634
635#define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)622#define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
636 623
637#define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE |\624#define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE |\
@@ -1188,11 +1175,6 @@ static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq) · 550 unmodified lines
1188 return container_of(msrq, struct mlx5_ib_srq, msrq);1175 return container_of(msrq, struct mlx5_ib_srq, msrq);
1189}1176}
1190 1177
1191static inline struct mlx5_ib_dm *to_mdm(struct ib_dm *ibdm)
1192{
1193 return container_of(ibdm, struct mlx5_ib_dm, ibdm);
1194}
1195
1196static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)1178static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
1197{1179{
1198 return container_of(ibmr, struct mlx5_ib_mr, ibmr);1180 return container_of(ibmr, struct mlx5_ib_mr, ibmr);
@@ -1347,11 +1329,6 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, · 148 unmodified lines
1347 struct ib_rwq_ind_table_init_attr *init_attr,1329 struct ib_rwq_ind_table_init_attr *init_attr,
1348 struct ib_udata *udata);1330 struct ib_udata *udata);
1349int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);1331int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
1350struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
1351 struct ib_ucontext *context,
1352 struct ib_dm_alloc_attr *attr,
1353 struct uverbs_attr_bundle *attrs);
1354int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs);
1355struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,1332struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
1356 struct ib_dm_mr_attr *attr,1333 struct ib_dm_mr_attr *attr,
1357 struct uverbs_attr_bundle *attrs);1334 struct uverbs_attr_bundle *attrs);
drivers/infiniband/hw/mlx5/mr.c+1−0
@@ -42,6 +42,7 @@ · 41 unmodified lines
42#include <rdma/ib_umem.h>42#include <rdma/ib_umem.h>
43#include <rdma/ib_umem_odp.h>43#include <rdma/ib_umem_odp.h>
44#include <rdma/ib_verbs.h>44#include <rdma/ib_verbs.h>
45#include "dm.h"
45#include "mlx5_ib.h"46#include "mlx5_ib.h"
46 47
47/*48/*
include/linux/mlx5/mlx5_ifc.h+41−1
@@ -133,6 +133,7 @@ enum { · 132 unmodified lines
133 MLX5_CMD_OP_PAGE_FAULT_RESUME = 0x204,133 MLX5_CMD_OP_PAGE_FAULT_RESUME = 0x204,
134 MLX5_CMD_OP_ALLOC_MEMIC = 0x205,134 MLX5_CMD_OP_ALLOC_MEMIC = 0x205,
135 MLX5_CMD_OP_DEALLOC_MEMIC = 0x206,135 MLX5_CMD_OP_DEALLOC_MEMIC = 0x206,
136 MLX5_CMD_OP_MODIFY_MEMIC = 0x207,
136 MLX5_CMD_OP_CREATE_EQ = 0x301,137 MLX5_CMD_OP_CREATE_EQ = 0x301,
137 MLX5_CMD_OP_DESTROY_EQ = 0x302,138 MLX5_CMD_OP_DESTROY_EQ = 0x302,
138 MLX5_CMD_OP_QUERY_EQ = 0x303,139 MLX5_CMD_OP_QUERY_EQ = 0x303,
@@ -1017,7 +1018,11 @@ struct mlx5_ifc_device_mem_cap_bits { · 878 unmodified lines
1017 1018
1018 u8 header_modify_sw_icm_start_address[0x40];1019 u8 header_modify_sw_icm_start_address[0x40];
1019 1020
1020 u8 reserved_at_180[0x680];1021 u8 reserved_at_180[0x80];
1022
1023 u8 memic_operations[0x20];
1024
1025 u8 reserved_at_220[0x5e0];
1021};1026};
1022 1027
1023struct mlx5_ifc_device_event_cap_bits {1028struct mlx5_ifc_device_event_cap_bits {
@@ -10417,6 +10422,41 @@ struct mlx5_ifc_destroy_vport_lag_in_bits { · 9393 unmodified lines
10417 u8 reserved_at_40[0x40];10422 u8 reserved_at_40[0x40];
10418};10423};
10419 10424
10425enum {
10426 MLX5_MODIFY_MEMIC_OP_MOD_ALLOC,
10427 MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC,
10428};
10429
10430struct mlx5_ifc_modify_memic_in_bits {
10431 u8 opcode[0x10];
10432 u8 uid[0x10];
10433
10434 u8 reserved_at_20[0x10];
10435 u8 op_mod[0x10];
10436
10437 u8 reserved_at_40[0x20];
10438
10439 u8 reserved_at_60[0x18];
10440 u8 memic_operation_type[0x8];
10441
10442 u8 memic_start_addr[0x40];
10443
10444 u8 reserved_at_c0[0x140];
10445};
10446
10447struct mlx5_ifc_modify_memic_out_bits {
10448 u8 status[0x8];
10449 u8 reserved_at_8[0x18];
10450
10451 u8 syndrome[0x20];
10452
10453 u8 reserved_at_40[0x40];
10454
10455 u8 memic_operation_addr[0x40];
10456
10457 u8 reserved_at_c0[0x140];
10458};
10459
10420struct mlx5_ifc_alloc_memic_in_bits {10460struct mlx5_ifc_alloc_memic_in_bits {
10421 u8 opcode[0x10];10461 u8 opcode[0x10];
10422 u8 reserved_at_10[0x10];10462 u8 reserved_at_10[0x10];
include/rdma/uverbs_named_ioctl.h+1−1
@@ -20,7 +20,7 @@ · 19 unmodified lines
20 20
21/* These are static so they do not need to be qualified */21/* These are static so they do not need to be qualified */
22#define UVERBS_METHOD_ATTRS(method_id) _method_attrs_##method_id22#define UVERBS_METHOD_ATTRS(method_id) _method_attrs_##method_id
23#define UVERBS_OBJECT_METHODS(object_id) _object_methods_##object_id23#define UVERBS_OBJECT_METHODS(object_id) _UVERBS_NAME(_object_methods_##object_id, __LINE__)
24 24
25#define DECLARE_UVERBS_NAMED_METHOD(_method_id, ...) \25#define DECLARE_UVERBS_NAMED_METHOD(_method_id, ...) \
26 static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \26 static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \
include/uapi/rdma/mlx5_user_ioctl_cmds.h+19−0
@@ -41,6 +41,25 @@ enum mlx5_ib_create_flow_action_attrs { · 40 unmodified lines
41 MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS = (1U << UVERBS_ID_NS_SHIFT),41 MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS = (1U << UVERBS_ID_NS_SHIFT),
42};42};
43 43
44enum mlx5_ib_dm_methods {
45 MLX5_IB_METHOD_DM_MAP_OP_ADDR = (1U << UVERBS_ID_NS_SHIFT),
46 MLX5_IB_METHOD_DM_QUERY,
47};
48
49enum mlx5_ib_dm_map_op_addr_attrs {
50 MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
51 MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP,
52 MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
53 MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
54};
55
56enum mlx5_ib_query_dm_attrs {
57 MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
58 MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
59 MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
60 MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
61};
62
44enum mlx5_ib_alloc_dm_attrs {63enum mlx5_ib_alloc_dm_attrs {
45 MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET = (1U << UVERBS_ID_NS_SHIFT),64 MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET = (1U << UVERBS_ID_NS_SHIFT),
46 MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,65 MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,