Merge branch 'mlx5_memic_ops' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux

Maor Gottlieb says: ==================== This series from Maor extends MEMIC to support atomic operations from the host in addition to already supported regular read/write. ==================== * 'memic_ops': RDMA/mlx5: Expose UAPI to query DM RDMA/mlx5: Add support in MEMIC operations RDMA/mlx5: Add support to MODIFY_MEMIC command RDMA/mlx5: Re-organize the DM code RDMA/mlx5: Move all DM logic to separate file RDMA/uverbs: Make UVERBS_OBJECT_METHODS to consider line number net/mlx5: Add MEMIC operations related bits

Jason Gunthorpe committed 5 years ago

Commit fe73f962 parents 780278c 187316411 files · +720 −369Browse files

Unified · Split

11 files

drivers/infiniband/hw/mlx5/Makefile +1 −0
drivers/infiniband/hw/mlx5/cmd.c +0 −101
drivers/infiniband/hw/mlx5/cmd.h +0 −3
drivers/infiniband/hw/mlx5/dm.c +584 −0
drivers/infiniband/hw/mlx5/dm.h +68 −0
drivers/infiniband/hw/mlx5/main.c +4 −239
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 −24
drivers/infiniband/hw/mlx5/mr.c +1 −0
include/linux/mlx5/mlx5_ifc.h +41 −1
include/rdma/uverbs_named_ioctl.h +1 −1
include/uapi/rdma/mlx5_user_ioctl_cmds.h +19 −0

drivers/infiniband/hw/mlx5/Makefile+1−0

@@ -6,6 +6,7 @@ mlx5_ib-y := ah.o \ · 5 unmodified lines
6	6	cong.o \
7	7	counters.o \
8	8	cq.o \
	9	+ dm.o \
9	10	doorbell.o \
10	11	gsi.o \
11	12	ib_virt.o \

drivers/infiniband/hw/mlx5/cmd.c+0−101

@@ -47,107 +47,6 @@ int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, · 46 unmodified lines
47	47	return mlx5_cmd_exec_inout(dev, query_cong_params, in, out);
48	48	}
49	49
50		−int mlx5_cmd_alloc_memic(struct mlx5_dm dm, phys_addr_t addr,
51		− u64 length, u32 alignment)
52		−{
53		− struct mlx5_core_dev *dev = dm->dev;
54		− u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)
55		− >> PAGE_SHIFT;
56		− u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
57		− u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);
58		− u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
59		− u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};
60		− u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};
61		− u32 mlx5_alignment;
62		− u64 page_idx = 0;
63		− int ret = 0;
64		−
65		− if (!length \|\| (length & MLX5_MEMIC_ALLOC_SIZE_MASK))
66		− return -EINVAL;
67		−
68		− /* mlx5 device sets alignment as 64*2^driver_value
69		− * so normalizing is needed.
70		− */
71		− mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 :
72		− alignment - MLX5_MEMIC_BASE_ALIGN;
73		− if (mlx5_alignment > max_alignment)
74		− return -EINVAL;
75		−
76		− MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);
77		− MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);
78		− MLX5_SET(alloc_memic_in, in, memic_size, length);
79		− MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,
80		− mlx5_alignment);
81		−
82		− while (page_idx < num_memic_hw_pages) {
83		− spin_lock(&dm->lock);
84		− page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,
85		− num_memic_hw_pages,
86		− page_idx,
87		− num_pages, 0);
88		−
89		− if (page_idx < num_memic_hw_pages)
90		− bitmap_set(dm->memic_alloc_pages,
91		− page_idx, num_pages);
92		−
93		− spin_unlock(&dm->lock);
94		−
95		− if (page_idx >= num_memic_hw_pages)
96		− break;
97		−
98		− MLX5_SET64(alloc_memic_in, in, range_start_addr,
99		− hw_start_addr + (page_idx * PAGE_SIZE));
100		−
101		− ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);
102		− if (ret) {
103		− spin_lock(&dm->lock);
104		− bitmap_clear(dm->memic_alloc_pages,
105		− page_idx, num_pages);
106		− spin_unlock(&dm->lock);
107		−
108		− if (ret == -EAGAIN) {
109		− page_idx++;
110		− continue;
111		− }
112		−
113		− return ret;
114		− }
115		−
116		− *addr = dev->bar_addr +
117		− MLX5_GET64(alloc_memic_out, out, memic_start_addr);
118		−
119		− return 0;
120		− }
121		−
122		− return -ENOMEM;
123		−}
124		−
125		−void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)
126		−{
127		− struct mlx5_core_dev *dev = dm->dev;
128		− u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
129		− u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
130		− u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};
131		− u64 start_page_idx;
132		− int err;
133		−
134		− addr -= dev->bar_addr;
135		− start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT;
136		−
137		− MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);
138		− MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);
139		− MLX5_SET(dealloc_memic_in, in, memic_size, length);
140		−
141		− err = mlx5_cmd_exec_in(dev, dealloc_memic, in);
142		− if (err)
143		− return;
144		−
145		− spin_lock(&dm->lock);
146		− bitmap_clear(dm->memic_alloc_pages,
147		− start_page_idx, num_pages);
148		− spin_unlock(&dm->lock);
149		−}
150		−
151	50	void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)
152	51	{
153	52	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};

drivers/infiniband/hw/mlx5/cmd.h+0−3

@@ -41,9 +41,6 @@ int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev dev, u32 mkey); · 40 unmodified lines
41	41	int mlx5_cmd_null_mkey(struct mlx5_core_dev dev, u32 null_mkey);
42	42	int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
43	43	void *out);
44		−int mlx5_cmd_alloc_memic(struct mlx5_dm dm, phys_addr_t addr,
45		− u64 length, u32 alignment);
46		−void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);
47	44	int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);
48	45	void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid);
49	46	void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid);

drivers/infiniband/hw/mlx5/dm.c+584−0

@@ -0,0 +1,584 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	+/*
	3	+ * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.
	4	+ */
	5	+
	6	+#include <rdma/uverbs_std_types.h>
	7	+#include "dm.h"
	8	+
	9	+#define UVERBS_MODULE_NAME mlx5_ib
	10	+#include <rdma/uverbs_named_ioctl.h>
	11	+
	12	+static int mlx5_cmd_alloc_memic(struct mlx5_dm dm, phys_addr_t addr,
	13	+ u64 length, u32 alignment)
	14	+{
	15	+ struct mlx5_core_dev *dev = dm->dev;
	16	+ u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)
	17	+ >> PAGE_SHIFT;
	18	+ u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
	19	+ u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);
	20	+ u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
	21	+ u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};
	22	+ u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};
	23	+ u32 mlx5_alignment;
	24	+ u64 page_idx = 0;
	25	+ int ret = 0;
	26	+
	27	+ if (!length \|\| (length & MLX5_MEMIC_ALLOC_SIZE_MASK))
	28	+ return -EINVAL;
	29	+
	30	+ /* mlx5 device sets alignment as 64*2^driver_value
	31	+ * so normalizing is needed.
	32	+ */
	33	+ mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 :
	34	+ alignment - MLX5_MEMIC_BASE_ALIGN;
	35	+ if (mlx5_alignment > max_alignment)
	36	+ return -EINVAL;
	37	+
	38	+ MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);
	39	+ MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);
	40	+ MLX5_SET(alloc_memic_in, in, memic_size, length);
	41	+ MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,
	42	+ mlx5_alignment);
	43	+
	44	+ while (page_idx < num_memic_hw_pages) {
	45	+ spin_lock(&dm->lock);
	46	+ page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,
	47	+ num_memic_hw_pages,
	48	+ page_idx,
	49	+ num_pages, 0);
	50	+
	51	+ if (page_idx < num_memic_hw_pages)
	52	+ bitmap_set(dm->memic_alloc_pages,
	53	+ page_idx, num_pages);
	54	+
	55	+ spin_unlock(&dm->lock);
	56	+
	57	+ if (page_idx >= num_memic_hw_pages)
	58	+ break;
	59	+
	60	+ MLX5_SET64(alloc_memic_in, in, range_start_addr,
	61	+ hw_start_addr + (page_idx * PAGE_SIZE));
	62	+
	63	+ ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);
	64	+ if (ret) {
	65	+ spin_lock(&dm->lock);
	66	+ bitmap_clear(dm->memic_alloc_pages,
	67	+ page_idx, num_pages);
	68	+ spin_unlock(&dm->lock);
	69	+
	70	+ if (ret == -EAGAIN) {
	71	+ page_idx++;
	72	+ continue;
	73	+ }
	74	+
	75	+ return ret;
	76	+ }
	77	+
	78	+ *addr = dev->bar_addr +
	79	+ MLX5_GET64(alloc_memic_out, out, memic_start_addr);
	80	+
	81	+ return 0;
	82	+ }
	83	+
	84	+ return -ENOMEM;
	85	+}
	86	+
	87	+void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,
	88	+ u64 length)
	89	+{
	90	+ struct mlx5_core_dev *dev = dm->dev;
	91	+ u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
	92	+ u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
	93	+ u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};
	94	+ u64 start_page_idx;
	95	+ int err;
	96	+
	97	+ addr -= dev->bar_addr;
	98	+ start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT;
	99	+
	100	+ MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);
	101	+ MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);
	102	+ MLX5_SET(dealloc_memic_in, in, memic_size, length);
	103	+
	104	+ err = mlx5_cmd_exec_in(dev, dealloc_memic, in);
	105	+ if (err)
	106	+ return;
	107	+
	108	+ spin_lock(&dm->lock);
	109	+ bitmap_clear(dm->memic_alloc_pages,
	110	+ start_page_idx, num_pages);
	111	+ spin_unlock(&dm->lock);
	112	+}
	113	+
	114	+void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
	115	+ u8 operation)
	116	+{
	117	+ u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};
	118	+ struct mlx5_core_dev *dev = dm->dev;
	119	+
	120	+ MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);
	121	+ MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC);
	122	+ MLX5_SET(modify_memic_in, in, memic_operation_type, operation);
	123	+ MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr);
	124	+
	125	+ mlx5_cmd_exec_in(dev, modify_memic, in);
	126	+}
	127	+
	128	+static int mlx5_cmd_alloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
	129	+ u8 operation, phys_addr_t *op_addr)
	130	+{
	131	+ u32 out[MLX5_ST_SZ_DW(modify_memic_out)] = {};
	132	+ u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {};
	133	+ struct mlx5_core_dev *dev = dm->dev;
	134	+ int err;
	135	+
	136	+ MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC);
	137	+ MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_ALLOC);
	138	+ MLX5_SET(modify_memic_in, in, memic_operation_type, operation);
	139	+ MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr);
	140	+
	141	+ err = mlx5_cmd_exec_inout(dev, modify_memic, in, out);
	142	+ if (err)
	143	+ return err;
	144	+
	145	+ *op_addr = dev->bar_addr +
	146	+ MLX5_GET64(modify_memic_out, out, memic_operation_addr);
	147	+ return 0;
	148	+}
	149	+
	150	+static int add_dm_mmap_entry(struct ib_ucontext *context,
	151	+ struct mlx5_user_mmap_entry *mentry, u8 mmap_flag,
	152	+ size_t size, u64 address)
	153	+{
	154	+ mentry->mmap_flag = mmap_flag;
	155	+ mentry->address = address;
	156	+
	157	+ return rdma_user_mmap_entry_insert_range(
	158	+ context, &mentry->rdma_entry, size,
	159	+ MLX5_IB_MMAP_DEVICE_MEM << 16,
	160	+ (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1);
	161	+}
	162	+
	163	+static void mlx5_ib_dm_memic_free(struct kref *kref)
	164	+{
	165	+ struct mlx5_ib_dm_memic *dm =
	166	+ container_of(kref, struct mlx5_ib_dm_memic, ref);
	167	+ struct mlx5_ib_dev *dev = to_mdev(dm->base.ibdm.device);
	168	+
	169	+ mlx5_cmd_dealloc_memic(&dev->dm, dm->base.dev_addr, dm->base.size);
	170	+ kfree(dm);
	171	+}
	172	+
	173	+static int copy_op_to_user(struct mlx5_ib_dm_op_entry *op_entry,
	174	+ struct uverbs_attr_bundle *attrs)
	175	+{
	176	+ u64 start_offset;
	177	+ u16 page_idx;
	178	+ int err;
	179	+
	180	+ page_idx = op_entry->mentry.rdma_entry.start_pgoff & 0xFFFF;
	181	+ start_offset = op_entry->op_addr & ~PAGE_MASK;
	182	+ err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
	183	+ &page_idx, sizeof(page_idx));
	184	+ if (err)
	185	+ return err;
	186	+
	187	+ return uverbs_copy_to(attrs,
	188	+ MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
	189	+ &start_offset, sizeof(start_offset));
	190	+}
	191	+
	192	+static int map_existing_op(struct mlx5_ib_dm_memic *dm, u8 op,
	193	+ struct uverbs_attr_bundle *attrs)
	194	+{
	195	+ struct mlx5_ib_dm_op_entry *op_entry;
	196	+
	197	+ op_entry = xa_load(&dm->ops, op);
	198	+ if (!op_entry)
	199	+ return -ENOENT;
	200	+
	201	+ return copy_op_to_user(op_entry, attrs);
	202	+}
	203	+
	204	+static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_MAP_OP_ADDR)(
	205	+ struct uverbs_attr_bundle *attrs)
	206	+{
	207	+ struct ib_uobject *uobj = uverbs_attr_get_uobject(
	208	+ attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE);
	209	+ struct mlx5_ib_dev *dev = to_mdev(uobj->context->device);
	210	+ struct ib_dm *ibdm = uobj->object;
	211	+ struct mlx5_ib_dm_memic *dm = to_memic(ibdm);
	212	+ struct mlx5_ib_dm_op_entry *op_entry;
	213	+ int err;
	214	+ u8 op;
	215	+
	216	+ err = uverbs_copy_from(&op, attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP);
	217	+ if (err)
	218	+ return err;
	219	+
	220	+ if (!(MLX5_CAP_DEV_MEM(dev->mdev, memic_operations) & BIT(op)))
	221	+ return -EOPNOTSUPP;
	222	+
	223	+ mutex_lock(&dm->ops_xa_lock);
	224	+ err = map_existing_op(dm, op, attrs);
	225	+ if (!err \|\| err != -ENOENT)
	226	+ goto err_unlock;
	227	+
	228	+ op_entry = kzalloc(sizeof(*op_entry), GFP_KERNEL);
	229	+ if (!op_entry)
	230	+ goto err_unlock;
	231	+
	232	+ err = mlx5_cmd_alloc_memic_op(&dev->dm, dm->base.dev_addr, op,
	233	+ &op_entry->op_addr);
	234	+ if (err) {
	235	+ kfree(op_entry);
	236	+ goto err_unlock;
	237	+ }
	238	+ op_entry->op = op;
	239	+ op_entry->dm = dm;
	240	+
	241	+ err = add_dm_mmap_entry(uobj->context, &op_entry->mentry,
	242	+ MLX5_IB_MMAP_TYPE_MEMIC_OP, dm->base.size,
	243	+ op_entry->op_addr & PAGE_MASK);
	244	+ if (err) {
	245	+ mlx5_cmd_dealloc_memic_op(&dev->dm, dm->base.dev_addr, op);
	246	+ kfree(op_entry);
	247	+ goto err_unlock;
	248	+ }
	249	+ /* From this point, entry will be freed by mmap_free */
	250	+ kref_get(&dm->ref);
	251	+
	252	+ err = copy_op_to_user(op_entry, attrs);
	253	+ if (err)
	254	+ goto err_remove;
	255	+
	256	+ err = xa_insert(&dm->ops, op, op_entry, GFP_KERNEL);
	257	+ if (err)
	258	+ goto err_remove;
	259	+ mutex_unlock(&dm->ops_xa_lock);
	260	+
	261	+ return 0;
	262	+
	263	+err_remove:
	264	+ rdma_user_mmap_entry_remove(&op_entry->mentry.rdma_entry);
	265	+err_unlock:
	266	+ mutex_unlock(&dm->ops_xa_lock);
	267	+
	268	+ return err;
	269	+}
	270	+
	271	+static struct ib_dm handle_alloc_dm_memic(struct ib_ucontext ctx,
	272	+ struct ib_dm_alloc_attr *attr,
	273	+ struct uverbs_attr_bundle *attrs)
	274	+{
	275	+ struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
	276	+ struct mlx5_ib_dm_memic *dm;
	277	+ u64 start_offset;
	278	+ u16 page_idx;
	279	+ int err;
	280	+ u64 address;
	281	+
	282	+ if (!MLX5_CAP_DEV_MEM(dm_db->dev, memic))
	283	+ return ERR_PTR(-EOPNOTSUPP);
	284	+
	285	+ dm = kzalloc(sizeof(*dm), GFP_KERNEL);
	286	+ if (!dm)
	287	+ return ERR_PTR(-ENOMEM);
	288	+
	289	+ dm->base.type = MLX5_IB_UAPI_DM_TYPE_MEMIC;
	290	+ dm->base.size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
	291	+ dm->base.ibdm.device = ctx->device;
	292	+
	293	+ kref_init(&dm->ref);
	294	+ xa_init(&dm->ops);
	295	+ mutex_init(&dm->ops_xa_lock);
	296	+ dm->req_length = attr->length;
	297	+
	298	+ err = mlx5_cmd_alloc_memic(dm_db, &dm->base.dev_addr,
	299	+ dm->base.size, attr->alignment);
	300	+ if (err) {
	301	+ kfree(dm);
	302	+ return ERR_PTR(err);
	303	+ }
	304	+
	305	+ address = dm->base.dev_addr & PAGE_MASK;
	306	+ err = add_dm_mmap_entry(ctx, &dm->mentry, MLX5_IB_MMAP_TYPE_MEMIC,
	307	+ dm->base.size, address);
	308	+ if (err) {
	309	+ mlx5_cmd_dealloc_memic(dm_db, dm->base.dev_addr, dm->base.size);
	310	+ kfree(dm);
	311	+ return ERR_PTR(err);
	312	+ }
	313	+
	314	+ page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF;
	315	+ err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
	316	+ &page_idx, sizeof(page_idx));
	317	+ if (err)
	318	+ goto err_copy;
	319	+
	320	+ start_offset = dm->base.dev_addr & ~PAGE_MASK;
	321	+ err = uverbs_copy_to(attrs,
	322	+ MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
	323	+ &start_offset, sizeof(start_offset));
	324	+ if (err)
	325	+ goto err_copy;
	326	+
	327	+ return &dm->base.ibdm;
	328	+
	329	+err_copy:
	330	+ rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
	331	+ return ERR_PTR(err);
	332	+}
	333	+
	334	+static struct ib_dm handle_alloc_dm_sw_icm(struct ib_ucontext ctx,
	335	+ struct ib_dm_alloc_attr *attr,
	336	+ struct uverbs_attr_bundle *attrs,
	337	+ int type)
	338	+{
	339	+ struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev;
	340	+ struct mlx5_ib_dm_icm *dm;
	341	+ u64 act_size;
	342	+ int err;
	343	+
	344	+ dm = kzalloc(sizeof(*dm), GFP_KERNEL);
	345	+ if (!dm)
	346	+ return ERR_PTR(-ENOMEM);
	347	+
	348	+ dm->base.type = type;
	349	+ dm->base.ibdm.device = ctx->device;
	350	+
	351	+ if (!capable(CAP_SYS_RAWIO) \|\| !capable(CAP_NET_RAW)) {
	352	+ err = -EPERM;
	353	+ goto free;
	354	+ }
	355	+
	356	+ if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner) \|\|
	357	+ MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner) \|\|
	358	+ MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) \|\|
	359	+ MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2))) {
	360	+ err = -EOPNOTSUPP;
	361	+ goto free;
	362	+ }
	363	+
	364	+ /* Allocation size must a multiple of the basic block size
	365	+ * and a power of 2.
	366	+ */
	367	+ act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev));
	368	+ act_size = roundup_pow_of_two(act_size);
	369	+
	370	+ dm->base.size = act_size;
	371	+ err = mlx5_dm_sw_icm_alloc(dev, type, act_size, attr->alignment,
	372	+ to_mucontext(ctx)->devx_uid,
	373	+ &dm->base.dev_addr, &dm->obj_id);
	374	+ if (err)
	375	+ goto free;
	376	+
	377	+ err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
	378	+ &dm->base.dev_addr, sizeof(dm->base.dev_addr));
	379	+ if (err) {
	380	+ mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size,
	381	+ to_mucontext(ctx)->devx_uid,
	382	+ dm->base.dev_addr, dm->obj_id);
	383	+ goto free;
	384	+ }
	385	+ return &dm->base.ibdm;
	386	+free:
	387	+ kfree(dm);
	388	+ return ERR_PTR(err);
	389	+}
	390	+
	391	+struct ib_dm mlx5_ib_alloc_dm(struct ib_device ibdev,
	392	+ struct ib_ucontext *context,
	393	+ struct ib_dm_alloc_attr *attr,
	394	+ struct uverbs_attr_bundle *attrs)
	395	+{
	396	+ enum mlx5_ib_uapi_dm_type type;
	397	+ int err;
	398	+
	399	+ err = uverbs_get_const_default(&type, attrs,
	400	+ MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
	401	+ MLX5_IB_UAPI_DM_TYPE_MEMIC);
	402	+ if (err)
	403	+ return ERR_PTR(err);
	404	+
	405	+ mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",
	406	+ type, attr->length, attr->alignment);
	407	+
	408	+ switch (type) {
	409	+ case MLX5_IB_UAPI_DM_TYPE_MEMIC:
	410	+ return handle_alloc_dm_memic(context, attr, attrs);
	411	+ case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
	412	+ return handle_alloc_dm_sw_icm(context, attr, attrs,
	413	+ MLX5_SW_ICM_TYPE_STEERING);
	414	+ case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
	415	+ return handle_alloc_dm_sw_icm(context, attr, attrs,
	416	+ MLX5_SW_ICM_TYPE_HEADER_MODIFY);
	417	+ default:
	418	+ return ERR_PTR(-EOPNOTSUPP);
	419	+ }
	420	+}
	421	+
	422	+static void dm_memic_remove_ops(struct mlx5_ib_dm_memic *dm)
	423	+{
	424	+ struct mlx5_ib_dm_op_entry *entry;
	425	+ unsigned long idx;
	426	+
	427	+ mutex_lock(&dm->ops_xa_lock);
	428	+ xa_for_each(&dm->ops, idx, entry) {
	429	+ xa_erase(&dm->ops, idx);
	430	+ rdma_user_mmap_entry_remove(&entry->mentry.rdma_entry);
	431	+ }
	432	+ mutex_unlock(&dm->ops_xa_lock);
	433	+}
	434	+
	435	+static void mlx5_dm_memic_dealloc(struct mlx5_ib_dm_memic *dm)
	436	+{
	437	+ dm_memic_remove_ops(dm);
	438	+ rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
	439	+}
	440	+
	441	+static int mlx5_dm_icm_dealloc(struct mlx5_ib_ucontext *ctx,
	442	+ struct mlx5_ib_dm_icm *dm)
	443	+{
	444	+ enum mlx5_sw_icm_type type =
	445	+ dm->base.type == MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM ?
	446	+ MLX5_SW_ICM_TYPE_STEERING :
	447	+ MLX5_SW_ICM_TYPE_HEADER_MODIFY;
	448	+ struct mlx5_core_dev *dev = to_mdev(dm->base.ibdm.device)->mdev;
	449	+ int err;
	450	+
	451	+ err = mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size, ctx->devx_uid,
	452	+ dm->base.dev_addr, dm->obj_id);
	453	+ if (!err)
	454	+ kfree(dm);
	455	+ return 0;
	456	+}
	457	+
	458	+static int mlx5_ib_dealloc_dm(struct ib_dm *ibdm,
	459	+ struct uverbs_attr_bundle *attrs)
	460	+{
	461	+ struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(
	462	+ &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
	463	+ struct mlx5_ib_dm *dm = to_mdm(ibdm);
	464	+
	465	+ switch (dm->type) {
	466	+ case MLX5_IB_UAPI_DM_TYPE_MEMIC:
	467	+ mlx5_dm_memic_dealloc(to_memic(ibdm));
	468	+ return 0;
	469	+ case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
	470	+ case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
	471	+ return mlx5_dm_icm_dealloc(ctx, to_icm(ibdm));
	472	+ default:
	473	+ return -EOPNOTSUPP;
	474	+ }
	475	+}
	476	+
	477	+static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_QUERY)(
	478	+ struct uverbs_attr_bundle *attrs)
	479	+{
	480	+ struct ib_dm *ibdm =
	481	+ uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE);
	482	+ struct mlx5_ib_dm *dm = to_mdm(ibdm);
	483	+ struct mlx5_ib_dm_memic *memic;
	484	+ u64 start_offset;
	485	+ u16 page_idx;
	486	+ int err;
	487	+
	488	+ if (dm->type != MLX5_IB_UAPI_DM_TYPE_MEMIC)
	489	+ return -EOPNOTSUPP;
	490	+
	491	+ memic = to_memic(ibdm);
	492	+ page_idx = memic->mentry.rdma_entry.start_pgoff & 0xFFFF;
	493	+ err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
	494	+ &page_idx, sizeof(page_idx));
	495	+ if (err)
	496	+ return err;
	497	+
	498	+ start_offset = memic->base.dev_addr & ~PAGE_MASK;
	499	+ err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
	500	+ &start_offset, sizeof(start_offset));
	501	+ if (err)
	502	+ return err;
	503	+
	504	+ return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
	505	+ &memic->req_length,
	506	+ sizeof(memic->req_length));
	507	+}
	508	+
	509	+void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev,
	510	+ struct mlx5_user_mmap_entry *mentry)
	511	+{
	512	+ struct mlx5_ib_dm_op_entry *op_entry;
	513	+ struct mlx5_ib_dm_memic *mdm;
	514	+
	515	+ switch (mentry->mmap_flag) {
	516	+ case MLX5_IB_MMAP_TYPE_MEMIC:
	517	+ mdm = container_of(mentry, struct mlx5_ib_dm_memic, mentry);
	518	+ kref_put(&mdm->ref, mlx5_ib_dm_memic_free);
	519	+ break;
	520	+ case MLX5_IB_MMAP_TYPE_MEMIC_OP:
	521	+ op_entry = container_of(mentry, struct mlx5_ib_dm_op_entry,
	522	+ mentry);
	523	+ mdm = op_entry->dm;
	524	+ mlx5_cmd_dealloc_memic_op(&dev->dm, mdm->base.dev_addr,
	525	+ op_entry->op);
	526	+ kfree(op_entry);
	527	+ kref_put(&mdm->ref, mlx5_ib_dm_memic_free);
	528	+ break;
	529	+ default:
	530	+ WARN_ON(true);
	531	+ }
	532	+}
	533	+
	534	+DECLARE_UVERBS_NAMED_METHOD(
	535	+ MLX5_IB_METHOD_DM_QUERY,
	536	+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE, UVERBS_OBJECT_DM,
	537	+ UVERBS_ACCESS_READ, UA_MANDATORY),
	538	+ UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
	539	+ UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
	540	+ UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
	541	+ UVERBS_ATTR_TYPE(u16), UA_MANDATORY),
	542	+ UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
	543	+ UVERBS_ATTR_TYPE(u64), UA_MANDATORY));
	544	+
	545	+ADD_UVERBS_ATTRIBUTES_SIMPLE(
	546	+ mlx5_ib_dm, UVERBS_OBJECT_DM, UVERBS_METHOD_DM_ALLOC,
	547	+ UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
	548	+ UVERBS_ATTR_TYPE(u64), UA_MANDATORY),
	549	+ UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
	550	+ UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),
	551	+ UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
	552	+ enum mlx5_ib_uapi_dm_type, UA_OPTIONAL));
	553	+
	554	+DECLARE_UVERBS_NAMED_METHOD(
	555	+ MLX5_IB_METHOD_DM_MAP_OP_ADDR,
	556	+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE,
	557	+ UVERBS_OBJECT_DM,
	558	+ UVERBS_ACCESS_READ,
	559	+ UA_MANDATORY),
	560	+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP,
	561	+ UVERBS_ATTR_TYPE(u8),
	562	+ UA_MANDATORY),
	563	+ UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
	564	+ UVERBS_ATTR_TYPE(u64),
	565	+ UA_MANDATORY),
	566	+ UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
	567	+ UVERBS_ATTR_TYPE(u16),
	568	+ UA_OPTIONAL));
	569	+
	570	+DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DM,
	571	+ &UVERBS_METHOD(MLX5_IB_METHOD_DM_MAP_OP_ADDR),
	572	+ &UVERBS_METHOD(MLX5_IB_METHOD_DM_QUERY));
	573	+
	574	+const struct uapi_definition mlx5_ib_dm_defs[] = {
	575	+ UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
	576	+ UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DM),
	577	+ {},
	578	+};
	579	+
	580	+const struct ib_device_ops mlx5_ib_dev_dm_ops = {
	581	+ .alloc_dm = mlx5_ib_alloc_dm,
	582	+ .dealloc_dm = mlx5_ib_dealloc_dm,
	583	+ .reg_dm_mr = mlx5_ib_reg_dm_mr,
	584	+};

drivers/infiniband/hw/mlx5/dm.h+68−0

@@ -0,0 +1,68 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	+/*
	3	+ * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.
	4	+ */
	5	+
	6	+#ifndef _MLX5_IB_DM_H
	7	+#define _MLX5_IB_DM_H
	8	+
	9	+#include "mlx5_ib.h"
	10	+
	11	+extern const struct ib_device_ops mlx5_ib_dev_dm_ops;
	12	+extern const struct uapi_definition mlx5_ib_dm_defs[];
	13	+
	14	+struct mlx5_ib_dm {
	15	+ struct ib_dm ibdm;
	16	+ u32 type;
	17	+ phys_addr_t dev_addr;
	18	+ size_t size;
	19	+};
	20	+
	21	+struct mlx5_ib_dm_op_entry {
	22	+ struct mlx5_user_mmap_entry mentry;
	23	+ phys_addr_t op_addr;
	24	+ struct mlx5_ib_dm_memic *dm;
	25	+ u8 op;
	26	+};
	27	+
	28	+struct mlx5_ib_dm_memic {
	29	+ struct mlx5_ib_dm base;
	30	+ struct mlx5_user_mmap_entry mentry;
	31	+ struct xarray ops;
	32	+ struct mutex ops_xa_lock;
	33	+ struct kref ref;
	34	+ size_t req_length;
	35	+};
	36	+
	37	+struct mlx5_ib_dm_icm {
	38	+ struct mlx5_ib_dm base;
	39	+ u32 obj_id;
	40	+};
	41	+
	42	+static inline struct mlx5_ib_dm to_mdm(struct ib_dm ibdm)
	43	+{
	44	+ return container_of(ibdm, struct mlx5_ib_dm, ibdm);
	45	+}
	46	+
	47	+static inline struct mlx5_ib_dm_memic to_memic(struct ib_dm ibdm)
	48	+{
	49	+ return container_of(ibdm, struct mlx5_ib_dm_memic, base.ibdm);
	50	+}
	51	+
	52	+static inline struct mlx5_ib_dm_icm to_icm(struct ib_dm ibdm)
	53	+{
	54	+ return container_of(ibdm, struct mlx5_ib_dm_icm, base.ibdm);
	55	+}
	56	+
	57	+struct ib_dm mlx5_ib_alloc_dm(struct ib_device ibdev,
	58	+ struct ib_ucontext *context,
	59	+ struct ib_dm_alloc_attr *attr,
	60	+ struct uverbs_attr_bundle *attrs);
	61	+void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev,
	62	+ struct mlx5_user_mmap_entry *mentry);
	63	+void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,
	64	+ u64 length);
	65	+void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,
	66	+ u8 operation);
	67	+
	68	+#endif /* _MLX5_IB_DM_H */

drivers/infiniband/hw/mlx5/main.c+4−239

@@ -34,6 +34,7 @@ · 33 unmodified lines
34	34	#include "ib_rep.h"
35	35	#include "cmd.h"
36	36	#include "devx.h"
	37	+#include "dm.h"
37	38	#include "fs.h"
38	39	#include "srq.h"
39	40	#include "qp.h"
@@ -2088,14 +2089,11 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry) · 2048 unmodified lines
2088	2089	struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
2089	2090	struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
2090	2091	struct mlx5_var_table *var_table = &dev->var_table;
2091		− struct mlx5_ib_dm *mdm;
2092	2092
2093	2093	switch (mentry->mmap_flag) {
2094	2094	case MLX5_IB_MMAP_TYPE_MEMIC:
2095		− mdm = container_of(mentry, struct mlx5_ib_dm, mentry);
2096		− mlx5_cmd_dealloc_memic(&dev->dm, mdm->dev_addr,
2097		− mdm->size);
2098		− kfree(mdm);
	2095	+ case MLX5_IB_MMAP_TYPE_MEMIC_OP:
	2096	+ mlx5_ib_dm_mmap_free(dev, mentry);
2099	2097	break;
2100	2098	case MLX5_IB_MMAP_TYPE_VAR:
2101	2099	mutex_lock(&var_table->bitmap_lock);
@@ -2220,19 +2218,6 @@ free_bfreg: · 118 unmodified lines
2220	2218	return err;
2221	2219	}
2222	2220
2223		−static int add_dm_mmap_entry(struct ib_ucontext *context,
2224		− struct mlx5_ib_dm *mdm,
2225		− u64 address)
2226		−{
2227		− mdm->mentry.mmap_flag = MLX5_IB_MMAP_TYPE_MEMIC;
2228		− mdm->mentry.address = address;
2229		− return rdma_user_mmap_entry_insert_range(
2230		− context, &mdm->mentry.rdma_entry,
2231		− mdm->size,
2232		− MLX5_IB_MMAP_DEVICE_MEM << 16,
2233		− (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1);
2234		−}
2235		−
2236	2221	static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
2237	2222	{
2238	2223	unsigned long idx;
@@ -2334,206 +2319,6 @@ static int mlx5_ib_mmap(struct ib_ucontext ibcontext, struct vm_area_struct vm · 95 unmodified lines
2334	2319	return 0;
2335	2320	}
2336	2321
2337		−static inline int check_dm_type_support(struct mlx5_ib_dev *dev,
2338		− u32 type)
2339		−{
2340		− switch (type) {
2341		− case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2342		− if (!MLX5_CAP_DEV_MEM(dev->mdev, memic))
2343		− return -EOPNOTSUPP;
2344		− break;
2345		− case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2346		− case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2347		− if (!capable(CAP_SYS_RAWIO) \|\|
2348		− !capable(CAP_NET_RAW))
2349		− return -EPERM;
2350		−
2351		− if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) \|\|
2352		− MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner) \|\|
2353		− MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2) \|\|
2354		− MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner_v2)))
2355		− return -EOPNOTSUPP;
2356		− break;
2357		− }
2358		−
2359		− return 0;
2360		−}
2361		−
2362		−static int handle_alloc_dm_memic(struct ib_ucontext *ctx,
2363		− struct mlx5_ib_dm *dm,
2364		− struct ib_dm_alloc_attr *attr,
2365		− struct uverbs_attr_bundle *attrs)
2366		−{
2367		− struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
2368		− u64 start_offset;
2369		− u16 page_idx;
2370		− int err;
2371		− u64 address;
2372		−
2373		− dm->size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
2374		−
2375		− err = mlx5_cmd_alloc_memic(dm_db, &dm->dev_addr,
2376		− dm->size, attr->alignment);
2377		− if (err)
2378		− return err;
2379		−
2380		− address = dm->dev_addr & PAGE_MASK;
2381		− err = add_dm_mmap_entry(ctx, dm, address);
2382		− if (err)
2383		− goto err_dealloc;
2384		−
2385		− page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF;
2386		− err = uverbs_copy_to(attrs,
2387		− MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
2388		− &page_idx,
2389		− sizeof(page_idx));
2390		− if (err)
2391		− goto err_copy;
2392		−
2393		− start_offset = dm->dev_addr & ~PAGE_MASK;
2394		− err = uverbs_copy_to(attrs,
2395		− MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
2396		− &start_offset, sizeof(start_offset));
2397		− if (err)
2398		− goto err_copy;
2399		−
2400		− return 0;
2401		−
2402		−err_copy:
2403		− rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
2404		−err_dealloc:
2405		− mlx5_cmd_dealloc_memic(dm_db, dm->dev_addr, dm->size);
2406		−
2407		− return err;
2408		−}
2409		−
2410		−static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,
2411		− struct mlx5_ib_dm *dm,
2412		− struct ib_dm_alloc_attr *attr,
2413		− struct uverbs_attr_bundle *attrs,
2414		− int type)
2415		−{
2416		− struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev;
2417		− u64 act_size;
2418		− int err;
2419		−
2420		− /* Allocation size must a multiple of the basic block size
2421		− * and a power of 2.
2422		− */
2423		− act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev));
2424		− act_size = roundup_pow_of_two(act_size);
2425		−
2426		− dm->size = act_size;
2427		− err = mlx5_dm_sw_icm_alloc(dev, type, act_size, attr->alignment,
2428		− to_mucontext(ctx)->devx_uid, &dm->dev_addr,
2429		− &dm->icm_dm.obj_id);
2430		− if (err)
2431		− return err;
2432		−
2433		− err = uverbs_copy_to(attrs,
2434		− MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
2435		− &dm->dev_addr, sizeof(dm->dev_addr));
2436		− if (err)
2437		− mlx5_dm_sw_icm_dealloc(dev, type, dm->size,
2438		− to_mucontext(ctx)->devx_uid, dm->dev_addr,
2439		− dm->icm_dm.obj_id);
2440		−
2441		− return err;
2442		−}
2443		−
2444		−struct ib_dm mlx5_ib_alloc_dm(struct ib_device ibdev,
2445		− struct ib_ucontext *context,
2446		− struct ib_dm_alloc_attr *attr,
2447		− struct uverbs_attr_bundle *attrs)
2448		−{
2449		− struct mlx5_ib_dm *dm;
2450		− enum mlx5_ib_uapi_dm_type type;
2451		− int err;
2452		−
2453		− err = uverbs_get_const_default(&type, attrs,
2454		− MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
2455		− MLX5_IB_UAPI_DM_TYPE_MEMIC);
2456		− if (err)
2457		− return ERR_PTR(err);
2458		−
2459		− mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",
2460		− type, attr->length, attr->alignment);
2461		−
2462		− err = check_dm_type_support(to_mdev(ibdev), type);
2463		− if (err)
2464		− return ERR_PTR(err);
2465		−
2466		− dm = kzalloc(sizeof(*dm), GFP_KERNEL);
2467		− if (!dm)
2468		− return ERR_PTR(-ENOMEM);
2469		−
2470		− dm->type = type;
2471		−
2472		− switch (type) {
2473		− case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2474		− err = handle_alloc_dm_memic(context, dm,
2475		− attr,
2476		− attrs);
2477		− break;
2478		− case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2479		− err = handle_alloc_dm_sw_icm(context, dm,
2480		− attr, attrs,
2481		− MLX5_SW_ICM_TYPE_STEERING);
2482		− break;
2483		− case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2484		− err = handle_alloc_dm_sw_icm(context, dm,
2485		− attr, attrs,
2486		− MLX5_SW_ICM_TYPE_HEADER_MODIFY);
2487		− break;
2488		− default:
2489		− err = -EOPNOTSUPP;
2490		− }
2491		−
2492		− if (err)
2493		− goto err_free;
2494		−
2495		− return &dm->ibdm;
2496		−
2497		−err_free:
2498		− kfree(dm);
2499		− return ERR_PTR(err);
2500		−}
2501		−
2502		−int mlx5_ib_dealloc_dm(struct ib_dm ibdm, struct uverbs_attr_bundle attrs)
2503		−{
2504		− struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(
2505		− &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2506		− struct mlx5_core_dev *dev = to_mdev(ibdm->device)->mdev;
2507		− struct mlx5_ib_dm *dm = to_mdm(ibdm);
2508		− int ret;
2509		−
2510		− switch (dm->type) {
2511		− case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2512		− rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);
2513		− return 0;
2514		− case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2515		− ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_STEERING,
2516		− dm->size, ctx->devx_uid, dm->dev_addr,
2517		− dm->icm_dm.obj_id);
2518		− if (ret)
2519		− return ret;
2520		− break;
2521		− case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2522		− ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_HEADER_MODIFY,
2523		− dm->size, ctx->devx_uid, dm->dev_addr,
2524		− dm->icm_dm.obj_id);
2525		− if (ret)
2526		− return ret;
2527		− break;
2528		− default:
2529		− return -EOPNOTSUPP;
2530		− }
2531		−
2532		− kfree(dm);
2533		−
2534		− return 0;
2535		−}
2536		−
2537	2322	static int mlx5_ib_alloc_pd(struct ib_pd ibpd, struct ib_udata udata)
2538	2323	{
2539	2324	struct mlx5_ib_pd *pd = to_mpd(ibpd);
@@ -3817,20 +3602,6 @@ DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR, · 1277 unmodified lines
3817	3602	&UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
3818	3603	&UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
3819	3604
3820		−ADD_UVERBS_ATTRIBUTES_SIMPLE(
3821		− mlx5_ib_dm,
3822		− UVERBS_OBJECT_DM,
3823		− UVERBS_METHOD_DM_ALLOC,
3824		− UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
3825		− UVERBS_ATTR_TYPE(u64),
3826		− UA_MANDATORY),
3827		− UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
3828		− UVERBS_ATTR_TYPE(u16),
3829		− UA_OPTIONAL),
3830		− UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
3831		− enum mlx5_ib_uapi_dm_type,
3832		− UA_OPTIONAL));
3833		−
3834	3605	ADD_UVERBS_ATTRIBUTES_SIMPLE(
3835	3606	mlx5_ib_flow_action,
3836	3607	UVERBS_OBJECT_FLOW_ACTION,
@@ -3853,10 +3624,10 @@ static const struct uapi_definition mlx5_ib_defs[] = { · 16 unmodified lines
3853	3624	UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
3854	3625	UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
3855	3626	UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
	3627	+ UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
3856	3628
3857	3629	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
3858	3630	&mlx5_ib_flow_action),
3859		− UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
3860	3631	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
3861	3632	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
3862	3633	UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
@@ -4032,12 +3803,6 @@ static const struct ib_device_ops mlx5_ib_dev_xrc_ops = { · 169 unmodified lines
4032	3803	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
4033	3804	};
4034	3805
4035		−static const struct ib_device_ops mlx5_ib_dev_dm_ops = {
4036		− .alloc_dm = mlx5_ib_alloc_dm,
4037		− .dealloc_dm = mlx5_ib_dealloc_dm,
4038		− .reg_dm_mr = mlx5_ib_reg_dm_mr,
4039		−};
4040		−
4041	3806	static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
4042	3807	{
4043	3808	struct mlx5_core_dev *mdev = dev->mdev;

drivers/infiniband/hw/mlx5/mlx5_ib.h+1−24

@@ -166,6 +166,7 @@ enum mlx5_ib_mmap_type { · 165 unmodified lines
166	166	MLX5_IB_MMAP_TYPE_VAR = 2,
167	167	MLX5_IB_MMAP_TYPE_UAR_WC = 3,
168	168	MLX5_IB_MMAP_TYPE_UAR_NC = 4,
	169	+ MLX5_IB_MMAP_TYPE_MEMIC_OP = 5,
169	170	};
170	171
171	172	struct mlx5_bfreg_info {
@@ -618,20 +619,6 @@ struct mlx5_user_mmap_entry { · 446 unmodified lines
618	619	u32 page_idx;
619	620	};
620	621
621		−struct mlx5_ib_dm {
622		− struct ib_dm ibdm;
623		− phys_addr_t dev_addr;
624		− u32 type;
625		− size_t size;
626		− union {
627		− struct {
628		− u32 obj_id;
629		− } icm_dm;
630		− /* other dm types specific params should be added here */
631		− };
632		− struct mlx5_user_mmap_entry mentry;
633		−};
634		−
635	622	#define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ \| MLX5_IB_MTT_WRITE)
636	623
637	624	#define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE \|\
@@ -1188,11 +1175,6 @@ static inline struct mlx5_ib_srq to_mibsrq(struct mlx5_core_srq msrq) · 550 unmodified lines
1188	1175	return container_of(msrq, struct mlx5_ib_srq, msrq);
1189	1176	}
1190	1177
1191		−static inline struct mlx5_ib_dm to_mdm(struct ib_dm ibdm)
1192		−{
1193		− return container_of(ibdm, struct mlx5_ib_dm, ibdm);
1194		−}
1195		−
1196	1178	static inline struct mlx5_ib_mr to_mmr(struct ib_mr ibmr)
1197	1179	{
1198	1180	return container_of(ibmr, struct mlx5_ib_mr, ibmr);
@@ -1347,11 +1329,6 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, · 148 unmodified lines
1347	1329	struct ib_rwq_ind_table_init_attr *init_attr,
1348	1330	struct ib_udata *udata);
1349	1331	int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
1350		−struct ib_dm mlx5_ib_alloc_dm(struct ib_device ibdev,
1351		− struct ib_ucontext *context,
1352		− struct ib_dm_alloc_attr *attr,
1353		− struct uverbs_attr_bundle *attrs);
1354		−int mlx5_ib_dealloc_dm(struct ib_dm ibdm, struct uverbs_attr_bundle attrs);
1355	1332	struct ib_mr mlx5_ib_reg_dm_mr(struct ib_pd pd, struct ib_dm *dm,
1356	1333	struct ib_dm_mr_attr *attr,
1357	1334	struct uverbs_attr_bundle *attrs);

drivers/infiniband/hw/mlx5/mr.c+1−0

@@ -42,6 +42,7 @@ · 41 unmodified lines
42	42	#include <rdma/ib_umem.h>
43	43	#include <rdma/ib_umem_odp.h>
44	44	#include <rdma/ib_verbs.h>
	45	+#include "dm.h"
45	46	#include "mlx5_ib.h"
46	47
47	48	/*

include/linux/mlx5/mlx5_ifc.h+41−1

@@ -133,6 +133,7 @@ enum { · 132 unmodified lines
133	133	MLX5_CMD_OP_PAGE_FAULT_RESUME = 0x204,
134	134	MLX5_CMD_OP_ALLOC_MEMIC = 0x205,
135	135	MLX5_CMD_OP_DEALLOC_MEMIC = 0x206,
	136	+ MLX5_CMD_OP_MODIFY_MEMIC = 0x207,
136	137	MLX5_CMD_OP_CREATE_EQ = 0x301,
137	138	MLX5_CMD_OP_DESTROY_EQ = 0x302,
138	139	MLX5_CMD_OP_QUERY_EQ = 0x303,
@@ -1017,7 +1018,11 @@ struct mlx5_ifc_device_mem_cap_bits { · 878 unmodified lines
1017	1018
1018	1019	u8 header_modify_sw_icm_start_address[0x40];
1019	1020
1020		− u8 reserved_at_180[0x680];
	1021	+ u8 reserved_at_180[0x80];
	1022	+
	1023	+ u8 memic_operations[0x20];
	1024	+
	1025	+ u8 reserved_at_220[0x5e0];
1021	1026	};
1022	1027
1023	1028	struct mlx5_ifc_device_event_cap_bits {
@@ -10417,6 +10422,41 @@ struct mlx5_ifc_destroy_vport_lag_in_bits { · 9393 unmodified lines
10417	10422	u8 reserved_at_40[0x40];
10418	10423	};
10419	10424
	10425	+enum {
	10426	+ MLX5_MODIFY_MEMIC_OP_MOD_ALLOC,
	10427	+ MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC,
	10428	+};
	10429	+
	10430	+struct mlx5_ifc_modify_memic_in_bits {
	10431	+ u8 opcode[0x10];
	10432	+ u8 uid[0x10];
	10433	+
	10434	+ u8 reserved_at_20[0x10];
	10435	+ u8 op_mod[0x10];
	10436	+
	10437	+ u8 reserved_at_40[0x20];
	10438	+
	10439	+ u8 reserved_at_60[0x18];
	10440	+ u8 memic_operation_type[0x8];
	10441	+
	10442	+ u8 memic_start_addr[0x40];
	10443	+
	10444	+ u8 reserved_at_c0[0x140];
	10445	+};
	10446	+
	10447	+struct mlx5_ifc_modify_memic_out_bits {
	10448	+ u8 status[0x8];
	10449	+ u8 reserved_at_8[0x18];
	10450	+
	10451	+ u8 syndrome[0x20];
	10452	+
	10453	+ u8 reserved_at_40[0x40];
	10454	+
	10455	+ u8 memic_operation_addr[0x40];
	10456	+
	10457	+ u8 reserved_at_c0[0x140];
	10458	+};
	10459	+
10420	10460	struct mlx5_ifc_alloc_memic_in_bits {
10421	10461	u8 opcode[0x10];
10422	10462	u8 reserved_at_10[0x10];

include/rdma/uverbs_named_ioctl.h+1−1

@@ -20,7 +20,7 @@ · 19 unmodified lines
20	20
21	21	/* These are static so they do not need to be qualified */
22	22	#define UVERBS_METHOD_ATTRS(method_id) _method_attrs_##method_id
23		−#define UVERBS_OBJECT_METHODS(object_id) _object_methods_##object_id
	23	+#define UVERBS_OBJECT_METHODS(object_id) _UVERBS_NAME(_object_methods_##object_id, __LINE__)
24	24
25	25	#define DECLARE_UVERBS_NAMED_METHOD(_method_id, ...) \
26	26	static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \

include/uapi/rdma/mlx5_user_ioctl_cmds.h+19−0

@@ -41,6 +41,25 @@ enum mlx5_ib_create_flow_action_attrs { · 40 unmodified lines
41	41	MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS = (1U << UVERBS_ID_NS_SHIFT),
42	42	};
43	43
	44	+enum mlx5_ib_dm_methods {
	45	+ MLX5_IB_METHOD_DM_MAP_OP_ADDR = (1U << UVERBS_ID_NS_SHIFT),
	46	+ MLX5_IB_METHOD_DM_QUERY,
	47	+};
	48	+
	49	+enum mlx5_ib_dm_map_op_addr_attrs {
	50	+ MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
	51	+ MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP,
	52	+ MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET,
	53	+ MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX,
	54	+};
	55	+
	56	+enum mlx5_ib_query_dm_attrs {
	57	+ MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
	58	+ MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET,
	59	+ MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX,
	60	+ MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH,
	61	+};
	62	+
44	63	enum mlx5_ib_alloc_dm_attrs {
45	64	MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET = (1U << UVERBS_ID_NS_SHIFT),
46	65	MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,