History

RDMA/core: Introduce generic buffer descriptor infrastructure for umem

Introduce a per-attribute UVERBS_ATTR_UMEM model so each uverbs command's umem set is explicit in its UAPI definition. Add driver-facing wrapper helpers that pin a umem on demand from an attribute or a VA addr; the driver owns the returned umem and releases it from its destroy/error paths.

Jiri Pirko committed 5 months ago

Commit 3cfdff41 parent 28fb7015 files · +346 −0Browse files

Unified · Split

5 files
  1. drivers/infiniband/core/ib_core_uverbs.c +25 −0
  2. drivers/infiniband/core/umem.c +228 −0
  3. include/rdma/ib_umem.h +36 −0
  4. include/rdma/uverbs_ioctl.h +30 −0
  5. include/uapi/rdma/ib_user_ioctl_verbs.h +27 −0
drivers/infiniband/core/ib_core_uverbs.c+25−0
@@ -708,6 +708,31 @@ int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, · 707 unmodified lines
708708 }
709709 EXPORT_SYMBOL(uverbs_get_flags32);
710710
711+/**
712+ * uverbs_get_buffer_desc - Read a buffer descriptor from a uverbs attr.
713+ * @attrs_bundle: uverbs attribute bundle.
714+ * @attr_id: id of an UVERBS_ATTR_UMEM-typed attribute.
715+ * @desc: descriptor to fill.
716+ *
717+ * Return: 0 on success, -ENOENT if @attr_id is not set, -EINVAL on a
718+ * malformed descriptor, or any other negative errno propagated from
719+ * uverbs_copy_from() (notably -EFAULT on copy_from_user() failure).
720+ */
721+int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle,
722+ u16 attr_id, struct ib_uverbs_buffer_desc *desc)
723+{
724+ int ret;
725+
726+ ret = uverbs_copy_from(desc, attrs_bundle, attr_id);
727+ if (ret)
728+ return ret;
729+ if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK)
730+ return -EINVAL;
731+ desc->optional_flags &= IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK;
732+ return 0;
733+}
734+EXPORT_SYMBOL(uverbs_get_buffer_desc);
735+
711736 /* Once called an abort will call through to the type's destroy_hw() */
712737 void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
713738 u16 idx)
drivers/infiniband/core/umem.c+228−0
@@ -269,6 +269,175 @@ umem_kfree: · 268 unmodified lines
269269 return ret ? ERR_PTR(ret) : umem;
270270 }
271271
272+/**
273+ * ib_umem_get_desc - Pin a umem from a buffer descriptor.
274+ * @device: IB device.
275+ * @desc: buffer descriptor (VA or DMABUF).
276+ * @access: IB access flags.
277+ *
278+ * Return: caller-owned umem on success, ERR_PTR(...) on error.
279+ */
280+struct ib_umem *ib_umem_get_desc(struct ib_device *device,
281+ const struct ib_uverbs_buffer_desc *desc,
282+ int access)
283+{
284+ struct ib_umem_dmabuf *umem_dmabuf;
285+
286+ if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK)
287+ return ERR_PTR(-EINVAL);
288+
289+ if (overflows_type(desc->addr, unsigned long) ||
290+ overflows_type(desc->length, size_t))
291+ return ERR_PTR(-EOVERFLOW);
292+
293+ switch (desc->type) {
294+ case IB_UVERBS_BUFFER_TYPE_DMABUF:
295+ umem_dmabuf = ib_umem_dmabuf_get_pinned(device, desc->addr,
296+ desc->length, desc->fd,
297+ access);
298+ if (IS_ERR(umem_dmabuf))
299+ return ERR_CAST(umem_dmabuf);
300+ return &umem_dmabuf->umem;
301+ case IB_UVERBS_BUFFER_TYPE_VA:
302+ return __ib_umem_get_va(device, desc->addr, desc->length,
303+ access);
304+ default:
305+ return ERR_PTR(-EINVAL);
306+ }
307+}
308+EXPORT_SYMBOL(ib_umem_get_desc);
309+
310+/*
311+ * Per-command legacy buffer-desc filler.
312+ * Returns 0 on success (desc filled), -ENODATA if no legacy attrs apply,
313+ * negative errno on validation failure.
314+ */
315+typedef int (*ib_umem_buf_desc_filler_t)(const struct uverbs_attr_bundle *attrs,
316+ struct ib_uverbs_buffer_desc *desc);
317+
318+/*
319+ * ib_umem_resolve_desc - Resolve a buffer descriptor from a per-command UMEM
320+ * attribute and/or a legacy attr filler.
321+ *
322+ * Return:
323+ * 0 @desc filled.
324+ * -ENOENT no source produced a buffer.
325+ * -EINVAL both the UMEM attribute and the legacy filler produced a buffer.
326+ * -errno propagated from attr read / filler validation.
327+ */
328+static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs,
329+ u16 attr_id,
330+ ib_umem_buf_desc_filler_t legacy_filler,
331+ struct ib_uverbs_buffer_desc *desc)
332+{
333+ bool have_desc = false;
334+ int ret;
335+
336+ if (!attrs)
337+ return -ENOENT;
338+
339+ ret = uverbs_get_buffer_desc(attrs, attr_id, desc);
340+ if (!ret)
341+ have_desc = true;
342+ else if (ret != -ENOENT)
343+ return ret;
344+
345+ if (legacy_filler) {
346+ struct ib_uverbs_buffer_desc legacy_desc = {};
347+
348+ ret = legacy_filler(attrs, &legacy_desc);
349+ if (!ret) {
350+ if (have_desc)
351+ return -EINVAL;
352+ *desc = legacy_desc;
353+ have_desc = true;
354+ } else if (ret != -ENODATA) {
355+ return ret;
356+ }
357+ }
358+
359+ return have_desc ? 0 : -ENOENT;
360+}
361+
362+/*
363+ * ib_umem_get_desc_check - Pin a umem from @desc and verify it meets
364+ * @min_size.
365+ */
366+static struct ib_umem *
367+ib_umem_get_desc_check(struct ib_device *device,
368+ const struct ib_uverbs_buffer_desc *desc,
369+ size_t min_size, int access)
370+{
371+ struct ib_umem *umem;
372+
373+ umem = ib_umem_get_desc(device, desc, access);
374+ if (IS_ERR(umem))
375+ return umem;
376+ if (umem->length < min_size) {
377+ ib_umem_release(umem);
378+ return ERR_PTR(-EINVAL);
379+ }
380+ return umem;
381+}
382+
383+/*
384+ * ib_umem_get_from_attrs - Pin a umem from a per-command UMEM attribute
385+ * and/or a legacy attr filler.
386+ *
387+ * Return: caller-owned umem on success; NULL when no source supplied a
388+ * buffer; ERR_PTR(...) on error.
389+ */
390+static struct ib_umem *
391+ib_umem_get_from_attrs(struct ib_device *device,
392+ const struct uverbs_attr_bundle *attrs,
393+ u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler,
394+ size_t size, int access)
395+{
396+ struct ib_uverbs_buffer_desc desc = {};
397+ int ret;
398+
399+ ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc);
400+ if (ret == -ENOENT)
401+ return NULL;
402+ if (ret)
403+ return ERR_PTR(ret);
404+ return ib_umem_get_desc_check(device, &desc, size, access);
405+}
406+
407+/*
408+ * ib_umem_get_from_attrs_or_va - Pin a umem from a per-command UMEM
409+ * attribute and/or a legacy attr filler,
410+ * falling back to a UHW VA when no source
411+ * matched.
412+ *
413+ * @size is always consumed: it is the length to pin on the VA fallback
414+ * path AND the post-pin minimum-length check on the attr / legacy paths.
415+ * Callers must always pass a meaningful, validated value.
416+ *
417+ * Return: caller-owned umem on success, ERR_PTR(...) on error.
418+ */
419+static struct ib_umem *
420+ib_umem_get_from_attrs_or_va(struct ib_device *device,
421+ const struct uverbs_attr_bundle *attrs,
422+ u16 attr_id,
423+ ib_umem_buf_desc_filler_t legacy_filler,
424+ u64 addr, size_t size, int access)
425+{
426+ struct ib_uverbs_buffer_desc desc = {};
427+ int ret;
428+
429+ ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc);
430+ if (ret == -ENOENT)
431+ desc = (struct ib_uverbs_buffer_desc){
432+ .type = IB_UVERBS_BUFFER_TYPE_VA,
433+ .addr = addr,
434+ .length = size,
435+ };
436+ else if (ret)
437+ return ERR_PTR(ret);
438+ return ib_umem_get_desc_check(device, &desc, size, access);
439+}
440+
272441 /**
273442 * ib_umem_get_va - Pin and DMA map userspace memory.
274443 *
@@ -284,6 +453,65 @@ struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, · 9 unmodified lines
284453 }
285454 EXPORT_SYMBOL(ib_umem_get_va);
286455
456+/**
457+ * ib_umem_get_attr - Pin a umem from a per-command UMEM attribute.
458+ * @device: IB device.
459+ * @attrs: uverbs attribute bundle (may be NULL).
460+ * @attr_id: per-command UMEM attribute id.
461+ * @size: minimum required umem length.
462+ * @access: IB access flags.
463+ *
464+ * Return: caller-owned umem on success; NULL when no source supplied
465+ * a buffer; ERR_PTR(...) on error.
466+ */
467+struct ib_umem *ib_umem_get_attr(struct ib_device *device,
468+ const struct uverbs_attr_bundle *attrs,
469+ u16 attr_id, size_t size, int access)
470+{
471+ return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size,
472+ access);
473+}
474+EXPORT_SYMBOL(ib_umem_get_attr);
475+
476+/**
477+ * ib_umem_get_attr_or_va - Pin a umem from a per-command UMEM attribute,
478+ * falling back to a UHW VA.
479+ * @device: IB device.
480+ * @attrs: uverbs attribute bundle (may be NULL).
481+ * @attr_id: per-command UMEM attribute id.
482+ * @addr: UHW user VA used when no per-command attribute matched.
483+ * @size: on the attr / legacy paths, the minimum required umem length
484+ * validated post-pin; on the VA fallback path, the length to pin.
485+ * @access: IB access flags.
486+ *
487+ * Like ib_umem_get_attr(), but pins @addr/@size when no per-command
488+ * UMEM attribute is supplied.
489+ *
490+ * IMPORTANT: @size is always consumed. On the attr / legacy paths it is
491+ * used as the post-pin minimum-length check; on the VA fallback path it
492+ * is the length to pin. Callers MUST pass a meaningful, validated value
493+ * even when they expect an attribute-supplied buffer to be used.
494+ *
495+ * Every in-tree caller passes the same value for the two roles of @size
496+ * because no driver today distinguishes a user-passed buffer length from
497+ * a driver-computed minimum. Drivers that currently accept a user-supplied
498+ * length without cross-checking it against a driver minimum (vmw_pvrdma
499+ * CQ/QP/SRQ, qedr CQ/QP/SRQ, mana WQ/QP, ionic CQ/QP), once tightened to
500+ * compute and check a real minimum, will want to introduce a separate
501+ * helper that passes these as distinct values.
502+ *
503+ * Return: caller-owned umem on success, ERR_PTR(...) on error.
504+ */
505+struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device,
506+ const struct uverbs_attr_bundle *attrs,
507+ u16 attr_id, u64 addr, size_t size,
508+ int access)
509+{
510+ return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr,
511+ size, access);
512+}
513+EXPORT_SYMBOL(ib_umem_get_attr_or_va);
514+
287515 /**
288516 * ib_umem_release - release pinned memory
289517 * @umem: umem struct to release
include/rdma/ib_umem.h+36−0
@@ -73,10 +73,26 @@ static inline size_t ib_umem_num_pages(struct ib_umem *umem) · 72 unmodified lines
7373 {
7474 return ib_umem_num_dma_blocks(umem, PAGE_SIZE);
7575 }
76+
77+struct ib_udata;
78+struct ib_uverbs_buffer_desc;
79+struct uverbs_attr_bundle;
80+
7681 #ifdef CONFIG_INFINIBAND_USER_MEM
7782
83+struct ib_umem *ib_umem_get_desc(struct ib_device *device,
84+ const struct ib_uverbs_buffer_desc *desc,
85+ int access);
7886 struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr,
7987 size_t size, int access);
88+struct ib_umem *ib_umem_get_attr(struct ib_device *device,
89+ const struct uverbs_attr_bundle *attrs,
90+ u16 attr_id, size_t size, int access);
91+struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device,
92+ const struct uverbs_attr_bundle *attrs,
93+ u16 attr_id, u64 addr, size_t size,
94+ int access);
95+
8096 void ib_umem_release(struct ib_umem *umem);
8197 int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
8298 size_t length);
@@ -160,12 +176,32 @@ void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf); · 77 unmodified lines
160176
161177 #include <linux/err.h>
162178
179+static inline struct ib_umem *
180+ib_umem_get_desc(struct ib_device *device,
181+ const struct ib_uverbs_buffer_desc *desc, int access)
182+{
183+ return ERR_PTR(-EOPNOTSUPP);
184+}
163185 static inline struct ib_umem *ib_umem_get_va(struct ib_device *device,
164186 unsigned long addr, size_t size,
165187 int access)
166188 {
167189 return ERR_PTR(-EOPNOTSUPP);
168190 }
191+static inline struct ib_umem *
192+ib_umem_get_attr(struct ib_device *device,
193+ const struct uverbs_attr_bundle *attrs, u16 attr_id,
194+ size_t size, int access)
195+{
196+ return ERR_PTR(-EOPNOTSUPP);
197+}
198+static inline struct ib_umem *
199+ib_umem_get_attr_or_va(struct ib_device *device,
200+ const struct uverbs_attr_bundle *attrs, u16 attr_id,
201+ u64 addr, size_t size, int access)
202+{
203+ return ERR_PTR(-EOPNOTSUPP);
204+}
169205 static inline void ib_umem_release(struct ib_umem *umem) { }
170206 static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
171207 size_t length) {
include/rdma/uverbs_ioctl.h+30−0
@@ -590,6 +590,28 @@ struct uapi_definition { · 589 unmodified lines
590590 UA_OPTIONAL, \
591591 .is_udata = 1)
592592
593+/*
594+ * Per-attribute UMEM descriptor. The payload is a single
595+ * struct ib_uverbs_buffer_desc identifying a memory region backed by
596+ * dma-buf or user virtual address. _access selects UA_OPTIONAL or
597+ * UA_MANDATORY. Drivers obtain a umem from the attribute via the
598+ * ib_umem_get_*() wrapper helpers.
599+ */
600+#define UVERBS_ATTR_UMEM(_attr_id, _access) \
601+ UVERBS_ATTR_PTR_IN(_attr_id, \
602+ UVERBS_ATTR_TYPE(struct ib_uverbs_buffer_desc), \
603+ _access)
604+
605+/*
606+ * Bit masks of the @flags / @optional_flags fields of struct
607+ * ib_uverbs_buffer_desc that the kernel understands. @flags is strict:
608+ * any bit outside the known mask makes the call fail with -EINVAL.
609+ * @optional_flags is advisory: bits outside the known mask are silently
610+ * dropped. Both masks are extended as new bits are introduced.
611+ */
612+#define IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK 0U
613+#define IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK 0U
614+
593615 /* =================================================
594616 * Parsing infrastructure
595617 * =================================================
@@ -862,6 +884,8 @@ int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, · 266 unmodified lines
862884 size_t idx, u64 allowed_bits);
863885 int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, size_t idx,
864886 const void *from, size_t size);
887+int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle,
888+ u16 attr_id, struct ib_uverbs_buffer_desc *desc);
865889 __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
866890 gfp_t flags);
867891
@@ -920,6 +944,12 @@ static inline int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, · 52 unmodified lines
920944 {
921945 return -EINVAL;
922946 }
947+static inline int
948+uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle,
949+ u16 attr_id, struct ib_uverbs_buffer_desc *desc)
950+{
951+ return -EINVAL;
952+}
923953 static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle,
924954 size_t size)
925955 {
include/uapi/rdma/ib_user_ioctl_verbs.h+27−0
@@ -273,4 +273,31 @@ struct ib_uverbs_gid_entry { · 272 unmodified lines
273273 __u32 netdev_ifindex; /* It is 0 if there is no netdev associated with it */
274274 };
275275
276+enum ib_uverbs_buffer_type {
277+ IB_UVERBS_BUFFER_TYPE_DMABUF,
278+ IB_UVERBS_BUFFER_TYPE_VA,
279+};
280+
281+/*
282+ * Describes a single buffer backed by dma-buf or user virtual address.
283+ * Used as the payload of a per-attribute UVERBS_ATTR_UMEM-typed attribute.
284+ *
285+ * @type: buffer type from enum ib_uverbs_buffer_type
286+ * @fd: dma-buf file descriptor (valid for IB_UVERBS_BUFFER_TYPE_DMABUF)
287+ * @flags: required flags; the kernel rejects the call with -EINVAL if any
288+ * bit is not understood. No bits are defined yet.
289+ * @optional_flags: advisory flags; bits the kernel does not understand are
290+ * silently ignored. No bits are defined yet.
291+ * @addr: offset within dma-buf, or user virtual address for VA
292+ * @length: buffer length in bytes
293+ */
294+struct ib_uverbs_buffer_desc {
295+ __u32 type;
296+ __s32 fd;
297+ __u32 flags;
298+ __u32 optional_flags;
299+ __aligned_u64 addr;
300+ __aligned_u64 length;
301+};
302+
276303 #endif